Search This Blog

2012-09-14

bash remote server file copy listener

#!/bin/bash
# file: get-files-listener.sh v.1.1.2 docs at the end 
umask 000 ; 
 
# print the commands 
# set -x
# print each input line as well
set -v
# exit the script if any statement returns a non-true return value. gotcha !!!
# set -e 
 
 
# set the variables from the $0.hostname.conf file which has ini like syntax
doSetVars(){
     export PATH=/cygdrive/c/cygwin/bin/:$PATH
     MyDir=`dirname $(readlink -f $0)`
     Tmp="$MyDir/tmp/tmp.$$"
     mkdir -p $Tmp           #create the tmp dir if it does not exist 
     ( set -o posix ; set ) >$Tmp/variables.before 
     MyName=`basename $0`
     IniFile="$MyDir/$MyName.`hostname`.conf"
     IniSection=MainSection
     LogFile="$MyDir/$MyName.log"
     # get the machine / host specific configuration 
     doParseIniFile
     #source $0
     ( set -o posix ; set ) >$Tmp/variables.after
     doLog " Using the following vars :"
     cmd="comm --nocheck-order -3 $Tmp/variables.before $Tmp/variables.after"
     doRunCmdAndLog $cmd
     rm -fr $Tmp
}
#eof function doSetVars
 
 
# parse the ini like $0.hostname.conf and set the variables
doParseIniFile(){
  eval `sed -e 's/[[:space:]]*\=[[:space:]]*/=/g' \
      -e 's/;.*$//' \
      -e 's/[[:space:]]*$//' \
      -e 's/^[[:space:]]*//' \
      -e "s/^\(.*\)=\([^\"']*\)$/\1=\"\2\"/" \
     < $IniFile \
      | sed -n -e "/^\[$IniSection\]/,/^\s*\[/{/^[^;].*\=.*/p;}"`
}
#eof function parseIniFile
 
 
# run a command and log the call and its output to the LogFile
# usage: doRunCmdAndLog $cmd
doRunCmdAndLog(){
    cmd="$*" ; 
    doLog " DEBUG running cmd : \"$cmd\""
    Msg=$($cmd 2>&1)
    [ $? -eq 0 ]  || doLog "ERROR : Failed to run the following command \"$cmd\" with the following output \"$Msg\" !!!"
    doLog " DEBUG : cmdoutput : \"$Msg\""
}
#eof function doRunCmdAndLog
 
 
# run a command on failure exit with message
# usage: doRunCmdOrExit $cmd
doRunCmdOrExit(){
   cmd="$*" ; 
   doLog " DEBUG running cmd : \"$cmd\""
   Msg=$($cmd 2>&1)
   # if error occured during the execution exit with error
   [ $? -eq 0 ] || doExit "ERROR : FATAL : Failed to run the following command \"$cmd\" with the following output \"$Msg\" !!!"
   #if no error occured just log the message
   doLog " DEBUG : cmdoutput : \"$Msg\""
}
#eof function doRunCmdOrExit
 
 
# echo pass params and print them to a log file
doLog(){
   # check terminal if exists echo 
   test -t 1 && echo "`date +%Y.%m.%d-%H:%M:%S` [$$] $*"
   # check LogFile and 
   test -z $LogFile || {
    echo "`date +%Y.%m.%d-%H:%M:%S` [$$] $*" >> $LogFile
   } #eof test
} 
# eof function doLog 
 
 
# exit with passed status and message
doExit(){
     ExitStatus=0
     case $1 in
        [0-9]) ExitStatus="$1"; shift 1;;
  esac
     Msg="$*"
     test "$ExitStatus" = "0" || Msg=" ERROR: $Msg"
     doLog " $Msg"
     exit $ExitStatus
}
#eof function Exit
 
 
#
# waits for the loop to occur 
doWaitLoop(){
 
   count=0
   while [[ $count -lt $NUM_OF_SECONDS ]] ; do
    #debug echo $count ;
    count=$((count + 1)) ;
    sleep 1;
    # go to the server and copy the file flag
    doScpFlagFile
    # if the file flag exist locally 
    test -f "$FLAG_FILE_LOCAL" && {
        # do copy the whole server
        doScpFiles
        doRemoveFilesDir
        doRecreateFilesDir
        # set the time flag
        flag_file_time=`cat $FLAG_FILE_LOCAL`
        mv $FLAG_FILE_LOCAL $FLAG_FILE_LOCAL.$flag_file_time
        continue 
 }
done ; 
#eof while
#debug echo "Value of count after loop is: $count"
}
#eof doWaitLoop
 
 
doScpFlagFile(){ 
   doLog "START  doScpFlagFile"
   cmd="scp  $USER@$SERVER:$FLAG_FILE_SERVER $LOCAL_DIR"
   doRunCmdAndLog "$cmd"
   doLog "STOP doScpFlagFile"
}
#eof func doScp
 
 
doScpFiles(){ 
   doLog "START  doScpFiles"
   scp -rv $USER@$SERVER:$SERVER_DIR $LOCAL_DIR
   doLog "STOP doScpFiles"
}
#eof func doScp
 
 
doRemoveFilesDir(){ 
   doLog "START  doRemoveFilesDir"
   #scp -v user@server:/dir/data/*.txt .
   cmd="ssh $USER@$SERVER nohup bash -c \"rm -fR $SERVER_DIR\""
   doRunCmdAndLog "$cmd"
   doLog "STOP doRemoveFilesDir"
}
#eof func doRemoveFilesDir
 
 
doRecreateFilesDir(){ 
   doLog "START  doRecreateFilesDir"
   #scp -v user@server.domain.fi:/dir/data/*.txt .
   cmd="ssh $USER@$SERVER nohup bash -c \"mkdir -p $SERVER_DIR\""
   doRunCmdAndLog $cmd
   cmd="ssh $USER@$SERVER nohup bash -c \"chmod -R 777 $SERVER_DIR\""
   doRunCmdAndLog $cmd
   doLog "STOP doRecreateFilesDir"
}
#eof func doRecreateFilesDir
 
 
# the main function called 
main(){
  doSetVars
  doWaitLoop
  Exit 0 "MAIN END"
 }
#eof function main
 
 
# Action !!!
main
 
 
# Purpose:
# to listen for file flag on an ssh server and copy all the files 
# from a predifined dir on the server to a pre-defined dir locally 
# to provide an easy starting template for writing bash and sh scripts
# Prerequisites: ppk authentication between the ssh client and the server 
# with  the following features: 
# - prints the set in the script variables
# - separation of host specific vars into $0.`hostname`.conf file
# - doLog function for both xterm and log file printing
# - for loop examples with head removal and inline find and replace 
# 
# EOF File:get-files-listener.sh
# ==========================================================
# VersionHistory
# 1.0.0 --- ysg ---  2012-09-14 13:39:24 --- Initial creation 
pre

Labels

perl (41) Cheat Sheet (25) how-to (24) windows (14) sql server 2008 (13) linux (12) oracle (12) sql (12) Unix (11) cmd windows batch (10) mssql (10) cmd (9) script (9) textpad (9) netezza (8) sql server 2005 (8) cygwin (7) meta data mssql (7) metadata (7) bash (6) code generation (6) Informatica (5) cheatsheet (5) energy (5) tsql (5) utilities (5) excel (4) future (4) generic (4) html (4) perl modules (4) programs (4) settings (4) sh (4) shortcuts (4) поуки (4) принципи (4) Focus Fusion (3) Solaris (3) cool programs (3) development (3) economy (3) example (3) freeware (3) fusion (3) git cheat sheet (3) logging (3) morphus (3) mssql 2005 (3) nuclear (3) nz (3) parse (3) python (3) sftp (3) sofware development (3) source (3) sqlplus (3) table (3) vim (3) .Net (2) C# (2) China (2) GUI (2) Google (2) GoogleCL (2) Solaris Unix (2) ascii (2) awk (2) batch (2) cas (2) chrome extensions (2) code2html (2) columns (2) configuration (2) conversion (2) duplicates (2) excel shortcuts (2) export (2) file (2) free programs (2) informatica sql repository (2) linux cheat sheet (2) mssql 2008 (2) mysql (2) next big future (2) nsis (2) nz netezza cheat sheet (2) nzsql (2) ora (2) prediction (2) publish (2) release management (2) report (2) security (2) single-click (2) sqlserver 2005 (2) sqlserver 2008 (2) src (2) ssh (2) template (2) tools (2) vba (2) video (2) xlt (2) xml (2) youtube videos (2) *nix (1) .vimrc (1) .virmrc vim settings configs (1) BSD license (1) Bulgaria (1) Dallas (1) Database role (1) Dense plasma focus (1) Deployment (1) ERP (1) ExcelToHtml (1) GD (1) GDP (1) HP-UX (1) Hosting (1) INC (1) IT general (1) ITIL management bullshit-management (1) IZarc (1) Java Web Start (1) JavaScript anchor html jquery (1) Khan Academy (1) LINUX UNIX BASH AND CYGWIN TIPS AND TRICKS (1) Linux Unix rpm cpio build install configure (1) Linux git source build .configure make (1) ListBox (1) MIT HYDROGEN VIRUS (1) OO (1) Obama (1) PowerShell (1) Run-time (1) SDL (1) SIWA (1) SOX (1) Scala (1) Services (1) Stacks (1) SubSonic (1) TED (1) abstractions (1) ansible hosts linux bash (1) ansible linux deployment how-to (1) ansible yum pip python (1) apache (1) apache 2.2 (1) application life cycle (1) architecture (1) archive (1) arguments (1) avatar (1) aws cheat sheet cli (1) aws cli (1) aws cli amazon cheat sheet (1) aws elb (1) backup (1) bash Linux open-ssh ssh ssh_server ssh_client public-private key authentication (1) bash perl search and replace (1) bash stub (1) bin (1) biofuels (1) biology (1) books (1) browser (1) bubblesort (1) bugs (1) build (1) byte (1) cas_sql_dev (1) chennai (1) chrome (1) class (1) claut (1) cmdow (1) code generation sqlserver (1) command (1) command line (1) conf (1) confluence (1) console (1) convert (1) cool programs windows free freeware (1) copy-paste (1) csv (1) ctags (1) current local time (1) cygwin X11 port-forwarding mintty xclock Linux Unix X (1) cygwin bash how-to tips_n_tricks (1) cygwin conf how-to (1) data (1) data types (1) db2 cheat sheet (1) db2 starter ibm bash Linux (1) debt (1) diagram (1) dictionaries (1) digital (1) disk (1) disk space (1) documentation (1) dos (1) dubai (1) e-cars (1) electric cars (1) electricity (1) emulate (1) errors (1) exponents (1) export workflow (1) extract (1) fast export (1) fexp (1) file extension (1) file permissions (1) findtag (1) firewall (1) for loop (1) freaky (1) functions (1) fusion research (1) german (1) git gitlab issues handling system (1) google cli (1) google code (1) google command line interface (1) gpg (1) ha (1) head (1) helsinki (1) history (1) hop or flop (1) host-independant (1) how-to Windows cmd time date datetime (1) ibm db2 cognos installation example db deployment provisioning (1) ideas (1) image (1) informatica oracle sql (1) informatica repo sql workflows sessions file source dir (1) informatica source files etl (1) install (1) isg-pub issue-tracker architecture (1) it management best practices (1) java (1) jump to (1) keyboard shortcuts (1) ksh (1) level (1) linkedin (1) linux bash ansible hosts (1) linux bash commands (1) linux bash how-to shell expansion (1) linux bash shell grep xargs (1) linux bash tips and t ricks (1) linux bash unix cygwin cheatsheet (1) linux bash user accounts password (1) linux bash xargs space (1) linux cheat-sheet (1) linux cheatsheet cheat-sheet revised how-to (1) linux how-to non-root vim (1) linux ssh hosts parallel subshell bash oneliner (1) london (1) make (1) me (1) metacolumn (1) metadata functions (1) metaphonre (1) method (1) model (1) movie (1) multithreaded (1) mysql cheat sheet (1) mysql how-to table datatypes (1) n900 (1) nano (1) neteza (1) netezza bash linux nps (1) netezza nps (1) netezza nps nzsql (1) netezza nz Linux bash (1) netezza nz bash linux (1) netezza nz nzsql sql (1) netezza nzsql database db sizes (1) non-password (1) nord pol (1) nps backup nzsql schema (1) number formatting (1) nz db size (1) nz table count rows (1) nzsql date timestamp compare bigint to_date to_char now (1) on-lier (1) one-liners (1) one-to-many (1) oneliners (1) open (1) open source (1) openrowset (1) openssl (1) oracle PL/SQL (1) oracle Perl perl (1) oracle installation usability (1) oracle number formatting format-model ora-sql oracle (1) oracle templates create table (1) oracle trigger generic autoincrement (1) oracle vbox virtual box cheat sheet (1) oracle virtual box cheat sheet (1) outlook (1) parser (1) password (1) paths (1) perl @INC compile-time run-time (1) perl disk usage administration Linux Unix (1) perl modules configuration management (1) permissions (1) php (1) picasa (1) platform (1) postgreSQL how-to (1) powerShell cmd cygwin mintty.exe terminal (1) ppm (1) predictions (1) prices (1) principles (1) productivity (1) project (1) prompt (1) proxy account (1) public private key (1) publishing (1) putty (1) qt (1) read file (1) registry (1) relationship (1) repository (1) rm (1) scp (1) scripts (1) scsi (1) search and replace (1) sed (1) sendEmail (1) sh stub (1) shortcuts Windows sql developer Oracle (1) sidebar (1) silicon (1) smtp (1) software procurement (1) sofware (1) sort (1) sql script (1) sql_dev (1) sqlcmd (1) sqlite (1) sqlite3 (1) sshd (1) sshd cygwin (1) stackoverflow (1) stored procedure (1) stub (1) stupidity (1) subroutines (1) svn (1) sysinternals (1) tail (1) tar (1) temp table (1) templates (1) teradata (1) terminal (1) test (1) testing (1) theory (1) thorium (1) time (1) tip (1) title (1) tmux .tmux.conf configuration (1) tmux efficiency bash (1) tool (1) ui code prototyping tips and tricks (1) umask Linux Unix bash file permissions chmod (1) url (1) urls (1) user (1) utility (1) utils (1) vb (1) vbox virtual box cheat sheet (1) vim perl regex bash search for string (1) vim recursively hacks (1) vim starter (1) vim-cheat-sheet vim cheat-sheet (1) vimeo (1) visual stuio (1) warsaw (1) wiki (1) wikipedia (1) window (1) windows 7 (1) windows 8 (1) windows programs (1) windows reinstall (1) windows utility batch perl space Windows::Clipboard (1) wisdoms (1) workflow (1) worth-reading (1) wrapper (1) xp_cmdshell (1) xslt (1) youtube (1)

Blog Archive

Translate with Google Translate

My Blog List

VideoBar

This content is not yet available over encrypted connections.