tuhol.de

User Tools

Site Tools

Trace: • backup
interessen:computer:pc_technik:backup

Differences

This shows you the differences between two versions of the page.

Link to this comparison view

Next revision 		Previous revision
interessen:computer:pc_technik:backup [2015/01/30 12:58]
tomtom created 		interessen:computer:pc_technik:backup [2015/02/15 11:54] (current)
tomtom
Line 7: Line 7:
 Damit hatte ich auch die Möglichkeit beim Backup gleichzeitig auf zwei (oder mehr) verschiedenen Server meine Backups zu schreiben. Damit hatte ich auch die Möglichkeit beim Backup gleichzeitig auf zwei (oder mehr) verschiedenen Server meine Backups zu schreiben.
  
 +Da ich das neue Script so flexibel wie möglich machen wollte habe ich mir überlegt, alle Parameter in ein extra File auszulagern. Ich habe wegen der einfachen Lesbarkeit -json als Format gewählt.
 +
 +===== Backup-Script =====
 +Es sind noch viele "echo" drinn damit man beim manuellen starten des Scriptes sehen kann, was gerade passiert und wie die folgenden Kommandos dann zusammen gesetzt werden.
 +
 +  #!/bin/bash
 +  #############################################################################
 +  #                                                                           #
 +  # backup script                                                             #
 +  # Author : theta-my -> thomas@tuhol.de                                      #
 +  #                                                                           #
 +  # can run with different backup methods,                                    #
 +  # please take a look in backup_<host>.jason file for further informations   #
 +  #                                                                           #
 +  # to make my work easier, i have all my urgent data located in one folder   #
 +  # and mount this folder or sub folders to the right location in file system #
 +  #                                                                           #
 +  #############################################################################
 +  
 +  setup()
 +  {
 +  tstemp=`date +%Y%m%d%H%M`
 +  host=`hostname`
 + 
 +  # define definition file and check availability
 +  DATA_FILE=/opt/backup/backup_$host.json
 +  if [ ! -f $DATA_FILE ]
 +  then
 +  logger " <error> : definition file /opt/backup/backup_$host.json not reachable, check file location"
 +  exit
 +  fi
 + 
 +  WORK_DIR=`jq .work_dir $DATA_FILE|sed 's/\"//g'`/
 +  cd $WORK_DIR
 +  echo "working and running in : "$WORK_DIR
 +  MOUNT_FS=""
 +  message=""
 +  ERR_STOP=0
 +  ERR_COUNT=0
 +  ERR_HIGH_COUNT=0
 +  ERR_MEDIUM_COUNT=0
 +  ERR_LOW_COUNT=0
 +  LOG_DIR=`jq .log_dir $DATA_FILE|sed 's/\"//g'`
 +  LOG_FILE=$LOG_DIR/$tstemp.backup.log
 +  LOG_CHECK_FILE=$LOG_DIR/$tstemp.backup.check.log
 +  }
 +  
 +  error_check()
 +  {
 +  echo "job 'error_check' is running"
 +  IGNOR=`jq .jobs.error_check.err_ignore $DATA_FILE|sed 's/\"//g'`
 +  echo "error to ignor : "$IGNOR
 +  
 +  HIGH=`jq .jobs.error_check.err_high $DATA_FILE|sed 's/\"//g'`
 +  MEDIUM=`jq .jobs.error_check.err_medium $DATA_FILE|sed 's/\"//g'`
 +  LOW=`jq .jobs.error_check.err_low $DATA_FILE|sed 's/\"//g'`
 +  
 +  for FILE in `cat $LOG_CHECK_FILE | grep '<f'|cut -d\/ -f2-`
 +  do
 +  echo "check line "$FILE
 +  if [[ $IGNOR = "" ]]
 +  then 
 +  error_check_sub
 +  else
 +  for TYPE in ${IGNOR[@]}
 +  do
 +  if ( ! $FILE| grep $TYPE )
 +  then
 +  error_check_sub
 +  fi
 +  done
 +  fi
 +  done
 +  }
 +  
 +  error_check_sub()
 +  {
 +  if ( cat $LOG_FILE | grep $FILE )
 +  then
 +  echo "check line "$FILE
 +  for i in ${HIGH[@]}
 +  do
 +  if ( $FILE|grep $i )
 +  then
 +  (($ERR_HIGH_COUNT++))
 +  fi
 +  done
 + 
 +  for i in ${MEDIUM[@]}
 +  do
 +  if ( $FILE|grep $i )
 +  then
 +  (($ERR_MEDIUM_COUNT++))
 +  fi
 +  done
 + 
 +  for i in ${LOW[@]}
 +  do
 +  if ( $FILE|grep $i )
 +  then
 +  (($ERR_LOW_COUNT++))
 +  fi
 +  done
 +  
 +  (($ERR_COUNT++))
 +  echo $ERR_COUNT
 +  fi
 +  }
 +  
 +  messaging()
 +  {
 +  echo "job 'messaging' is running, generate massage for syslog"
 +  if [ $ERR_COUNT -gt 0 ]
 +  then
 +  if [ $ERR_HIGH_COUNT -eq 0 ]
 +  then
 +  pre_massage=' <error>'
 +  else
 +  pre_message=' <warning>'
 +  fi
 +  message=$pre_message' : backup has '$ERR_COUNT' failures: '$ERR_HIGH_COUNT' high, '$ERR_MEDIUM_COUNT' medium and '$ERR_LOW_COUNT' low; check errorlog in '$LOG_DIR
 +  echo "backup has "$ERR_COUNT
 +  else
 +  message=' <info> : backup successfully completed'
 +  
 +  if [[ `jq .log_remove $DATA_FILE|sed 's/\"//g'` = "yes" ]]
 +  then
 +  rm $LOG_DIR/*
 +  fi
 +  fi
 +  }
 +  
 +  replicate()
 +  {
 +  echo "job 'replicate' is running"
 +  TARGET_DIR=`jq .jobs.replicate.remote_folder $DATA_FILE|sed 's/\"//g'`
 +  REPL_OPT=`jq .jobs.replicate.repli_opt $DATA_FILE|sed 's/\"//g'`
 +  METHOD=`jq .jobs.replicate.method $DATA_FILE|sed 's/\"//g'`
 +  if [[ ! $REPL_OPT = "" ]]
 +  then
 +  if [[ $REPL_OPT = "local" ]]
 +  then
 +  DESTIN=$MOUNT_P/$TARGET_DIR
 +  PASSFILE=""
 +  if [ ! -d $DESTIN ]
 +  then
 +  echo $DESTIN" not exist, will create"
 +  mkdir $DESTIN
 +  fi
 +  DESTIN=$DESTIN/
 +  echo "local sync target "$DESTIN
 +  else
 +  SERVER=`jq .jobs.replicate.sync_server $DATA_FILE`
 +  if( ! ping -c 3  $SERVER > /dev/null )
 +  then
 +  (($ERR_STOP++))
 +  message="<info> : server "$SERVER" not reachable"
 +  break 
 +  fi
 + 
 +  USER=`jq .jobs.replicate.sync_user $DATA_FILE|sed 's/\"//g'`
 +  PASS_OPT=`jq .jobs.replicate.pass_option $DATA_FILE`
 +  DESTIN=$USER@$SERVER::/$MOUNT_P/$TARGET_DIR/
 +  echo "remote sync with "$DESTIN
 +  fi
 + 
 +  FOLDERS=`jq .jobs.replicate.sync_folders $DATA_FILE|sed 's/\"//g'`
 +  echo $FOLDERS
 + 
 +  date >> $LOG_FILE
 +  date >> $LOG_CHECK_FILE
 + 
 +  for DIR in ${FOLDERS[@]}
 +  do
 +  echo $DIR
 +  echo "run : "$METHOD `jq .jobs.replicate.run_options $DATA_FILE|sed 's/\"//g'` $DIR $DESTIN $PASS_OPT
 +  echo "{"$DIR"()" >> $LOG_FILE
 +  $METHOD `jq .jobs.replicate.run_options $DATA_FILE|sed 's/\"//g'` $DIR $DESTIN $PASS_OPT >> $LOG_FILE
 +  echo "{"$DIR"()" >> $LOG_CHECK_FILE
 +  $METHOD `jq .jobs.replicate.log_options $DATA_FILE|sed 's/\"//g'` $DIR $DESTIN $PASS_OPT >> $LOG_CHECK_FILE
 +  echo "}" >> $LOG_FILE
 +  echo "}" >> $LOG_CHECK_FILE
 +  done
 + 
 +  date >> $LOG_FILE
 +  date >> $LOG_CHECK_FILE
 +  error_check
 +  else
 +  (($ERR_STOP++))
 +  message=" <warning> : no method defined"
 +  fi
 +  }
 +  
 +  mount_fs()
 +  {
 +  echo "job 'mount_fs' is running"
 +  # check if target host available
 +  TARGET_SRV=`jq .jobs.mount_fs.ping_target $DATA_FILE|sed 's/\"//g'`
 +  echo "check, if you'r in right location, try to ping "$TARGET_SRV
 +  MOUNT_P=`jq .jobs.mount_fs.mount_point $DATA_FILE|sed 's/\"//g'`
 +  if( ping -c 3  $TARGET_SRV > /dev/null )
 +  then
 +  echo $TARGET_SRV" reachable, you'r in right location to backup"
 +  if [[ `jq .jobs.mount_fs.mount_def $DATA_FILE|sed 's/\"//g'` = "file" ]]
 +  then
 +  echo "mount "`jq .jobs.mount_fs.mount_opt $DATA_FILE|sed 's/\"//g'` `jq .jobs.mount_fs.mount_file $DATA_FILE|sed 's/\"//g'` $MOUNT_P
 +  mount `jq .jobs.mount_fs.mount_opt $DATA_FILE|sed 's/\"//g'` `jq .jobs.mount_fs.mount_file $DATA_FILE|sed 's/\"//g'` $MOUNT_P
 +  else
 +  CONNECT=`jq .jobs.mount_fs.mount_srv $DATA_FILE|sed 's/\"//g'`:`jq .jobs.mount_fs.mount_share $DATA_FILE|sed 's/\"//g'`
 +  echo "mount "`jq .jobs.mount_fs.mount_opt $DATA_FILE|sed 's/\"//g'` $CONNECT $MOUNT_P
 +  mount `jq .jobs.mount_fs.mount_opt $DATA_FILE|sed 's/\"//g'` $CONNECT $MOUNT_P
 +  fi
 +  sleep 10 # be sure mount is completed or timed out
 +  if ( mount | grep $MOUNT_P > /dev/null )
 +  then
 +  MOUNT_FS=`jq .jobs.mount_fs.mount_point $DATA_FILE|sed 's/\"//g'`
 +  echo "umount "$MOUNT_FS" needed"
 +  else
 +  (($ERR_STOP++))
 +  message=" <warning> : Can not mount backup target volume"
 +  fi
 +  else
 +  (($ERR_STOP++))
 +  message=" <info> : "$TARGET_SRV" not reachable, server down or you are not at right location"
 +  fi
 +  }
 +  
 +  # master function
 +  main()
 +  {
 +  # select job sequence
 + 
 +  SEQUENCE=`jq .job_seq $DATA_FILE|sed 's/\"//g'`
 +  echo "job sequence are : "$SEQUENCE
 + 
 +  if [[ $SEQUENCE = "" ]]
 +  then
 +  message=" <info> no jobs defined"
 +  else
 +  for JOB in ${SEQUENCE[@]}
 +  do
 +  if ( echo $JOB | grep add_tasks )
 +  then
 +  SUB_TASK=${JOB##*.}
 +  TARGET=$WORK_DIR`jq .jobs.add_tasks.$SUB_TASK.folder $DATA_FILE|sed 's/\"//g'`/
 +  echo "sub task target "$TARGET
 +  echo "run sub task : "`jq .jobs.$JOB.task $DATA_FILE|sed 's/\"//g'` $TARGET
 +  `jq .jobs.$JOB.task $DATA_FILE|sed 's/\"//g'` $TARGET
 +  else
 +  echo "run main job : "$JOB
 +  $JOB
 +  fi
 +  if [[ $ERR_STOP -gt 0 ]]
 +  then
 +  logger $message
 +  exit
 +  fi
 +  done
 +  fi
 + 
 +  echo "clean up"
 + 
 +  if [[ ! $MOUNT_FS = "" ]]
 +  then
 +  echo "umount "$MOUNT_FS
 +  umount $MOUNT_FS
 +  fi
 + 
 +  logger $message
 +  }
 +  
 +  # don't change sequence below
 +  setup
 +  main
 +  exit
 +
 +===== Parameter-File =====
 +Das dazu gehörende ParameterFile sieht dann so aus bei mir:
 +
 +  {
 +  "description": "json-file to backup and restore per backup-script",
 +  "work_dir":"/data",
 +  "log_dir":"/data/backuplog",
 +  "log_remove":"yes",
 +  "job_seq":"mount_fs add_tasks.backup_opt replicate error_check messaging",
 +  "jobs":
 +  {
 +  "mount_fs":
 +  {
 +  "@":"'ping_target' is used to verify location; 'mount_def' can be 'file' or 'server'; in case of 'file', 'mount_file' needed",
 +  "ping_target":"192.168.2.1",
 +  "mount_def":"file",
 +  "mount_opt":"-t glusterfs",
 +  "mount_srv":"<file server>",
 +  "mount_share":"<file server share>",
 +  "@":"'mount_file' describe a method to connect to a server and his share(s), 'mount_srv' and 'mount_share' not needed in this case",
 +  "mount_file":"/etc/glusterfs/gluster1.vol",
 +  "mount_point":"/gluster"
 +  },
 +  "replicate":
 +  {
 +  "@":"'repli_opt' can be 'local' or 'remote', 'remote' needs 'pass_option', 'sync_server' and 'sync_user'",
 +  "repli_opt":"local",
 +  "@":"only rsync implemented yet, can try scp, but auth method must implemented by your self; use 'AX' to safe links",
 +  "method":"rsync",
 +     "run_options":"-avh -partial",
 +      "log_options":"-avhn -partial",
 +  "@":"pass_option : file content format 'username:password'",
 +  "pass_option":"--password-file=<file location>",
 +  "sync_server":"<rsyncd server>",
 +  "sync_user":"<username>",
 +  "@":"if absolute path used in 'sync_folders', not 'work_dir' needed",
 +  "sync_folders":"Carrier Closed Documentation Downloads notes privat Projekts Proposals transcripts backup transfer",
 +  "@":"'remote_folder' means the backup folder at the target file system",
 +  "remote_folder":"Arbeit"
 +  },
 +  "error_check":
 +  {
 +  "@":"'err_ignore' means witch files or search words can be ignore in error log or errors at backup this files are normal (i.e. log-files)",
 +  "err_ignore":"",
 +  "@":"error level definitions",
 +  "err_high":"",
 +  "err_medium":"notes",
 +  "err_low":""
 +  },
 +  "messaging":"<no further parameter needed, all included in backup script>",
 +  "add_tasks":
 +  {
 +  "<task name>":
 +  {
 +  "folder":"<target folder for task>",
 +  "task":"<can be external(!) script or command, no sub routine in backup script>"
 +  },
 +  "backup_db":
 +  {
 +  "folder":"mysql_backup",
 +  "task":"mysqldump -u root --password=<PASSWORD> --all-databases > $TARGET"
 +  },
 +  "backup_opt":
 +  {
 +  "folder":"backup",
 +  "task":"cp -r /opt/backup/*"
 +  }
 +  }
 +  }
 +  }
 +
 +
 +
 +Die Jobs "mount_fs" und "replicate" sind als feste Optionen, siehe "job_seq" (Job Sequence), fertig definierte Schritte im BackupScript. Wobei auch "mount_fs" nur eine Option ist. Ich habe bei mir "mount_fs" gewählt, damit ich bei der Replikation keine Authentifizierung benötige und auch gleichzeitig auf zwei verschiedene Platten, bei mir zwei Server, schreiben kann. Wenn also bei mir ein Server, wegen was auch immer, nicht zur Verfügung steht, dann kann ich trotzdem mein Backup machen.
 +
 +"add_tasks" sind frei definierbare zusätzliche Aufgaben deren Auführungszeitpunkt in "job_seq" definiert wird.
interessen/computer/pc_technik/backup.1422619130.txt.gz · Last modified: 2015/01/30 12:58 by tomtom

Page Tools

Except where otherwise noted, content on this wiki is licensed under the following license: Public Domain
Public Domain Donate Powered by PHP Valid HTML5 Valid CSS Driven by DokuWiki