tuhol.de

This page is read only. You can view the source, but not change it. Ask your administrator if you think this is wrong.
====== Backup ======

Einige Zeit hat das Backup mit "rsync" gut funktioniert, aber ich habe mitr überlegt, dass ein einzelnes Backup bzw. das Backup auf eine einzelne Platte eventuell doch nicht so optimal ist. Daher habe ich mich ein wenig umgesehen wie man das besser machen kann ohne gleich das Backup-Script mehrmals mit verschiedenen Zielen aufzurufen.

Die Lösung war dann die Umstellung meiner derzeitigen gluster Konfiguration hin zu einer gespiegelten Konfiguration.

Damit hatte ich auch die Möglichkeit beim Backup gleichzeitig auf zwei (oder mehr) verschiedenen Server meine Backups zu schreiben.

Da ich das neue Script so flexibel wie möglich machen wollte habe ich mir überlegt, alle Parameter in ein extra File auszulagern. Ich habe wegen der einfachen Lesbarkeit -json als Format gewählt.

===== Backup-Script =====
Es sind noch viele "echo" drinn damit man beim manuellen starten des Scriptes sehen kann, was gerade passiert und wie die folgenden Kommandos dann zusammen gesetzt werden.

  #!/bin/bash
  #############################################################################
  #                                                                           #
  # backup script                                                             #
  # Author : theta-my -> thomas@tuhol.de                                      #
  #                                                                           #
  # can run with different backup methods,                                    #
  # please take a look in backup_<host>.jason file for further informations   #
  #                                                                           #
  # to make my work easier, i have all my urgent data located in one folder   #
  # and mount this folder or sub folders to the right location in file system #
  #                                                                           #
  #############################################################################
  
  setup()
  {
  	tstemp=`date +%Y%m%d%H%M`
  	host=`hostname`
  	
  	# define definition file and check availability
  	DATA_FILE=/opt/backup/backup_$host.json
  	if [ ! -f $DATA_FILE ]
  	then
  		logger " <error> : definition file /opt/backup/backup_$host.json not reachable, check file location"
  		exit
  	fi
  	
  	WORK_DIR=`jq .work_dir $DATA_FILE|sed 's/\"//g'`/
  	cd $WORK_DIR
  	echo "working and running in : "$WORK_DIR
  	MOUNT_FS=""
  	message=""
  	ERR_STOP=0
  	ERR_COUNT=0
  	ERR_HIGH_COUNT=0
  	ERR_MEDIUM_COUNT=0
  	ERR_LOW_COUNT=0
  	LOG_DIR=`jq .log_dir $DATA_FILE|sed 's/\"//g'`
  	LOG_FILE=$LOG_DIR/$tstemp.backup.log
  	LOG_CHECK_FILE=$LOG_DIR/$tstemp.backup.check.log
  }
  
  error_check()
  {
  	echo "job 'error_check' is running"
  	IGNOR=`jq .jobs.error_check.err_ignore $DATA_FILE|sed 's/\"//g'`
  	echo "error to ignor : "$IGNOR
  
  	HIGH=`jq .jobs.error_check.err_high $DATA_FILE|sed 's/\"//g'`
  	MEDIUM=`jq .jobs.error_check.err_medium $DATA_FILE|sed 's/\"//g'`
  	LOW=`jq .jobs.error_check.err_low $DATA_FILE|sed 's/\"//g'`
  
  	for FILE in `cat $LOG_CHECK_FILE | grep '<f'|cut -d\/ -f2-`
  	do
  		echo "check line "$FILE
  		if [[ $IGNOR = "" ]]
  		then 
  			error_check_sub
  		else
  			for TYPE in ${IGNOR[@]}
  			do
  				if ( ! $FILE| grep $TYPE )
  				then
  					error_check_sub
  				fi
  			done
  		fi
  	done
  }
  
  error_check_sub()
  {
  	if ( cat $LOG_FILE | grep $FILE )
  	then
  		echo "check line "$FILE
  		for i in ${HIGH[@]}
  		do
  			if ( $FILE|grep $i )
  			then
  				(($ERR_HIGH_COUNT++))
  			fi
  		done
  		
  		for i in ${MEDIUM[@]}
  		do
  			if ( $FILE|grep $i )
  			then
  				(($ERR_MEDIUM_COUNT++))
  			fi
  		done
  		
  		for i in ${LOW[@]}
  		do
  			if ( $FILE|grep $i )
  			then
  				(($ERR_LOW_COUNT++))
  			fi
  		done
  
  		(($ERR_COUNT++))
  		echo $ERR_COUNT
  	fi
  }
  
  messaging()
  {
  	echo "job 'messaging' is running, generate massage for syslog"
  	if [ $ERR_COUNT -gt 0 ]
  	then
  		if [ $ERR_HIGH_COUNT -eq 0 ]
  		then
  			pre_massage=' <error>'
  		else
  			pre_message=' <warning>'
  		fi
  		message=$pre_message' : backup has '$ERR_COUNT' failures: '$ERR_HIGH_COUNT' high, '$ERR_MEDIUM_COUNT' medium and '$ERR_LOW_COUNT' low; check errorlog in '$LOG_DIR
  		echo "backup has "$ERR_COUNT
  	else
  		message=' <info> : backup successfully completed'
  
  		if [[ `jq .log_remove $DATA_FILE|sed 's/\"//g'` = "yes" ]]
  		then
  			rm $LOG_DIR/*
  		fi
  	fi
  }
  
  replicate()
  {
  	echo "job 'replicate' is running"
  	TARGET_DIR=`jq .jobs.replicate.remote_folder $DATA_FILE|sed 's/\"//g'`
  	REPL_OPT=`jq .jobs.replicate.repli_opt $DATA_FILE|sed 's/\"//g'`
  	METHOD=`jq .jobs.replicate.method $DATA_FILE|sed 's/\"//g'`
  	if [[ ! $REPL_OPT = "" ]]
  	then
  		if [[ $REPL_OPT = "local" ]]
  		then
  			DESTIN=$MOUNT_P/$TARGET_DIR
  			PASSFILE=""
  			if [ ! -d $DESTIN ]
  			then
  				echo $DESTIN" not exist, will create"
  				mkdir $DESTIN
  			fi
  			DESTIN=$DESTIN/
  			echo "local sync target "$DESTIN
  		else
  			SERVER=`jq .jobs.replicate.sync_server $DATA_FILE`
  			if( ! ping -c 3  $SERVER > /dev/null )
  			then
  				(($ERR_STOP++))
  				message="<info> : server "$SERVER" not reachable"
  				break 
  			fi
  	
  			USER=`jq .jobs.replicate.sync_user $DATA_FILE|sed 's/\"//g'`
  			PASS_OPT=`jq .jobs.replicate.pass_option $DATA_FILE`
  			DESTIN=$USER@$SERVER::/$MOUNT_P/$TARGET_DIR/
  			echo "remote sync with "$DESTIN
  		fi
  	
  		FOLDERS=`jq .jobs.replicate.sync_folders $DATA_FILE|sed 's/\"//g'`
  		echo $FOLDERS
  		
  		date >> $LOG_FILE
  		date >> $LOG_CHECK_FILE
  		
  		for DIR in ${FOLDERS[@]}
  		do
  			echo $DIR
  			echo "run : "$METHOD `jq .jobs.replicate.run_options $DATA_FILE|sed 's/\"//g'` $DIR $DESTIN $PASS_OPT
  			echo "{"$DIR"()" >> $LOG_FILE
  			$METHOD `jq .jobs.replicate.run_options $DATA_FILE|sed 's/\"//g'` $DIR $DESTIN $PASS_OPT >> $LOG_FILE
  			echo "{"$DIR"()" >> $LOG_CHECK_FILE
  			$METHOD `jq .jobs.replicate.log_options $DATA_FILE|sed 's/\"//g'` $DIR $DESTIN $PASS_OPT >> $LOG_CHECK_FILE
  			echo "}" >> $LOG_FILE
  			echo "}" >> $LOG_CHECK_FILE
  		done
  		
  		date >> $LOG_FILE
  		date >> $LOG_CHECK_FILE
  		error_check
  	else
  		(($ERR_STOP++))
  		message=" <warning> : no method defined"
  	fi
  }
  
  mount_fs()
  {
  	echo "job 'mount_fs' is running"
  	# check if target host available
  	TARGET_SRV=`jq .jobs.mount_fs.ping_target $DATA_FILE|sed 's/\"//g'`
  	echo "check, if you'r in right location, try to ping "$TARGET_SRV
  	MOUNT_P=`jq .jobs.mount_fs.mount_point $DATA_FILE|sed 's/\"//g'`
  	if( ping -c 3  $TARGET_SRV > /dev/null )
  	then	
  		echo $TARGET_SRV" reachable, you'r in right location to backup"
  		if [[ `jq .jobs.mount_fs.mount_def $DATA_FILE|sed 's/\"//g'` = "file" ]]
  		then
  			echo "mount "`jq .jobs.mount_fs.mount_opt $DATA_FILE|sed 's/\"//g'` `jq .jobs.mount_fs.mount_file $DATA_FILE|sed 's/\"//g'` $MOUNT_P
  			mount `jq .jobs.mount_fs.mount_opt $DATA_FILE|sed 's/\"//g'` `jq .jobs.mount_fs.mount_file $DATA_FILE|sed 's/\"//g'` $MOUNT_P
  		else
  			CONNECT=`jq .jobs.mount_fs.mount_srv $DATA_FILE|sed 's/\"//g'`:`jq .jobs.mount_fs.mount_share $DATA_FILE|sed 's/\"//g'`
  			echo "mount "`jq .jobs.mount_fs.mount_opt $DATA_FILE|sed 's/\"//g'` $CONNECT $MOUNT_P
  			mount `jq .jobs.mount_fs.mount_opt $DATA_FILE|sed 's/\"//g'` $CONNECT $MOUNT_P
  		fi
  		sleep 10 # be sure mount is completed or timed out
  		if ( mount | grep $MOUNT_P > /dev/null )
  		then
  			MOUNT_FS=`jq .jobs.mount_fs.mount_point $DATA_FILE|sed 's/\"//g'`
  			echo "umount "$MOUNT_FS" needed"
  		else
  			(($ERR_STOP++))
  			message=" <warning> : Can not mount backup target volume"
  		fi
  	else
  		(($ERR_STOP++))
  		message=" <info> : "$TARGET_SRV" not reachable, server down or you are not at right location"	
  	fi
  }
  
  # master function
  main()
  {
  	# select job sequence
  	
  	SEQUENCE=`jq .job_seq $DATA_FILE|sed 's/\"//g'`
  	echo "job sequence are : "$SEQUENCE
  	
  	if [[ $SEQUENCE = "" ]]
  	then
  		message=" <info> no jobs defined"
  	else
  		for JOB in ${SEQUENCE[@]}
  		do
  			if ( echo $JOB | grep add_tasks )
  			then
  				SUB_TASK=${JOB##*.}
  				TARGET=$WORK_DIR`jq .jobs.add_tasks.$SUB_TASK.folder $DATA_FILE|sed 's/\"//g'`/
  				echo "sub task target "$TARGET
  				echo "run sub task : "`jq .jobs.$JOB.task $DATA_FILE|sed 's/\"//g'` $TARGET
  				`jq .jobs.$JOB.task $DATA_FILE|sed 's/\"//g'` $TARGET
  			else
  				echo "run main job : "$JOB
  				$JOB
  			fi
  			if [[ $ERR_STOP -gt 0 ]]
  			then
  				logger $message
  				exit
  			fi
  		done
  	fi
  	
  	echo "clean up"
  	
  	if [[ ! $MOUNT_FS = "" ]]
  	then
  		echo "umount "$MOUNT_FS
  		umount $MOUNT_FS
  	fi
  	
  	logger $message
  }
  
  # don't change sequence below
  setup
  main
  exit

===== Parameter-File =====
Das dazu gehörende ParameterFile sieht dann so aus bei mir:

  {
  	"description": "json-file to backup and restore per backup-script",
  	"work_dir":"/data",
  	"log_dir":"/data/backuplog",
  	"log_remove":"yes",	
  	"job_seq":"mount_fs add_tasks.backup_opt replicate error_check messaging",
  	"jobs":
  	{
  		"mount_fs":
  		{
  			"@":"'ping_target' is used to verify location; 'mount_def' can be 'file' or 'server'; in case of 'file', 'mount_file' needed",		
  			"ping_target":"192.168.2.1",
  			"mount_def":"file",
  			"mount_opt":"-t glusterfs",
  			"mount_srv":"<file server>",
  			"mount_share":"<file server share>",
  			"@":"'mount_file' describe a method to connect to a server and his share(s), 'mount_srv' and 'mount_share' not needed in this case",
  			"mount_file":"/etc/glusterfs/gluster1.vol",
  			"mount_point":"/gluster"
  		},
  		"replicate":
  		{
  			"@":"'repli_opt' can be 'local' or 'remote', 'remote' needs 'pass_option', 'sync_server' and 'sync_user'",		
  			"repli_opt":"local",
  			"@":"only rsync implemented yet, can try scp, but auth method must implemented by your self; use 'AX' to safe links",
  			"method":"rsync",
  		   "run_options":"-avh -partial",
  	   	"log_options":"-avhn -partial",
  			"@":"pass_option : file content format 'username:password'",
  			"pass_option":"--password-file=<file location>",
  			"sync_server":"<rsyncd server>",
  			"sync_user":"<username>",
  			"@":"if absolute path used in 'sync_folders', not 'work_dir' needed",
  			"sync_folders":"Carrier Closed Documentation Downloads notes privat Projekts Proposals transcripts backup transfer",
  			"@":"'remote_folder' means the backup folder at the target file system",
  			"remote_folder":"Arbeit"
  		},
  		"error_check":
  		{
  			"@":"'err_ignore' means witch files or search words can be ignore in error log or errors at backup this files are normal (i.e. log-files)",
  			"err_ignore":"",
  			"@":"error level definitions",
  			"err_high":"",
  			"err_medium":"notes",
  			"err_low":""
  		},
  		"messaging":"<no further parameter needed, all included in backup script>",
  		"add_tasks":
  		{
  			"<task name>":
  			{
  				"folder":"<target folder for task>",
  				"task":"<can be external(!) script or command, no sub routine in backup script>"
  			},
  			"backup_db":
  			{
  				"folder":"mysql_backup",
  				"task":"mysqldump -u root --password=<PASSWORD> --all-databases > $TARGET"
  			},
  			"backup_opt":
  			{
  				"folder":"backup",
  				"task":"cp -r /opt/backup/*"
  			}
  		}
  	}
  }



Die Jobs "mount_fs" und "replicate" sind als feste Optionen, siehe "job_seq" (Job Sequence), fertig definierte Schritte im BackupScript. Wobei auch "mount_fs" nur eine Option ist. Ich habe bei mir "mount_fs" gewählt, damit ich bei der Replikation keine Authentifizierung benötige und auch gleichzeitig auf zwei verschiedene Platten, bei mir zwei Server, schreiben kann. Wenn also bei mir ein Server, wegen was auch immer, nicht zur Verfügung steht, dann kann ich trotzdem mein Backup machen.

"add_tasks" sind frei definierbare zusätzliche Aufgaben deren Auführungszeitpunkt in "job_seq" definiert wird.
tuhol.de

User Tools

Site Tools

Page Tools
