Restructure, and rewrite substantial partsHEADmasterdevel

A notable problem (to me) with the former version was, that upon simple
movements in the source file tree, these would in general cause
duplicated data in the backup forest.  This problem has been resolved
by essentially tracking file movements, using hard links.

Also, the code has been divided into several different files, extranous
code removed, the organization of remote code execution simplified.

In the process of simplification, all parts requiring direct interaction
of the user with the program have been replaced.  In most cases, this
means that the program now just terminates with an error instead of
allowing the user to confirm deletion of an unexpected (remnant) file.
This may be considered a drawback, but actually these interactive
options were anyways suboptimal solutions in most cases - where they
occured, which was due to remnant files of aborted or failed former
backups.

In general, there have been a lot of changes.  Which are not thoroughly
documented here.  Since there is often no strong relation to the old
code, this is not deemed necessary, as the in-code documentation is
expected to be of sufficient help.

Also, there has been added a little general documentation in the README
and particularly the NOTES file.

It is to be noted that this new version is far from sophisticated.  It
has been tested with and is in use for real data, however lacks a lot of
convenience.  In particular, if a backup fails unexpectedly, the next
backup will in the very most cases loudly fail without manual
intervention.  Also, the program is not able to continue the growth of a
backup tree built with former versions of this program, by itself.  This
can however be arranged by hand.

1 files changed, 14 insertions, 634 deletions

diff --git a/rsync-backup b/rsync-backup
index 0a7df33..86bff28 100755
--- a/rsync-backup
+++ b/rsync-backup
@@ -21,642 +21,22 @@
 #
 
 
-typeset -r exec_name="${0##*/}"
+typeset -r full_exec_name="$(realpath "$0")"
+typeset -r exec_name="${full_exec_name##*/}"
+typeset -r source_code_dir="${full_exec_name%/*}"
 
+cd "$source_code_dir"
 
-# CONSTANTS
-typeset -r CONFIG=/etc/rsync-backup/config
-typeset -r RET_TYPES='
-	RET_SUCCESS RET_FAILURE RET_BADSYNTAX RET_FILE_ERR RET_ERROR'
-typeset -r RET_SUCCESS=0
-typeset -r RET_FAILURE=1
-typeset -r RET_BADSYNTAX=2
-typeset -r RET_FILE_ERR=4
-typeset -r RET_ERROR=8
-typeset -r DATEFMT='%Y-%m-%d_%H%M%S'
-
-# Shell options:
-#   nullglob: In case of a non-matching glob, return nothing.
-#   dotglob:  Include dotfiles (except . and ..) in glob matching.
-typeset -r SHOPTS='dotglob'
-typeset -r SHOPTS_AFTER_GET_ARGS='nullglob'
-
-
-# Global program variables.
-typeset src  src_path  src_host
-typeset dest dest_path dest_host
-typeset -a filter_args
-typeset -i n
-typeset bakdate
-
-
-# Variables that may be configured.
-typeset RSYNC=rsync
-typeset -a rsync_args
-typeset -a rsync_inc_args
-typeset -A bakpath
-typeset filter_file_all			# Applied after specific filter.
-typeset -A filter_file
-
-# Default arguments for rsync (-rptogAXlHS, --timeout=, --info=).
-rsync_args=(
-	--recursive
-	--perms --times --owner --group
-	--acls --xattrs
-	--links
-	--hard-links
-	--sparse
-	--timeout=60
-	#-vv
-	--info=progress2
-	)
-
-# Supplemental arguments for rsync; only used when an actual incremental
-# update is performed, i.e. when --link-dest is used.
-rsync_inc_args=(
-	--fuzzy --fuzzy
-	)
-
-filter_file_all=/etc/rsync-backup/filter
-
-source "$CONFIG"
-
-
-# RSYNC args.
-# basic args:  --recursive --perms --times --owner --group --links
-# extra args:  --sparse --acls --xattrs --hard-links --timeout=60
-#              --fuzzy --fuzzy
-#              --filter="merge <file>"
-# likely:      --compress
-#              --partial{,dir=DIR}
-#              --progress
-# possibly:    --devices --specials
-#              --max-size=SIZE-OF-FILE
-#              --one-file-system
-#              --log-file=FILE --log-file-format=FORMAT
-#              --human-readable
-# unlikely:    --omit-{dir,link}-times --update (!--inplace) --delete
-#              --exclude
-# great:       --link-dest=DIR (timestamps?)
-# testing:     --verbose --dry-run (-vn)
-# interesting: --sockopts --itemize-changes --out-format=FORMAT
-#              --stats
-
-function main
-{
-	shopt -s $SHOPTS
-
-	# Set $src, $dest_path and $filter_args.
-	get_args "$@" || return $?
-
-	shopt -s $SHOPTS_AFTER_GET_ARGS
-
-	# Verify $src_path exists.
-	run_function "$src_host" src_check_path '' 'src_path src' || return $?
-
-	# Make sure $dest_path exists.
-	run_function "$dest_host" dest_check_path '' 'dest_path dest' || return $?
-
-	# Populate the global $dirs array.
-	dirs=( $(
-		run_function "$dest_host" dest_find_existing_backup_dirs '' 'dest_path' \
-			|| return $?
-		) )
-
-	n=${#dirs[@]}
-
-	run_function "$dest_host" dest_prepare 'dest_clear_file' \
-		'dest dirs[@] n dest_host dest_path' "$(date +$DATEFMT)" \
-		|| return $?
-
-	perform_backup || return $?
-
-	run_function "$dest_host" dest_finalize \
-		'dest_refresh_symlinks dest_clear_file' \
-		'dest_host dest_path bakdate n' \
-		|| return $?
-
-	dest_refresh_symlinks
-
-	return $RET_SUCCESS
-}
-
-
-# Set the globabl variables $src, $dest_path and $filter_args according to
-# command line parameters and the configuration.
-#
-# $@: command line parameters
-#
-# requires the shell option nullglob to be _unset_.  (for test -v to work)
-#
-function get_args
-{
-	if [ $# -eq 0 ]
-	then
-		echo "Usage: $exec_name <src> [<dest>]"
-		return $RET_BADSYNTAX
-	fi
-
-	local l_src="$1"
-	local l_dest
-
-	# Set $src_path, $src_host and $src.
-	resolve_host_path src "$l_src" || return $?
-
-	if [ $# -gt 1 ]
-	then
-		l_dest="$2"
-	elif test -v bakpath[$src]		# $# -eq 1
-	then
-		l_dest="${bakpath[$src]}"
-	else
-		echo "Error: Backup path for $src neither configured"
-		echo "       nor specified on command line."
-
-		return $RET_BADSYNTAX
-	fi
-
-	# Set $dest_path, $dest_host and $dest.
-	resolve_host_path dest "$l_dest" || return $?
-
-	filter_args=()
-	if test -v filter_file[$src] && test -f "${filter_file[$src]}"
-	then
-		filter_args+=("--filter=merge ${filter_file[$src]}")
-	fi
-	if test -v filter_file_all -a -f "$filter_file_all"
-	then
-		filter_args+=("--filter=merge ${filter_file_all}")
-	fi
-
-	return $RET_SUCCESS
-}
-
-
-# Split host:target like rsync and convert local (!) relative paths to
-# absolute ones.  This is needed at least for $src_path which is used as
-# key for associative arrays.
-#
-# The results are written to the global variables $<vtype>, $<vtype>_host
-# and $<vtype>_path, where <vtype> is either src or dest.  $<vtype> gets
-# set to the original to be resolved string, unless we are local, i.e.
-# there is no host.
-#
-# Note: This is not a place for bash to shine.  Perl for instance would do
-#       this much more easily.
-#
-# Examples:
-# - a:b -> (a, b)
-# - a/b:c -> ('', a/b:c)
-#
-# $1: which global variables to set: (src|dest)
-# $2: path string potentially containing host prefix
-#
-# sets: ${$1}_host: host name (may be empty)
-#       ${$1}_path: path (absolute, if local, i.e. ${$1}_host='')
-#       ${$1}:      ${$1}_path,                   if ${$1}_host='',
-#                   ${$1}_host:${$1}_path (= $2), elsewise.
-#
-function resolve_host_path
-{
-	local vtype="$1"
-	local combined="$2"
-	local host path
-
-	# Split host and path.
-	host=${combined%%:*}
-	if [[ "$host" == "$combined" || "$host" =~ '/' ]]
-	then
-		host=''
-		path="$combined"
-
-		# Resolve to absolute directory.
-		# pwd -P avoids symlinks.
-		# Do not resolve symlimks in ${path} or test for its existance.
-		#  `- realpath could do that.
-		if ! [[ "$path" =~ / ]]
-		then
-			path="$(pwd -P)/${path}"
-		fi
-
-		combined="$path"
-	else
-		path="${combined#*:}"
-	fi
-
-	# Set global variables.
-	if [[ "$vtype" == 'src' ]]
-	then
-		src_host="$host"
-		src_path="$path"
-		src="$combined"
-	else # "$vtype" == 'dest'
-		dest_host="$host"
-		dest_path="$path"
-		dest="$combined"
-	fi
-
-	return $RET_SUCCESS
-}
-
-
-# Run a function on a remote host.
-# If the host parameter is empty, run locally.
-#
-# $1: host: name
-# $2: function to run: name
-# $3: other functions to export: [name (. ' ' . name)*]
-# $4: variables to export: [name (. '[@]')? (. ' ' . name (. '[@]')?)*]
-# ${@:5}: function arguments: string+
-#
-function run_function
-{
-	local host="$1"
-	local fun=$2
-	local funs="$fun $3"
-	local vars="$4"
-	local vars_set
-
-	shift 4
-
-	if [[ -z "$host" ]]
-	then
-		$fun "$@"
-	else
-		# Create string to export variables.
-		vars_set=
-		for var in $RET_TYPES $vars
-		do
-			vars_set+="$(eval echo \${${var}@A}); "
-		done
-
-		# Use ssh to run command remotely.
-		# Make sure to properly escape the strings.
-		ssh "$host" "shopt -s ${SHOPTS}; ${vars_set} $(typeset -f $funs);" \
-			"$fun $(printf "'%s' " "$@")"
-	fi
-
-	return $?
-}
-
-
-# Run command remotely.
-#
-# Not used.
-#
-# $1: host: name
-# ${@:2}: command with arguments: string+
-#
-function do_thing
-{
-	local host="$1"
-	shift
-
-	if [[ -z "$host" ]]
-	then
-		"$@"
-	else
-		# Use ssh to run command remotely.
-		# Make sure to properly escape the strings.
-		ssh "$host" "shopt -s ${SHOPTS}; $(printf "'%s' " "$@")"
-	fi
-
-	return $?
-}
-
-
-# Check if $src exists.
-# Must be run on source host.
-#
-# requires: $src_path, $src
-#
-function src_check_path
-{
-	if ! test -d "$src_path"
-	then
-		echo "Source (\"$src\") does not exist."
-		return $RET_FILE_ERR
-	fi
-
-	return $RET_SUCCESS
-}
-
-
-# Check if $dest_path exists and try to create it if necessary.
-# Must be run on destination host.
-#
-# requires: $dest_path, $dest
-# interactive.
-#
-function dest_check_path
-{
-	local -l reply
-
-	if ! test -d "$dest_path"
-	then
-		echo "Target (\"$dest\") does not exist."
-		echo -n "Do you want do create it [y/N]? "
-		read reply
-		if ! [[ "$reply" =~ ^y(es)?$ ]]
-		then
-			return $RET_FAILURE
-		fi
-		echo
-
-		if ! mkdir -p "$dest_path"
-		then
-			echo "Backup creation has failed."
-			echo "\"$dest\" could not be created."
-			return $RET_FILE_ERR
-		fi
-	fi
-
-	return $RET_SUCCESS
-}
-
-
-# Print existing backup directories.
-# Must be run on destination host.
-#
-# requires: $dest_path
-#
-function dest_find_existing_backup_dirs
-{
-	find "$dest_path" -mindepth 1 -maxdepth 1 -type d \
-		-regextype posix-extended -regex \
-		'.*/[0-9]{4}-[0-1][0-9]-[0-3][0-9]_[0-2][0-9]([0-5][0-9]){2}.backup' \
-		-printf '%f\n' \
-		| LC_ALL=C sort
-
-	if [[ ${PIPESTATUS[0]} -eq 0 && ${PIPESTATUS[1]} -eq 0 ]]
-	then
-		return $RET_SUCCESS
-	else
-		return $RET_ERROR
-	fi
-}
-
-
-# Prepare destination for the backup process.
-# Must be run on destination host.
-#
-# requires: $dirs[@], $n
-#           $dest_host (self, <- dest_clear_file)
-#           $dest_path
-#
-# $1: current date on local host in format $DATEFMT
-#
-# interactive (<- dest_clear_file)
-#
-function dest_prepare
-{
-	local current_date="$1"
-
-	# Check for dates in the future, which might mess with the yet to be
-	# determined new backup's date.
-	if [[ $n -gt 0 ]]
-	then
-		local last_date=${dirs[n-1]%.backup}
-
-		if [[ "$current_date" == "$last_date" ]]
-		then
-			echo "Last backup was run in the same second."
-			echo "Exiting."
-			return $RET_FAILURE
-		elif
-			[[ "$current_date" < "$last_date" ]]
-		then
-			echo "Last backup lies in the future.  Fix your clocks."
-			echo "Exiting."
-			return $RET_FAILURE
-		fi
-	fi
-
-	dest_clear_file "${dest_path}/new.backup" "Backup creation has failed." \
-		|| return $?
-
-	return $RET_SUCCESS
-}
-
-
-# Perform the actual backup.
-# To be run locally.
-#
-# requires: $DATEFMT,
-#           $dest_path, $dest, $src_path, $src,
-#           $n, $dirs[@],
-#           $rsync_args[@], $rsync_inc_args[@], $filter_args[@]
-#
-# sets: $bakdate
-#
-function perform_backup
-{
-	bakdate=`date +"$DATEFMT"`
-
-	printf 'Starting backup nr. %u.\n' "$n"
-	if [ $n -eq 0 ]
-	then
-		# Perform first backup, not incremental.
-		$RSYNC "${rsync_args[@]}" \
-			"${filter_args[@]}" \
-			"$src"/ "${dest}/new.backup" \
-			|| return $RET_ERROR
-	else
-		# Perform incremental backup on the basis of the last.
-		$RSYNC "${rsync_args[@]}" \
-			"${rsync_inc_args[@]}" --link-dest="../${dirs[n-1]}" \
-			"${filter_args[@]}" \
-			"$src"/ "${dest}/new.backup" \
-			|| return $RET_ERROR
-	fi
-
-	return $RET_SUCCESS
-}
-
-
-# Finalize the backup creation on the destination host.
-#
-# requires: $dest_host (<- dest_refresh_symlinks),
-#           $dest_path (self, <- dest_refresh_symlinks),
-#           $bakdate (self, <- dest_refresh_symlinks),
-#           $n (self, <- dest_refresh_symlinks),
-#
-# interactive (<- dest_refresh_symlinks)
-#
-function dest_finalize
-{
-	mv "${dest_path}/new.backup" "${dest_path}/${bakdate}.backup" \
-		|| return $RET_FILE_ERR
-
-	if
-		dest_refresh_symlinks
-	then
-		printf '\nCreating of backup nr. %u successfully executed.\n' "$n"
-		return $RET_SUCCESS
-	else
-		local ret=$?
-		printf '\nCreating of backup nonetheless successfully executed.\n'
-		return $ret
-	fi
-}
-
-
-# Create symbolic links to backup directories.
-# Must be run on the destination host.
-#
-# requires: $dest_host, $dest_path, $bakdate, $n
-# interactive (<- dest_clear_file)
-#
-function dest_refresh_symlinks
-{
-	local -i i
-	local host_prefix="${dest_host}${dest_host:+:}"
-	local linkdir ret
-
-	linkdir="${dest_path}/latest"
-	if test -L "$linkdir"
-	then
-		rm "$linkdir"
-		ln -s "${bakdate}.backup" "$linkdir" || return $RET_ERROR
-	elif
-		dest_clear_file "$linkdir" "Creation of softlink \"latest\" failed." \
-			|| return $?
-	then
-		ln -s "${bakdate}.backup" "$linkdir" || return $RET_ERROR
-	fi
-
-	# by_number
-	numdir="${dest_path}/by_number"
-	if ! test -d "$numdir"
-	then
-		dest_clear_file "$dest_host" "$numdir" \
-			"Setting up numbered softlinks failed." || return $?
-
-		mkdir "$numdir" || return $RET_ERROR
-	fi
-
-	# Remove old symlinks.
-	find "$numdir" -mindepth 1 -maxdepth 1 -type l -regextype posix-extended \
-		-regex '.*/[0-9]*' -delete || return $RET_ERROR
-
-	# TODO: Simplify.
-	for file in "$numdir"/*
-	do
-		echo $file
-		dest_clear_file "$file" "Setting up numbered softlinks failed." \
-			|| return $?
-	done
-
-	# Set last dir on the new one.
-	dirs[$n]="${bakdate}.backup"
-
-	for (( i = 0; i <= n; i++ ))
-	do
-		ln -s "../${dirs[i]}" "$(printf "%s/%.${#n}u" "$numdir" $i)"
-	done
-
-	return $RET_SUCCESS
-}
-
-
-# Make sure a file does not exist on the destination host.
-# Must be run on destination host.
-#
-# $1: file: path
-# $2: error message: string
-#
-# requires: $dest_host
-# interactive.
-#
-function dest_clear_file
-{
-	local file_path="$1"
-	local err_msg="$2"
-	local file="${dest_host}${dest_host:+:}${file_path}"
-
-	local -l reply
-
-	if test -e "$file_path" -o -L "$file_path"
-	then
-		printf "%s %s\n %s\n%s\n" \
-			"It seems like the last backup process failed or you have created" \
-		 	"the file" "$file" "manually."
-
-		echo -n "Do you want to delete it [y/N]? "
-		read reply
-
-		if [[ "$reply" =~ ^y(es)?$ ]]
-		then
-			printf "Deleting.\n"
-
-			if rm -rf "$file_path"
-			then
-				return $RET_SUCCESS
-			else
-				printf '%s\n' "$err_msg"
-				return $RET_FILE_ERR
-			fi
-		else
-			printf '%s\n' "$err_msg"
-			return $RET_FAILURE
-		fi
-	fi
-
-	return $RET_SUCCESS
-}
-
-
-# Ask a yes/no question.
-# Note, that ^D is considered as a default answer.
-#
-# Not used.
-#
-# $1 default answer: (y|n|x)
-# $2 question: string
-#
-# returns yes / no: (0|1)
-#
-# interactive.
-#
-function ask
-{
-	# -l auto-converts everything to lower case.
-	local -l default="$1"
-	local question="$2"
-
-	while true
-	do
-		if [[ "$default" == "y" ]]
-		then
-			printf "%s [Y/n]? " "$question"
-		elif [[ "$default" == "n" ]]
-		then
-			printf "%s [y/N]? " "$question"
-		else
-			printf "%s [y/n]? " "$question"
-			default="x"
-		fi
-
-		local -l reply	# -l converts to lowercase
-		read reply
-
-		if [[ "$reply" =~ ^y(es)?$ ]]
-		then
-			return 0
-		elif [[ "$reply" =~ ^no?$ ]]
-		then
-			return 1
-		elif [[ "$default" == "x" || "$reply" != "" ]]
-		then
-			continue
-		elif [[ "$default" == "y" ]]
-		then
-			return 0
-		else # "$default" == "n"
-			return 1
-		fi
-	done
-}
+# Include the other files.
+. ./variables
+. ./files
+. ./write
+. ./remote
+. ./local
+. ./source
+. ./destination
 
 
 main "$@"
+
+# vi: ft=bash ts=2 sw=2 noet