#!/usr/bin/perl -w
#
# Delta-backup: Simple tool for cross system backup with differential archiving.
#
# Documentation can be read with perldoc (perldoc delta-backup.pl) or directly in this script.
#
# Details of my motivations to create this script can be found here :
# http://kwartik.wordpress.com/delta-backup
#
# Revision: 003
# Author: kwartik@gmail.com
# Date of creation : 2010-01-01
# Last revision date : 2010-03-21
#
# History of revisions
# - 2010-01-05, rev002 - bug fix in black list check
# - 2010-03-21, rev003 - bug fix in black list check
#
# LICENCE: Copyright 2009 kwartik@gmail.com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################################
=pod
=encoding utf8
=head1 NAME
Delta-backup - Simple tool for cross system backup with differential archiving.
=head1 SYNOPSIS
./delta-backup.pl [ -h|--help ] [ -i|--init] \
[ -s|--source host:source_directory_path ] [ -d|--dest host:destination_directory_path ] \
[ -l|--delta host:differiential_directory_path ] [ -x|--exclude list_of_directories_and_files_to_exclude_in_the_delta ] \
[ -z|--zip ] [ -e|--encryption ] [ -p|--encryption_password ] \
[ -t|--test ]
=head1 DESCRIPTION
The principle of B password to use to encrypt the differential backup.
Don't forget it, you will need it the day you will restore your backups !
-B test mode, only display what would be backup
=back
=head2 LIMITATIONS
Although this script works with any combination of local or remote values for source, destination and delta directories, the recommended combination (to have best performances) is :
- local directory for source
- remote or local directory for dest
- local directory for delta
Exlude list: if file /dir/file is in exclude list, all /dir/file* will aussi be in it. Also if file myfile is in exclude list all files and directories
myfile* wille be excluded
Files that are moved on the system are considered to be modified so they will appear in the delta.
No ~ should be specified in paths.
=head2 SYSTEM-REQUIREMENTS AND DEPENDENCIES
=over 1
- LINUX or traditionnal UNIX based environment of both source and destination hosts. This scripts have been tested on a Mandriva Linux 2010 distribution but there is no reason why it should not work on other distributions.
- In case the source and destination directories are not on the same hosts, the host on which the script is executed must be able to connect on the other hosts through SSH without any password (generate for this a SSH key with ssh-keygen -t rsa command, choose an empty password and copy-paste the content of the ~/.ssh/id_rsa.pub at the end of the ~/.ssh/authorized_keys file on the remote host).
- Getopt::Long module (installed by default on most Linux distributions)
- rsync (installed by default on most Linux distributions)
- optionnaly gzip and openssl to compress and encrypt the differential archives
=back
=head2 EXAMPLES
=over 1
- See what files would be copied with the initial full backup : ./delta-backup.pl --test --init --source local:/data/rsync-test --dest neon:/data/rsync-test
- Run the initial full backup : ./delta-backup.pl --init --source local:/data/rsync-test --dest neon:/data/rsync-test
- See what files would be synchronized between source and dest and archived in the delta after the initial full backup: ./delta-backup.pl --test --source local:/data/rsync-test --dest neon:/data/rsync-test --delta local:/data/rsync-delta
- Synchronize the source and the destination, archive delta using compression and encryption: ./delta-backup.pl --source local:/data/rsync-test --dest neon:/data/rsync-test --delta local:/data/rsync-delta --zip --encryption --encryption_password='averystrongpassword'
=back
=head1 AUTHOR
kwartik@gmail.com
=head1 LICENCE
Copyright 2009 kwartik@gmail.com
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
=cut
##############################################################################
use Getopt::Long;
### BEGIN OF CONFIGURABLE SECTION ############################################
# Dependencies (path to binaries), change them only if their location is not
# listed in your $PATH
$rsync = "rsync";
$date = "date";
$rm = "rm";
$mv = "mv";
$cp = "cp";
$du = "du";
$tar = "tar";
$gzip = "gzip";
$ssh = "ssh";
$scp = "scp";
$openssl = "openssl";
# tempory directory on local host
# Eg: $tmp_dir="/tmp";
$tmp_dir="/tmp";
# The following options can be overriden with the command line parameters
# $source: hostname:/path/to/root
# if local "local" shoud be specified for the hostname
# Eg: $source = "local:/data/rsync-test";
$source = "";
# hostname:/path/to/root
# if local "local" shoud be specified for the hostname
# Warning: source and dest directories can't be both remote
$dest = "";
# $delta: path of the directory where the differential backups must be archived
# Eg: $delta = "local:/home/delta-backup";
$delta = "";
# $exclude: path of the list which contains all files and directories
# on the source system to exlude in the delta (one file or directory per line).
# The list must be on the local system, even if it can reference files on a
# remote system (if the source system is remote).
# The paths listed in this file must be listed RELATIVELY to the source root directory.
# Eg: $exclude = "/data/rsync-delta/files-to-exclude-in-delta.lst";
$exclude = "";
# $test: If test mode is set, delta-backup will only display what it would archived
# To be set to 1 to activate test mode, 0 otherwise
# Eg: $test=1
$test=0;
$init=0;
# $gzip: Set to '1' to compress the delta
$zip=0;
# Optional encryption
# If you choose encryption, it is highly advised to also activate compression
$encryption=0;
$encryption_password="astrongpassword";
### END OF CONFIGURABLE SECTION ############################################
my $debug = 0; #0: normal mode; 1: debug mode
$help=0;
$now = `$date +%Y%m%d-%s`;
chomp($now);
#chomp($delta);
@exclude_list = ();
GetOptions (
"h|help" => \$help,
"i|init" => \$init,
"s|source=s" => \$source,
"d|dest=s" => \$dest,
"l|delta=s" => \$delta,
"x|exclude=s" => \$exclude,
"z|zip" => \$zip,
"e|encryption" => \$encryption,
"p|encryption_password=s" => \$encryption_password,
"t|test" => \$test
);
sub help() {
print "
DELTA-BACKUP: backup tool allowing rsync and differential secure archiving
Syntax: delta-backup.pl \
[ -h|--help ] print this help\
[ -i|--init] use this to create the first backup (no differential will be created, only source and dest need to be specified)
[ -s|--source host:source_directory_path ] \
[ -d|--dest host:destination_directory_path (where the total rsync based backup should be saved) ] \
[ -l|--delta host:differiential_directory_path (where the difference with last backup should be archived) ] \
[ -x|--exclude list_of_directories_and_files_to_exclude_in_the_delta ] \
[ -z|--zip ] compress the differential backup \
[ -e|--encryption ] encrypt the differential backup \
[ -p|--encryption_password ] password to use to encrypt the differential backup.
Don't forget it, you will need it the day you will restore your backups ! \
[ -t|--test ] test mode, only display what would be backup.
Execute \"perldoc delta-backup.pl\" for more detailed help.
Details of my motivations to create this scripts can be found here: http://kwartik.wordpress.com/delta-backup\n";
return 0;
}
if ( $help ne 0 ) {
help();
exit 1;
}
sub isBlackListed($) {
my $file_to_check = $_[0];
if ( $debug eq '1' ) { print "file to check in blacklist: |$file_to_check| => RES="; }
my $res = 0;
# This first method seems to work only for a exact match
# my $is_in_exclude_list = grep(/^$file_to_check/, @exclude_list);
# if ( $is_in_exclude_list >= 1 ) {
# print "$file_to_check is in exclude list\n";
# return 1;
# }
# Alternative method that works:
# PB: if /rep/fic is in exclude list, /rep/fic* will aussi be in it...
foreach (@exclude_list) {
my $item=$_;
#suppression of final /
$item =~ s/\/$//;
#if ( $debug eq '1' ) {print ":$item:"};
if ( $file_to_check =~ /^$item/ ) {
if ( $debug eq '1' ) { print "YES \n"; }
return 1;
}
}
if ( $debug eq '1' ) { print "NO \n"; }
return $res;
}
$source_host = "";
$source_dir = "";
$dest_host = "";
$dest_dir = "";
$delta_host = "";
$delta_dir = "";
if ( $source =~ /(\w+):(.*)$/ ) {
($source_host, $source_dir) = ( $source =~ /(\w+):(.*)$/ )
} else {
print "ERROR: source option ($source) does not respect the semantic host:directory_path (host should be set to 'local' for local host)\n";
exit 3;
}
if ( $dest =~ /(\w+):(.*)$/ ) {
($dest_host, $dest_dir) = ( $dest =~ /(\w+):(.*)$/ )
} else {
print "ERROR: dest option ($dest) does not respect the semantic host:directory_path (host should be set to 'local' for local host)\n";
exit 4;
}
if ( $delta =~ /(\w+):(.*)$/ ) {
($delta_host, $delta_dir) = ( $delta =~ /(\w+):(.*)$/ )
} elsif ( ! $init ) {
print "ERROR: delta option ($delta $init) does not respect the semantic host:directory_path (host should be set to 'local' for local host)\n";
exit 5;
}
if ( ! $init && $delta_host !~ /local|$source_host|$dest_host/ ) {
print "ERROR: delta directory must be either on the source or dest host.\n";
exit 6;
}
printf("\nDelta backup is being executed with the following parameters :\n");
print "\n";
print ">> init : $init\n";
print ">> source : $source\n";
print ">> dest : $dest\n";
print ">> delta : $delta\n";
print ">> exclude : $exclude\n";
print ">> zip : $zip\n";
print ">> encryption : $encryption\n";
print ">> encryption_password : $encryption_password\n";
print ">> test : $test\n\n";
# if ( ! -d $delta ) {
# print "ERROR: delta directory $delta does not exist\n";
# exit 2;
# }
if ( $exclude ne '' ) {
print "Exclude list for the delta ($exclude):\n";
open (LIST, ";
if ( defined $line ) {
($line) = ($line =~ /(.*)\n$/);
if ( $line !~ /^#/ ) {
if ( $line ne '' ) {
push(@exclude_list, $line );
print "->|$line|\n";
}
}
}
}
close (LIST);
print "\n";
}
if ( $source_host eq 'local' ) {
$source = $source_dir;
}
if ( $dest_host eq 'local' ) {
$dest = $dest_dir;
}
# We had a trailing / to $source if non existant (important for rsync)
if ( $source !~ /.*\/$/ ) {
$source = $source."/";
}
$fic_output_deleted_files_list="$tmp_dir/$now-delta-backup-deleted";
$fic_output_rsync = "$tmp_dir/$now-delta-backup-rsync.o";
$rsync_test_option = '';
if ( $test ) { $rsync_test_option = '--dry-run'; }
$command_rsync = "";
$command_rsync_init = "";
if ( $source_host ne 'local' && $dest_host ne 'local' ) {
$command_rsync = "$ssh $source_host \"$rsync -avz --delete $rsync_test_option $source_dir/ $dest\" > $fic_output_rsync";
$command_rsync_init = "$ssh $source_host \"$rsync -avz --delete $rsync_test_option $source_dir/ $dest\"";
}
else {
$command_rsync = "$rsync -avz --delete $rsync_test_option $source $dest > $fic_output_rsync";
$command_rsync_init = "$rsync -avz --delete $rsync_test_option $source $dest";
}
print "RSYNC:\n\n";
if ( $init ) {
$output_rsync_init=`$command_rsync_init`;
if ( $debug ) { print ">>>> $command_rsync_init\n\n" };
print "$output_rsync_init\n";
exit 0;
}
if ( $debug ) { print "rsync command: $command_rsync\n"; };
`$command_rsync`;
if ( ! $test ) { open(DEL, ">$fic_output_deleted_files_list") or die "ERROR: unable to create \"$fic_output_deleted_files_list\"", $!; }
# if delta is local => tar of local rsync directory between local and remote (local if both local)
# if delta is remote => tar of rsync directory between local and remote (local if both local)
$ref_host = "";
$ref_dir = "";
$cmd_prefix = "";
$cmd_suffix = "";
$is_fic_cmd_prefix = "";
$is_fic_cmd_suffix = "";
if ( $delta_host eq 'local' ) {
if ( $source_host eq 'local' ) {
$ref_dir = $source_dir;
$is_fic_cmd_prefix = "test -f '$source_dir/";
$is_fic_cmd_suffix = "'";
}
elsif ( $dest_host eq 'local' ) {
$ref_dir = $dest_dir;
$is_fic_cmd_prefix = "$ssh $source_host \"test -f '$source_dir/";
$is_fic_cmd_suffix = "'\"";
}
$cmd_suffix = "";
}
else {
if ( $dest_host ne 'local' ) {
$ref_host = $dest_host;
$ref_dir = $dest_dir;
$is_fic_cmd_prefix = "test -f '$source_dir/";
$is_fic_cmd_suffix = "'";
}
elsif ( $source_host ne 'local' ) {
$ref_host = $source_host;
$ref_dir = $source_dir;
$is_fic_cmd_prefix = "$ssh $source_host \"test -f '$source_dir/";
$is_fic_cmd_suffix = "'\"";
}
$cmd_prefix = "$ssh $ref_host \"";
$cmd_suffix = "\"";
}
open(RSYNC, " ) {
if ( $_ !~ /^deleting /) {
$fic=$_;
chomp($fic);
$rc=system("$is_fic_cmd_prefix$fic$is_fic_cmd_suffix");
if ( $debug ) { print "|system($is_fic_cmd_prefix$fic$is_fic_cmd_suffix)|=>rc=$rc\n"; }
if ( $rc == 0 && ! isBlackListed("$fic") ) {
$command = "$cmd_prefix cd $ref_dir;$tar -rpf $delta_dir/$now-delta-backup-new.tar '$fic';cd ->/dev/null$cmd_suffix";
print ">> $fic\n";
if ( $debug ) { print " ($command)" };
print "\n";
if ( ! $test ) { `$command`; }
}
}
elsif ( $_ =~ /^deleting / ) {
($fic) = ($_ =~ /^deleting (.*)$/);
if ( ! $test ) { print DEL "$fic\n"; }
print ">>> $command\n\n" };
if ( ! $test ) { `$command`; }
}
else {
print "\nCOMPRESSION: no differential to compress (differential is probably null)\n";
}
# Next step extension can only be .tar.gz
$extension = ".tar.gz";
}
# Optional Encryption
# openssl enc -bf-cbc -salt -in $1 -out $2 -k 'pass phrase'
# openssl enc -bf-cbc -d -salt -in $1 -out $2
if ( $encryption && ! $test ) {
#$command ="$cmd_prefix$openssl enc -bf-cbc -salt -in '$delta_dir/$now-delta-backup-new.tar.gz' -out '$delta_dir/$now-delta-backup-new.tar.gz.bf-cbc' -k '$encryption_password'$cmd_suffix";
$rc_test = system("$cmd_prefix test -f '$delta_dir/$now-delta-backup-new$extension' $cmd_suffix");
#$command ="$cmd_prefix if [ -f $delta_dir/$now-delta-backup-new$extension ]; then $openssl enc -bf-cbc -salt -in '$delta_dir/$now-delta-backup-new$extension' -out '$delta_dir/$now-delta-backup-new$extension.bf-cbc' -k '$encryption_password'; fi; if [ $? == 0 ]; then $rm -f $delta_dir/$now-delta-backup-new$extension; fi $cmd_suffix";
$command ="$cmd_prefix $openssl enc -bf-cbc -salt -in '$delta_dir/$now-delta-backup-new$extension' -out '$delta_dir/$now-delta-backup-new$extension.bf-cbc' -k '$encryption_password'; if [ $? == 0 ]; then $rm -f $delta_dir/$now-delta-backup-new$extension; fi $cmd_suffix";
if ( $rc_test == 0 ) {
print "\nENCRYPTION: creating $delta_dir/$now-delta-backup-new$extension.bf-cbc\n";
if ( $debug ) { print ">>>> $command\n\n" };
if ( ! $test ) {
`$command`;
print "\nTo decrypt delta archive use following command: openssl enc -bf-cbc -d -salt -in $delta_dir/$now-delta-backup-new$extension.bf-cbc -out $delta_dir/$now-delta-backup-new$extension\n";
}
}
else {
print "\nENCRYPTION: no differential to encrypt (differential is probably null)\n";
}
$rc_test = system("$cmd_prefix test -f '$delta_dir/$now-delta-backup-deleted' $cmd_suffix");
#$command ="$cmd_prefix if [ -f $delta_dir/$now-delta-backup-deleted ]; then $openssl enc -bf-cbc -salt -in '$delta_dir/$now-delta-backup-deleted' -out '$delta_dir/$now-delta-backup-deleted.bf-cbc' -k '$encryption_password'; fi; if [ $? == 0 ]; then $rm -f $delta_dir/$now-delta-backup-deleted; fi $cmd_suffix";
$command ="$cmd_prefix $openssl enc -bf-cbc -salt -in '$delta_dir/$now-delta-backup-deleted' -out '$delta_dir/$now-delta-backup-deleted.bf-cbc' -k '$encryption_password'; if [ $? == 0 ]; then $rm -f $delta_dir/$now-delta-backup-deleted; fi $cmd_suffix";
if ( $rc_test == 0 ) {
print "\nENCRYPTION: creating $delta_dir/$now-delta-backup-deleted.bf-cbc\n";
if ( $debug ) { print ">>>> $command\n\n" };
if ( ! $test ) {
`$command`;
print "\nTo decrypt list of deleted files use following command: openssl enc -bf-cbc -d -salt -in $delta_dir/$now-delta-backup-deleted.bf-cbc -out $delta_dir/$now-delta-backup-deleted\n";
}
}
else {
print "\nENCRYPTION: no list of deleted files to encrypt\n";
}
$rc_test = system("$cmd_prefix test -f '$delta_dir/$now-delta-backup-rsync.o' $cmd_suffix");
#$command ="$cmd_prefix if [ -f $delta_dir/delta-backup-rsync.out ]; then $openssl enc -bf-cbc -salt -in '$delta_dir/delta-backup-rsync.out' -out '$delta_dir/delta-backup-rsync.out.bf-cbc' -k '$encryption_password'; fi; if [ $? == 0 ]; then $rm -f $delta_dir/delta-backup-rsync.out; fi $cmd_suffix";
$command ="$cmd_prefix if [ -f $delta_dir/$now-delta-backup-rsync.o ]; then $openssl enc -bf-cbc -salt -in '$delta_dir/$now-delta-backup-rsync.o' -out '$delta_dir/$now-delta-backup-rsync.o.bf-cbc' -k '$encryption_password'; fi; if [ $? == 0 ]; then $rm -f $delta_dir/$now-delta-backup-rsync.o; fi $cmd_suffix";
if ( $rc_test == 0 ) {
print "\nENCRYPTION: creating $delta_dir/$now-delta-backup-rsync.o.bf-cbc\n";
if ( $debug ) { print ">>>> $command\n\n" };
if ( ! $test ) {
`$command`;
print "\nTo decrypt the rsync output file use following command: openssl enc -bf-cbc -d -salt -in $delta_dir/$now-delta-backup-rsync.o.bf-cbc -out $delta_dir/$now-delta-backup-rsync.o\n";
}
}
else {
print "\nENCRYPTION: no rsync output file to encrypt\n";
}
}
`$rm -f $fic_output_rsync`;
if ( ! $test && ! $init ) {
$command_du="$cmd_prefix $du -sh $delta_dir $cmd_suffix";
$command_du_res=`$command_du`;
print "\nCurrent size of delta directory : ", $command_du_res, "\n";
print "Don't forget to backup the delta directory on a non-rewritable medium.\n";
}
print "\n";
host:source_directory_path (the directory to backup). If host is the local host, then "local" should be specified for the host.
-B