handlers/maildir

   1 ###############################################################
   2 #
   3 #  This handler slowly creates a backup of each user's maildir
   4 #  to a remote server. It is designed to be run with low overhead
   5 #  in terms of cpu and bandwidth so it runs pretty slow.
   6 #
   7 #  if destdir is /backup/maildir/, then it will contain the files
   8 #    daily.1
   9 #    daily.2
  10 #    daily.3
  11 #    weekly.1
  12 #    weekly.2
  13 #    monthly.1
  14 #  if keepdaily is 3, keepweekly is 2, and keepmonthly is 1.
  15 #
  16 ##############################################################
  17
  18 getconf rotate yes
  19 getconf remove yes
  20
  21 getconf loadlimit 5
  22 getconf speedlimit 0
  23 getconf keepdaily 5
  24 getconf keepweekly 3
  25 getconf keepmonthly 1
  26
  27 getconf srcdir /var/maildir
  28 getconf destdir
  29 getconf desthost
  30 getconf destport 22
  31 getconf destuser
  32
  33 # strip trailing /
  34 destdir=${destdir%/}
  35 srcdir=${srcdir%/}
  36
  37 # used for testing
  38 getconf letter
  39 getconf user
  40
  41 [ -d $srcdir ] || fatal "source directory $srcdir doesn't exist"
  42
  43 [ ! $test ] || testflags="--dry-run -v"
  44 rsyncflags="$testflags -e 'ssh -p $destport'"
  45 flags_mail="$rsyncflags --archive --ignore-existing --delete --numeric-ids --size-only --bwlimit=$speedlimit"
  46 flags_folders="$rsyncflags --archive --delete --numeric-ids"
  47 excludes='--exclude ".Trash/*" --exclude ".Mistakes/*" --exclude ".Spam/*"'
  48
  49 # see if we can login
  50 debug "ssh -o PasswordAuthentication=no $desthost -l $destuser 'echo -n 1'"
  51 if [ ! $test ]; then
  52         result=`ssh -o PasswordAuthentication=no $desthost -l $destuser 'echo -n 1' 2>&1`
  53         if [ "$result" != "1" ]; then
  54                 fatal "Can't connect to $desthost as $destuser."
  55         fi
  56 fi
  57
  58 ##################################################################
  59 ### FUNCTIONS
  60
  61 # remote run the args remotely
  62 function rrun() {
  63         debug ssh -o PasswordAuthentication=no $desthost -l $destuser $@
  64         if [ ! $test ]; then
  65                 debug ssh -o PasswordAuthentication=no $desthost -l $destuser $@
  66         fi
  67 }
  68
  69 function do_letters() {
  70         for i in a b c d e f g h i j k l m n o p q r s t u v w x y z; do
  71                 do_maildirs "$srcdir/$i"
  72         done
  73 }
  74
  75 function do_maildirs() {
  76         local dir=$1
  77         [ -d $dir ] || fatal "directory $dir not found."
  78         for userdir in `ls -1 $dir`; do
  79                 do_userdir $userdir
  80         done
  81 }
  82
  83 function do_user() {
  84         local user=$1
  85         local letter=${user:0:1}
  86         local dir="$srcdir/$letter/$user"
  87         [ -d $dir ] || fatal "maildir $dir not found".
  88
  89         while 1; do
  90                 load=`uptime | sed 's/^.*load average: \\([^,]*\\).*$/\\1/'`
  91                 if [ $load -lt $loadlimit ]; then
  92                         info "load $load, sleeping..."
  93                         sleep 600
  94                 else
  95                         break
  96                 fi
  97         done
  98
  99         cmd="rsync $maildirrsyncflags $excludes '$dir' '$destuser@$desthost:$destdir/maildir/$letter'"
 100         debug $cmd
 101         # ret=`rsync $maildirrsyncflags $excludes '$dir' '$destuser@$desthost:$destdir/maildir/$letter' 2>&1`
 102 }
 103
 104 # remove any maildirs from backup which might have been deleted
 105 # and add new ones which have just been created.
 106
 107 function do_remove() {
 108         local tmp1=/tmp/maildirtmpfile$$
 109         local tmp2=/tmp/maildirtmpfile$$
 110
 111         for i in a b c d e f g h i j k l m n o p q r s t u v w x y z; do
 112                 ls -1 "$srcdir/$i" | sort > $tmp1
 113                 ssh -p $destport $desthost ls -1 '$destdir/maildir/$i' | sort > $tmp2
 114                 for deluser in `join -v 2 $tmp1 $tmp2`; do
 115                         cmd="ssh -p $destport $desthost rm -vr '$destdir/maildir/$i/$deluser/'"
 116                         debug $cmd
 117                 done
 118         done
 119         rm $tmp1
 120         rm $tmp2
 121 }
 122
 123 function do_rotate() {
 124         backuproot=$destdir
 125
 126         ssh -T -o PasswordAuthentication=no $desthost -l $destuser <<EOF
 127 ##### BEGIN REMOTE SCRIPT #####
 128         seconds_daily=86400
 129         seconds_weekly=604800
 130         seconds_monthly=2628000
 131         keepdaily=$keepdaily
 132         keepweekly=$keepweekly
 133         keepmonthly=$keepmonthly
 134         now=\`date +%s\`
 135
 136         for rottype in daily weekly monthly; do
 137                 seconds=\$((seconds_\${rottype}))
 138
 139                 dir="$backuproot/\$rottype"
 140                 if [ ! -d \$dir.1 ]; then
 141                         echo "Info: \$dir.1 does not exist. This backup is missing, so we are skipping the rotation."
 142                         continue 1
 143                 elif [ ! -f \$dir.1/created ]; then
 144                         echo "Warning: \$dir.1/created does not exist. This backup may be only partially completed. Skipping rotation."
 145                         continue 1
 146                 fi
 147
 148                 # Rotate the current list of backups, if we can.
 149                 oldest=\`ls -d \$dir.* | sed 's/^.*\.//' | sort -n | tail -1\`
 150                 [ "\$oldest" == "" ] && oldest=0
 151                 for (( i=\$oldest; i > 0; i-- )); do
 152                         if [ -d \$dir.\$i ]; then
 153                                 if [ -f \$dir.\$i/rotated ]; then
 154                                         rotated=\`tail -1 \$dir.\$i/rotated\`
 155                                 else
 156                                         rotated=0
 157                                 fi
 158                                 cutoff_time=\$(( now - (seconds*i) ))
 159                                 if [ \$rotated -lt \$cutoff_time ]; then
 160                                         next=\$(( i + 1 ))
 161                                         if [ ! -d \$dir.\$next ]; then
 162                                                 echo "mv \$dir.\$i \$dir.\$next"
 163                                                 mv \$dir.\$i \$dir.\$next
 164                                                 date +%c%n%s > \$dir.\$next/rotated
 165                                         else
 166                                                 echo "Info: skipping rotation of \$dir.\$i because \$dir.\$next already exists."
 167                                         fi
 168                                 else
 169                                         echo "Info: skipping rotation of \$dir.\$i because it was rotated" \$(( (now-rotated)/86400)) "days ago ("\$(( (now-cutoff_time)/86400))" needed)."
 170                                 fi
 171                         fi
 172                 done
 173         done
 174
 175         max=\$((keepdaily+1))
 176         if [ \( \$keepweekly -gt 0 -a -d $backuproot/daily.\$max \) -a ! -d $backuproot/weekly.1 ]; then
 177                 echo mv $backuproot/daily.\$max $backuproot/weekly.1
 178                 mv $backuproot/daily.\$max $backuproot/weekly.1
 179                 date +%c%n%s > $backuproot/weekly.1/rotated
 180         fi
 181
 182         max=\$((keepweekly+1))
 183         if [ \( \$keepmonthly -gt 0 -a -d $backuproot/weekly.\$max \) -a ! -d $backuproot/monthly.1 ]; then
 184                 echo mv $backuproot/weekly.\$max $backuproot/monthly.1
 185                 mv $backuproot/weekly.\$max $backuproot/monthly.1
 186                 date +%c%n%s > $backuproot/monthly.1/rotated
 187         fi
 188
 189         for rottype in daily weekly monthly; do
 190                 max=\$((keep\${rottype}+1))
 191                 dir="$backuproot/\$rottype"
 192                 oldest=\`ls -d \$dir.* | sed 's/^.*\.//' | sort -n | tail -1\`
 193                 [ "\$oldest" == "" ] && oldest=0
 194                 # if we've rotated the last backup off the stack, remove it.
 195                 for (( i=\$oldest; i >= \$max; i-- )); do
 196                         if [ -d \$dir.\$i ]; then
 197                                 echo "Info: removing \$dir.\$i"
 198                                 rm -rf \$dir.\$i
 199                         fi
 200                 done
 201         done
 202 ####### END REMOTE SCRIPT #######
 203 EOF
 204 }
 205
 206
 207 ###
 208 ##################################################################
 209
 210 ### ROTATE BACKUPS ###
 211
 212 if [ "$rotate" == "yes" ]; then
 213         do_rotate
 214 fi
 215
 216 ### REMOVE OLD MAILDIRS ###
 217
 218 if [ "$remove" == "yes" ]; then
 219         debug remove
 220 fi
 221
 222 ### ROTATE BACKUPS ###
 223
 224 if [ "$letter" != "" ]; then
 225         debug letter
 226 fi
 227
 228 if [ "$user" != "" ]; then
 229         debug user
 230 fi