Skip to content

Instantly share code, notes, and snippets.

@vanzhiganov
Forked from udienz/rsync-debian.sh
Created July 19, 2023 19:53
Show Gist options
  • Save vanzhiganov/5f515fc85244d662e4df2183b62f62f0 to your computer and use it in GitHub Desktop.
Save vanzhiganov/5f515fc85244d662e4df2183b62f62f0 to your computer and use it in GitHub Desktop.
#! /bin/sh
set -e
# This script originates from http://www.debian.org/mirror/anonftpsync
# CVS: cvs.debian.org:/cvs/webwml - webwml/english/mirror/anonftpsync
# Version: $Id: anonftpsync,v 1.43 2008-06-15 18:16:04 spaillar Exp $
# Note: You MUST have rsync 2.6.4 or newer, which is available in sarge
# and all newer Debian releases, or at http://rsync.samba.org/
# Don't forget:
# chmod u+x anonftpsync
# Set the variables below to fit your site. You can then use cron to have
# this script run daily to automatically update your copy of the archive.
# TO is the destination for the base of the Debian mirror directory
# (the dir that holds dists/ and ls-lR).
# (mandatory)
TO=/ftp/debian/
# RSYNC_HOST is the site you have chosen from the mirrors file.
# (http://www.debian.org/mirror/list-full)
# (mandatory)
RSYNC_HOST=mirror.unej.ac.id
# RSYNC_DIR is the directory given in the "Packages over rsync:" line of
# the mirrors file for the site you have chosen to mirror.
# (mandatory)
RSYNC_DIR=debian
# LOGDIR is the directory where the logs will be written to
# (mandatory)
LOGDIR=/var/log/mirror
# ARCH_EXCLUDE can be used to exclude a complete architecture from
# mirrorring. Please use as space seperated list.
# Possible values are:
# alpha, amd64, arm, armel, hppa, hurd-i386, i386, ia64, m68k, mipsel, mips, powerpc, s390, sh and sparc
#
# There is one special value: source
# This is not an architecture but will exclude all source code in /pool
#
# eg.
# ARCH_EXCLUDE="alpha arm armel hppa hurd-i386 ia64 m68k mipsel mips s390 sparc"
#
# With a blank ARCH_EXCLUDE you will mirror all available architectures
# (optional)
ARCH_EXCLUDE="alpha arm armel hppa hurd-i386 ia64 m68k mipsel mips powerpc s390 sparc"
# EXCLUDE is a list of parameters listing patterns that rsync will exclude, in
# addition to the architectures excluded by ARCH_EXCLUDE.
#
# Use ARCH_EXCLUDE to exclude specific architectures or all sources
#
# --exclude stable, testing, unstable options DON'T remove the packages of
# the given distribution. If you want do so, use debmirror instead.
#
# The following example would exclude mostly everything:
#EXCLUDE="\
# --exclude stable/ --exclude testing/ --exclude unstable/ \
# --exclude source/ \
# --exclude *.orig.tar.gz --exclude *.diff.gz --exclude *.dsc \
# --exclude /contrib/ --exclude /non-free/ \
# "
# With a blank EXCLUDE you will mirror the entire archive, except the
# architectures excluded by ARCH_EXCLUDE.
# (optional)
EXCLUDE=
# MAILTO is the address to send logfiles to;
# if it is not defined, no mail will be sent
# (optional)
MAILTO=webmaster@$HOSTNAME
# LOCK_TIMEOUT is a timeout in minutes. Defaults to 360 (6 hours).
# This program creates a lock to ensure that only one copy
# of it is mirroring any one archive at any one time.
# Locks held for longer than the timeout are broken, unless
# a running rsync process appears to be connected to $RSYNC_HOST.
LOCK_TIMEOUT=360
# You may establish the connection via a web proxy by setting the environment
# variable RSYNC_PROXY to a hostname:port pair pointing to your web proxy. Note
# that your web proxy’s configuration must support proxy connections to port 873.
#
# RSYNC_PROXY="IP:PORT"
# export RSYNC_PROXY=$RSYNC_PROXY
# There should be no need to edit anything below this point, unless there
# are problems.
#-----------------------------------------------------------------------------#
# If you are accessing a rsync server/module which is password-protected,
# uncomment the following lines (and edit the other file).
# . ftpsync.conf
# export RSYNC_PASSWORD
# RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST
#-----------------------------------------------------------------------------#
# Check for some environment variables
if [ -z "$TO" ] || [ -z "$RSYNC_HOST" ] || [ -z "$RSYNC_DIR" ] || [ -z "$LOGDIR" ]; then
echo "One of the following variables seems to be empty:"
echo "TO, RSYNC_HOST, RSYNC_DIR or LOGDIR"
exit 2
fi
# Note: on some non-Debian systems, hostname doesn't accept -f option.
# If that's the case on your system, make sure hostname prints the full
# hostname, and remove the -f option. If there's no hostname command,
# explicitly replace `hostname -f` with the hostname.
HOSTNAME=`hostname -f`
# The hostname must match the "Site" field written in the list of mirrors.
# If hostname doesn't returns the correct value, fill and uncomment below
# HOSTNAME=mirror.domain.tld
LOCK="${TO}/Archive-Update-in-Progress-${HOSTNAME}"
# The temp directory used by rsync --delay-updates is not
# world-readable remotely. It must be excluded to avoid errors.
TMP_EXCLUDE="--exclude .~tmp~/"
# Exclude architectures defined in $ARCH_EXCLUDE
for ARCH in $ARCH_EXCLUDE; do
EXCLUDE=$EXCLUDE"\
--exclude binary-$ARCH/ \
--exclude disks-$ARCH/ \
--exclude installer-$ARCH/ \
--exclude Contents-$ARCH.gz \
--exclude Contents-$ARCH.diff/ \
--exclude arch-$ARCH.files \
--exclude arch-$ARCH.list.gz \
--exclude *_$ARCH.deb \
--exclude *_$ARCH.udeb "
if [ "$ARCH" = "source" ]; then
SOURCE_EXCLUDE="\
--exclude source/ \
--exclude *.tar.gz \
--exclude *.diff.gz \
--exclude *.dsc "
fi
done
# Logfile
LOGFILE=$LOGDIR/debian-mirror.log
# optionally, use the rsync module name in the log file name:
# LOGFILE=$LOGDIR/$(echo $RSYNC_DIR | tr / _)-mirror.log
# LOGFILE=$LOGDIR/${RSYNC_DIR/\//_}-mirror.log
# Get in the right directory and set the umask to be group writable
#
cd $HOME
umask 002
# If we are running mirror script for the first time, create the
# destination directory and the trace directory underneath it
if [ ! -d "${TO}/project/trace/" ]; then
mkdir -p ${TO}/project/trace
fi
# Check to see if another sync is in progress
if [ -f "$LOCK" ]; then
# Note: this requires the findutils find; for other finds, adjust as necessary
if [ "`find $LOCK -maxdepth 1 -cmin -$LOCK_TIMEOUT`" = "" ]; then
# Note: this requires the procps ps; for other ps', adjust as necessary
if ps ax | grep '[r]'sync | grep -q $RSYNC_HOST; then
echo "stale lock found, but a rsync is still running, aiee!"
exit 1
else
echo "stale lock found (not accessed in the last $LOCK_TIMEOUT minutes), forcing update!"
rm -f $LOCK
fi
else
echo "current lock file exists, unable to start rsync!"
exit 1
fi
fi
touch $LOCK
# Note: on some non-Debian systems, trap doesn't accept "exit" as signal
# specification. If that's the case on your system, try using "0".
trap "rm -f $LOCK" exit
set +e
# First sync stage
rsync -rltvHSB8192 --timeout 3600 --stats --partial --times \
--exclude Packages* --exclude Sources* \
--exclude Release* --exclude ls-lR* --exclude .~tmp~/ \
$TMP_EXCLUDE $EXCLUDE $SOURCE_EXCLUDE \
$RSYNC_HOST::$RSYNC_DIR $TO >> $LOGFILE 2>&1
result=$?
if [ "$result" = 0 ]; then
# Second sync stage
rsync -rltvHSB8192 --timeout 3600 --stats \
--exclude project/trace/$HOSTNAME --max-delete=40000 --delay-updates --delete \
--delete-after --delete-excluded --exclude .~tmp~/ \
--exclude "Archive-Update-in-Progress-${HOSTNAME}" \
--exclude "project/trace/${HOSTNAME}" \
$TMP_EXCLUDE $EXCLUDE $SOURCE_EXCLUDE \
$RSYNC_HOST::$RSYNC_DIR/ $TO >> $LOGFILE 2>&1
LANG=C date -u > "${TO}/project/trace/${HOSTNAME}"
else
echo "ERROR: Help, something weird happened" | tee -a $LOGFILE
echo "mirroring /pool exited with exitcode" $result | tee -a $LOGFILE
fi
if [ -n "$MAILTO" ]; then
mail -s "debian archive synced" $MAILTO < $LOGFILE
fi
savelog $LOGFILE >/dev/null
rm $LOCK
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment