Using rsync to maintain local FC10 updates repo

[Gordon Messmer]

Robert Moskowitz wrote:
> 
> So I wanted to do a simple rsync against one of the mirrors, and tried 
> the script:
> 
> #!/bin/sh
> rsync -auv 
> rsync://mirrors.usc.edu/pub/linux/distributions/fedora/linux/updates/10/i386/ 
> \
> --delete --exclude=debug/ /repos/fedora/10/updates/i386

The rsync paths aren't like the ftp or http paths.  You want:

#!/bin/sh
rsync -auv rsync://mirrors.usc.edu/fedora/linux/updates/10/i386/ \
--delete --exclude=debug/ /repos/fedora/10/updates/i386

Except that this mirror's rsync seems to be broken!  ;)

Since Bruno included his rsync script, I thought I'd throw mine in for 
fun.  You can create a configuration file like the attached 
fedora10.conf and then run "update_mirror fedora10.conf init" to build a 
new local mirror, and then run "update_mirror fedora10.conf update" in 
the future to pull in new updates.  It's very space efficient, as it'll 
remove packages from the base when they appear in updates.  The "result" 
directory under the MIRROR base will have a tree from which you can install.

I'm still using this to build CentOS mirrors myself, but I've given it 
up in favor of Revisor for Fedora mirrors, and will be dropping it soon 
for my CentOS mirrors.  Revisor is a much better tool.  It takes 
slightly longer to update than the rsync script, but it uses about the 
same amount of bandwidth, since it only downloads new packages, and it's 
more exact than the rsync script.
-------------- next part --------------
#!/bin/sh


MIRROR=/home/gordon/deploy/mirrors
SYNC="rsync -avH --delete --exclude debug --exclude '*-debuginfo-*.rpm'"

fail_cd () {
    echo "Could not cd to $1"
    exit 1
}

usage () {
	cat <<EOF
Use: update_mirror [config_file] [command]
Command must be one of "init", or "update".
EOF
	exit
}

init () {
	# Init will reinitialize all of the mirrors.
	for ARCH in $ARCHLIST ; do
		echo "Initializing ${NAME} release ${RELEASE} for ${ARCH}"
		BASEPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/${BASE}/"
		echo "Downloading base OS to ${BASEPATH}"
		mkdir -p ${MIRROR}/${NAME}-${RELEASE}-${ARCH}
		eval URL="\$URL_${BASE}"
		eval $SYNC ${URL} ${BASEPATH}
		# Build a list of packages in the base for update-only repos
		BASEPKGSYNCLIST=${BASEPATH%/}.synclist
		find ${BASEPATH} -type f -a -name '*.rpm' \
			| xargs rpm -qp --qf "+ %{NAME}-*.rpm\n" \
			> ${BASEPKGSYNCLIST}
		for REPO in $ADDITIONAL ; do
			REPOPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/${REPO}/"
			sync_additional 0
		done
		for REPO in $UPDATES ; do
			REPOPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/${REPO}/"
			sync_additional 1
		done
	done
}

sync_additional () {
	UPDATEONLY="$1"
	if [ "$UPDATEONLY" = "1" -a -r "${BASEPKGSYNCLIST}" ]; then
		SYNCLIST="--include-from ${BASEPKGSYNCLIST} --include '*/' --include 'repodata/**' --exclude '*'"
	fi
	echo "Updating \"${REPO}\" in ${REPOPATH}"
	eval URL="\$URL_${REPO}"
	eval $SYNC ${SYNCLIST} ${URL} ${REPOPATH}
	if test $? -ne 0 ; then
		echo "Update for \"${REPO}\" failed, aborting script"
		exit 0
	fi
	unset SYNCLIST
	# Set the marker if anything was actually downloaded.  This marker
	# and the beacon file were created in the update function.
	UPDATECOUNT=$(find "${REPOPATH}" -type f -cnewer "${BEACONPATH}" | wc -l)
	test $UPDATECOUNT -gt 0 && UPDATESFOUND=1
}

update () {
	for ARCH in $ARCHLIST ; do
		echo "Updating ${NAME} release ${RELEASE} for ${ARCH}"
		BASEPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/${BASE}/"
		BASEPKGSYNCLIST=${BASEPATH%/}.synclist
		RESULTPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/result/"
		BEACONPATH="$(mktemp /tmp/beacon.XXXXXX)"
		UPDATESFOUND=0
		for REPO in $ADDITIONAL ; do
			REPOPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/${REPO}/"
			sync_additional 0
		done
		for REPO in $UPDATES ; do
			REPOPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/${REPO}/"
			sync_additional 1
		done
		rm "${BEACONPATH}"
		if test "${UPDATESFOUND}" -eq 0 ; then
			echo "No new packages were found.  Skipping repomd update."
			continue
		fi
		echo "Cleaning up result directory"
		test -e ${RESULTPATH}/RPMS && rm -rf ${RESULTPATH}/RPMS
		mkdir -p ${RESULTPATH}/{repodata,images/xen,RPMS}
		ln -f ${BASEPATH}/repodata/*comps.xml ${RESULTPATH}/repodata/
		ln -f ${BASEPATH}/images/*.img ${RESULTPATH}/images/
		ln -f ${BASEPATH}/images/xen/* ${RESULTPATH}/images/xen/
		for REPO in $ADDITIONAL $UPDATES ; do
			REPOPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/${REPO}/"
			echo "Removing packages which exist in \"${REPO}\" from base"
			# If any package in this repository exists in the base, remove it.
			BASEPKGLIST=$(mktemp /tmp/baselist.XXXXXX)
			REPOPKGLIST=$(mktemp /tmp/repolist.XXXXXX)
			find ${BASEPATH} -type f -a -name '*.rpm' \
				| xargs rpm -qp --qf "%{NAME}:%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}.rpm\n" \
				> ${BASEPKGLIST}
			find ${REPOPATH} -type f -a -name '*.rpm' \
				| xargs rpm -qp --qf "%{NAME}\n" \
				| sort | uniq \
				> ${REPOPKGLIST}
			egrep "^("$(tr '\n' '|' < ${REPOPKGLIST})"):" ${BASEPKGLIST} \
				| cut -f2- -d: \
				| while read file ; do
					find ${BASEPATH} -type f -a -name "${file}"
				done \
				| xargs --no-run-if-empty rm
			rm -f ${BASEPKGLIST} ${REPOPKGLIST}
			find ${REPOPATH} -type f -a -name '*.rpm' \
				| xargs --no-run-if-empty ln -t ${RESULTPATH}/RPMS/
		done
		find ${BASEPATH} -type f -a -name '*.rpm' \
			| xargs --no-run-if-empty ln -t ${RESULTPATH}/RPMS/
		echo "Updating repomd data in ${RESULTPATH}"
		# Update the repodata that anaconda uses.
		pushd ${RESULTPATH}
			createrepo -g repodata/*comps.xml ${RESULTPATH}
		popd
	done
}

test -f "$1" || usage
. $1

case $2 in
	init)
		init
		;;
	update)
		update
		;;
	*)
		usage
		;;
esac

exit

-------------- next part --------------
NAME=fedora
RELEASE=10
ARCHLIST="i386 x86_64"
ARCHLIST="i386"
BASE="os"
UPDATES="updates"
URL_os='rsync://mirrors.kernel.org/fedora/releases/test/10-Preview/Fedora/${ARCH}/os/'
URL_updates='rsync://mirrors.kernel.org/fedora/development/${ARCH}/os/'
#URL_os='rsync://mirrors.kernel.org/fedora/releases/${RELEASE}/Fedora/${ARCH}/os/'
#URL_updates='rsync://mirrors.kernel.org/fedora/updates/${RELEASE}/${ARCH}/'
#URL_os='rsync://mirror.anl.gov/fedora/linux/releases/${RELEASE}/Fedora/${ARCH}/os/'
#URL_updates='rsync://mirror.anl.gov/fedora/linux/updates/${RELEASE}/${ARCH}/'