Wiki Table of Contents

Rsync mirror script

Current Rsync Server Script 01/23/2019 for syncing from source forge with direct uploads to head rsync mirror.
Builds rsync mirror directory structure.
Compares and downloads with Source Forge
Sets directly uploaded files to mirror.

#######################################################################################
############################### Start File #############################################
#######################################################################################
#!/bin/bash
# File Name: rsync-server-script
# Dependencies: bash, wget, diff, cat, sed, tee, grep, rm, mount (bind), unmount, cut, rsync
# Version: 2.0
# Purpose: Sync local directory with SourceForge
# Authors: Dave ([email protected])
# Copyright (C) Tuesday, Feb. 7, 2011 by Dave / [email protected]
# License: gplv2
# This file is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#############################################################################
#File extensions that you would like to be synced
FILE_TYPES=(".iso" ".md5" ".delta" ".zsync" ".sig" ".sha256")
#Directory to log in, place temporary list files. Set full path.
WORK_DIR="/MIRROR/TEMP"
#Root directory that the apache/rsyncd service points to. Set as full path.
SYNC_DIR="/MIRROR/SYNC"
#Source Forge Directory Local Storage Director
SF_DIR="/MIRROR/SF"
#Direct FTP Upload Directory
DIRECT_DIR="/MIRROR/UPLOADS"
#Mirmon Time File location
MIRMON_TIME="/MIRROR/mirmon/TIME.txt"
#The 4 lines below should not need to be changed manually
PID_FILE="$WORK_DIR/running.pid"
LOG_FILE="$WORK_DIR/log.txt"
TODAY=$(date)
PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
#Set Variables For antiX
function set_vbs_antix(){
DIST='ANTIX'
#Paths set as subdirectories in SourceForge
#Compare below to here to vizualize (https://sourceforge.net/projects/antix-linux/files/?source=navbar)
PATHS=("/Final/antiX-17/" "/Final/antiX-17.1/" "/Final/antiX-17.2/" "/Final/antiX-17.3/")
#Source Forge Reference URLS
SOURCEFORGE_RSS='https://sourceforge.net/projects/antix-linux/rss?path='
SOURCEFORGE_FILES='https://sourceforge\.net/projects/antix-linux/files'
}
#Set Variables For antiX
function set_vbs_mx(){
DIST='MX'
#Paths set as subdirectories in SourceForge
#Compare below to here to vizualize (https://sourceforge.net/projects/antix-linux/files/?source=navbar)
PATHS=("/Final/antiX-17/" "/Final/antiX-17.1/" "/Final/antiX-17.2/" "/Final/antiX-17.3/")
#Source Forge Reference URLS
SOURCEFORGE_RSS='https://sourceforge.net/projects/mx-linux/rss?path='
SOURCEFORGE_FILES='https://sourceforge\.net/projects/mx-linux/files'
}
function log() {
echo "[email protected]" |tee -a $LOG_FILE
}
#Pass array as ( dir1 dir2 )
function check_dir() {
for i in "[email protected]"; do
log "Checking for directory $i";
if [ ! -d "$i" ]; then
log "Making directory $i";
mkdir -p "$i";
fi
done
}
#Pass array as ( source|mp source2|mp2 )
function mount_dir() {
for i in "[email protected]"; do
to=${i/*|/ }
from=${i/|*/ }
log "Bind mounting $from to $to";
mount --bind $from $to;
done
}
#Pass array as ( mp1 mp2 mp3 )
function unmount_dir() {
for i in "[email protected]"; do
log "unmounting $i";
umount -l $i;
done
}
function get_sflist() {
for path in "${PATHS[@]}"
do
log "Getting Sourceforge RSS for $path..."
wget --no-check-certificate -qO- $SOURCEFORGE_RSS/$path > $WORK_DIR/sfraw-$LIST_BASENAME
log "Sorting Sourceforge RSS to usable list"
for filetype in ${FILE_TYPES[@]}
do
log " - Searching for $filetype"
cat $WORK_DIR/sfraw-$LIST_BASENAME | grep "
$SOURCEFORGE_FILES$path.*$filetype"|sed 's|
|'$path'\||ig'|sed "s///ig" >> $WORK_DIR/sfsorted-$LIST_BASENAME
sed -i "s|
$SOURCEFORGE_FILES$path.*$filetype||g" $WORK_DIR/sfraw-$LIST_BASENAME
done
rm $WORK_DIR/sfraw-$LIST_BASENAME;
done
log "Building basename comparable list for Source Forge"
for line in `cat "$WORK_DIR/sfsorted-$LIST_BASENAME"`
do
base_path=$(echo "$line" |cut -d "|" -f1)
download_url=$(echo "$line" |cut -d "|" -f2)
clean_download_url=$(echo "$download_url" | sed "s|/download||ig")
basename=$(basename "$clean_download_url")
echo "$base_path|$basename|$download_url" >> $WORK_DIR/sf-$LIST_BASENAME
done
rm $WORK_DIR/sfsorted-$LIST_BASENAME;
}
function get_local_list() {
log "Checking local directory layout"
if [ ! -f "$SF_DIR/$DIST" ]; then mkdir -p "$SF_DIR/$DIST"; fi
for path in ${PATHS[@]}
do
if [ ! -f "$SF_DIR/$DIST$path" ]; then mkdir -p "$SF_DIR/$DIST$path"; fi
log "Getting Local listing for $path..."
find "$SF_DIR/$DIST$path" >> $WORK_DIR/raw-$LIST_BASENAME
log "Sorting local listing to usable list"
for filetype in ${FILE_TYPES[@]}
do
log " - Searching for $filetype"
cat $WORK_DIR/raw-$LIST_BASENAME | grep ".*$filetype" >> $WORK_DIR/sorted-$LIST_BASENAME
sed -i "s|.*$filetype.*||g" $WORK_DIR/raw-$LIST_BASENAME
done
rm $WORK_DIR/raw-$LIST_BASENAME;
done
log "Building basename comparable list for Local System"
for line in `cat "$WORK_DIR/sorted-$LIST_BASENAME"`
do
basename=$(basename "$line")
base_path=$(echo "$line" |sed "s|$basename||ig" |sed "s|$SF_DIR/$DIST||ig")
echo "$base_path|$basename|$line" >> $WORK_DIR/local-$LIST_BASENAME
done
rm $WORK_DIR/sorted-$LIST_BASENAME;
}
function diff_compare(){
log "Comparing differences between Source Forge and Local System"
cat $WORK_DIR/local-$LIST_BASENAME|cut -d "|" -f1,2 > $WORK_DIR/local-diff-$DIST.txt
sort $WORK_DIR/local-diff-$DIST.txt -o $WORK_DIR/local-diff-$DIST.txt
cat $WORK_DIR/sf-$LIST_BASENAME|cut -d "|" -f1,2 > $WORK_DIR/sf-diff-$DIST.txt
sort $WORK_DIR/sf-diff-$DIST.txt -o $WORK_DIR/sf-diff-$DIST.txt
diff $WORK_DIR/sf-diff-$DIST.txt $WORK_DIR/local-diff-$DIST.txt > $WORK_DIR/diff-$DIST.txt
sed -i "s| sed -i "s|>\ |>|ig" $WORK_DIR/diff-$DIST.txt
rm $WORK_DIR/local-diff-$DIST.txt
rm $WORK_DIR/sf-diff-$DIST.txt
if grep -q -E "^" $WORK_DIR/diff-$DIST.txt; then
for line in `cat $WORK_DIR/diff-$DIST.txt`
do
if [[ "$line" == " log "Copying file from SourceForge"
path=`echo "$line" |cut -d "|" -f1 |sed "s| filename=`echo "$line" |cut -d "|" -f2`
download_url=`cat $WORK_DIR/sf-$LIST_BASENAME | grep "^$path|$filename|" | cut -d "|" -f3`
wget --no-check-certificate --output-document=$SF_DIR/$DIST$path$filename $download_url
#echo "Downloading $SF_DIR/$DIST$path$filename"
elif [[ "$line" == ">"* ]]; then
log "Removing file from Local System as not on SourceForge"
path=`echo "$line" |cut -d "|" -f1 |sed "s|>||ig"`
filename=`echo "$line" |cut -d "|" -f2`
file=`cat $WORK_DIR/local-$LIST_BASENAME | grep "^$path|$filename|" | cut -d "|" -f3`
rm $file
#echo "Removing $file"
fi
done
else
log "File is empty, nothing to do";
fi
}
function do_work(){
log "Making / checking work environment directory structure"
LIST_BASENAME="$DIST-file-list.txt"
array=("$SYNC_DIR" "$SF_DIR" "$SF_DIR/$DIST" "$DIRECT_DIR" "$SYNC_DIR/$DIST" "$SYNC_DIR/$DIST/Direct")
check_dir ${array[@]}
if [ -f "$WORK_DIR/sf-$LIST_BASENAME" ]; then rm "$WORK_DIR/sf-$LIST_BASENAME"; fi
if [ -f "$WORK_DIR/local-$LIST_BASENAME" ]; then rm "$WORK_DIR/local-$LIST_BASENAME"; fi
if [ -f "$WORK_DIR/diff-$DIST.txt" ]; then rm "$WORK_DIR/diff-$DIST.txt"; fi
touch "$WORK_DIR/sf-$LIST_BASENAME";
touch "$WORK_DIR/local-$LIST_BASENAME";
touch "$WORK_DIR/diff-$DIST.txt";
log "Removing Bind Mounts"
array=("$SYNC_DIR/$DIST/Direct" "$SYNC_DIR/$DIST")
unmount_dir ${array[@]}
log "Syncing Sourceforge"
get_sflist
get_local_list
diff_compare
log "Adding bind mounts";
array=("$SF_DIR/$DIST/|$SYNC_DIR/$DIST" "$DIRECT_DIR/$DIST/|$SYNC_DIR/$DIST/Direct")
mount_dir ${array[@]}
}
check_dir "$WORK_DIR";
if [ -f "$PID_FILE" ]; then
log "Script already running of stale pid file.";
log "Please check why another instance is still running";
exit 1;
fi
if [ -f "$LOG_FILE" ]; then rm "$LOG_FILE"; fi
touch "$LOG_FILE";
log "Starting sync $TODAY"
log "Recording PID"
echo "$$" > $PID_FILE;
log "Stopping rsync Service"
service rsync stop
log "#########################"
log "Building Mirror for antiX"
log "#########################"
set_vbs_antix;
do_work;
log "#########################"
log " Building Mirror for MX "
log "#########################"
set_vbs_mx;
do_work;
log "Starting rsync service"
service rsync start
log "Recording time stamp for mirmon"
date +%s > $SYNC_DIR/TIME.txt
if [ ! -f "$WORK_DIR/sf-$LIST_BASENAME" ]; then ln -s $SYNC_DIR/TIME.txt $MIRMON_TIME; fi
log "Removing PID file before exiting"
rm $PID_FILE
#######################################################################################
############################### End File #############################################
#######################################################################################

Leave a Comment

Do NOT follow this link or you will be banned from the site!