contrib/obs-mirror: import

Import scripts from OS#4862 to have them properly versioned. We use this
to replicate Osmocom related packages from OBS to:
https://download.osmocom.org/obs-mirror

Put it in contrib/, because it doesn't need to get installed to the
jenkins nodes (as it would happen with everything in scripts/).

Related: OS#4862
Change-Id: I3f47cddbcee6c9b85fde77a039dcd5b54dd89f2d

5 files changed, 99 insertions, 0 deletions

diff --git a/contrib/obs-mirror/README b/contrib/obs-mirror/README
new file mode 100644
index 0000000..4cab14b
--- /dev/null
+++ b/contrib/obs-mirror/README
@@ -0,0 +1,7 @@
+The scripts in this directory are used to create an archive of Osmocom related
+packages from OBS, at: https://downloads.osmocom.org/obs-mirror/
+
+There is no mechanism in place to deploy updated scripts after updating them
+here in osmo-ci.git, so this must be done manually.
+
+See OS#4862 for more information.
diff --git a/contrib/obs-mirror/obs-mirror-include.txt b/contrib/obs-mirror/obs-mirror-include.txt
new file mode 100644
index 0000000..037ce3f
--- /dev/null
+++ b/contrib/obs-mirror/obs-mirror-include.txt
@@ -0,0 +1,9 @@
+latest/CentOS_8
+latest/Debian_10
+latest/Raspbian_10
+latest/xUbuntu_21.04
+
+nightly/CentOS_8
+nightly/Debian_10
+nightly/Raspbian_10
+nightly/xUbuntu_21.04
diff --git a/contrib/obs-mirror/obs-mirror.sh b/contrib/obs-mirror/obs-mirror.sh
new file mode 100755
index 0000000..76317c4
--- /dev/null
+++ b/contrib/obs-mirror/obs-mirror.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Mirror script to create a local archive of the OBS packages in the network:osmocom
+# repositories / projects
+#
+# We are using hard-links to perform de-duplication on the local side; only
+# those files that changed compoared to the previous run will be rsync'ed over
+# from the remote side
+#
+# See also: OS#4862
+
+set -e -x
+
+# base directory on the local side
+BASE_DIR="/downloads/obs-mirror/"
+# sync remote where to find the osmocom packages
+REMOTE="rsync.opensuse.org::opensuse-full-really-everything-including-repositories/opensuse/repositories/network:/osmocom:"
+
+cd "$BASE_DIR"
+
+RSYNC_ARGS="-av --delete"
+RSYNC_ARGS="$RSYNC_ARGS --files-from /home/pkgmirror/obs-mirror-include.txt --recursive"
+DATE=`date +%Y%m%d-%H%M%S`
+
+# create output directory
+DIR="$BASE_DIR/$DATE"
+mkdir -p "$DIR"
+
+PREVIOUS="$BASE_DIR/.previous"
+if [ -d "$PREVIOUS" ]; then
+	RSYNC_ARGS+=" --link-dest=$PREVIOUS"
+fi
+
+# finally, perform rsync
+# || true: don't stop here if one of the dirs from the include list does not exist
+rsync $RSYNC_ARGS "$REMOTE"/ "$DIR"/ || true
+
+# update '.previous' for the next run
+rm -f "$PREVIOUS"
+ln -sf "$DATE" "$PREVIOUS"
diff --git a/contrib/obs-mirror/rm-old-nightly-archives.sh b/contrib/obs-mirror/rm-old-nightly-archives.sh
new file mode 100755
index 0000000..3a5f276
--- /dev/null
+++ b/contrib/obs-mirror/rm-old-nightly-archives.sh
@@ -0,0 +1,31 @@
+#!/bin/sh -e
+# Remove nightly archives older than one month (OS#4862)
+DRY=0
+
+# Get removal date in seconds since epoch and display it
+DATE_RM_SEC=$(expr $(date +%s) - 3600 \* 24 \* 32)
+DATE_RM_STR=$(date -d "@$DATE_RM_SEC" +"%Y-%m-%d")
+echo "Removing nightly archives from $DATE_RM_STR and older"
+
+cd /downloads/obs-mirror
+
+for i in */nightly; do
+        # "Last modified" isn't set to the date of the dir name for some
+        # archives, so parse the date from the dir name instead
+        DATE_DIR="$(basename "$(dirname "$i")")"  # e.g. "20210604-002301"
+        DATE_DIR_SEC="$(date -d "$(echo "$DATE_DIR" | cut -d "-" -f 1)" +%s)"
+        if [ -z "$DATE_DIR_SEC" ]; then
+                echo "ERROR: $i: failed to parse date from dir name"
+                continue
+        fi
+
+        if [ "$DATE_DIR_SEC" -lt "$DATE_RM_SEC" ]; then
+                DATE_DIR_STR="$(date -d "@$DATE_DIR_SEC" +"%Y-%m-%d")"
+                echo "Removing $i ($DATE_DIR_STR)..."
+                if [ "$DRY" = 0 ]; then
+                        rm -r "$i"
+                fi
+        fi
+done
+
+echo "Done"
diff --git a/contrib/obs-mirror/test-include-list.sh b/contrib/obs-mirror/test-include-list.sh
new file mode 100755
index 0000000..6766403
--- /dev/null
+++ b/contrib/obs-mirror/test-include-list.sh
@@ -0,0 +1,12 @@
+#!/bin/sh -ex
+BASE_DIR="/downloads/obs-mirror/"
+
+cd "$BASE_DIR"
+
+rsync \
+        -a \
+        --list-only \
+        --files-from /home/pkgmirror/obs-mirror-include.txt \
+        --recursive \
+        "$(realpath .previous)"/ \
+        new-backup-dir/