jobs/osmocom-release-tarballs: modernize

Rewrite the osmocom-release-manuals script (previous version is in
docker-playground Ic35a28a386170b85d32aab8f2bd33e48e6d45392):

* Instead of using a separate docker container for this, that also lists
  all dependencies for all packages (as needed to pass ./configure), use
  debian-bookworm-build and install missing packages at time of
  generating the tarballs with "apt-get build-dep". Missing dependencies
  are typically other Osmocom libraries.

* This allows removing the debian 11 based release-tarball-build-dist
  container. As the script doesn't depend on a separate docker container
  anymore, move it to osmo-ci.git.

* Make it similar to scripts/manuals/publish-manuals-for-tags.sh, so it
  is easier to maintain both.

Related: OS#6057
Change-Id: I9f8b671b9780da500637a64fc4dbc72b450f9d11

4 files changed, 351 insertions, 6 deletions

diff --git a/.gitignore b/.gitignore
index 3fcbe85..517a620 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,4 +13,5 @@ _release_tarballs/
 _docker_playground
 _repo_install_test_data/
 _temp_manuals/
+_temp_releases/
 scripts/repo-install-test/run-inside-env.sh
diff --git a/contrib/known_hosts b/contrib/known_hosts
new file mode 100644
index 0000000..de29e93
--- /dev/null
+++ b/contrib/known_hosts
@@ -0,0 +1,3 @@
+[ftp.osmocom.org]:48 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDDgQ9HntlpWNmh953a2Gc8NysKE4orOatVT1wQkyzhARnfYUerRuwyNr1GqMyBKdSI9amYVBXJIOUFcpV81niA7zQRUs66bpIMkE9/rHxBd81SkorEPOIS84W4vm3SZtuNqa+fADcqe88Hcb0ZdTzjKILuwi19gzrQyME2knHY71EOETe9Yow5RD2hTIpB5ecNxI0LUKDq+Ii8HfBvndPBIr0BWYDugckQ3Bocf+yn/tn2/GZieFEyFpBGF/MnLbAAfUKIdeyFRX7ufaiWWz5yKAfEhtziqdAGZaXNaLG6gkpy3EixOAy6ZXuTAk3b3Y0FUmDjhOHllbPmTOcKMry9
+[ftp.osmocom.org]:48 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPdWn1kEousXuKsZ+qJEZTt/NSeASxCrUfNDW3LWtH+d8Ust7ZuKp/vuyG+5pe5pwpPOgFu7TjN+0lVjYJVXH54=
+[ftp.osmocom.org]:48 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK8iivY70EiR5NiGChV39gRLjNpC8lvu1ZdHtdMw2zuX
diff --git a/jobs/osmocom-release-tarballs.yml b/jobs/osmocom-release-tarballs.yml
index 2e6bb04..557898a 100644
--- a/jobs/osmocom-release-tarballs.yml
+++ b/jobs/osmocom-release-tarballs.yml
@@ -19,19 +19,18 @@
           num-to-keep: 30
     parameters:
       - string:
-          name: BRANCH_DOCKER_PLAYGROUND
+          name: BRANCH_OSMO_CI
           description: |
-                docker-playground.git branch where the Dockerfile gets pulled from
+                osmo-ci.git branch with scripts/tarballs/publish-tarballs-for-tags.sh
           default: 'master'
     builders:
       - shell: |
-          cd release-tarball-build-dist
-          ./jenkins.sh
+          PUBLISH=1 scripts/tarballs/publish-tarballs-for-tags.sh
     scm:
       - git:
           branches:
-            - '$BRANCH_DOCKER_PLAYGROUND'
-          url: https://gerrit.osmocom.org/docker-playground
+            - '$BRANCH_OSMO_CI'
+          url: https://gerrit.osmocom.org/osmo-ci
           git-config-name: 'Jenkins Builder'
           git-config-email: 'jenkins@osmocom.org'
           wipe-workspace: true
diff --git a/scripts/tarballs/publish-tarballs-for-tags.sh b/scripts/tarballs/publish-tarballs-for-tags.sh
new file mode 100755
index 0000000..052e445
--- /dev/null
+++ b/scripts/tarballs/publish-tarballs-for-tags.sh
@@ -0,0 +1,342 @@
+#!/bin/sh -e
+. "$(dirname "$0")/../common.sh"
+OSMO_CI_DIR="$(realpath $(dirname "$0")/../..)"
+TEMP="$OSMO_CI_DIR/_temp_releases"
+WEB_PATH="/downloads/home/releases/web-files"
+SSH_COMMAND="ssh -o UserKnownHostsFile=$OSMO_CI_DIR/contrib/known_hosts -p 48"
+DOCKER_IMAGE="$USER/debian-bookworm-build"
+LOG_PREFIX="::"
+
+OSMO_RELEASE_REPOS="
+	libasn1c
+	libgtpnl
+	libosmo-abis
+	libosmo-netif
+	libosmo-pfcp
+	libosmo-sccp
+	libosmocore
+	libsmpp34
+	libusrp
+	osmo-bsc
+	osmo-bts
+	osmo-cbc
+	osmo-e1d
+	osmo-gbproxy
+	osmo-ggsn
+	osmo-gsm-manuals
+	osmo-hlr
+	osmo-hnbgw
+	osmo-hnodeb
+	osmo-iuh
+	osmo-mgw
+	osmo-msc
+	osmo-pcap
+	osmo-pcu
+	osmo-remsim
+	osmo-sgsn
+	osmo-sip-connector
+	osmo-smlc
+	osmo-sysmon
+	osmo-trx
+	osmo-uecups
+	osmocom-bb
+	simtrace2
+"
+
+# Old release tags that are duplicates or where generating the tarball fails
+TAGS_IGNORE="
+	libosmocore:0.5.0,
+	libosmocore:0.5.1,
+
+	libsmpp34:1.12,
+
+	osmo-bsc:1.0.1,
+	osmo-bsc:1.1.0,
+	osmo-bsc:1.1.1,
+	osmo-bsc:1.1.2,
+	osmo-bsc:1.2.0,
+	osmo-bsc:1.2.1,
+	osmo-bsc:1.2.2,
+
+	osmo-bts:0.2.0,
+	osmo-bts:0.3.0,
+
+	osmo-hlr:0.0.1,
+
+	osmo-mgw:1.0.1,
+
+	osmo-msc:1.0.1,
+
+	osmo-pcap:0.0.3,
+
+	osmo-pcu:0.1,
+	osmo-pcu:0.2,
+
+	osmo-sgsn:0.9.0,
+	osmo-sgsn:0.9.1,
+	osmo-sgsn:0.9.2,
+	osmo-sgsn:0.9.3,
+	osmo-sgsn:0.9.4,
+	osmo-sgsn:0.9.5,
+	osmo-sgsn:0.9.6,
+	osmo-sgsn:0.9.8,
+	osmo-sgsn:0.9.9,
+	osmo-sgsn:0.9.10,
+	osmo-sgsn:0.9.11,
+	osmo-sgsn:0.9.12,
+	osmo-sgsn:0.9.13,
+	osmo-sgsn:0.9.14,
+	osmo-sgsn:0.9.15,
+	osmo-sgsn:0.9.16,
+	osmo-sgsn:0.10.0,
+	osmo-sgsn:0.10.1,
+	osmo-sgsn:0.11.0,
+	osmo-sgsn:0.12.0,
+	osmo-sgsn:0.13.0,
+	osmo-sgsn:0.14.0,
+	osmo-sgsn:0.15.0,
+	osmo-sgsn:1.0.1,
+
+	osmo-sip-connector:0.0.1,
+
+	osmo-trx:0.2.0,
+	osmo-trx:0.3.0,
+	osmo-trx:1.3.0,
+"
+
+mkdir -p \
+	"$TEMP" \
+	"$TEMP/src"
+
+check_ssh_auth_sock() {
+	if [ -z "$SSH_AUTH_SOCK" ]; then
+		echo "ERROR: SSH_AUTH_SOCK is not set"
+		exit 1
+	fi
+}
+
+# $1: path on server, e.g. "/releases/osmo-bsc"
+get_server_ls() {
+	local dir="$1"
+	local out="$TEMP/ls$(echo "$dir" | tr / _)"
+
+	echo "$LOG_PREFIX Listing files on server: $dir"
+
+	if [ -e "$out" ]; then
+		echo "Skipped, file exists: $out"
+		return
+	fi
+
+	dir="$(echo "$dir" | sed "s.^/releases.$WEB_PATH.")"
+	$SSH_COMMAND releases@ftp.osmocom.org "ls -1 $dir" >"$out"
+}
+
+# $1: repository
+get_git_tags() {
+	local repo="$1"
+	local out="$TEMP/git_tags_$repo"
+
+	echo "$LOG_PREFIX Getting git tags"
+
+	if [ -e "$out" ]; then
+		echo "Skipped, file exists: $out"
+		return
+	fi
+
+	osmo_git_last_tags "$repo" "all" >"$out"
+}
+
+# $1: repository
+# $2: tag
+tarball_exists() {
+	local repo="$1"
+	local tag="$2"
+
+	grep -q "^$repo-$tag\.tar\." "$TEMP"/ls_releases_"$repo"
+}
+
+# $1: repository
+# $2: tag
+is_tag_ignored() {
+	local repo="$1"
+	local tag="$2"
+
+	case "$TAGS_IGNORE" in
+	*"$repo:$tag,"*)
+		return 0
+		;;
+	esac
+
+	return 1
+}
+
+# $1: repository
+# $2: tag
+clone_repo() {
+	local repo="$1"
+	local tag="$2"
+	local gitdir="$TEMP/src/$repo"
+
+	if ! [ -d "$gitdir" ]; then
+		local url="$(osmo_git_clone_url "$repo")"
+		echo "$LOG_PREFIX Cloning $url"
+		git -C "$TEMP/src" clone "$url" "$repo"
+	fi
+
+	echo "$LOG_PREFIX Checkout $tag"
+	cd "$gitdir"
+	git reset --hard HEAD
+	git checkout "$tag"
+	git submodule update --init
+	git clean -dxf
+
+	# Fix depends on packages that don't exist anymore
+	if [ -e debian/control ]; then
+		sed -i 's/dh-systemd \(.*\),//g' debian/control
+		sed -i 's/python-minimal,//g' debian/control
+	fi
+}
+
+# Get the desired tarball name, replace / with - in $1.
+# $1: project path (osmo-bsc, osmo-msc, simtrace/host, etc.)
+# $2: tag
+get_tarball_name() {
+	local project_path="$1"
+	local tag="$2"
+
+	echo "$(echo "$project_path" | tr / -)-$tag.tar.bz2"
+}
+
+# $1: repository
+# $2: project path (osmo-bsc, osmo-msc, simtrace/host, etc.)
+# $3: tag
+build_tarball() {
+	local repo="$1"
+	local project_path="$2"
+	local tag="$3"
+	local tarball_name="$(get_tarball_name "$project_path" "$tag")"
+	local uid_user="$(id -u)"
+	echo "$LOG_PREFIX Building release tarball: $tarball_name"
+
+	if ! docker run \
+		--rm \
+		-e "DEBIAN_FRONTEND=noninteractive" \
+		-v "$OSMO_CI_DIR/scripts/tarballs:/tarballs" \
+		-v "$TEMP/src:/src" \
+		"$DOCKER_IMAGE" \
+		sh -ex -c "
+			cd /src/$repo
+
+			if [ -e debian/control ]; then
+				apt-get update
+				apt-get -y build-dep .
+			fi
+
+			cd /src/$project_path
+
+			if [ -e configure.ac ]; then
+				su build -c \"autoreconf -fi\"
+				case \"$repo\" in
+				osmo-trx)
+					su build -c \"autoreconf -fi osmocom-bb/src/host/trxcon\"
+					su build -c \"./configure --with-mstrx\"
+					;;
+				*)
+					su build -c \"./configure\"
+					;;
+				esac
+				su build -c \"make dist-bzip2\"
+			else
+				su build -c \"git archive -o $tarball_name $tag\"
+			fi
+	"; then
+		echo "$LOG_PREFIX Building tarball failed!"
+		exit 1
+	fi
+
+	cd "$TEMP/src/$project_path"
+
+	# Adjust the tarball name, e.g. for simtrace2-host-*.tar.bz2
+	if ! [ -e "$tarball_name" ]; then
+		echo
+		mv -v *.tar.bz2 "$tarball_name"
+		echo
+	fi
+}
+
+# $1: repository
+# $2: tarball path within the repository dir
+publish_tarball() {
+	local repo="$1"
+	local tarball="$2"
+	local tarball_path="$TEMP/src/$repo/$tarball"
+	local tarball_path_remote="releases@ftp.osmocom.org:$WEB_PATH/$repo/$(basename "$tarball")"
+
+	echo "$LOG_PREFIX Publishing $tarball"
+
+	if [ ! -e "$tarball_path" ]; then
+		echo "$LOG_PREFIX ERROR: tarball not found: $tarball_path"
+		exit 1
+	fi
+
+	if [ "$PUBLISH" != 1 ]; then
+		echo "$LOG_PREFIX Skipping, PUBLISH != 1"
+		return
+	fi
+
+	rsync -vz -e "$SSH_COMMAND" "$tarball_path" $tarball_path_remote
+}
+
+# $1: repository
+# $2: tag
+build_publish_tarballs() {
+	local repo="$1"
+	local tag="$2"
+	local tarballs="$repo-$tag.tar.bz2"
+
+	build_tarball "$repo" "$repo" "$tag"
+
+	case "$repo" in
+	simtrace2)
+		if [ -e "$TEMP"/src/simtrace2/host/configure.ac ]; then
+			build_tarball "simtrace2" "simtrace2/host" "$tag"
+			tarballs="$tarballs host/simtrace2-host-$tag.tar.bz2"
+		fi
+		;;
+	esac
+
+	for tarball in $tarballs; do
+		publish_tarball "$repo" "$tarball"
+	done
+}
+
+check_ssh_auth_sock
+get_server_ls "/releases"
+
+for repo in $OSMO_RELEASE_REPOS; do
+	LOG_PREFIX=":: ($repo)"
+
+	if grep -q "^$repo$" "$TEMP/ls_releases"; then
+		get_server_ls "/releases/$repo"
+	else
+		echo "$LOG_PREFIX No release directory on server"
+		touch "$TEMP/ls_releases_$repo"
+	fi
+
+	get_git_tags "$repo"
+
+	echo "$LOG_PREFIX Building missing tarballs"
+	for tag in $(cat "$TEMP"/git_tags_"$repo"); do
+		LOG_PREFIX=":: ($repo, $tag)"
+		if tarball_exists "$repo" "$tag"; then
+			echo "$LOG_PREFIX Skipping, tarball exists"
+			continue
+		elif is_tag_ignored "$repo" "$tag"; then
+			echo "$LOG_PREFIX Skipping, tag is ignored"
+			continue
+		fi
+
+		clone_repo "$repo" "$tag"
+		build_publish_tarballs "$repo" "$tag"
+	done
+done