obs: add build_binpkg.py

Add a script to build deb/rpm packages, as it would be done on
obs.osmocom.org. This will be used by jenkins to verify deb and rpm
builds for each submitted gerrit patch.

I have attempted to use 'osc build' instead of directly calling
apt-get build-dep and dpkg-buildpackage (and rpm equivalents). Using
'osc build' would have the advantage that the build works as close to
the OBS build as possible. However it would try to install dependencies
with sudo, so we would need to have sudo available in the docker
container that builds the untrusted code from gerrit. Let's not do that.

Related: OS#2385
Change-Id: I4c6b5d61af35df98cbc70d9ddc8ad36d38a9ce18

9 files changed, 275 insertions, 3 deletions

diff --git a/scripts/obs/build_binpkg.py b/scripts/obs/build_binpkg.py
new file mode 100755
index 0000000..b670af9
--- /dev/null
+++ b/scripts/obs/build_binpkg.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import argparse
+import multiprocessing
+import os
+import lib
+import lib.binpkg_deb
+import lib.config
+import lib.docker
+import lib.git
+import lib.metapkg
+import lib.srcpkg
+
+
+def main():
+    distro_default = lib.config.docker_distro_default
+    distro_choices = [distro_default] + lib.config.docker_distro_other
+    jobs_default = multiprocessing.cpu_count() + 1
+
+    parser = argparse.ArgumentParser(
+        description="Build a deb or rpm package as it would be done on"
+                    " obs.osmocom.org. Use after building a source package"
+                    " with build_srcpkg.py."
+                    f" Output dir: {lib.config.path_temp}/binpkgs")
+    parser.add_argument("-d", "--docker", choices=distro_choices,
+                        const=distro_default, nargs="?",
+                        help="build the package in docker for a specific"
+                             f" distro (default: {distro_default})")
+    parser.add_argument("-j", "--jobs", type=int, default=jobs_default,
+                        help=f"parallel running jobs (default: {jobs_default})")
+    parser.add_argument("-v", "--verbose", action="store_true",
+                        help="always print shell commands and their output,"
+                             " instead of only printing them on error")
+    parser.add_argument("package",
+                        help="package name, e.g. libosmocore")
+    args = parser.parse_args()
+
+    lib.set_cmds_verbose(args.verbose)
+
+    srcdir = f"{lib.config.path_temp}/srcpkgs/{args.package}"
+    if not os.path.exists(srcdir):
+        print(f"ERROR: {args.package}: no srcpkg found, run build_srcpkg.py"
+              " first!")
+        exit(1)
+
+    bindir = f"{lib.config.path_temp}/binpkgs"
+    lib.run_cmd(["rm", "-rf", bindir])
+    os.makedirs(bindir)
+
+    distro = args.docker if args.docker else distro_default
+
+    env = {"JOBS": str(args.jobs),
+           "PACKAGE": args.package,
+           "BUILDUSER": os.environ["USER"]}
+
+    script_path = "data/build_deb.sh"
+    if not distro.startswith("debian:"):
+        script_path = "data/build_rpm.sh"
+
+    if args.docker:
+        env["BUILDUSER"] = "user"
+        lib.docker.run_in_docker_and_exit(script_path,
+                                          image_type="build_binpkg",
+                                          distro=distro,
+                                          pass_argv=False, env=env)
+    else:
+        lib.run_cmd(["sudo", "-E", script_path], env=env,
+                    cwd=lib.config.path_top)
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/obs/data/build_binpkg.Dockerfile b/scripts/obs/data/build_binpkg.Dockerfile
new file mode 100644
index 0000000..feacbc9
--- /dev/null
+++ b/scripts/obs/data/build_binpkg.Dockerfile
@@ -0,0 +1,72 @@
+ARG	DISTRO_FROM
+FROM	${DISTRO_FROM}
+ARG	DISTRO
+ARG	UID
+
+COPY	Release.key /tmp/Release.key
+
+RUN	useradd --uid=${UID} -m user
+
+# Only install build-essential here, and what's needed to add the Osmocom
+# repository. Everything else must be defined as dependency in the package
+# build recipe. For rpm-based distributions, there is no build-essential or
+# similar package. Instead add relevant packages from prjconf, e.g.:
+# https://build.opensuse.org/projects/CentOS:CentOS-8/prjconf
+RUN	case "$DISTRO" in \
+	debian*) \
+		apt-get update && \
+		apt-get install -y --no-install-recommends \
+			build-essential \
+			ca-certificates \
+			fakeroot \
+			git \
+			gnupg2 \
+			&& \
+		apt-get clean \
+		;; \
+	centos*) \
+		dnf -y install \
+			autoconf \
+			automake \
+			binutils \
+			dnf-utils \
+			gcc \
+			gcc-c++ \
+			glibc-devel \
+			libtool \
+			make \
+			redhat-rpm-config \
+			rpm-build \
+			rpmdevtools \
+			wget && \
+		yum config-manager --set-enabled powertools && \
+		su user -c rpmdev-setuptree \
+		;; \
+	esac
+
+# Add master repository, where packages immediately get updated after merging
+# patches to master.
+RUN	case "$DISTRO" in \
+	debian:11) \
+		apt-key add /tmp/Release.key && \
+		rm /tmp/Release.key && \
+		echo "deb https://downloads.osmocom.org/packages/osmocom:/master/Debian_11/ ./" \
+			> /etc/apt/sources.list.d/osmocom-master.list \
+		;; \
+	centos:8) \
+		{ echo "[network_osmocom_master]"; \
+		  echo "name=Nightly packages of the Osmocom project (CentOS_8)"; \
+		  echo "type=rpm-md"; \
+		  echo "baseurl=https://downloads.osmocom.org/packages/osmocom:/master/CentOS_8/"; \
+		  echo "gpgcheck=1"; \
+		  echo "gpgkey=https://downloads.osmocom.org/packages/osmocom:/master/CentOS_8/repodata/repomd.xml.key"; \
+		  echo "enabled=1"; \
+		} > /etc/yum.repos.d/network:osmocom:master.repo \
+		;; \
+	*) \
+		echo "can't install repo for $DISTRO" && \
+		exit 1 \
+		;; \
+	esac
+
+WORKDIR	/obs/
diff --git a/scripts/obs/data/build_deb.sh b/scripts/obs/data/build_deb.sh
new file mode 100755
index 0000000..0030278
--- /dev/null
+++ b/scripts/obs/data/build_deb.sh
@@ -0,0 +1,14 @@
+#!/bin/sh -ex
+
+apt_get="apt-get"
+if [ -n "$INSIDE_DOCKER" ]; then
+	export DEBIAN_FRONTEND=noninteractive
+	apt_get="apt-get -y"
+fi
+
+su "$BUILDUSER" -c "tar -C _temp/binpkgs -xvf _temp/srcpkgs/$PACKAGE/*.tar.*"
+cd _temp/binpkgs/*
+
+$apt_get update
+$apt_get build-dep .
+su "$BUILDUSER" -c "dpkg-buildpackage -us -uc -j$JOBS"
diff --git a/scripts/obs/data/build_rpm.sh b/scripts/obs/data/build_rpm.sh
new file mode 100755
index 0000000..39f49ee
--- /dev/null
+++ b/scripts/obs/data/build_rpm.sh
@@ -0,0 +1,30 @@
+#!/bin/sh -ex
+
+if ! [ -d /home/$BUILDUSER/rpmbuild/SOURCES ]; then
+	set +x
+	echo "ERROR: rpmdev-setuptree did not run"
+	echo "If this is an rpm based system and you want to build the package"
+	echo "here, run rpmdev-setuptree. Otherwise consider building the"
+	echo "package in docker (-d)."
+	exit 1
+fi
+
+yum_builddep="yum-builddep"
+if [ -n "$INSIDE_DOCKER" ]; then
+	yum_builddep="yum-builddep -y"
+fi
+
+spec="$(basename "$(find _temp/srcpkgs/"$PACKAGE" -name '*.spec')")"
+
+su "$BUILDUSER" -c "cp _temp/srcpkgs/$PACKAGE/$spec ~/rpmbuild/SPECS"
+su "$BUILDUSER" -c "cp _temp/srcpkgs/$PACKAGE/*.tar.* ~/rpmbuild/SOURCES"
+su "$BUILDUSER" -c "cp _temp/srcpkgs/$PACKAGE/rpmlintrc ~/rpmbuild/SOURCES"
+
+$yum_builddep "/home/$BUILDUSER/rpmbuild/SPECS/$spec"
+
+su "$BUILDUSER" -c "rpmbuild -bb ~/rpmbuild/SPECS/$spec"
+
+# Make built rpms available outside of docker
+if [ -n "$INSIDE_DOCKER" ]; then
+	su "$BUILDUSER" -c "mv ~/rpmbuild/RPMS/*/*.rpm _temp/binpkgs/"
+fi
diff --git a/scripts/obs/lib/__init__.py b/scripts/obs/lib/__init__.py
index 8f52dba..5979623 100644
--- a/scripts/obs/lib/__init__.py
+++ b/scripts/obs/lib/__init__.py
@@ -14,6 +14,7 @@ cmds_verbose = False
 
 
 def add_shared_arguments(parser):
+    """ Arguments shared between build_srcpkg.py and update_obs_project.py. """
     parser.add_argument("-f", "--feed",
 			help="package feed (default: nightly). The feed"
 			" determines the git revision to be built:"
diff --git a/scripts/obs/lib/binpkg_deb.py b/scripts/obs/lib/binpkg_deb.py
new file mode 100644
index 0000000..b26623c
--- /dev/null
+++ b/scripts/obs/lib/binpkg_deb.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import os
+import glob
+import lib.config
+
+
+def extract_source(srcdir, bindir):
+    tarball = glob.glob(f"{srcdir}/*.tar.*")[0]
+
+    print(f"extracting {os.path.basename(tarball)}")
+    lib.run_cmd(["tar", "-xf", tarball], cwd=bindir)
+
+    return glob.glob(f"{bindir}/*/")[0]
+
+
+def build(srcdir, jobs):
+    bindir = f"{lib.config.path_temp}/binpkg"
+    extractdir = extract_source(srcdir, bindir)
+
+    lib.set_cmds_verbose(True)
+
+    # install deps
+    lib.run_cmd(["apt-get", "-y", "build-dep", "."], cwd=extractdir)
+
+    print("running dpkg-buildpackage")
+    lib.run_cmd(["dpkg-buildpackage", "-us", "-uc", f"-j{jobs}"],
+                cwd=extractdir)
diff --git a/scripts/obs/lib/binpkg_rpm.py b/scripts/obs/lib/binpkg_rpm.py
new file mode 100644
index 0000000..b26623c
--- /dev/null
+++ b/scripts/obs/lib/binpkg_rpm.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import os
+import glob
+import lib.config
+
+
+def extract_source(srcdir, bindir):
+    tarball = glob.glob(f"{srcdir}/*.tar.*")[0]
+
+    print(f"extracting {os.path.basename(tarball)}")
+    lib.run_cmd(["tar", "-xf", tarball], cwd=bindir)
+
+    return glob.glob(f"{bindir}/*/")[0]
+
+
+def build(srcdir, jobs):
+    bindir = f"{lib.config.path_temp}/binpkg"
+    extractdir = extract_source(srcdir, bindir)
+
+    lib.set_cmds_verbose(True)
+
+    # install deps
+    lib.run_cmd(["apt-get", "-y", "build-dep", "."], cwd=extractdir)
+
+    print("running dpkg-buildpackage")
+    lib.run_cmd(["dpkg-buildpackage", "-us", "-uc", f"-j{jobs}"],
+                cwd=extractdir)
diff --git a/scripts/obs/lib/config.py b/scripts/obs/lib/config.py
index 52b84fb..393251b 100644
--- a/scripts/obs/lib/config.py
+++ b/scripts/obs/lib/config.py
@@ -112,3 +112,6 @@ git_latest_tag_pattern_other = {
 }
 
 docker_distro_default = "debian:11"
+docker_distro_other = [
+    "centos:8",
+]
diff --git a/scripts/obs/lib/docker.py b/scripts/obs/lib/docker.py
index 2944cb3..81464bb 100644
--- a/scripts/obs/lib/docker.py
+++ b/scripts/obs/lib/docker.py
@@ -14,11 +14,23 @@ def get_image_name(distro, image_type):
     return ret
 
 
+def get_distro_from(distro):
+    # CentOS 8 is EOL (SYS#5818)
+    if distro == "centos:8":
+        return "almalinux:8"
+
+    return distro
+
+
 def build_image(distro, image_type):
-    image_name = get_image_name(distro, image_type),
+    image_name = get_image_name(distro, image_type)
+    distro_from = get_distro_from(distro)
+
     print(f"docker: building image {image_name}")
+
     lib.run_cmd(["docker", "build",
                  "--build-arg", f"DISTRO={distro}",
+                 "--build-arg", f"DISTRO_FROM={distro_from}",
                  "--build-arg", f"UID={os.getuid()}",
                  "-t", image_name,
                  "-f", f"{lib.config.path_top}/data/{image_type}.Dockerfile",
@@ -38,12 +50,15 @@ def get_oscrc():
 
 
 def run_in_docker_and_exit(script_path, add_oscrc=False,
-                           image_type="build_srcpkg", distro=None):
+                           image_type="build_srcpkg", distro=None,
+                           pass_argv=True, env={}):
     """
     :param script_path: what to run inside docker, relative to scripts/obs/
     :param add_oscrc: put user's oscrc in docker (contains obs credentials!)
     :param image_type: which Dockerfile to use (data/{image_type}.Dockerfile)
     :param distro: which Linux distribution to use, e.g. "debian:11"
+    :param pass_argv: pass arguments from sys.argv to the script
+    :param env: dict of environment variables
     """
     if "INSIDE_DOCKER" in os.environ:
         return
@@ -72,10 +87,17 @@ def run_in_docker_and_exit(script_path, add_oscrc=False,
            "-e", "PYTHONUNBUFFERED=1",
            "-v", f"{lib.config.path_top}:/obs"]
 
+
+    for env_key, env_val in env.items():
+        cmd += ["-e", f"{env_key}={env_val}"]
+
     if oscrc:
         cmd += ["-v", f"{oscrc}:/home/user/.oscrc"]
 
-    cmd += [image_name, f"/obs/{script_path}"] + sys.argv[1:]
+    cmd += [image_name, f"/obs/{script_path}"]
+
+    if pass_argv:
+        cmd += sys.argv[1:]
 
     print(f"docker: running: {script_path} inside docker")
     ret = subprocess.run(cmd)