aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--.editorconfig12
-rw-r--r--.gitignore9
-rw-r--r--README.adoc30
-rw-r--r--README.md63
-rw-r--r--ansible/.gitignore2
-rw-r--r--ansible/README.md2
-rw-r--r--ansible/ansible.cfg8
-rw-r--r--ansible/hosts40
-rw-r--r--ansible/roles/apt-allow-relinfo-change/README.md12
-rw-r--r--ansible/roles/apt-allow-relinfo-change/tasks/main.yml7
-rw-r--r--ansible/roles/docker/defaults/main.yml4
-rw-r--r--ansible/roles/docker/files/Dockerfile33
-rw-r--r--ansible/roles/docker/files/docuum.service11
-rwxr-xr-xansible/roles/docker/files/docuum.sh43
-rw-r--r--ansible/roles/docker/tasks/docuum.yml40
-rw-r--r--ansible/roles/docker/tasks/main.yml8
-rw-r--r--ansible/roles/gsm-tester-bts/files/Release.key38
-rw-r--r--ansible/roles/gsm-tester-bts/tasks/main.yml2
-rw-r--r--ansible/roles/gsm-tester/tasks/main.yml6
-rw-r--r--ansible/roles/install-coverity/defaults/main.yml2
-rw-r--r--ansible/roles/install-coverity/tasks/main.yml67
-rw-r--r--ansible/roles/install-poky-sdk/tasks/main.yml74
-rw-r--r--ansible/roles/osmocom-jenkins-slave/README.md1
-rw-r--r--ansible/roles/osmocom-jenkins-slave/defaults/main.yml3
-rw-r--r--ansible/roles/osmocom-jenkins-slave/tasks/debian.yml15
-rw-r--r--ansible/roles/osmocom-jenkins-slave/tasks/main.yml13
-rw-r--r--ansible/roles/osmocom-jenkins-slave/tasks/osmo-ci.yml12
-rw-r--r--ansible/roles/osmocom-jenkins-slave/tasks/osmocom_build_deps.yml190
-rw-r--r--ansible/roles/osmocom-jenkins-slave/tasks/ttcn3-slave.yml38
-rw-r--r--ansible/roles/osmocom-obs-worker/files/obs-ca.crt20
-rw-r--r--ansible/roles/osmocom-obs-worker/files/obs.conf15
-rw-r--r--ansible/roles/osmocom-obs-worker/tasks/main.yml8
-rw-r--r--ansible/roles/osmocom-obs-worker/tasks/obs.yml52
-rw-r--r--ansible/roles/osmocom-obs-worker/tasks/openvpn.yml21
-rw-r--r--ansible/roles/osmocom-obs-worker/templates/obs-server442
-rw-r--r--ansible/roles/osmocom-workstation/tasks/git-repos.yml7
-rw-r--r--ansible/roles/osmocom-workstation/tasks/main.yml4
-rw-r--r--ansible/roles/podman/tasks/debian.yml33
-rw-r--r--ansible/roles/podman/tasks/main.yml12
-rwxr-xr-xansible/roles/qemu/files/qemu-create-vms.sh85
-rw-r--r--ansible/roles/qemu/tasks/main.yml22
-rw-r--r--ansible/setup-gsm-tester.yml8
-rw-r--r--ansible/setup-jenkins-slave.yml15
-rw-r--r--ansible/setup-obs-worker.yml21
-rw-r--r--ansible/setup-simtest.yml12
-rw-r--r--ansible/util/known_hosts10
-rw-r--r--ansible/util/ssh_config3
-rw-r--r--ansible/vars/Debian-11.yml2
-rw-r--r--ansible/vars/Debian-12.yml2
-rw-r--r--ansible/vars/Suse-15.yml2
-rwxr-xr-xcontrib/jenkins.sh17
-rw-r--r--contrib/known_hosts3
-rw-r--r--contrib/obs-mirror/README7
-rw-r--r--contrib/obs-mirror/obs-mirror-include.txt18
-rwxr-xr-xcontrib/obs-mirror/obs-mirror.sh49
-rwxr-xr-xcontrib/obs-mirror/rm-old-nightly-archives.sh34
-rwxr-xr-xcontrib/obs-mirror/test-include-list.sh13
-rwxr-xr-xcontrib/obs-upgrade-eclipse-titan.sh49
-rwxr-xr-xcoverity/badge_status.sh28
-rwxr-xr-xcoverity/build_Osmocom.sh29
l---------coverity/common.sh1
-rwxr-xr-xcoverity/jenkins.sh11
-rwxr-xr-xcoverity/local_test.sh17
-rwxr-xr-xcoverity/prepare_source_Osmcocom.sh45
-rwxr-xr-xcoverity/prepare_source_Osmocom.sh86
-rw-r--r--jobs/README.adoc86
-rw-r--r--jobs/coverity-status.yml46
-rw-r--r--jobs/coverity.yml37
-rw-r--r--jobs/gerrit-binpkgs.yml91
-rw-r--r--jobs/gerrit-lint.yml150
-rw-r--r--jobs/gerrit-verifications-comment.yml45
-rw-r--r--jobs/gerrit-verifications-dahdi.yml103
-rw-r--r--jobs/gerrit-verifications-parameters.yaml.inc53
-rw-r--r--jobs/gerrit-verifications.yml545
-rw-r--r--jobs/master-builds-dahdi.yml82
-rw-r--r--jobs/master-builds.yml384
-rw-r--r--jobs/octsim_osmo-ccid-firmware.yml53
-rw-r--r--jobs/osmo-gsm-manuals-trigger.yml7
-rw-r--r--jobs/osmo-gsm-tester-builder.yml6
-rw-r--r--jobs/osmo-gsm-tester-runner.yml67
-rw-r--r--jobs/osmo-gsm-tester-trigger.yml4
-rw-r--r--jobs/osmocom-api.yml46
-rw-r--r--jobs/osmocom-build-tags-against-master.yml19
-rw-r--r--jobs/osmocom-depcheck.yml10
-rw-r--r--jobs/osmocom-list-commits.yml10
-rw-r--r--jobs/osmocom-obs-check-builders.yml32
-rw-r--r--jobs/osmocom-obs-check-new-distros.yml39
-rw-r--r--jobs/osmocom-obs-ewild.yml72
-rw-r--r--jobs/osmocom-obs-nhofmeyr.yml73
-rw-r--r--jobs/osmocom-obs-rhizomatica.yml73
-rw-r--r--jobs/osmocom-obs-sync.yml54
-rw-r--r--jobs/osmocom-obs-wireshark.yml42
-rw-r--r--jobs/osmocom-obs.yml39
-rw-r--r--jobs/osmocom-release-manuals.yml45
-rw-r--r--jobs/osmocom-release-tarballs.yml46
-rw-r--r--jobs/registry-rebuild-upload-fpga-build.yml10
-rw-r--r--jobs/registry-rebuild-upload-titan.yml59
-rw-r--r--jobs/registry-triggers.yml9
-rw-r--r--jobs/registry-update-base-images.yml17
-rw-r--r--jobs/repo-install-test.yml26
-rw-r--r--jobs/ttcn3-testsuites-kernel-git.yml20
-rw-r--r--jobs/ttcn3-testsuites.yml422
-rw-r--r--jobs/update-osmo-ci-on-slaves.yml43
-rw-r--r--jobs/update-osmo-python-on-slaves.yml14
-rwxr-xr-xlint/checkpatch/checkpatch.pl24
-rwxr-xr-xlint/checkpatch/checkpatch_json.py60
-rwxr-xr-xlint/checkpatch/checkpatch_osmo.sh48
-rw-r--r--lint/checkpatch/spelling.txt4
-rw-r--r--lint/checkpatch/typedefs_osmo.txt5
-rwxr-xr-xlint/docker_run_rm.sh17
-rwxr-xr-xlint/lint_diff.sh54
-rwxr-xr-xqemu-kvm/ggsn-kernel-gtp/postinst.sh8
-rw-r--r--qemu-kvm/ggsn-kernel-gtp/postinst/Release.key38
-rw-r--r--qemu-kvm/ggsn-kernel-gtp/postinst/authorized_keys3
-rw-r--r--scripts/common-obs-conflict.sh146
-rw-r--r--scripts/common-obs.sh313
-rw-r--r--scripts/common.sh83
-rwxr-xr-xscripts/docker-cleanup.sh51
-rwxr-xr-xscripts/jenkins-gerrit/comment_generate.py260
-rwxr-xr-xscripts/jenkins-gerrit/comment_send.sh33
-rw-r--r--scripts/manuals/0001-build-custom-dblatex.sty-remove-DRAFT.patch32
-rwxr-xr-xscripts/manuals/publish-manuals-for-tags.sh434
-rw-r--r--scripts/obs/README141
-rwxr-xr-xscripts/obs/build_binpkg.py107
-rwxr-xr-xscripts/obs/build_srcpkg.py52
-rwxr-xr-xscripts/obs/check_builders.sh19
-rwxr-xr-xscripts/obs/check_new_distros.py110
-rw-r--r--scripts/obs/data/Release.key26
-rwxr-xr-xscripts/obs/data/build.sh15
-rw-r--r--scripts/obs/data/build_binpkg.Dockerfile90
-rw-r--r--scripts/obs/data/build_binpkg_manuals.Dockerfile20
-rwxr-xr-xscripts/obs/data/build_deb.sh25
-rwxr-xr-xscripts/obs/data/build_rpm.sh44
-rw-r--r--scripts/obs/data/build_srcpkg.Dockerfile31
-rw-r--r--scripts/obs/data/rpmlintrc5
-rw-r--r--scripts/obs/data/rpmmacros5
-rw-r--r--scripts/obs/lib/__init__.py197
-rw-r--r--scripts/obs/lib/binpkg_deb.py29
-rw-r--r--scripts/obs/lib/binpkg_rpm.py29
-rw-r--r--scripts/obs/lib/config.py152
-rw-r--r--scripts/obs/lib/debian.py133
-rw-r--r--scripts/obs/lib/docker.py119
-rw-r--r--scripts/obs/lib/git.py157
-rw-r--r--scripts/obs/lib/metapkg.py108
-rw-r--r--scripts/obs/lib/osc.py182
-rw-r--r--scripts/obs/lib/rpm_spec.py84
-rw-r--r--scripts/obs/lib/srcpkg.py233
-rwxr-xr-xscripts/obs/sync_obs_projects.py322
-rwxr-xr-xscripts/obs/update_obs_project.py251
-rwxr-xr-xscripts/obs/update_obs_wireshark.sh47
-rw-r--r--scripts/osmo-build.sh230
-rwxr-xr-xscripts/osmo-ci-docker-rebuild.sh17
-rw-r--r--scripts/osmo-depcheck/config.py2
-rw-r--r--scripts/osmo-depcheck/dependencies.py4
-rwxr-xr-xscripts/osmo-deps.sh6
-rwxr-xr-xscripts/osmo-layer1-headers.sh4
-rwxr-xr-xscripts/osmocom-api-doxygen.sh56
-rwxr-xr-xscripts/osmocom-build-old-tags-against-master.sh7
-rwxr-xr-xscripts/osmocom-latest-packages.sh240
-rwxr-xr-xscripts/osmocom-list-commits.sh3
-rwxr-xr-xscripts/osmocom-next-packages.sh6
-rwxr-xr-xscripts/osmocom-nightly-packages.sh274
-rwxr-xr-xscripts/osmocom-packages-docker.sh2
-rwxr-xr-xscripts/osmocom-release-tarballs.sh216
-rwxr-xr-xscripts/repo-install-test.sh294
-rw-r--r--scripts/repo-install-test/blacklist_centos8.txt11
-rw-r--r--scripts/repo-install-test/blacklist_debian10.txt63
-rw-r--r--scripts/repo-install-test/blacklist_debian9.txt27
-rwxr-xr-xscripts/repo-install-test/run-inside.sh (renamed from scripts/repo-install-test/run-inside-docker.sh)250
-rwxr-xr-xscripts/tarballs/install-depends.sh31
-rwxr-xr-xscripts/tarballs/prefer-configure.sh34
-rwxr-xr-xscripts/tarballs/publish-tarballs-for-tags.sh365
-rwxr-xr-xscripts/verify_endian_header.sh23
-rwxr-xr-xscripts/verify_value_string_arrays_are_terminated.py2
-rwxr-xr-xscripts/xgoldmon-jenkins.sh23
175 files changed, 8536 insertions, 2967 deletions
diff --git a/.editorconfig b/.editorconfig
new file mode 100644
index 0000000..23fbc8d
--- /dev/null
+++ b/.editorconfig
@@ -0,0 +1,12 @@
+; This file is for unifying the coding style for different editors and IDEs.
+; See http://editorconfig.org for details.
+
+# Top-most EditorConfig file.
+root = true
+
+[**/*.yml]
+indent_size = 2
+indent_style = space
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
diff --git a/.gitignore b/.gitignore
index 78e48b5..5f47674 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,17 +1,18 @@
cov-analysis-*
source-*
install-*
+!install-depends.sh
tokens.txt
.*.sw?
jenkins_jobs.ini
jenkins-jobs.ini
__pycache__/
+_cache/
_temp/
_deps/
_release_tarballs/
_docker_playground
_repo_install_test_data/
-_repo_install_test_cache/
-
-# osmocom-nightly-packages.sh
-nightly-3g_*
+_temp_manuals/
+_temp_releases/
+scripts/repo-install-test/run-inside-env.sh
diff --git a/README.adoc b/README.adoc
deleted file mode 100644
index bd4ef2a..0000000
--- a/README.adoc
+++ /dev/null
@@ -1,30 +0,0 @@
-Scripts used for the CI handling and coverity uploads
-
-coverity: scripts used to submit the osmocom sources for coverity scan.
-This depends on these, which are not included in osmo-ci:
-- a tokens.txt file in coverity/ -- see coverity/get_token.sh
-- a cov-analysis-linux64-8.5.0 in coverity/
- (or the like, may need to adjust some scripts to match)
-
-jobs: Jenkins Job Builder YAML files defining jenkins jobs. Read jobs/README.adoc
-for more information about deployment.
-
-scripts: used by jenkins jobs. Various osmo*/contrib/jenkins.sh scripts assume
-osmo-ci to be checked out in the build slave user's home, i.e. using a PATH of
-$HOME/osmo-ci/scripts.
-
-obs-patches: patches to build projects for various debian distributions, e.g.
-a patch for limesuite that fixes the libwxgtk3.0-dev => libwxgtk3.0-gtk3-dev
-rename in control/debian for debian10. Used by osmo_obs_distro_specific_patch()
-in scripts/common-obs.sh.
-
-_docker_playground: Clone of docker-playground.git, so the scripts can build
-required docker images. This dir gets created on demand by scripts/common.sh,
-and automatically fetched and reset to "origin/master" (override with
-$OSMO_BRANCH_DOCKER_PLAYGROUND). The fetch and reset gets skipped if
-_docker_playground is a symlink. For development, set it up as follows:
-
- $ git clone https://git.osmocom.org/docker-playground
- $ git clone https://git.osmocom.org/osmo-ci
- $ cd osmo-ci
- $ ln -s ../docker-playground _docker_playground
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..f0a3dc9
--- /dev/null
+++ b/README.md
@@ -0,0 +1,63 @@
+# Osmocom CI and infrastructure files
+
+## ansible
+Ansible rules for setting up machines of the Osmocom infrastructure.
+See `ansible/README.md`.
+
+## contrib
+Scripts and files that did not fit into other directories.
+
+## coverity
+Scripts used to submit the osmocom sources for coverity scan. This depends on a
+`tokens.txt`, see `coverity/get_token.sh`.
+
+### how to upgrade
+* Download the latest version from [here](https://scan.coverity.com/download)
+* Find the "upgrade considerations" [here](https://sig-product-docs.synopsys.com/bundle/coverity-docs/page/upgrade-guide/topics/important_upgrade_considerations.html)
+* Place the resulting `cov-analysis-linux64-$VERSION.tar.gz` in `ansible/files`
+* Adjust `coverity_version` in `ansible/roles/install-coverity/defaults/main.yml`
+* Deploy the ansible playbook:
+```
+$ cd ansible
+$ ansible-playbook -v -i hosts setup-jenkins-slave.yml -l coverity_slaves -t coverity
+```
+* On success, only one task should be skipped (the "Please download..." task):
+```
+PLAY RECAP ********************************************************************************************************************
+build2-deb11build-ansible : ok=8 changed=4 unreachable=0 failed=0 skipped=1 rescued=0 ignored=0
+build3-deb11build-ansible : ok=9 changed=4 unreachable=0 failed=0 skipped=1 rescued=0 ignored=0
+build4-deb12build-ansible : ok=9 changed=4 unreachable=0 failed=0 skipped=1 rescued=0 ignored=0
+```
+
+## jobs
+Jenkins Job Builder YAML files defining jenkins jobs. Read `jobs/README.adoc`
+for more information about deployment.
+
+## lint
+The linter running on patches submitted via gerrit. See the wiki page
+[Linting](https://osmocom.org/projects/cellular-infrastructure/wiki/Linting)
+for more information.
+
+## qemu-kvm
+A script to create a virtual machine with kernel gtp ggsn for qemu-kvm.
+
+## scripts
+Scripts used by jenkins jobs. Various `osmo*/contrib/jenkins.sh` scripts assume
+osmo-ci to be checked out in the build slave user's home, i.e. using a PATH of
+`$HOME/osmo-ci/scripts`.
+
+## _docker_playground
+A clone of
+[docker-playground](https://gitea.osmocom.org/osmocom/docker-playground),
+so the scripts can build required docker images. This dir gets created on
+demand by scripts/common.sh, and automatically fetched and reset to
+"origin/master" (override with `$OSMO_BRANCH_DOCKER_PLAYGROUND`). The fetch and
+reset gets skipped if _docker_playground is a symlink. For development, set it
+up as follows:
+
+```
+$ git clone https://gitea.osmocom.org/osmocom/docker-playground
+$ git clone https://gitea.osmocom.org/osmocom/osmo-ci
+$ cd osmo-ci
+$ ln -s ../docker-playground _docker_playground
+```
diff --git a/ansible/.gitignore b/ansible/.gitignore
index 2fa603c..b23a1b1 100644
--- a/ansible/.gitignore
+++ b/ansible/.gitignore
@@ -1,4 +1,4 @@
-poky-*
+poky-*toolchain*
cov-analysis-*.tar.gz
*.retry
*.swo
diff --git a/ansible/README.md b/ansible/README.md
index e5d146c..9085c74 100644
--- a/ansible/README.md
+++ b/ansible/README.md
@@ -33,4 +33,4 @@ User root
# how to make slaves log-in to registry.osmocom.org:
-`ansible jenkins-slaves -u root -a "su - osmocom-build -c 'docker login -u jenkins-slave -p PASSWD https://registry.osmocom.org/'"`
+`ansible jenkins_slaves -u root -a "su - osmocom-build -c 'docker login -u jenkins_slave -p PASSWD https://registry.osmocom.org/'"`
diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg
new file mode 100644
index 0000000..6c361a3
--- /dev/null
+++ b/ansible/ansible.cfg
@@ -0,0 +1,8 @@
+[defaults]
+stdout_callback=debug
+
+remote_user = root
+
+[ssh_connection]
+ssh_args = -F util/ssh_config -o ControlMaster=auto -o ControlPersist=60s -o StrictHostKeyChecking=accept-new
+
diff --git a/ansible/hosts b/ansible/hosts
index d88147a..51e26b9 100644
--- a/ansible/hosts
+++ b/ansible/hosts
@@ -1,22 +1,32 @@
-[gsm-tester]
+[gsm_tester]
osmo-gsm-tester-rnd ansible_host=10.9.25.101
osmo-gsm-tester-prod ansible_host=10.9.25.107
-[coverity-slaves]
-admin2-deb9build ansible_host=2a01:4f8:13b:828::1:300
-build2-deb9build-ansible ansible_host=2a01:4f8:10b:2ad9::1:6
-host2-deb9build-ansible ansible_host=2a01:4f8:120:8470::1:3
+[coverity_slaves]
+build2-deb11build-ansible ansible_host=2a01:4f8:10b:2ad9::1:13
+build3-deb11build-ansible ansible_host=2a01:4f8:212:3762::1:3
+build4-deb12build-ansible ansible_host=2a01:4f8:121:200a::1:3
-[jenkins-slaves]
-admin2-deb9build ansible_host=2a01:4f8:13b:828::1:300 osmocom_jenkins_slave_fstrim=True
-build2-deb9build-ansible ansible_host=2a01:4f8:10b:2ad9::1:6 osmocom_jenkins_slave_fstrim=True
-build2-deb10build-ansible ansible_host=2a01:4f8:10b:2ad9::1:10 osmocom_jenkins_slave_fstrim=True
-host2-deb9build-ansible ansible_host=2a01:4f8:120:8470::1:3 osmocom_jenkins_slave_fstrim=True
-gtp0-deb9build ansible_host=10.34.2.102 osmocom_jenkins_slave_fstrim=True
-gtp0-deb10build ansible_host=10.34.2.101 osmocom_jenkins_slave_fstrim=True
-rpi4-raspbian10build-ansible-1 ansible_host=10.9.25.50
-rpi4-raspbian10build-ansible-2 ansible_host=10.9.25.60
-rpi4-raspbian10build-ansible-3 ansible_host=10.9.25.70
+[jenkins_slaves]
+build2-deb10build-ansible ansible_host=2a01:4f8:10b:2ad9::1:10 osmocom_jenkins_slave_fstrim=True docker_max_image_space="80 GB"
+build2-deb11build-ansible ansible_host=2a01:4f8:10b:2ad9::1:13 osmocom_jenkins_slave_fstrim=True docker_max_image_space="80 GB"
+build3-deb11build-ansible ansible_host=2a01:4f8:212:3762::1:3 osmocom_jenkins_slave_fstrim=True docker_max_image_space="80 GB"
+build4-deb12build-ansible ansible_host=2a01:4f8:121:200a::1:3 osmocom_jenkins_slave_fstrim=True docker_max_image_space="80 GB"
+host2-deb11build-ansible ansible_host=2a01:4f8:120:8470::1:4 osmocom_jenkins_slave_fstrim=True docker_max_image_space="80 GB"
+rpi4-raspbian11build-ansible-1 ansible_host=10.9.25.50 docker_max_image_space="20 GB"
+rpi4-raspbian11build-ansible-2 ansible_host=10.9.25.60 docker_max_image_space="20 GB"
+rpi4-raspbian11build-ansible-3 ansible_host=10.9.25.70 docker_max_image_space="20 GB"
+lx2-raspbian11build-ansible ansible_host=10.34.2.153 docker_max_image_space="20 GB"
[simtester]
simtest ansible_host=10.9.25.80
+
+[obs_workers]
+build3-obsworker1 ansible_host=2a01:4f8:212:3762::1:2 obs_worker_instances=8 obs_worker_jobs=8
+build4-obsworker1 ansible_host=2a01:4f8:121:200a::1:2 obs_worker_instances=8 obs_worker_jobs=8
+lx2-1-obs-aarch64-1 ansible_host=2a02:8106:13:1e34:4e9:e7ff:fe97:fe92 obs_worker_instances=1 obs_worker_jobs=4
+
+# Need to have kvm inside lxc, see OS#5365
+[qemu]
+build3-deb11build-ansible ansible_host=2a01:4f8:212:3762::1:3
+build4-deb12build-ansible ansible_host=2a01:4f8:121:200a::1:3
diff --git a/ansible/roles/apt-allow-relinfo-change/README.md b/ansible/roles/apt-allow-relinfo-change/README.md
new file mode 100644
index 0000000..cd00a46
--- /dev/null
+++ b/ansible/roles/apt-allow-relinfo-change/README.md
@@ -0,0 +1,12 @@
+---
+When the suite of one debian release changes from stable to oldstable, apt
+stops working with the following error:
+
+```
+W:This must be accepted explicitly before updates for this repository can be applied. See apt-secure(8) manpage for details.
+E:Repository 'http://raspbian.raspberrypi.org/raspbian buster InRelease' changed its 'Suite' value from 'stable' to 'oldstable'
+```
+
+This role configures apt to allow the release info change.
+
+Related: https://github.com/ansible/ansible/issues/48352
diff --git a/ansible/roles/apt-allow-relinfo-change/tasks/main.yml b/ansible/roles/apt-allow-relinfo-change/tasks/main.yml
new file mode 100644
index 0000000..098f5ee
--- /dev/null
+++ b/ansible/roles/apt-allow-relinfo-change/tasks/main.yml
@@ -0,0 +1,7 @@
+---
+- name: "configure to allow release info change"
+ lineinfile:
+ path: /etc/apt/apt.conf.d/99relinfochange
+ state: present
+ create: yes
+ line: Acquire::AllowReleaseInfoChange::Suite "true";
diff --git a/ansible/roles/docker/defaults/main.yml b/ansible/roles/docker/defaults/main.yml
index 2c03d90..9faf286 100644
--- a/ansible/roles/docker/defaults/main.yml
+++ b/ansible/roles/docker/defaults/main.yml
@@ -2,3 +2,7 @@
# Adds this user to the group docker which is allowed to access docker
jenkins_user: jenkins
+
+# Keep amount of stored docker images below this size
+# https://osmocom.org/projects/osmocom-servers/wiki/Docker_cache_clean_up
+docker_max_image_space: "40 GB"
diff --git a/ansible/roles/docker/files/Dockerfile b/ansible/roles/docker/files/Dockerfile
new file mode 100644
index 0000000..d0ddf9c
--- /dev/null
+++ b/ansible/roles/docker/files/Dockerfile
@@ -0,0 +1,33 @@
+ARG REGISTRY=docker.io
+FROM ${REGISTRY}/alpine:3.15
+ARG DOCKER_GID
+
+RUN apk add \
+ cargo \
+ docker-cli
+
+# Create user and docker group with same group-id as on host system, create
+# /opt/docuum dir owned by user
+RUN set -x && \
+ if getent group "${DOCKER_GID}"; then \
+ delgroup $(getent group "${DOCKER_GID}" | cut -d: -f1); \
+ fi && \
+ addgroup -g "${DOCKER_GID}" docker && \
+ adduser -D -u 1000 -G docker user && \
+ mkdir /opt/docuum && \
+ chown user /opt/docuum
+
+USER user
+
+ARG DOCUUM_VER=0.20.4
+
+RUN set -x && \
+ cd /opt/docuum && \
+ wget https://github.com/stepchowfun/docuum/archive/refs/tags/v${DOCUUM_VER}.tar.gz \
+ -O docuum.tar.gz && \
+ tar -xf docuum.tar.gz && \
+ cd docuum-${DOCUUM_VER} && \
+ cargo build --release && \
+ cd .. && \
+ mv docuum-${DOCUUM_VER}/target/release/docuum . && \
+ rm -rf ~/.cargo docuum-${DOCUUM_VER} docuum.tar.gz
diff --git a/ansible/roles/docker/files/docuum.service b/ansible/roles/docker/files/docuum.service
new file mode 100644
index 0000000..8c62973
--- /dev/null
+++ b/ansible/roles/docker/files/docuum.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=Docuum
+After=docker.service
+Wants=docker.service
+
+[Service]
+ExecStart=/opt/docuum/docuum.sh
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
diff --git a/ansible/roles/docker/files/docuum.sh b/ansible/roles/docker/files/docuum.sh
new file mode 100755
index 0000000..e7d5e28
--- /dev/null
+++ b/ansible/roles/docker/files/docuum.sh
@@ -0,0 +1,43 @@
+#!/bin/sh -ex
+
+# Maximum amount of storage that docker images may consume
+THRESHOLD="$(cat /opt/docuum/docker_max_image_space)"
+
+DIR="$(dirname "$(realpath "$0")")"
+IMG="osmo-ci-docuum"
+DOCUUM_UID="1000"
+DOCKER_GID="$(getent group docker | cut -d : -f 3)"
+PULL_ARG=""
+
+if [ -z "$THRESHOLD" ]; then
+ set +x
+ echo "ERROR: failed to read threshold from /opt/docuum/docker_max_image_space"
+ exit 1
+fi
+
+if [ "$INITIAL_BUILD" = 1 ]; then
+ PULL_ARG="--pull"
+fi
+
+mkdir -p /var/cache/docuum
+chown "$DOCUUM_UID" /var/cache/docuum
+
+cd "$DIR"
+docker build \
+ --build-arg DOCKER_GID="$DOCKER_GID" \
+ $PULL_ARG \
+ -t "$IMG" \
+ .
+
+if [ "$INITIAL_BUILD" = 1 ]; then
+ exit 0
+fi
+
+docker run \
+ --rm \
+ --init \
+ --name docuum \
+ -v /var/run/docker.sock:/var/run/docker.sock \
+ -v /var/cache/docuum:/home/user \
+ "$IMG" \
+ sh -c "exec /opt/docuum/docuum --threshold '$THRESHOLD'"
diff --git a/ansible/roles/docker/tasks/docuum.yml b/ansible/roles/docker/tasks/docuum.yml
new file mode 100644
index 0000000..594c010
--- /dev/null
+++ b/ansible/roles/docker/tasks/docuum.yml
@@ -0,0 +1,40 @@
+---
+- name: "docuum : set docker_max_image_space to {{ docker_max_image_space }}"
+ lineinfile:
+ path: /opt/docuum/docker_max_image_space
+ state: present
+ create: yes
+ line: "{{ docker_max_image_space }}"
+ regexp: ".*" # replace previous content if changed
+
+- name: "docuum : copy Dockerfile"
+ copy:
+ src: Dockerfile
+ dest: /opt/docuum/
+ mode: 0644
+
+- name: "docuum : copy docuum.sh"
+ copy:
+ src: docuum.sh
+ dest: /opt/docuum/
+ mode: 0755
+
+- name: "docuum : build container"
+ shell: INITIAL_BUILD=1 /opt/docuum/docuum.sh
+
+- name: "docuum : copy docuum.service"
+ copy:
+ src: docuum.service
+ dest: /lib/systemd/system/docuum.service
+ register: docuumservice
+
+- name: "docuum : systemctl daemon-reload"
+ systemd:
+ daemon_reload: yes
+ when: docuumservice is changed
+
+- name: "docuum : ensure the systemd service is installed"
+ systemd:
+ name: docuum.service
+ state: started
+ enabled: yes
diff --git a/ansible/roles/docker/tasks/main.yml b/ansible/roles/docker/tasks/main.yml
index a268d2b..f64ae96 100644
--- a/ansible/roles/docker/tasks/main.yml
+++ b/ansible/roles/docker/tasks/main.yml
@@ -40,6 +40,7 @@
- name: cleanup old docker images
cron:
name: cleanup-docker-images
+ disabled: false
minute: 0
hour: '*/3'
user: "{{ jenkins_user }}"
@@ -50,3 +51,10 @@
src: daemon.json
dest: /etc/docker/daemon.json
notify: restart docker
+
+# daemon.json configures overlay2 storage driver, clean up vfs leftovers
+- name: cleanup vfs storage dir
+ shell: "docker info | grep -q 'Storage Driver: overlay2' || rm -rf /var/lib/docker/vfs"
+
+# After docker is set up, add docuum to clean old docker images
+- import_tasks: docuum.yml
diff --git a/ansible/roles/gsm-tester-bts/files/Release.key b/ansible/roles/gsm-tester-bts/files/Release.key
index 4257562..ecca084 100644
--- a/ansible/roles/gsm-tester-bts/files/Release.key
+++ b/ansible/roles/gsm-tester-bts/files/Release.key
@@ -1,20 +1,26 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: GnuPG v1.4.5 (GNU/Linux)
-mQENBFJBt/wBCADAht3d/ilNuyzaXYw/QwTRvmjyoDvfXw+H/3Fvk1zlDZoiKPPc
-a1wCVBINUZl7vYM2OXqbJwYa++JP2Q48xKSvC6thbRc/YLievkbcvTemf7IaREfl
-CTjoYpoqXHa9kHMw1aALDm8CNU88jZmnV7v9L6hKkbYDxie+jpoj7D6B9JlxgNJ4
-5dQyRNsFGVcIl4Vplt1HyGc5Q5nQI/VgS2rlF/IOXmhRQBc4LEDdU8R2IKnkU4ee
-S7TWanAigGAQhxGuCkS39/CWzc1DhLhjlNhBl/+RTPejkqJtAy00ZLps3+RqUN1Y
-CU/Fsr7aRlYVGqQ/BlptwV0XQ2VVYJX2oEBBABEBAAG0MG5ldHdvcmsgT0JTIFBy
-b2plY3QgPG5ldHdvcmtAYnVpbGQub3BlbnN1c2Uub3JnPokBPAQTAQIAJgUCXoL5
-awIbAwUJEF/xbwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEGLrGgkXKA3fBFkH
-/Ry9IT3WkpyrKLi6wlTKBZTDGz58xDEd6i4fKr1sepgfexzmu/t1125A4Jyuy5Qp
-wolWdVIxmo8Mo9/u16ssGV/Iiy4y+3v+yfOExUeaM5XS4eW5cWcySBi+y3PGL17o
-Qf/iZDnzJtx16kMgD+lybTFSRDhRg4BV1egMMtOV/XIhC7mp0Gt6oOU72fGuWFKs
-3prY+XmL20x8+FfI35sXRAQu2L7OBDnUJoHHuHIT1s5no5JSYubctY/KNT/9YGme
-DSspyOxXJTo6YgOJStyMQZ3DaE4XxSnGpzPV27OnfMWjRf+KLvgFjx9n/zPIzBTh
-Q9gL7GfQUOkJ5L0GrOvVxBiIRgQTEQIABgUCUkG3/AAKCRA7MBG3a51lIzhdAJ9v
-d6XPffMZRcCGgDEY5OaTn/MsCQCgrXbeZpFJgnirSrc8rRonvzYFiF4=
-=zwAO
+mQENBGKzE1QBCADFcM3ZzggvgxNRNNqDGWf5xIDAiK5qzFLdGes7L6F9VCHdaPy0
+RAOB5bFb/Q1tSDFNEBLtaauXKz+4iGL6qMVjZcyjzpB5w4jKN+kkrFRhjDNUv/SH
+BX6d+P7v5WBGSNArNgA8D1BGzckp5a99EZ0okMJFEqIcN40PD6OGugpq5XnVV5Nk
+e93fLa2Cu8vhFBcVn6CuHeEhsmuMf6NLbQRNfNNCEEUYaZn7beMYtpZ7t1djsKx5
+1xGm50OzI22FLu8lELQ9d7qMVGRG3WHYawX9BDteRybiyqxfwUHm1haWazRJtlGt
+UWyzvwAb80BK1J2Nu5fbAa3w5CoEPAbUuCyrABEBAAG0JW9zbW9jb20gT0JTIFBy
+b2plY3QgPG9zbW9jb21Ab3Ntb2NvbT6JAVQEEwEIAD4WIQRrKp83ktFetw1Oao+G
+pzC2U3JZcwUCYrMV4wIbAwUJBB6yjwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAK
+CRCGpzC2U3JZc4FRCACQQkKIrnvQ7n2u7GSmyVZa3I+oLoFXSGqaGyey5TW/nrMm
+vFDKU3qliHiuNSmUY35SnAhXUsvqOYppxVRoO1MLrqUvzMOnIWqkJpf8mtjGUnsW
+jyVeto7Rsjs75y2i1Hk+e7ljb/V65J3NlfrfEYWbqR9AKd53ReNXTdrQ0J05A38N
+GdI4Ld/2lNISAwaBmGhqdeKsLHpQw/JERU1TApVJR1whFiIwDF1rOCg9GPnNKIk7
+yRZdK267XzztrainX/cbPILyzUZEDhYs6wQuyACyQ1YUxZIxrwVfk7PMNay8CrLH
+z42B73Ne5IAj8+op/3iJafFONLm7YXiDUFN+QDYAiQEzBBMBCAAdFiEExoiYhHND
+S7aVYlnqa51NyAUyjdsFAmKzE1UACgkQa51NyAUyjdvuZgf+OXmr//i7u7Gg7eWB
+7e0qUsyCId9lXS8J437x3K6ciJfD7/6RSy8TFW5Nglm/uSkbyq582I8t+SoOirMD
+E6cg9U/5+h5s46bAf+Kd2XS/6tLGeNLM18i4el8CP06NpFzDrsKu76uYFpyRiiHD
+otBdtgxeLJ83LugGfZslF+/5cigJkAJMhAdVvGO8h85R6fba8ZSOKtMKkaQRfi76
+nhyOrJPlLuS+DLEnHwdkOFgtKnxHdjM97K+Tx0gisb6uwaWroXfSLnhP8RTLLZZy
+Z+noU1Hw3c+mn4c/NYbcC/uwHYHKRzuf9gHnQ3dGgv0Z5sbeLRVo92hjGj7Ftlyd
+4hmKBg==
+=HxK4
-----END PGP PUBLIC KEY BLOCK-----
diff --git a/ansible/roles/gsm-tester-bts/tasks/main.yml b/ansible/roles/gsm-tester-bts/tasks/main.yml
index f1377f0..7ada1d8 100644
--- a/ansible/roles/gsm-tester-bts/tasks/main.yml
+++ b/ansible/roles/gsm-tester-bts/tasks/main.yml
@@ -42,7 +42,7 @@
- name: add apt-repo of OBS network:osmocom:nightly
apt_repository:
filename: obs_network_osmocom_latest
- repo: "deb http://download.opensuse.org/repositories/network:/osmocom:/nightly/Debian_{{ ansible_distribution_major_version }}.0/ ./"
+ repo: "deb https://downloads.osmocom.org/packages/osmocom:/nightly/Debian_{{ ansible_distribution_major_version }}.0/ ./"
update_cache: yes
- name: install osmocom utilities
diff --git a/ansible/roles/gsm-tester/tasks/main.yml b/ansible/roles/gsm-tester/tasks/main.yml
index 7d28558..a1fb6cd 100644
--- a/ansible/roles/gsm-tester/tasks/main.yml
+++ b/ansible/roles/gsm-tester/tasks/main.yml
@@ -66,7 +66,7 @@
- name: install srsLTE runtime extra dependencies (Debian)
apt:
name:
- - soapysdr0.7-module-lms7
+ - soapysdr-module-lms7
when: ansible_distribution == 'Debian'
tags:
- srs
@@ -132,7 +132,7 @@
- name: setup ofono repository
git:
- repo: 'https://git.sysmocom.de/sysmocom/ofono'
+ repo: 'https://gitea.sysmocom.de/sysmocom/ofono'
dest: /root/ofono
version: osmo-gsm-tester
clone: yes
@@ -354,7 +354,7 @@
- name: setup osmo-gsm-tester repository
git:
- repo: 'git://git.osmocom.org/osmo-gsm-tester'
+ repo: 'https://gitea.osmocom.org/cellular-infrastructure/osmo-gsm-tester'
dest: /root/osmo-gsm-tester
version: master
clone: yes
diff --git a/ansible/roles/install-coverity/defaults/main.yml b/ansible/roles/install-coverity/defaults/main.yml
index 6b97781..7be5be6 100644
--- a/ansible/roles/install-coverity/defaults/main.yml
+++ b/ansible/roles/install-coverity/defaults/main.yml
@@ -1,4 +1,4 @@
---
-coverity_version: 2019.03
+coverity_version: 2023.6.2
coverity_installer_file: "cov-analysis-linux64-{{ coverity_version }}.tar.gz"
diff --git a/ansible/roles/install-coverity/tasks/main.yml b/ansible/roles/install-coverity/tasks/main.yml
index 5baf6f3..ca4a24c 100644
--- a/ansible/roles/install-coverity/tasks/main.yml
+++ b/ansible/roles/install-coverity/tasks/main.yml
@@ -3,41 +3,46 @@
apt:
name:
- curl
-
-- name: copy coverity installer
- copy:
- src: "{{ coverity_installer_file }}"
- dest: "/tmp/{{ coverity_installer_file }}"
- mode: 0750
- register: coverity_copy
- ignore_errors: yes
tags: [coverity]
-- name: create /opt/coverity
- file:
- path: /opt/coverity/
- state: directory
- when: coverity_copy.failed == False
+- name: check if coverity needs to be installed
+ stat:
+ path: "/opt/coverity/{{ coverity_version }}/"
+ register: coverity_stat
tags: [coverity]
-- name: unpack coverity
- unarchive:
- src: "/tmp/{{ coverity_installer_file }}"
- dest: /opt/coverity/
- remote_src: yes
- when: coverity_copy.failed == False
+- name: install coverity if required
+ when: coverity_stat.stat.exists == False
tags: [coverity]
+ block:
+ - name: copy coverity installer
+ copy:
+ src: "{{ coverity_installer_file }}"
+ dest: "/tmp/{{ coverity_installer_file }}"
+ mode: 0750
+ register: coverity_copy
+ ignore_errors: yes
-- name: create link /opt/coverity/last
- file:
- src: /opt/coverity/cov-analysis-linux64-{{ coverity_version }}
- dest: /opt/coverity/current
- state: link
- when: coverity_copy.failed == False
- tags: [coverity]
+ - name: create /opt/coverity/{{ coverity_version }}/
+ file:
+ path: /opt/coverity/{{ coverity_version }}/
+ state: directory
+ when: coverity_copy.failed == False
-- name: "Please download {{ coverity_installer_file }} to your ansible directory to allow ansible to install coverity"
- debug:
- msg: "Ansible can not find {{ coverity_installer_file }}"
- when: coverity_copy.failed
- tags: [coverity]
+ - name: unpack coverity
+ unarchive:
+ src: "/tmp/{{ coverity_installer_file }}"
+ dest: /opt/coverity/{{ coverity_version }}
+ remote_src: yes
+ when: coverity_copy.failed == False
+
+ - name: create link /opt/coverity/current
+ shell: rm -vf /opt/coverity/current && ln -svf /opt/coverity/{{ coverity_version }}/* /opt/coverity/current
+ args:
+ warn: false
+ when: coverity_copy.failed == False
+
+ - name: "Please download {{ coverity_installer_file }} to your ansible directory to allow ansible to install coverity"
+ debug:
+ msg: "Ansible can not find {{ coverity_installer_file }}"
+ when: coverity_copy.failed
diff --git a/ansible/roles/install-poky-sdk/tasks/main.yml b/ansible/roles/install-poky-sdk/tasks/main.yml
index 85b3c94..57d0137 100644
--- a/ansible/roles/install-poky-sdk/tasks/main.yml
+++ b/ansible/roles/install-poky-sdk/tasks/main.yml
@@ -1,46 +1,54 @@
---
-- name: install bzip2 and tar
+- name: install dependencies for poky installer
apt:
name:
- - tar
+ - build-essential
- bzip2
+ - file
+ - tar
cache_valid_time: 3600
update_cache: yes
-
-- name: copy poky installer
- copy:
- src: "{{ poky_installer_file }}"
- dest: "/tmp/{{ poky_installer_file }}"
- mode: 0750
- register: poky_copy
- ignore_errors: yes
- tags: [poky]
-
-- name: execute poky installer
- command: "/tmp/{{ poky_installer_file }} -y -d '{{ poky_dest }}'"
- args:
- creates: "{{ poky_dest }}"
- when: poky_copy.failed == False
tags: [poky]
-- name: change owner/group to jenkins user
- file:
+- name: check if poky needs to be installed
+ stat:
path: "{{ poky_dest }}"
- owner: "{{ jenkins_user }}"
- group: "{{ jenkins_user }}"
- recurse: yes
- when: poky_copy.failed == False
+ register: poky_stat
tags: [poky]
-- name: remove poky installer
- file:
- path: "/tmp/{{ poky_installer_file }}"
- state: absent
- when: poky_copy.failed == False
+- name: install poky if required
tags: [poky]
+ when: poky_stat.stat.exists == False
+ block:
+ - name: copy poky installer
+ copy:
+ src: "{{ poky_installer_file }}"
+ dest: "/tmp/{{ poky_installer_file }}"
+ mode: 0750
+ register: poky_copy
+ ignore_errors: yes
-- name: "Please download {{ poky_installer_file }} to your ansible directory to allow ansible to install poky"
- debug:
- msg: "Ansible can not find or copy {{ poky_installer_file }}"
- when: poky_copy.failed == True
- tags: [poky]
+ - name: execute poky installer
+ command: "/tmp/{{ poky_installer_file }} -y -d '{{ poky_dest }}'"
+ args:
+ creates: "{{ poky_dest }}"
+ when: poky_copy.failed == False
+
+ - name: change owner/group to jenkins user
+ file:
+ path: "{{ poky_dest }}"
+ owner: "{{ jenkins_user }}"
+ group: "{{ jenkins_user }}"
+ recurse: yes
+ when: poky_copy.failed == False
+
+ - name: remove poky installer
+ file:
+ path: "/tmp/{{ poky_installer_file }}"
+ state: absent
+ when: poky_copy.failed == False
+
+ - name: "Please download {{ poky_installer_file }} to your ansible directory to allow ansible to install poky"
+ debug:
+ msg: "Ansible can not find or copy {{ poky_installer_file }}"
+ when: poky_copy.failed == True
diff --git a/ansible/roles/osmocom-jenkins-slave/README.md b/ansible/roles/osmocom-jenkins-slave/README.md
index 50aed16..81f68b3 100644
--- a/ansible/roles/osmocom-jenkins-slave/README.md
+++ b/ansible/roles/osmocom-jenkins-slave/README.md
@@ -3,7 +3,6 @@
Support the following variables:
* `install_jenkins_utilities`: (true) install usefull utilities including qemu/debootstrap/fakeroot
-* `install_osmocom_build_deps`: (true) install all osmocom runtime and build time dependencies
* `generic_slave`: (true) contains tasks used by the most osmocom jenkins slaves
* `osmocom_jenkins_slave_fstrim`: (false) calls fstrim periodical
* `ttcn3_slave`: (true) install titan ttcn3 compiler and prepares the docker-playground
diff --git a/ansible/roles/osmocom-jenkins-slave/defaults/main.yml b/ansible/roles/osmocom-jenkins-slave/defaults/main.yml
index 4e1f179..a4ce8ac 100644
--- a/ansible/roles/osmocom-jenkins-slave/defaults/main.yml
+++ b/ansible/roles/osmocom-jenkins-slave/defaults/main.yml
@@ -6,9 +6,6 @@ jenkins_user: jenkins
# install usefull utilities including qemu/debootstrap/fakeroot
install_jenkins_utilities: true
-# install all osmocom runtime and build time dependencies
-install_osmocom_build_deps: true
-
# E.g. a generic_slave requires different tasks as a
# special slave including the gsm-tester
generic_slave: true
diff --git a/ansible/roles/osmocom-jenkins-slave/tasks/debian.yml b/ansible/roles/osmocom-jenkins-slave/tasks/debian.yml
index 0c762e9..62beb8e 100644
--- a/ansible/roles/osmocom-jenkins-slave/tasks/debian.yml
+++ b/ansible/roles/osmocom-jenkins-slave/tasks/debian.yml
@@ -3,14 +3,15 @@
- name: install jenkins utilities
apt:
name:
- - htop
- - virt-manager
- - qemu
- - qemu-user-static
- - qemu-system-arm
- - proot
+ - bc
- debootstrap
- fakeroot
+ - htop
+ - proot
+ - python3-setuptools
+ - qemu-system-arm
+ - qemu-user-static
+ - virt-manager
cache_valid_time: 3600
update_cache: yes
install_recommends: no
@@ -50,7 +51,7 @@
- name: install java for buster and later
apt:
name:
- - openjdk-11-jre-headless
+ - default-jre-headless
cache_valid_time: 3600
update_cache: yes
install_recommends: no
diff --git a/ansible/roles/osmocom-jenkins-slave/tasks/main.yml b/ansible/roles/osmocom-jenkins-slave/tasks/main.yml
index 2623201..1868d97 100644
--- a/ansible/roles/osmocom-jenkins-slave/tasks/main.yml
+++ b/ansible/roles/osmocom-jenkins-slave/tasks/main.yml
@@ -22,19 +22,22 @@
shell: /bin/bash
uid: 1000
-- name: setup jenkins ssh key
+- name: setup jenkins ssh key (authorized_keys)
authorized_key:
user: "{{ jenkins_user }}"
key: "{{ lookup('file', 'jenkins.osmocom.org.pub') }}"
+- name: setup gerrit ssh key (known_hosts)
+ known_hosts:
+ name: '[gerrit.osmocom.org]:29418'
+ key: '[gerrit.osmocom.org]:29418 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQDI4wZwtcGz2V98iZWWEtsA162PolLtgQTBjnC14ONzZGQkTh9bTPsoSxnM5TKOm4F2bzq7gb+Qrsj4ZECD19qztmD37kMP9jn7/2i1V8oLAy9ojyklXAdBVTliQteVI7ieDOyKCnGbszvFWXY2isoO1k7yJLv0QyL7F5AuZlxeHw=='
+ path: /etc/ssh/ssh_known_hosts
+ state: present
+
- name: install ttcn3 dependencies
include_tasks: ttcn3-slave.yml
when: ttcn3_slave and ansible_distribution == 'Debian' and ansible_distribution_version|int >= 9 and not (ansible_architecture == "armv7l" or ansible_architecture == "arm64")
-- name: include osmocom_build_deps.yml
- include_tasks: osmocom_build_deps.yml
- when: install_osmocom_build_deps
-
- name: include generic slave
include_tasks: generic-slave.yml
when: generic_slave
diff --git a/ansible/roles/osmocom-jenkins-slave/tasks/osmo-ci.yml b/ansible/roles/osmocom-jenkins-slave/tasks/osmo-ci.yml
index b26d6ab..a2f2797 100644
--- a/ansible/roles/osmocom-jenkins-slave/tasks/osmo-ci.yml
+++ b/ansible/roles/osmocom-jenkins-slave/tasks/osmo-ci.yml
@@ -4,18 +4,26 @@
# we need to initially install osmo-ci and the docker images *before*
# that jenkins job can execute subsequent updates
+- name: install osmo-ci build dependencies
+ apt:
+ name:
+ - make
+ cache_valid_time: 3600
+ update_cache: yes
+ install_recommends: no
+
- name: deploy osmo-ci to home directory
become: yes
become_user: "{{ jenkins_user }}"
git:
- repo: "https://git.osmocom.org/osmo-ci"
+ repo: "https://gerrit.osmocom.org/osmo-ci"
dest: "/home/{{ jenkins_user }}/osmo-ci"
- name: deploy osmo-gsm-manuals to home directory
become: yes
become_user: "{{ jenkins_user }}"
git:
- repo: "https://git.osmocom.org/osmo-gsm-manuals"
+ repo: "https://gerrit.osmocom.org/osmo-gsm-manuals"
dest: "/home/{{ jenkins_user }}/osmo-gsm-manuals"
- name: rebuild osmocom docker images for jenkins build tests
diff --git a/ansible/roles/osmocom-jenkins-slave/tasks/osmocom_build_deps.yml b/ansible/roles/osmocom-jenkins-slave/tasks/osmocom_build_deps.yml
deleted file mode 100644
index e2b5a98..0000000
--- a/ansible/roles/osmocom-jenkins-slave/tasks/osmocom_build_deps.yml
+++ /dev/null
@@ -1,190 +0,0 @@
----
-- name: install build utilities
- apt:
- name:
- - asciidoc
- - autoconf
- - autoconf-archive
- - automake
- - bc
- - bison
- - coccinelle
- - cppcheck
- - debhelper
- - devscripts
- - dh-autoreconf
- - dh-systemd
- - doxygen
- - flex
- - g++
- - gawk
- - gcc
- - gcc-arm-none-eabi
- - git
- - git-buildpackage
- - graphviz
- - libcsv-dev
- - libtool
- - libboost-all-dev
- - make
- - mscgen
- - osc
- - pkgconf
- - python-minimal
- - python-setuptools
- - python3
- - python3-setuptools
- - stow
- - texinfo
- # by gsm-manuals
- - python3-nwdiag
- - python-pychart
- - inkscape
- # by gapk
- - libasound-dev
- - libopencore-amrnb-dev
- # by LimeSuite
- - cmake
- - patchelf
- # by libusrp
- - sdcc
- # by mingw upload job
- - rsync
- cache_valid_time: 3600
- update_cache: yes
-
-- name: install build dependencies and libraries
- apt:
- name:
- - dahdi-source
- - docbook5-xml
- - libboost-dev
- - libc-ares-dev
- - libdbd-sqlite3
- - libdbi-dev
- - libffi-dev
- - libffi-dev
- - libfftw3-dev
- - libgmp-dev
- - libgnutls28-dev
- - libgps-dev
- - libgsm1-dev
- - libnl-3-dev
- - libnl-route-3-dev
- - libmnl-dev
- # by osmo-sysmon
- - liboping-dev
- - libncurses5-dev
- - libortp-dev
- - libpcap-dev
- - libpcsclite-dev
- - libreadline-dev
- - libsctp-dev
- - libsigsegv-dev
- - libsnmp-dev
- - libsofia-sip-ua-glib-dev
- - libsqlite3-dev
- - libssl-dev
- - libsystemd-dev
- - libtalloc-dev
- - libuhd-dev
- - libusb-1.0-0-dev
- - libusb-dev
- - libzmq3-dev
- - sqlite3
- - libaio-dev
- cache_valid_time: 3600
- update_cache: yes
- install_recommends: no
-
-- name: install liblua for jessie
- apt:
- name: liblua5.3-dev
- cache_valid_time: 3600
- update_cache: yes
- install_recommends: no
- default_release: jessie-backports
- when: ansible_distribution == 'Debian' and ansible_distribution_release == 'jessie'
-
-- name: install liblua for stretch or later
- apt:
- name:
- - liblua5.3-dev
- cache_valid_time: 3600
- update_cache: yes
- install_recommends: no
- when: ansible_distribution == 'Debian' and ansible_distribution_version|int >= 9
-
-- name: install liblimesuite-dev for stretch or later
- apt:
- name:
- - liblimesuite-dev
- cache_valid_time: 3600
- update_cache: yes
- install_recommends: no
- when: ansible_distribution == 'Debian' and ansible_distribution_version|int >= 9
-
-- name: install osmo-remsim dependencies
- apt:
- name:
- - libjansson-dev
- - libulfius-dev
- cache_valid_time: 3600
- update_cache: yes
- when: ansible_distribution == 'Debian' and ansible_distribution_version|int >= 9
-
-- name: install srsLTE build dependencies
- apt:
- name:
- - cmake
- - libfftw3-dev
- - libmbedtls-dev
- - libboost-program-options-dev
- - libconfig++-dev
- - libsctp-dev
- - libpcsclite-dev
- - libuhd-dev
- - libczmq-dev
- - libsoapysdr-dev
- cache_valid_time: 3600
- update_cache: yes
- when: ansible_distribution == 'Debian' and ansible_distribution_version|int >= 9
-
-- name: install open5gs build dependencies
- apt:
- name:
- - meson
- - ninja-build
- - libgcrypt-dev
- - libidn11-dev
- - libmongoc-dev
- - libbson-dev
- - libyaml-dev
- - libnghttp2-dev
- cache_valid_time: 3600
- update_cache: yes
- when: ansible_distribution == 'Debian' and ansible_distribution_version|int >= 10
-
-- name: download osmo-python-tests for py2
- git:
- repo: "https://gerrit.osmocom.org/python/osmo-python-tests"
- dest: "/home/{{ jenkins_user }}/osmo-python-tests2"
- version: "fb4b813d4df62b7b2445bdced961eb1847267eed"
-
-- name: install osmo-python-tests for py2
- shell:
- cmd: "python2 setup.py clean build install && date > installed"
- chdir: "/home/{{ jenkins_user }}/osmo-python-tests2"
- creates: "/home/{{ jenkins_user }}/osmo-python-tests2/installed"
-
-- name: download osmo-python-tests for py3
- git:
- repo: "https://gerrit.osmocom.org/python/osmo-python-tests"
- dest: "/home/{{ jenkins_user }}/osmo-python-tests3"
- version: "be7fcf5f28a5d0f9ad35510f09b5979268d76598"
-
-- name: install osmo-python-tests for py3
- shell:
- cmd: "python3 setup.py clean build install && date > installed"
- chdir: "/home/{{ jenkins_user }}/osmo-python-tests3"
- creates: "/home/{{ jenkins_user }}/osmo-python-tests3/installed"
diff --git a/ansible/roles/osmocom-jenkins-slave/tasks/ttcn3-slave.yml b/ansible/roles/osmocom-jenkins-slave/tasks/ttcn3-slave.yml
index fbfc1ec..4237073 100644
--- a/ansible/roles/osmocom-jenkins-slave/tasks/ttcn3-slave.yml
+++ b/ansible/roles/osmocom-jenkins-slave/tasks/ttcn3-slave.yml
@@ -1,4 +1,12 @@
---
+- name: install programs used in jenkins.sh
+ apt:
+ name:
+ - netcat-openbsd
+ - sudo
+ cache_valid_time: 3600
+ update_cache: yes
+ install_recommends: no
- name: ensure jenkins folder exists
become: yes
@@ -11,33 +19,27 @@
become: yes
become_user: "{{ jenkins_user }}"
git:
- repo: "https://git.osmocom.org/docker-playground"
+ repo: "https://gerrit.osmocom.org/docker-playground"
dest: "/home/{{ jenkins_user }}/jenkins/docker-playground"
- name: add apt.key
- apt_key:
- url: https://download.opensuse.org/repositories/network:/osmocom:/latest/{{ obs_distro }}/Release.key
+ get_url:
+ url: https://obs.osmocom.org/projects/osmocom/public_key
+ dest: /etc/apt/trusted.gpg.d/obs.osmocom.org.asc
+ # OBS webserver needs this header, returns 404 otherwise
+ # https://github.com/ansible/ansible/issues/50436#issuecomment-451205256
+ headers:
+ Accept: '*/*'
- name: add apt repository
apt_repository:
- repo: "deb http://download.opensuse.org/repositories/network:/osmocom:/latest/{{ obs_distro }}/ ./"
+ repo: "deb [signed-by=/etc/apt/trusted.gpg.d/obs.osmocom.org.asc] https://downloads.osmocom.org/packages/osmocom:/latest/{{ obs_distro }}/ ./"
filename: obs_osmocom
update_cache: yes
-- name: install titan
+- name: install libosmocore-utils
apt:
- name: eclipse-titan
+ name:
+ - libosmocore-utils
update_cache: yes
cache_valid_time: 3600
-
-- name: Get DEB architecture
- shell: dpkg --print-architecture
- register: deb_architecture
-
-- name: Install libfftranscode0
- apt:
- deb: https://ftp.osmocom.org/binaries/libfftranscode/libfftranscode0_0.3_{{deb_architecture.stdout}}.deb
-
-- name: Install libfftranscode-dev
- apt:
- deb: https://ftp.osmocom.org/binaries/libfftranscode/libfftranscode-dev_0.3_{{deb_architecture.stdout}}.deb
diff --git a/ansible/roles/osmocom-obs-worker/files/obs-ca.crt b/ansible/roles/osmocom-obs-worker/files/obs-ca.crt
new file mode 100644
index 0000000..271d168
--- /dev/null
+++ b/ansible/roles/osmocom-obs-worker/files/obs-ca.crt
@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDSzCCAjOgAwIBAgIUcLDeyxhcMpcccZ1i7AIKqvwTJ2gwDQYJKoZIhvcNAQEL
+BQAwFjEUMBIGA1UEAwwLRWFzeS1SU0EgQ0EwHhcNMjIwNTEwMDkzMjQwWhcNNDIw
+NTA1MDkzMjQwWjAWMRQwEgYDVQQDDAtFYXN5LVJTQSBDQTCCASIwDQYJKoZIhvcN
+AQEBBQADggEPADCCAQoCggEBALUsvHoy4oqpOHuzyoIG3qyoTGQW8Zpse7RaYAWJ
+CLgcAGelXWrOhPy40xAWHpLzAnAIxYR4ZSlD2Ebaj08esz4VQs2CKQ0mJEVZA7oq
+84YO3WK02HzoYFLee4ySvI967FCn6BwVZQVdz/A8wOIlAbe7Hmucx6dNLTLTR+Pw
+YwHMSdnj+OlC2oX2UO+ZIR7jFbSYp7jGF9V8zk74NoAFAuv5rs1UeP01ZF9xgXll
+KHUajaUKMKs+6KTo4d2l9ha45LHcoAuDUURzf4GMvO7zVVGw4mFwy/YsPoLV5+vX
+YT4GqkBgk9iZXirgTk6xCkiP0NUw5UNhtarUDKvR/U8wLw8CAwEAAaOBkDCBjTAd
+BgNVHQ4EFgQUHdDZVpUSwtRrSKp+PuMRK12TlQYwUQYDVR0jBEowSIAUHdDZVpUS
+wtRrSKp+PuMRK12TlQahGqQYMBYxFDASBgNVBAMMC0Vhc3ktUlNBIENBghRwsN7L
+GFwylxxxnWLsAgqq/BMnaDAMBgNVHRMEBTADAQH/MAsGA1UdDwQEAwIBBjANBgkq
+hkiG9w0BAQsFAAOCAQEAPGqIfE172P1NaQGRIHdsRJRot9NnJzZih3q9m+T/YIE8
+8T3uoxhEsMPLzP19w8wAbRy+CjpLwTuYO44kOIVLYhFdx3BTVRhw+YU2RH4PSttE
+JDVyYTFPGzjxwprpfui1aGom8u0oPL5MeutMWrPH6qVlIv7JGR3EbDG0iYwE5MZJ
+6kW8VW6o5fL02uT+75vNkHTipLUO6S4l6h9NrrgR0ijZ5dhyKC0FZnLnowtMCXXl
+2rfHSnGKjMGcq8Jlrwo6eMVPbZM6rrU/tvrjJkqESZY0jMktd0fV903XJ6+X4U9U
+stu/HUtKUQC8d3wua1PuI1uPvpr8AQDcf5LuA/STHg==
+-----END CERTIFICATE-----
diff --git a/ansible/roles/osmocom-obs-worker/files/obs.conf b/ansible/roles/osmocom-obs-worker/files/obs.conf
new file mode 100644
index 0000000..7dc2a84
--- /dev/null
+++ b/ansible/roles/osmocom-obs-worker/files/obs.conf
@@ -0,0 +1,15 @@
+client
+remote 2a01:4f8:201:344a::1:4 1194
+ca "/etc/openvpn/obs-ca.crt"
+cert "/etc/openvpn/obs.crt"
+key "/etc/openvpn/obs.key"
+comp-lzo yes
+cipher AES-256-CBC
+dev tun
+proto udp6
+nobind
+auth-nocache
+script-security 2
+persist-key
+persist-tun
+remote-cert-tls server
diff --git a/ansible/roles/osmocom-obs-worker/tasks/main.yml b/ansible/roles/osmocom-obs-worker/tasks/main.yml
new file mode 100644
index 0000000..b350163
--- /dev/null
+++ b/ansible/roles/osmocom-obs-worker/tasks/main.yml
@@ -0,0 +1,8 @@
+---
+
+- name: install OBS
+ include_tasks: obs.yml
+
+- name: install openvpn
+ include_tasks: openvpn.yml
+
diff --git a/ansible/roles/osmocom-obs-worker/tasks/obs.yml b/ansible/roles/osmocom-obs-worker/tasks/obs.yml
new file mode 100644
index 0000000..a0a6227
--- /dev/null
+++ b/ansible/roles/osmocom-obs-worker/tasks/obs.yml
@@ -0,0 +1,52 @@
+- name: add OBS rpm repository
+ zypper_repository:
+ repo: https://download.opensuse.org/repositories/OBS:/Server:/2.10/{{ ansible_distribution_version }}/OBS:Server:2.10.repo
+ autorefresh: yes
+ auto_import_keys: yes
+
+- name: install obs-worker
+ zypper:
+ name:
+ - obs-worker
+
+- name: Make sure an entry in /etc/hosts exists
+ lineinfile:
+ path: /etc/hosts
+ regexp: "^172.16.31.1"
+ line: "172.16.31.1 obs-backend.osmocom.org obs obs.osmocom.org"
+ state: present
+
+- name: Create /srv/obs/cache
+ file:
+ path: /srv/obs/cache
+ owner: obsrun
+ group: obsrun
+ recurse: yes
+
+- name: generate obs-server config file
+ template:
+ src: obs-server
+ dest: /etc/sysconfig/obs-server
+
+- name: install lxc
+ when: obs_vm_type == "lxc"
+ zypper:
+ name:
+ - lxc
+
+- name: install qemu
+ when: obs_vm_type != "lxc"
+ zypper:
+ name:
+ - qemu-arm
+
+- name: install qemu-arm
+ when: obs_vm_type == "emulator:armv7l" or obs_vm_type == "emulator:aarch64"
+ zypper:
+ name:
+ - qemu-arm
+
+- name: ensure the systemd service is installed
+ systemd:
+ name: obsworker.service
+ enabled: yes
diff --git a/ansible/roles/osmocom-obs-worker/tasks/openvpn.yml b/ansible/roles/osmocom-obs-worker/tasks/openvpn.yml
new file mode 100644
index 0000000..da53fa5
--- /dev/null
+++ b/ansible/roles/osmocom-obs-worker/tasks/openvpn.yml
@@ -0,0 +1,21 @@
+- name: Install openvpn
+ zypper:
+ name:
+ - openvpn
+
+- name: copy openvpn config
+ copy:
+ src: obs.conf
+ dest: /etc/openvpn/
+ mode: 0600
+
+- name: copy openvpn CA certificate
+ copy:
+ src: obs-ca.crt
+ dest: /etc/openvpn/
+ mode: 0600
+
+- name: ensure the systemd service is installed
+ systemd:
+ name: openvpn@obs.service
+ enabled: yes
diff --git a/ansible/roles/osmocom-obs-worker/templates/obs-server b/ansible/roles/osmocom-obs-worker/templates/obs-server
new file mode 100644
index 0000000..f042cd1
--- /dev/null
+++ b/ansible/roles/osmocom-obs-worker/templates/obs-server
@@ -0,0 +1,442 @@
+#
+# NOTE: all these options can be also declared in /etc/buildhost.config on each worker differently.
+#
+
+## Path: Applications/OBS
+## Description: The OBS backend code directory
+## Type: string
+## Default: ""
+## Config: OBS
+#
+# An empty dir will lead to the fall back directory, typically /usr/lib/obs/server/
+#
+OBS_BACKENDCODE_DIR=""
+
+## Path: Applications/OBS
+## Description: The base for OBS communication directory
+## Type: string
+## Default: ""
+## Config: OBS
+#
+# An empty dir will lead to the fall back directory, typically /srv/obs/run
+#
+OBS_RUN_DIR=""
+
+## Path: Applications/OBS
+## Description: The base for OBS logging directory
+## Type: string
+## Default: ""
+## Config: OBS
+#
+# An empty dir will lead to the fall back directory, typically /srv/obs/log
+#
+OBS_LOG_DIR=""
+
+## Path: Applications/OBS
+## Description: The base directory for OBS
+## Type: string
+## Default: ""
+## Config: OBS
+#
+# An empty dir will lead to the fall back directory, typically /srv/obs
+#
+OBS_BASE_DIR=""
+
+## Path: Applications/OBS
+## Description: Automatically setup api and webui for OBS server, be warned, this will replace config files !
+## Type: ("yes" | "no")
+## Default: "no"
+## Config: OBS
+#
+# This is usally only enabled on the OBS Appliance
+#
+OBS_API_AUTOSETUP="no"
+#
+# NOTE: all these options can be also declared in /etc/buildhost.config on each worker differently.
+#
+
+## Path: Applications/OBS
+## Description: define source server host to be used
+## Type: string
+## Default: ""
+## Config: OBS
+#
+# An empty setting will point to localhost:5352 by default
+#
+OBS_SRC_SERVER="obs-backend.osmocom.org:5352"
+
+## Path: Applications/OBS
+## Description: define repository server host to be used
+## Type: string
+## Default: ""
+## Config: OBS
+#
+# An empty setting will point to localhost:5252 by default
+#
+OBS_REPO_SERVERS="obs-backend.osmocom.org:5252"
+
+## Path: Applications/OBS
+## Description: define number of build instances
+## Type: integer
+## Default: 0
+## Config: OBS
+#
+# 0 instances will automatically use the number of CPU's
+#
+OBS_WORKER_INSTANCES="{{obs_worker_instances}}"
+
+## Path: Applications/OBS
+## Description: define names of build instances for z/VM
+## Type: string
+## Default: ""
+## Config: OBS
+#
+# The names of the workers as defined in z/VM. These must have two minidisks
+# assigned, and have a secondary console configured to the local machine:
+# 0150 is the root device
+# 0250 is the swap device
+#
+#OBS_WORKER_INSTANCE_NAMES="LINUX075 LINUX076 LINUX077"
+OBS_WORKER_INSTANCE_NAMES=""
+
+## Path: Applications/OBS
+## Description: The base directory, where sub directories for each worker will get created
+## Type: string
+## Default: ""
+## Config: OBS
+#
+#
+OBS_WORKER_DIRECTORY=""
+
+## Path: Applications/OBS
+## Description: The base for port numbers used by worker instances
+## Type: integer
+## Default: "0"
+## Config: OBS
+#
+# 0 means let the operating system assign a port number
+#
+OBS_WORKER_PORTBASE="0"
+
+## Path: Applications/OBS
+## Description: Number of parallel compile jobs per worker
+## Type: integer
+## Default: "1"
+## Config: OBS
+#
+# this maps usually to "make -j1" during build
+#
+OBS_WORKER_JOBS="{{obs_worker_jobs}}"
+
+## Path: Applications/OBS
+## Description: Run in test mode (build results will be ignore, no job blocking)
+## Type: ("yes" | "")
+## Default: ""
+## Config: OBS
+#
+OBS_WORKER_TEST_MODE=""
+
+## Path: Applications/OBS
+## Description: define one or more labels for the build host.
+## Type: string
+## Default: ""
+## Config: OBS
+#
+# A label can be used to build specific packages only on dedicated hosts.
+# For example for benchmarking.
+#
+OBS_WORKER_HOSTLABELS=""
+
+## Path: Applications/OBS
+## Description: can be used to define a security level of the worker
+## Type: string
+## Default: ""
+## Config: OBS
+#
+# This will extend the hostlabels and can be used to limit the workers
+# to the hosts which have all security fixes deployed.
+#
+OBS_WORKER_SECURITY_LEVEL=""
+
+## Path: Applications/OBS
+## Description: Register in SLP server
+## Type: ("yes" | "no")
+## Default: "yes"
+## Config: OBS
+#
+#
+OBS_USE_SLP="yes"
+
+## Path: Applications/OBS
+## Description: Use a common cache directory for downloaded packages
+## Type: string
+## Default: ""
+## Config: OBS
+#
+# Enable caching requires a given directory here. Be warned, content will be
+# removed there !
+#
+OBS_CACHE_DIR="/srv/obs/cache"
+
+## Path: Applications/OBS
+## Description: Defines the package cache size
+## Type: size in MB
+## Default: ""
+## Config: OBS
+#
+# Set the size to 50% of the maximum usable size of this partition
+#
+OBS_CACHE_SIZE="100000"
+
+## Path: Applications/OBS
+## Description: Defines the nice level of running workers
+## Type: integer
+## Default: 18
+## Config: OBS
+#
+# Nicenesses range from -20 (most favorable scheduling) to 19 (least
+# favorable).
+# Default to 18 as some testsuites depend on being able to switch to
+# one priority below (19) _and_ having changed the numeric level
+# (so going from 19->19 makes them fail).
+#
+OBS_WORKER_NICE_LEVEL=18
+
+## Path: Applications/OBS
+## Description: Set used VM type by worker
+## Type: ("auto" | "xen" | "kvm" | "lxc" | "zvm" | "emulator:$arch" | "emulator:$arch:$script" | "none" | "openstack")
+## Default: "auto"
+## Config: OBS
+#
+#
+OBS_VM_TYPE="{{obs_vm_type}}"
+
+## Path: Applications/OBS
+## Description: Set kernel used by worker (kvm)
+## Type: ("none" | "/boot/vmlinuz" | "/foo/bar/vmlinuz)
+## Default: "none"
+## Config: OBS
+#
+# For z/VM this is normally /boot/image
+#
+OBS_VM_KERNEL="none"
+
+## Path: Applications/OBS
+## Description: Set initrd used by worker (kvm)
+## Type: ("none" | "/boot/initrd" | "/foo/bar/initrd-foo)
+## Default: "none"
+## Config: OBS
+#
+# for KVM, you have to create with (example for openSUSE 11.2):
+#
+# export rootfstype="ext4"
+# mkinitrd -d /dev/null -m "ext4 binfmt_misc virtio_pci virtio_blk" -k vmlinuz-2.6.31.12-0.2-default -i initrd-2.6.31.12-0.2-default-obs_worker
+#
+# a working initrd file which includes virtio and binfmt_misc for OBS in order to work fine
+#
+# for z/VM, the build script will create a initrd at the given location if
+# it does not yet exist.
+#
+OBS_VM_INITRD="none"
+
+## Path: Applications/OBS
+## Description: Autosetup for XEN/KVM/TMPFS disk (root) - Filesize in MB
+## Type: integer
+## Default: "4096"
+## Config: OBS
+#
+#
+OBS_VM_DISK_AUTOSETUP_ROOT_FILESIZE="4096"
+
+## Path: Applications/OBS
+## Description: Autosetup for XEN/KVM disk (swap) - Filesize in MB
+## Type: integer
+## Default: "1024"
+## Config: OBS
+#
+#
+OBS_VM_DISK_AUTOSETUP_SWAP_FILESIZE="1024"
+
+## Path: Applications/OBS
+## Description: Default filesystem to use for autosetup.
+## Type: ("ext2" | "ext3" | "ext4" | "reiserfs" | "btrfs" | "xfs")
+## Default: "ext3"
+## Config: OBS
+#
+#
+# Buildflag vmfstype may overwrite this for a specific job.
+OBS_VM_DISK_AUTOSETUP_FILESYSTEM="ext3"
+
+## Path: Applications/OBS
+## Description: Filesystem mount options to use for autosetup
+## Type: string
+## Default: ""
+## Config: OBS
+#
+#
+OBS_VM_DISK_AUTOSETUP_MOUNT_OPTIONS=""
+
+## Path: Applications/OBS
+## Description: Enable build in memory
+## Type: ("yes" | "")
+## Default: ""
+## Config: OBS
+#
+# WARNING: this requires much memory!
+#
+OBS_VM_USE_TMPFS="yes"
+
+## Path: Applications/OBS
+## Description: Specify custom options for VM handler
+## Type: string
+## Default: ""
+## Config: OBS
+#
+# Can be used to workaround problems with VM handler and should not be needed usually
+#
+OBS_VM_CUSTOM_OPTION=""
+
+## Path: Applications/OBS
+## Description: Memory allocated for each VM (512) if not set
+## Type: integer
+## Default: ""
+## Config: OBS
+#
+#
+OBS_INSTANCE_MEMORY=""
+
+## Path: Applications/OBS
+## Description: Enable storage auto configuration
+## Type: ("yes" | "")
+## Default: ""
+## Config: OBS
+#
+# WARNING: this may destroy data on your hard disk !
+# This is usually only used on mass deployed worker instances
+#
+OBS_STORAGE_AUTOSETUP=""
+
+## Path: Applications/OBS
+## Description: Setup LVM via obsstoragesetup
+## Type: ("take_all" | "use_obs_vg" | "none")
+## Default: "use_obs_vg"
+## Config: OBS
+#
+# take_all: WARNING: all LVM partitions will be used and all data erased !
+# use_obs_vg: A lvm volume group named "OBS" will be re-setup for the workers.
+#
+OBS_SETUP_WORKER_PARTITIONS="use_obs_vg"
+
+## Path: Applications/OBS
+## Description: Size in MB when creating LVM partition for cache partition
+## Type: integer
+## Default: ""
+## Config: OBS
+#
+#
+OBS_WORKER_CACHE_SIZE=""
+
+## Path: Applications/OBS
+## Description: Size in MB when creating LVM partition for each worker root partition
+## Type: integer
+## Default: ""
+## Config: OBS
+#
+#
+OBS_WORKER_ROOT_SIZE=""
+
+## Path: Applications/OBS
+## Description: Size in MB when creating LVM partition for each worker swap partition
+## Type: integer
+## Default: ""
+## Config: OBS
+#
+#
+OBS_WORKER_SWAP_SIZE=""
+
+## Path: Applications/OBS
+## Description: URL to a proxy service for caching binaries used by worker
+## Type: string
+## Default: ""
+## Config: OBS
+#
+#
+OBS_WORKER_BINARIES_PROXY=""
+
+## Path: Applications/OBS
+## Description: URL to a ssh pub key to allow root user login
+## Type: string
+## Default: ""
+## Config: OBS
+#
+# This is usually used on mass (PXE) deployed workers)
+#
+OBS_ROOT_SSHD_KEY_URL=""
+
+## Path: Applications/OBS
+## Description: URL to a script to be downloaded and executed
+## Type: string
+## Default: ""
+## Config: OBS
+#
+# This is a hook for doing special things in your setup at boot time
+#
+OBS_WORKER_SCRIPT_URL=""
+
+## Path: Applications/OBS
+## Description: If chroot/lxc is used for build, empty it after build is finished
+## Type: ("yes" | "")
+## Default: ""
+## Config: OBS
+#
+#
+OBS_WORKER_CLEANUP_CHROOT=""
+
+##Path: Application/OBS
+## Description: wipes the build environment of the worker after the build
+## Type: ("yes" | "")
+## Default: ""
+## Config: OBS
+#
+#
+OBS_WORKER_WIPE_AFTER_BUILD=""
+
+##Path: Application/OBS
+## Description: name or id of openstack instance that controls the worker (building) instances
+## Type: ("yes" | "")
+## Default: ""
+## Config: OBS
+#
+#
+OBS_WORKER_CONTROL_INSTANCE=""
+
+##Path: Application/OBS
+## Description: name or id flavor to create openstack worker (building) instance
+## Type: ("yes" | "")
+## Default: ""
+## Config: OBS
+#
+#
+OBS_WORKER_OS_FLAVOR=""
+
+##Path: Application/OBS
+## Description: openstack environment variables. Only used when OBS_VM_TYPE=openstack
+## Type: ("yes" | "")
+## Default: ""
+## Config: OBS
+#
+#
+OS_AUTH_URL=""
+
+OS_PROJECT_ID=""
+OS_PROJECT_NAME=""
+OS_USER_DOMAIN_NAME=""
+OS_USERNAME=""
+OS_PASSWORD=""
+OS_REGION_NAME=""
+
+OBS_WORKER_PREFIX=""
+
+OBS_OPENSTACK_DISK_SIZE=""
+OBS_OPENSTACK_SWAP_SIZE=""
+OBS_OPENSTACK_MEMORY_SIZE=""
diff --git a/ansible/roles/osmocom-workstation/tasks/git-repos.yml b/ansible/roles/osmocom-workstation/tasks/git-repos.yml
index edef6dd..719b6eb 100644
--- a/ansible/roles/osmocom-workstation/tasks/git-repos.yml
+++ b/ansible/roles/osmocom-workstation/tasks/git-repos.yml
@@ -4,7 +4,7 @@
become: yes
become_user: "{{ osmocom_user }}"
git:
- repo: git://git.osmocom.org/{{ item }}.git
+ repo: https://git.osmocom.org/{{ item }}
dest: "{{ git_destination }}/{{ item }}"
accept_hostkey: yes
force: no
@@ -37,6 +37,7 @@
- osmo-gtp-kernel
- osmo-hlr
- osmo-hnodeb
+ - osmo-hnbgw
- osmo-iuh
- osmo-mgw
- osmo-msc
@@ -105,7 +106,7 @@
chdir: "{{ git_destination }}/osmo-ttcn3-hacks"
- name: install sysmocom git repos
- git: repo=https://git.sysmocom.de/{{ item }}.git
+ git: repo=https://gitea.sysmocom.de/{{ item }}.git
dest={{ git_destination }}/{{ item }}
accept_hostkey=yes
force=yes
@@ -118,7 +119,7 @@
- name: install github git repos
- git: repo=git://github.com/{{ item }}.git
+ git: repo=https://github.com/{{ item }}.git
dest={{ git_destination }}/{{ item }}
accept_hostkey=yes
force=yes
diff --git a/ansible/roles/osmocom-workstation/tasks/main.yml b/ansible/roles/osmocom-workstation/tasks/main.yml
index ae29111..0331207 100644
--- a/ansible/roles/osmocom-workstation/tasks/main.yml
+++ b/ansible/roles/osmocom-workstation/tasks/main.yml
@@ -9,8 +9,8 @@
shell: /usr/bin/zsh
groups: docker
-- name: include git repos
- include: git-repos.yml
+- name: import git repos
+ import_tasks: git-repos.yml
- name: install distribution packages
apt:
diff --git a/ansible/roles/podman/tasks/debian.yml b/ansible/roles/podman/tasks/debian.yml
deleted file mode 100644
index e261a3f..0000000
--- a/ansible/roles/podman/tasks/debian.yml
+++ /dev/null
@@ -1,33 +0,0 @@
----
-
-- name: add apt https support
- apt:
- cache_valid_time: 3600
- update_cache: yes
- pkg:
- - apt-transport-https
- - gnupg
- - cron
- - ca-certificates
- when: ansible_distribution_major_version < 11
-
-- name: add docker gpg key to apt keyring
- apt_key:
- id: 2472D6D0D2F66AF87ABA8DA34D64390375060AA4
- url: https://ftp.uni-stuttgart.de/opensuse/repositories/devel:/kubic:/libcontainers:/stable/{{ obs_distro }}/Release.key
- when: ansible_distribution_major_version < 11
-
-- apt_repository:
- repo: "deb https://ftp.uni-stuttgart.de/opensuse/repositories/devel:/kubic:/libcontainers:/stable/{{ obs_distro }}/ /"
- state: present
- update_cache: yes
- when: ansible_distribution_major_version < 11
-
-- name: install podman
- apt:
- pkg:
- - podman
- - runc
- - buildah
- - skopeo
- - dnsmasq
diff --git a/ansible/roles/podman/tasks/main.yml b/ansible/roles/podman/tasks/main.yml
index 049f5c2..45a712d 100644
--- a/ansible/roles/podman/tasks/main.yml
+++ b/ansible/roles/podman/tasks/main.yml
@@ -1,7 +1,9 @@
---
-- name: Include tasks for Debian
- include_tasks: debian.yml
- when: (ansible_distribution in ['Debian', 'Raspbian'])
-
-# TODO: add other distributions below
+- name: install podman
+ apt:
+ pkg:
+ - podman
+ - buildah
+ # Has reasonable recommends needed for dns resolving etc.
+ install_recommends: yes
diff --git a/ansible/roles/qemu/files/qemu-create-vms.sh b/ansible/roles/qemu/files/qemu-create-vms.sh
new file mode 100755
index 0000000..a6ee8e7
--- /dev/null
+++ b/ansible/roles/qemu/files/qemu-create-vms.sh
@@ -0,0 +1,85 @@
+#!/bin/sh -e
+# Create qcow2 images with ssh root login enabled for repo-install-test and
+# store them in /opt/qemu.
+# Set KEEP_CACHE=1 during development, so virt-builder only needs to download
+# the image once.
+
+# Distribution names, as in the base images from here:
+# https://builder.libguestfs.org/
+DISTROS="
+ alma-8.5
+ debian-10
+ debian-11
+ debian-12
+"
+TEMP_SCRIPT="$(mktemp)"
+
+if [ "$(id -u)" != 0 ]; then
+ echo "ERROR: run this as root"
+ exit 1
+fi
+
+mkdir -p /opt/qemu
+
+for distro in $DISTROS; do
+ img="/opt/qemu/$distro.qcow2"
+
+ echo
+ echo "# $distro"
+ echo
+
+ if [ -e "$img" ]; then
+ echo "=> File exists, skipping."
+ continue
+ fi
+
+ case "$distro" in
+ alma-*)
+ # Install SCTP kernel module
+ # https://forums.centos.org/viewtopic.php?t=71818
+ cat <<- EOF > "$TEMP_SCRIPT"
+ #!/bin/sh -ex
+ dnf upgrade -y kernel
+ dnf install -y kernel-modules-extra
+ rm -f /etc/modprobe.d/sctp-blacklist.conf
+ EOF
+ ;;
+ debian-*)
+ # Generate SSH server keys and allow login as root
+ cat <<- EOF > "$TEMP_SCRIPT"
+ #!/bin/sh -ex
+ ssh-keygen -A
+ echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
+ EOF
+ ;;
+ esac
+
+ EXTRA_ARGS=""
+ case "$distro" in
+ debian-12)
+ # repo-install-test runs out of space with the default size
+ EXTRA_ARGS="--size 8G"
+ ;;
+ esac
+
+ virt-builder \
+ "$distro" \
+ -o "/opt/qemu/$distro.qcow2" \
+ --format qcow2 \
+ --root-password password:root \
+ --run "$TEMP_SCRIPT" \
+ --verbose \
+ $EXTRA_ARGS
+
+ if [ -z "$KEEP_CACHE" ]; then
+ virt-builder --delete-cache
+ fi
+done
+
+rm "$TEMP_SCRIPT"
+
+# Marker for ansible main.yml to skip the script
+touch /opt/qemu/.qemu-create-vms-done-v2
+
+echo
+echo "Done!"
diff --git a/ansible/roles/qemu/tasks/main.yml b/ansible/roles/qemu/tasks/main.yml
new file mode 100644
index 0000000..1678217
--- /dev/null
+++ b/ansible/roles/qemu/tasks/main.yml
@@ -0,0 +1,22 @@
+---
+
+- name: install packages
+ apt:
+ name:
+ - libguestfs-tools
+ - netcat-openbsd
+ - qemu-system-x86
+ - sshpass
+ cache_valid_time: 3600
+ update_cache: yes
+
+- name: run qemu-create-vms.sh
+ ansible.builtin.script: qemu-create-vms.sh
+ args:
+ creates: /opt/qemu/.qemu-create-vms-done-v2
+
+- name: add jenkins to the kvm group
+ user:
+ name: "{{ jenkins_user }}"
+ groups: kvm
+ append: yes
diff --git a/ansible/setup-gsm-tester.yml b/ansible/setup-gsm-tester.yml
index 26285df..526b621 100644
--- a/ansible/setup-gsm-tester.yml
+++ b/ansible/setup-gsm-tester.yml
@@ -1,6 +1,6 @@
---
- name: setup osmo-gsm-tester apu
- hosts: gsm-tester
+ hosts: gsm_tester
user: root
tasks:
- name: install common utilities
@@ -14,6 +14,12 @@
- curl
cache_valid_time: 3600
update_cache: yes
+ - name: deploy osmo-ci to home directory
+ become: yes
+ become_user: "jenkins"
+ git:
+ repo: "https://gerrit.osmocom.org/osmo-ci"
+ dest: "/home/jenkins/osmo-ci"
roles:
- name: gsm-tester
tags:
diff --git a/ansible/setup-jenkins-slave.yml b/ansible/setup-jenkins-slave.yml
index 07414aa..5813b40 100644
--- a/ansible/setup-jenkins-slave.yml
+++ b/ansible/setup-jenkins-slave.yml
@@ -1,6 +1,6 @@
---
- name: setup jenkins slaves
- hosts: jenkins-slaves
+ hosts: jenkins_slaves
user: root
vars_files:
- "vars/{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version']}}.yml"
@@ -15,6 +15,8 @@
cache_valid_time: 3600
update_cache: yes
roles:
+ - name: apt-allow-relinfo-change
+
- name: docker
jenkins_user: osmocom-build
tags:
@@ -45,9 +47,18 @@
- name: ntpd
- name: setup coverity slaves
- hosts: coverity-slaves
+ hosts: coverity_slaves
user: root
roles:
- name: install-coverity
tags:
- coverity
+
+- name: setup qemu images
+ hosts: qemu
+ user: root
+ roles:
+ - name: qemu
+ jenkins_user: osmocom-build
+ tags:
+ - qemu
diff --git a/ansible/setup-obs-worker.yml b/ansible/setup-obs-worker.yml
new file mode 100644
index 0000000..b79e1a5
--- /dev/null
+++ b/ansible/setup-obs-worker.yml
@@ -0,0 +1,21 @@
+---
+- name: setup OBS worker
+ hosts: obs_workers
+ user: root
+ vars_files:
+ - "vars/{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version']}}.yml"
+ tasks:
+ - name: install common utilities
+ zypper:
+ name:
+ - vim
+ - tmux
+ - screen
+ - tcpdump
+ - strace
+ - ltrace
+
+ roles:
+ - name: osmocom-obs-worker
+ tags:
+ - obs-worker
diff --git a/ansible/setup-simtest.yml b/ansible/setup-simtest.yml
index 6eb67fc..eb37186 100644
--- a/ansible/setup-simtest.yml
+++ b/ansible/setup-simtest.yml
@@ -18,14 +18,24 @@
- python3-pip
- python3-serial
- python3-pyscard
+ - tshark
- virtualenv
install_recommends: no
+ - name: "pysim: get requirements.txt"
+ get_url:
+ url: https://gitea.osmocom.org/sim-card/pysim/raw/branch/master/requirements.txt
+ dest: /tmp/pysim_requirements.txt
+
+ - name: "pysim: install deps from requirements.txt"
+ pip:
+ executable: pip3
+ requirements: /tmp/pysim_requirements.txt
+
roles:
- name: osmocom-jenkins-slave
generic_slave: false
install_jenkins_utilities: false
- install_osmocom_build_deps: false
ttcn3_slave: false
tags:
- jenkins-slave
diff --git a/ansible/util/known_hosts b/ansible/util/known_hosts
new file mode 100644
index 0000000..76e2a64
--- /dev/null
+++ b/ansible/util/known_hosts
@@ -0,0 +1,10 @@
+2a01:4f8:10b:2ad9::1:10 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHtUqYQ2qE2w2et8frRXfauWiocse2sdqRLo1gksFaSS
+2a01:4f8:10b:2ad9::1:13 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEfg7NNBKjVJGoL3oUDrJCccI2D8podsSPKnRn1aTUge
+2a01:4f8:212:3762::1:3 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICl7tuOgN64+iYuYCtfCKaUbN9IWzKycpKVRgzsjVCPY
+2a01:4f8:120:8470::1:4 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIPKRz2Eg+h6RTREn52n/ip3pzUjRu9pw1XdjWUrvvi/
+10.9.25.50 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPowg8lZeIGJ6LCLA8U4eSHnnPX90FaVgInDuRi5U9Zq
+10.9.25.60 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOAK6SQXmifMamE1VHL/1GmPkk6KhplzT20jSOwb8PyZ
+10.9.25.70 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILioWTUt6CyZTceSKqy09mtlNUif+ETKTF6whnWuIROx
+2a01:4f8:121:200a::1:3 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGafw2VdeZo4YNvbu8VEBjDrsCFL+ZSbHtvhspNIQ9PR
+2a01:4f8:121:200a::1:2 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFyp0daWkfsRHErCVCcP/FZmtiRvFNAsK2A5TlQ6Ja+P
+10.34.2.153 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIG0rQGxSxWJHwNNT5XdeuMPSYCAXpcDnFDzvyw5+sH3l
diff --git a/ansible/util/ssh_config b/ansible/util/ssh_config
new file mode 100644
index 0000000..9738e83
--- /dev/null
+++ b/ansible/util/ssh_config
@@ -0,0 +1,3 @@
+ UserKnownHostsFile=util/known_hosts
+ HashKnownHosts=no
+
diff --git a/ansible/vars/Debian-11.yml b/ansible/vars/Debian-11.yml
new file mode 100644
index 0000000..8361e15
--- /dev/null
+++ b/ansible/vars/Debian-11.yml
@@ -0,0 +1,2 @@
+---
+obs_distro: Debian_11
diff --git a/ansible/vars/Debian-12.yml b/ansible/vars/Debian-12.yml
new file mode 100644
index 0000000..4db6ac1
--- /dev/null
+++ b/ansible/vars/Debian-12.yml
@@ -0,0 +1,2 @@
+---
+obs_distro: Debian_12
diff --git a/ansible/vars/Suse-15.yml b/ansible/vars/Suse-15.yml
new file mode 100644
index 0000000..0cf00c2
--- /dev/null
+++ b/ansible/vars/Suse-15.yml
@@ -0,0 +1,2 @@
+---
+obs_distro: fixme
diff --git a/contrib/jenkins.sh b/contrib/jenkins.sh
index e1564a7..96becc7 100755
--- a/contrib/jenkins.sh
+++ b/contrib/jenkins.sh
@@ -1,10 +1,22 @@
#!/bin/sh
set -e -x
+WORKSPACE_DIR="$(realpath "$(dirname "$0")/..")"
# Clone repository to ~/, or update existing
# $1: name of osmocom project
clone_repo() {
- cd ~/"$1" || (cd ~/ && git clone git://git.osmocom.org/"$1" && cd ~/"$1")
+ local project="$1"
+ local url="https://gerrit.osmocom.org/$project"
+
+ if [ -d ~/"$project" ]; then
+ cd ~/"$project"
+ git remote set-url origin "$url"
+ else
+ cd ~
+ git clone "$url"
+ cd "$project"
+ fi
+
git rev-parse HEAD
git status
@@ -12,12 +24,13 @@ clone_repo() {
git rev-parse HEAD
git status
+
+ cd "$WORKSPACE_DIR"
}
clone_repo osmo-ci
clone_repo osmo-gsm-manuals
if [ `uname` = "Linux" ] && [ "x${OSMO_CI_NO_DOCKER}" != "x1" ]; then
- cd ~/osmo-ci
scripts/osmo-ci-docker-rebuild.sh
fi
diff --git a/contrib/known_hosts b/contrib/known_hosts
new file mode 100644
index 0000000..de29e93
--- /dev/null
+++ b/contrib/known_hosts
@@ -0,0 +1,3 @@
+[ftp.osmocom.org]:48 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDDgQ9HntlpWNmh953a2Gc8NysKE4orOatVT1wQkyzhARnfYUerRuwyNr1GqMyBKdSI9amYVBXJIOUFcpV81niA7zQRUs66bpIMkE9/rHxBd81SkorEPOIS84W4vm3SZtuNqa+fADcqe88Hcb0ZdTzjKILuwi19gzrQyME2knHY71EOETe9Yow5RD2hTIpB5ecNxI0LUKDq+Ii8HfBvndPBIr0BWYDugckQ3Bocf+yn/tn2/GZieFEyFpBGF/MnLbAAfUKIdeyFRX7ufaiWWz5yKAfEhtziqdAGZaXNaLG6gkpy3EixOAy6ZXuTAk3b3Y0FUmDjhOHllbPmTOcKMry9
+[ftp.osmocom.org]:48 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPdWn1kEousXuKsZ+qJEZTt/NSeASxCrUfNDW3LWtH+d8Ust7ZuKp/vuyG+5pe5pwpPOgFu7TjN+0lVjYJVXH54=
+[ftp.osmocom.org]:48 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK8iivY70EiR5NiGChV39gRLjNpC8lvu1ZdHtdMw2zuX
diff --git a/contrib/obs-mirror/README b/contrib/obs-mirror/README
new file mode 100644
index 0000000..4cab14b
--- /dev/null
+++ b/contrib/obs-mirror/README
@@ -0,0 +1,7 @@
+The scripts in this directory are used to create an archive of Osmocom related
+packages from OBS, at: https://downloads.osmocom.org/obs-mirror/
+
+There is no mechanism in place to deploy updated scripts after updating them
+here in osmo-ci.git, so this must be done manually.
+
+See OS#4862 for more information.
diff --git a/contrib/obs-mirror/obs-mirror-include.txt b/contrib/obs-mirror/obs-mirror-include.txt
new file mode 100644
index 0000000..6458090
--- /dev/null
+++ b/contrib/obs-mirror/obs-mirror-include.txt
@@ -0,0 +1,18 @@
+latest/CentOS_7
+latest/CentOS_8
+latest/Debian_10
+latest/Debian_11
+latest/Debian_12
+latest/Raspbian_10
+latest/Raspbian_11
+latest/xUbuntu_18.04
+latest/xUbuntu_20.04
+latest/xUbuntu_21.04
+latest/xUbuntu_22.04
+latest/xUbuntu_23.04
+
+nightly/CentOS_8
+nightly/Debian_11
+nightly/Debian_12
+nightly/Raspbian_11
+nightly/xUbuntu_23.04
diff --git a/contrib/obs-mirror/obs-mirror.sh b/contrib/obs-mirror/obs-mirror.sh
new file mode 100755
index 0000000..85336ab
--- /dev/null
+++ b/contrib/obs-mirror/obs-mirror.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Mirror script to create a local archive of the OBS packages in the network:osmocom
+# repositories / projects
+#
+# We are using hard-links to perform de-duplication on the local side; only
+# those files that changed compoared to the previous run will be rsync'ed over
+# from the remote side
+#
+# See also: OS#4862
+
+DATE=`date +%Y%m%d-%H%M%S`
+LOGFILE="/home/pkgmirror/obs-mirror-$DATE.log"
+echo "Redirecting all output to: $LOGFILE"
+exec >$LOGFILE 2>&1
+
+set -e -x
+SCRIPT_DIR="$(realpath "$(dirname "$(realpath "$0")")")"
+
+# base directory on the local side
+BASE_DIR="/downloads/obs-mirror/"
+# sync remote where to find the osmocom packages
+REMOTE="downloads.osmocom.org::packages/osmocom:"
+
+cd "$BASE_DIR"
+
+RSYNC_ARGS="-av --delete"
+RSYNC_ARGS="$RSYNC_ARGS --files-from $SCRIPT_DIR/obs-mirror-include.txt --recursive --exclude='.*'"
+DIR="$BASE_DIR/$DATE"
+TEMP_DIR="$BASE_DIR/.temp"
+
+rm -rf "$TEMP_DIR"
+mkdir "$TEMP_DIR"
+
+PREVIOUS="$BASE_DIR/.previous"
+if [ -d "$PREVIOUS" ]; then
+ RSYNC_ARGS+=" --link-dest=$PREVIOUS"
+fi
+
+# finally, perform rsync
+if rsync $RSYNC_ARGS "$REMOTE"/ "$TEMP_DIR"/; then
+ mv "$TEMP_DIR" "$DIR"
+
+ # update '.previous' for the next run
+ rm -f "$PREVIOUS"
+ ln -sf "$DATE" "$PREVIOUS"
+else
+ exit 1
+fi
diff --git a/contrib/obs-mirror/rm-old-nightly-archives.sh b/contrib/obs-mirror/rm-old-nightly-archives.sh
new file mode 100755
index 0000000..485788f
--- /dev/null
+++ b/contrib/obs-mirror/rm-old-nightly-archives.sh
@@ -0,0 +1,34 @@
+#!/bin/sh -e
+# Remove nightly archives older than one month (OS#4862)
+echo "Redirecting all output to: /home/pkgmirror/rm-old-nightly-archives.log"
+exec >"/home/pkgmirror/rm-old-nightly-archives.log" 2>&1
+
+DRY=0
+
+# Get removal date in seconds since epoch and display it
+DATE_RM_SEC=$(expr $(date +%s) - 3600 \* 24 \* 365 \* 2)
+DATE_RM_STR=$(date -d "@$DATE_RM_SEC" +"%Y-%m-%d")
+echo "Removing nightly archives from $DATE_RM_STR and older (DRY=$DRY)"
+
+cd /downloads/obs-mirror
+
+for i in */nightly; do
+ # "Last modified" isn't set to the date of the dir name for some
+ # archives, so parse the date from the dir name instead
+ DATE_DIR="$(basename "$(dirname "$i")")" # e.g. "20210604-002301"
+ DATE_DIR_SEC="$(date -d "$(echo "$DATE_DIR" | cut -d "-" -f 1)" +%s)"
+ if [ -z "$DATE_DIR_SEC" ]; then
+ echo "ERROR: $i: failed to parse date from dir name"
+ continue
+ fi
+
+ if [ "$DATE_DIR_SEC" -lt "$DATE_RM_SEC" ]; then
+ DATE_DIR_STR="$(date -d "@$DATE_DIR_SEC" +"%Y-%m-%d")"
+ echo "Removing $i ($DATE_DIR_STR)..."
+ if [ "$DRY" = 0 ]; then
+ rm -r "$i"
+ fi
+ fi
+done
+
+echo "Done"
diff --git a/contrib/obs-mirror/test-include-list.sh b/contrib/obs-mirror/test-include-list.sh
new file mode 100755
index 0000000..829e8f7
--- /dev/null
+++ b/contrib/obs-mirror/test-include-list.sh
@@ -0,0 +1,13 @@
+#!/bin/sh -ex
+SCRIPT_DIR="$(realpath "$(dirname "$(realpath "$0")")")"
+BASE_DIR="/downloads/obs-mirror/"
+
+cd "$BASE_DIR"
+
+rsync \
+ -a \
+ --list-only \
+ --files-from "$SCRIPT_DIR"/obs-mirror-include.txt \
+ --recursive \
+ "$(realpath .previous)"/ \
+ new-backup-dir/
diff --git a/contrib/obs-upgrade-eclipse-titan.sh b/contrib/obs-upgrade-eclipse-titan.sh
new file mode 100755
index 0000000..a949241
--- /dev/null
+++ b/contrib/obs-upgrade-eclipse-titan.sh
@@ -0,0 +1,49 @@
+#!/bin/sh -e
+# SPDX-License-Identifier: GPL-3.0-or-later
+# Copyright 2023 sysmocom - s.f.m.c. GmbH
+# Author: Oliver Smith
+#
+# Helper script to upgrade the eclipse-titan package on obs.osmocom.org.
+# Usage instructions:
+# https://osmocom.org/projects/cellular-infrastructure/wiki/Upgrading_eclipse-titan_in_the_Osmocom_OBS
+
+old_dsc="$(find -maxdepth 1 -name 'eclipse-titan_*.dsc')"
+old_ver="$(grep '^Version: ' "$old_dsc" | cut -d ':' -f 2 | xargs | cut -d - -f 1)"
+new_ver="$1"
+
+if [ -z "$new_ver" ]; then
+ echo "usage: update.sh NEW_VERSION"
+ exit 1
+fi
+
+echo ":: upgrade from $old_ver to $new_ver"
+
+tarball="titan.core-$new_ver.tar.bz2"
+if ! [ -e "$tarball" ]; then
+ echo ":: download $tarball"
+ wget "https://gitlab.eclipse.org/eclipse/titan/titan.core/-/archive/$new_ver/$tarball"
+fi
+
+echo ":: extract $old_dsc"
+dpkg-source -x "$old_dsc"
+
+echo ":: update sourcedir with $tarball"
+cd "eclipse-titan-$old_ver"
+uupdate -v "$new_ver" ../"$tarball"
+
+echo ":: now modify eclipse-titan-$new_ver (e.g. adjust changelog) and press return when done"
+read foo
+
+echo ":: build new source package"
+cd "../eclipse-titan-$new_ver"
+dpkg-buildpackage -S -uc -us -d
+cd ".."
+
+echo ":: clean up extracted dirs"
+rm -rf "eclipse-titan-$old_ver" \
+ "eclipse-titan-$new_ver" \
+ "eclipse-titan-$new_ver.orig" \
+ *.buildinfo \
+ *.changes
+
+echo ":: done!"
diff --git a/coverity/badge_status.sh b/coverity/badge_status.sh
new file mode 100755
index 0000000..72fdcce
--- /dev/null
+++ b/coverity/badge_status.sh
@@ -0,0 +1,28 @@
+#!/bin/sh -e
+# Check if the coverity badge says "failed". This can happen for example if we
+# use a coverity version that is no longer supported, and it doesn't fail in
+# any other obvious way. (SYS#6685)
+
+# Download the status badge svg image, which contains the word "passed" when
+# it was successful, and "failed" when it failed.
+# Example: <text x="62" y="14">passed 91 new defects</text>
+
+BADGE="_temp/badge.svg"
+
+mkdir -p _temp
+rm -f "$BADGE"
+
+wget --no-verbose -O "$BADGE" "https://scan.coverity.com/projects/7523/badge.svg"
+
+if grep -q passed "$BADGE" && ! grep -q failed "$BADGE"; then
+ echo "Success"
+ exit 0
+fi
+
+echo
+echo "ERROR: coverity failed!"
+echo
+echo "Find the error details here:"
+echo "https://scan.coverity.com/projects/osmocom?tab=overview"
+echo
+exit 1
diff --git a/coverity/build_Osmocom.sh b/coverity/build_Osmocom.sh
index f3a903b..61bd2b7 100755
--- a/coverity/build_Osmocom.sh
+++ b/coverity/build_Osmocom.sh
@@ -12,7 +12,11 @@ export PKG_CONFIG_PATH="$prefix/lib/pkgconfig"
do_build() {
autoreconf --install --force
- ./configure --prefix="$prefix" --with-systemdsystemunitdir=no $*
+ ./configure \
+ --prefix="$prefix" \
+ --with-systemdsystemunitdir=no \
+ --disable-doxygen \
+ $*
make $PARALLEL_MAKE
make install
@@ -70,10 +74,14 @@ build_default libosmocore
build_libasn1c
build_libusrp
build_default libosmo-abis
+build_default libosmo-gprs
build_default libosmo-netif
+build_default libosmo-pfcp
build_default libosmo-sccp
build_default libsmpp34
-build_default osmo-ggsn
+build_default libgtpnl
+build_default osmo-ggsn \
+ --enable-gtp-linux
build_default osmo-gbproxy
build_default osmo-iuh
build_default osmo-pcu \
@@ -95,6 +103,9 @@ build_default osmo-sgsn \
build_default osmo-smlc
build_default osmo-cbc
build_default osmo-hnodeb
+build_default osmo-hnbgw \
+ --enable-pfcp
+build_default osmo-bsc-nat
# GMR
build_default libosmo-dsp
@@ -104,6 +115,7 @@ build_default osmo-gmr
build_default osmo-sip-connector
build_default osmo-trx \
+ --with-mstrx \
--with-usrp1 \
--with-uhd \
--with-lms \
@@ -113,3 +125,16 @@ build_default osmo-trx \
build_osmocombb
# SIMTrace host tools
build_simtrace2
+
+build_default osmo-pcap
+build_default osmo-e1d
+build_default osmo-e1-recorder
+build_default osmo-upf
+
+# OS#5886: doesn't build
+# build_default osmo-isdntap
+
+build_default osmo-uecups
+build_default osmo-remsim \
+ --with-usbdropdir=$prefix/usr/lib/pcsc/drivers \
+ --with-serialconfdir=$prefix/etc/reader.conf.d
diff --git a/coverity/common.sh b/coverity/common.sh
new file mode 120000
index 0000000..8f435c6
--- /dev/null
+++ b/coverity/common.sh
@@ -0,0 +1 @@
+../scripts/common.sh \ No newline at end of file
diff --git a/coverity/jenkins.sh b/coverity/jenkins.sh
index 33d4fa5..b33ae86 100755
--- a/coverity/jenkins.sh
+++ b/coverity/jenkins.sh
@@ -1,13 +1,18 @@
#!/usr/bin/env bash
+# Use 'local_test.sh' during development
set -e -x
-base_dir="$PWD"
-src_dir="$base_dir/source-Osmocom"
+readlink /opt/coverity/current
+
+export PATH=$PATH:/opt/coverity/current/bin
+
+base_dir="$HOME/osmo-ci/coverity"
+src_dir="$PWD/source-Osmocom"
cov_dir="$src_dir/cov-int"
rm -rf "$src_dir"
-./prepare_source_Osmcocom.sh
+./prepare_source_Osmocom.sh
export PATH="$base_dir/cov-analysis-linux64-8.5.0/bin/:$PATH"
diff --git a/coverity/local_test.sh b/coverity/local_test.sh
new file mode 100755
index 0000000..f2b89c3
--- /dev/null
+++ b/coverity/local_test.sh
@@ -0,0 +1,17 @@
+#!/bin/sh -ex
+# Use this script for local testing of the prepare source and build scripts
+SCRIPT_DIR="$(dirname "$(realpath "$0")")"
+
+export SRC_SKIP_FETCH=1
+export SRC_CLEAN=1
+export PARALLEL_MAKE="-j$(echo $(nproc) + 1 | bc)"
+
+mkdir -p /tmp/coverity
+rm -f /tmp/coverity/common.sh
+cp "$SCRIPT_DIR"/* /tmp/coverity
+ln -sf "$SCRIPT_DIR/../scripts/common.sh" /tmp/coverity/common.sh
+
+cd /tmp/coverity
+
+./prepare_source_Osmocom.sh
+./build_Osmocom.sh
diff --git a/coverity/prepare_source_Osmcocom.sh b/coverity/prepare_source_Osmcocom.sh
deleted file mode 100755
index 78661e1..0000000
--- a/coverity/prepare_source_Osmcocom.sh
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/bin/sh
-BASEDIR=source-Osmocom
-
-[ -d $BASEDIR ] || mkdir -p $BASEDIR
-cd $BASEDIR
-
-for proj in \
- asn1c \
- libasn1c \
- libosmo-abis \
- libosmocore \
- libosmo-dsp \
- libosmo-netif \
- libosmo-sccp \
- libsmpp34 \
- libusrp \
- osmo-bsc \
- osmo-msc \
- osmo-mgw \
- osmo-ggsn \
- osmo-gbproxy \
- osmo-sgsn \
- osmo-bts \
- osmo-gmr \
- osmo-iuh \
- osmo-pcu \
- osmo-sysmon \
- osmo-sip-connector \
- osmo-trx \
- osmo-hlr \
- osmocom-bb \
- osmo-smlc \
- osmo-cbc \
- simtrace2 \
- osmo-hnodeb \
- ; do
-
- if [ -d $proj ]; then
- (cd $proj && git fetch && git checkout -f -B master origin/master)
- else
- git clone git://git.osmocom.org/$proj
- fi
-done
-
-git clone https://git.sysmocom.de/sysmo-bts/layer1-api
diff --git a/coverity/prepare_source_Osmocom.sh b/coverity/prepare_source_Osmocom.sh
new file mode 100755
index 0000000..2c69356
--- /dev/null
+++ b/coverity/prepare_source_Osmocom.sh
@@ -0,0 +1,86 @@
+#!/bin/sh -xe
+. $(realpath common.sh)
+BASEDIR=source-Osmocom
+
+osmo_cmd_require sdcc
+
+# How to add a new project:
+# * add it to the list below
+# * add it to build_Osmocom.sh
+# * add it as component here:
+# https://scan.coverity.com/projects/osmocom?tab=analysis_settings
+
+PROJECTS="
+ libasn1c
+ libosmo-abis
+ libosmocore
+ libosmo-dsp
+ libosmo-gprs
+ libosmo-netif
+ libosmo-pfcp
+ libosmo-sccp
+ libsmpp34
+ libusrp
+ osmo-bsc
+ osmo-msc
+ osmo-mgw
+ osmo-ggsn
+ osmo-gbproxy
+ osmo-sgsn
+ osmo-bts
+ osmo-gmr
+ osmo-iuh
+ osmo-pcu
+ osmo-sysmon
+ osmo-sip-connector
+ osmo-trx
+ osmo-hlr
+ osmocom-bb
+ osmo-smlc
+ osmo-cbc
+ simtrace2
+ osmo-hnodeb
+ osmo-hnbgw
+ osmo-bsc-nat
+ osmo-pcap
+ osmo-e1d
+ osmo-e1-recorder
+ libgtpnl
+ osmo-upf
+ osmo-isdntap
+ osmo-uecups
+ osmo-remsim
+"
+
+PROJECTS_DONT_BUILD_TEST="
+ asn1c
+"
+
+mkdir -p $BASEDIR
+cd $BASEDIR
+
+for proj in $PROJECTS $PROJECTS_DONT_BUILD_TEST; do
+ if [ -d $proj ]; then
+ if [ -z "$SRC_SKIP_FETCH" ]; then
+ (cd $proj && git fetch && git checkout -f -B master origin/master && git submodule update --recursive --remote)
+ fi
+ if [ -n "$SRC_CLEAN" ]; then
+ git -C "$proj" clean -ffxd
+ fi
+ else
+ git clone --recursive "$(osmo_git_clone_url "$proj")"
+ fi
+done
+
+# We want to compile tests, but not execute them. Using 'noinst_PROGRAMS'
+# instead of 'check_PROGRAMS' allows building test binaries during 'make all'.
+for proj in $PROJECTS; do
+ files="$(git -C $proj grep -l check_PROGRAMS)" || true
+ if [ -n "$files" ]; then
+ (cd $proj && sed -i "s/check_PROGRAMS/noinst_PROGRAMS/" $files)
+ fi
+done
+
+if ! [ -d layer1-api ]; then
+ git clone https://gitea.sysmocom.de/sysmo-bts/layer1-api
+fi
diff --git a/jobs/README.adoc b/jobs/README.adoc
index bea1d9f..eb14152 100644
--- a/jobs/README.adoc
+++ b/jobs/README.adoc
@@ -5,8 +5,6 @@ https://docs.openstack.org/infra/jenkins-job-builder/index.html[Jenkins Job Buil
Install jenkins-job-builder:
-For debian 9, see troubleshooting below. Otherwise:
-
apt-get install jenkins-job-builder
Have a jenkins-job-builder.ini file. One of
@@ -74,30 +72,78 @@ individual job names.
Also be aware that jobs are only actually updated when anything changed.
-- debian 9
+*Jenkins labels*
-For jenkins-job-builder to work on debian 9:
+Most jenkins jobs should run a docker container and install all required
+dependencies inside that, so we don't need to install them on the jenkins node.
+These jobs don't need to set a label, they can just run on any generic jenkins
+node that has docker available. So if you add a new job, you probably don't
+need a label at all.
-add 'testing' to /etc/apt/sources.list like
+Existing jobs typically have a label set by the topic they belong to, e.g.:
+- osmocom-master
+- osmocom-gerrit
+- ttcn3
- deb http://ftp.de.debian.org/debian/ testing main
+Other labels indicate specific software/hardware works here, e.g.:
+- coverity
+- hdlc
+- osmo-gsm-tester
+- podman
-then
+*ccache*
- sudo apt-get update
- sudo apt-get -t testing install python-jenkins
- sudo apt-get install jenkins-job-builder
+The jobs from master-builds and gerrit-verifications use ccache. View the
+statistics with SSH on the build nodes with:
-and make sure to *not* pick the python3 versions
-(or you'll need to add catching of HTTPError to jenkins/__init__.py in
-'def maybe_add_crumb')
+ $ CCACHE_DIR=~/ccache/gerrit-verifications ccache -s
+ $ CCACHE_DIR=~/ccache/master-builds ccache -s
-*Jenkins labels*
+Note that running multiple jobs in parallel influence the ccache statistics,
+and it's impossible to tell which job caused which change in the stats (that's
+why they are not printed at the end of each job, it would be confusing).
+
+*timers*
+
+A lot of the jenkins jobs run daily with a timer:
+
+ triggers:
+ - timed: "H 20 * * *"
+
+or weekly:
+
+ triggers:
+ - timed: "H 20 * * H"
+
+Use H for the minute / day of week, to have it derivated as hash of the job
+name. Replace 20 with the hour (UTC) the job should run.
+
+The jobs follow this timetable, to ensure we don't attempt to use binary
+packages before they have been built (leading to failing jobs).
+
+ 18:00 - 21:00 OBS related
+ 18:XX osmocom-obs-sync (sync Debian:12 etc. with openSUSE OBS)
+ 19:XX osmocom-obs-check-new-distros
+ 19:XX osmocom-obs-wireshark
+ 20:XX osmocom-obs (new binary packages start building on OBS!)
+
+ 22:00 - 03:00 Jobs that don't need binary packages
+ 22:XX coverity
+ 22:XX octsim_osmo-ccid-firmware
+ 22:XX osmo-gsm-tester-runner (virtual)
+ 23:XX master-builds-dahdi
+ 00:XX osmocom-api
+ 00:XX registry-update-base-images
+ 01:XX osmocom-build-tags-against-master
+ 01:XX osmocom-list-commits
+ 02:XX registry-triggers
+ 02:XX registry-rebuild-upload-fpga-build (weekly)
-The build jobs are using labels to specify on which nodes the job should be run.
-Please use the following labels
+ 04:00 - 18:00 Jobs that need binary packages
+ ttcn3-testsuites
+ 04:XX osmocom-release-manuals
+ 05:XX osmocom-release-tarballs
+ 06:XX repo-install-test
+ 06:XX coverity-status (runs intentionally much later than the coverity job)
-- osmocom-master-debian8: are nodes which can build osmocom-master and is a debian 8
-- osmocom-master-debian9: are nodes which can build osmocom-master and is a debian 9
-- osmocom-gerrit-debian8: are nodes which can build osmocom-gerrit and is a debian 8
-- osmocom-gerrit-debian9: are nodes which can build osmocom-gerrit and is a debian 9
+master-builds: to avoid complexity, these run throughout the day (H H * * *).
diff --git a/jobs/coverity-status.yml b/jobs/coverity-status.yml
new file mode 100644
index 0000000..cb7522e
--- /dev/null
+++ b/jobs/coverity-status.yml
@@ -0,0 +1,46 @@
+# Check if the coverity badge says "failed". This can happen for example if we
+# use a coverity version that is no longer supported, and it doesn't fail in
+# any other obvious way. (SYS#6685)
+- project:
+ name: osmocom-coverity-status
+ jobs:
+ - osmocom-coverity-status
+
+- job:
+ name: 'osmocom-coverity-status'
+ project-type: freestyle
+ defaults: global
+ description: |
+ Verify that the coverity badge says "passed" and not "failed" (SYS#6685).
+ <br>
+ <br>
+ <img src="https://scan.coverity.com/projects/7523/badge.svg">
+ builders:
+ - shell: |
+ coverity/badge_status.sh
+ publishers:
+ - email:
+ recipients: 'jenkins-notifications@lists.osmocom.org'
+ notify-every-unstable-build: true
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
+ parameters:
+ - string:
+ name: BRANCH
+ description: osmo-ci.git branch
+ default: 'origin/master'
+ scm:
+ - git:
+ url: https://gerrit.osmocom.org/osmo-ci
+ git-config-name: 'Jenkins Builder'
+ git-config-email: 'jenkins@osmocom.org'
+ branches:
+ - '$BRANCH'
+ triggers:
+ - timed: "H 06 * * *"
+ publishers:
+ - email:
+ notify-every-unstable-build: true
+ recipients: 'jenkins-notifications@lists.osmocom.org'
diff --git a/jobs/coverity.yml b/jobs/coverity.yml
index 5614fc0..42c060a 100644
--- a/jobs/coverity.yml
+++ b/jobs/coverity.yml
@@ -2,47 +2,48 @@
- project:
name: osmocom-coverity
jobs:
- - osmocom-coverity-deps
- osmocom-coverity
- job:
- name: 'osmocom-coverity-deps'
- apt:
- name: "{{ item }}"
- cache_valid_time: 3600
- update_cache: yes
- with_items:
- - curl
-
-- job:
name: 'osmocom-coverity'
project-type: freestyle
defaults: global
description: 'Generated by job-builder'
- display-name: 'Coverity-Upload (jenkins-job-builder)'
node: coverity
builders:
- shell: |
- cd $HOME/osmo-ci/coverity
- export PATH=$PATH:/opt/coverity/current/bin
- ./jenkins.sh
+ docker run \
+ --rm=true \
+ -e PARALLEL_MAKE="$PARALLEL_MAKE" \
+ -u build \
+ -v "$PWD:/build" \
+ -v "$HOME/osmo-ci/coverity:/home/build/osmo-ci/coverity:ro" \
+ -v "/opt/coverity:/opt/coverity:ro" \
+ -w /build/coverity \
+ "$USER/debian-bookworm-build" \
+ ./jenkins.sh
publishers:
- email:
- recipients: openbsc@lists.osmocom.org holger@freyther.de
+ recipients: 'jenkins-notifications@lists.osmocom.org'
notify-every-unstable-build: true
properties:
- build-discarder:
days-to-keep: 30
num-to-keep: 30
+ parameters:
+ - string:
+ name: BRANCH
+ description: osmo-ci.git branch
+ default: 'origin/master'
scm:
- git:
- url: git://git.osmocom.org/osmo-ci
+ url: https://gerrit.osmocom.org/osmo-ci
git-config-name: 'Jenkins Builder'
git-config-email: 'jenkins@osmocom.org'
branches:
- - 'origin/master'
+ - '$BRANCH'
triggers:
- - timed: "@daily"
+ - timed: "H 22 * * *"
publishers:
- email:
notify-every-unstable-build: true
diff --git a/jobs/gerrit-binpkgs.yml b/jobs/gerrit-binpkgs.yml
new file mode 100644
index 0000000..7cf2230
--- /dev/null
+++ b/jobs/gerrit-binpkgs.yml
@@ -0,0 +1,91 @@
+# This file holds all gerrit verifications for building deb and rpm packages.
+# https://jenkins.osmocom.org/jenkins/view/Jenkins-Gerrit/.
+# One can simply add a gerrit job by adding project's repository to repos list.
+
+- project:
+ name: gerrit-binpkgs
+ jobs:
+ - 'gerrit-binpkgs'
+
+- job:
+ name: 'gerrit-binpkgs'
+ project-type: freestyle
+ node: osmocom-gerrit
+ concurrent: true
+ retry-count: 3 # scm checkout
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 1000
+ description: |
+ binary packages job (deb/rpm) of CI for patches sent to
+ <a href="https://gerrit.osmocom.org">gerrit</a>
+ </br></br>
+ Related issue: <a href="https://osmocom.org/issues/2385">OS#2385</a>
+
+ parameters:
+ !include: gerrit-verifications-parameters.yaml.inc
+
+ scm:
+ - git:
+ basedir: 'code-from-gerrit'
+ url: '$GERRIT_REPO_URL'
+ credentials-id: d5eda5e9-b59d-44ba-88d2-43473cb6e42d
+ branches:
+ - $GERRIT_BRANCH
+ refspec: $GERRIT_REFSPEC
+ name:
+ choosing-strategy: gerrit
+ wipe-workspace: true
+ skip-tag: true
+ submodule:
+ recursive: false
+
+ builders:
+ - shell: |-
+ # Make obvious which distro we build for, for humans and for parsing
+ # by scripts/jenkins-gerrit/comment_generate.py
+ set +x
+ echo
+ echo "Building binary packages for distro: '$DISTRO'"
+ echo
+ set -x
+
+ # Clone osmo-ci.git
+ rm -rf osmo-ci
+ git clone \
+ --depth=1 \
+ --branch="$BRANCH_CI" \
+ https://gerrit.osmocom.org/osmo-ci \
+ osmo-ci
+ git -C osmo-ci log --oneline
+
+ # Move code from gerrit to build_srcpkg.py's git cache
+ cache_dir=osmo-ci/scripts/obs/_cache
+ mkdir -p $cache_dir
+ mv code-from-gerrit "$cache_dir/$PROJECT_NAME"
+
+ # Set a known branch name
+ git -C "$cache_dir/$PROJECT_NAME" checkout -B "origin/gerrit"
+
+ # Build source package
+ cd osmo-ci/scripts/obs/
+ ./build_srcpkg.py \
+ --docker \
+ --feed master \
+ --git-branch gerrit \
+ --git-skip-fetch \
+ "$PROJECT_NAME"
+
+ # Build binary package
+ ./build_binpkg.py \
+ --docker "$DISTRO" \
+ "$PROJECT_NAME"
+ wrappers:
+ - ansicolor:
+ colormap: xterm
+ - ssh-agent-credentials:
+ users:
+ - d5eda5e9-b59d-44ba-88d2-43473cb6e42d
+
+# vim: expandtab tabstop=2 shiftwidth=2
diff --git a/jobs/gerrit-lint.yml b/jobs/gerrit-lint.yml
index db00509..3039829 100644
--- a/jobs/gerrit-lint.yml
+++ b/jobs/gerrit-lint.yml
@@ -1,148 +1,62 @@
# This file holds all gerrit lint verifications https://jenkins.osmocom.org/jenkins/view/Jenkins-Gerrit/.
-# One cane simply add a gerrit job by adding project's repository to repos list.
+# Add project's repositories to jobs/gerrit-verifications.yml. When
+# pipeline_lint is set for that project, the pipeline will call the gerrit-lint
+# job for it.
- project:
name: gerrit-lint
- # following default values can be overridden by each repo
- disabled: false
- cmd: '~/osmo-ci/lint/lint_diff.sh HEAD~1'
- gerrit_url: 'ssh://jenkins@gerrit.osmocom.org:29418'
- repos_url: '{gerrit_url}/{repos}'
- gerrit_project: '{repos}'
-
- # in alphabetical order
- repos:
- - asn1c
- - cellmgr-ng
- - libasn1c
- - libgtpnl
- - libosmo-abis
- - libosmo-netif
- - libosmo-sccp
- - libosmocore
- - libsmpp34
- - libtelnet
- - libusrp
- - openbsc
- - openggsn
- - osmo-bsc
- - osmo-bts
- - osmo-hnodeb
-
- - osmo-ci:
- # Run the lint script from the checked out branch
- cmd: 'lint/lint_diff.sh HEAD~1'
-
- - osmo_dia2gsup:
- repos_url: '{gerrit_url}/erlang/{repos}'
- gerrit_project: 'erlang/{repos}'
-
- - osmo-e1-hardware
- - osmo-ggsn
- - osmo-gsm-manuals
-
- - osmo_gsup:
- repos_url: '{gerrit_url}/erlang/{repos}'
- gerrit_project: 'erlang/{repos}'
-
- - osmo-hlr
- - osmo-iuh
- - osmo-mgw
- - osmo-msc
- - osmo-pcap
- - osmo-pcu
-
- - osmo-python-tests:
- repos_url: '{gerrit_url}/python/{repos}'
- gerrit_project: 'python/{repos}'
-
- - osmo-sgsn
-
- - osmo_ss7:
- repos_url: '{gerrit_url}/erlang/{repos}'
- gerrit_project: 'erlang/{repos}'
-
- - osmo-sip-connector
- - osmo-trx
- - osmocom-bb
- - osmo-tetra
- - osmo-sysmon
- - osmo-remsim
- - simtrace2
- - osmo-asf4-dfu
- - osmo-ccid-firmware
- - osmo-e1d
- - osmo-cbc
- - osmo-e1-recorder
- - gapk
- - osmo-uecups
- - osmo-el2tpd
- - osmo-smlc
- - osmo-gbproxy
- - pysim
-
- - osmo-ttcn3-hacks:
- repos_url: 'https://gerrit.osmocom.org/{repos}'
-
jobs:
- - 'gerrit-{repos}-lint'
+ - 'gerrit-lint'
-- job-template:
- name: 'gerrit-{repos}-lint'
+- job:
+ name: 'gerrit-lint'
project-type: freestyle
- node: osmocom-gerrit-debian9
- disabled: '{obj:disabled}'
+ node: osmocom-gerrit
retry-count: 3 # scm checkout
properties:
- build-discarder:
days-to-keep: 30
- num-to-keep: 120
- artifact-days-to-keep: -1
- artifact-num-to-keep: -1
+ num-to-keep: 1000
description: |
- Run linting on patches submitted to gerrit for
- <a href="https://gerrit.osmocom.org/#/q/status:open+project:{repos}">{repos}</a>
+ Lint job of CI for patches sent to <a href="https://gerrit.osmocom.org">gerrit</a>.
</br></br>
Related issue: <a href="https://osmocom.org/issues/5087">OS#5087</a>
+ parameters:
+ !include: gerrit-verifications-parameters.yaml.inc
+
scm:
- git:
- url: '{obj:repos_url}'
+ basedir: 'code-from-gerrit'
+ url: '$GERRIT_REPO_URL'
credentials-id: d5eda5e9-b59d-44ba-88d2-43473cb6e42d
branches:
- $GERRIT_BRANCH
refspec: $GERRIT_REFSPEC
- name:
choosing-strategy: gerrit
wipe-workspace: false
skip-tag: true
submodule:
recursive: false
- triggers:
- - gerrit:
- trigger-on:
- - patchset-created-event:
- exclude-drafts: true
- exclude-no-code-change: true
- projects:
- - project-compare-type: 'PLAIN'
- project-pattern: '{obj:gerrit_project}'
- branches:
- - branch-compare-type: 'ANT'
- branch-pattern: '**'
- skip-vote:
- successful: false
- failed: false
- unstable: false
- notbuilt: false
- silent: false
- escape-quotes: false
- no-name-and-email: false
- trigger-for-unreviewed-patches: true
- server-name: gerrit.osmocom.org
-
builders:
- - shell: '{obj:cmd}'
+ - shell: |
+ rm -rf osmo-ci
+ git clone \
+ --depth=1 \
+ --branch="$BRANCH_CI" \
+ https://gerrit.osmocom.org/osmo-ci \
+ osmo-ci
+ git -C osmo-ci log --oneline
+
+ cd code-from-gerrit
+ ../osmo-ci/lint/lint_diff.sh HEAD~1
+
+ wrappers:
+ - ansicolor:
+ colormap: xterm
+ - ssh-agent-credentials:
+ users:
+ - d5eda5e9-b59d-44ba-88d2-43473cb6e42d
# vim: expandtab tabstop=2 shiftwidth=2
diff --git a/jobs/gerrit-verifications-comment.yml b/jobs/gerrit-verifications-comment.yml
new file mode 100644
index 0000000..3ed54ad
--- /dev/null
+++ b/jobs/gerrit-verifications-comment.yml
@@ -0,0 +1,45 @@
+# This job runs at the start/end of the pipeline in gerrit-verifications.yml.
+# On start it posts a link to the pipeline to gerrit, and on end a list of
+# failed/successful job links together with a vote +V/-V.
+
+- project:
+ name: gerrit-verifications-comment
+ jobs:
+ - 'gerrit-verifications-comment'
+
+- job:
+ name: 'gerrit-verifications-comment'
+ project-type: freestyle
+ node: osmocom-gerrit
+ retry-count: 3 # scm checkout
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 120
+ description: |
+ Send start and result comments to <a href="https://gerrit.osmocom.org">gerrit</a>.
+ </br></br>
+ Related issue: <a href="https://osmocom.org/issues/2385">OS#2385</a>
+
+ parameters:
+ !include: gerrit-verifications-parameters.yaml.inc
+
+ scm:
+ - git:
+ url: 'https://gerrit.osmocom.org/osmo-ci'
+ credentials-id: d5eda5e9-b59d-44ba-88d2-43473cb6e42d
+ branches:
+ - '$BRANCH_CI'
+ wipe-workspace: true
+
+ builders:
+ - shell: 'cd scripts/jenkins-gerrit && ./comment_send.sh'
+
+ wrappers:
+ - ansicolor:
+ colormap: xterm
+ - ssh-agent-credentials:
+ users:
+ - d5eda5e9-b59d-44ba-88d2-43473cb6e42d
+
+# vim: expandtab tabstop=2 shiftwidth=2
diff --git a/jobs/gerrit-verifications-dahdi.yml b/jobs/gerrit-verifications-dahdi.yml
new file mode 100644
index 0000000..5a8b3a6
--- /dev/null
+++ b/jobs/gerrit-verifications-dahdi.yml
@@ -0,0 +1,103 @@
+- project:
+ name: gerrit-dahdi-linux-stable
+ type: stable
+ repo: 'https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git'
+ linux-ver:
+ - "linux-4.19.y"
+ - "linux-5.10.y"
+ - "linux-6.1.y"
+ jobs:
+ - 'gerrit-dahdi-linux-{type}-{linux-ver}'
+
+- project:
+ name: gerrit-dahdi-linux-torvalds
+ type: torvalds
+ repo: 'https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git'
+ linux-ver:
+ - "master"
+ jobs:
+ - 'gerrit-dahdi-linux-{type}-{linux-ver}'
+
+
+- job-template:
+ name: 'gerrit-dahdi-linux-{type}-{linux-ver}'
+ project-type: freestyle
+ node: osmocom-gerrit
+ concurrent: true
+ retry-count: 0 # scm checkout
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 120
+ description: |
+ Pending gerrit code reviews of
+ <a href="https://gerrit.osmocom.org/#/q/status:open+project:dahdi-linux">dahdi-linux</a>
+ built against linux {linux-ver}
+ (<a href="https://osmocom.org/issues/5407">OS#5407</a>)
+ <br/><br/>
+ <b>Auto-generated using Jenkins Job Builder. DO NOT EDIT MANUALLY!</b>
+
+ scm:
+ # Let jenkins clone the big linux repository and keep it between builds
+ # (wipe-workspace is false). Clone dahdi-linux with code from gerrit in
+ # the shell part below.
+ - git:
+ basedir: 'linux'
+ url: '{repo}'
+ branches:
+ - '{linux-ver}'
+ refspec: '{linux-ver}'
+ shallow-clone: true
+ wipe-workspace: false
+ clean:
+ before: true
+
+ triggers:
+ - gerrit:
+ trigger-on:
+ - patchset-created-event:
+ exclude-drafts: true
+ exclude-no-code-change: true
+ projects:
+ - project-compare-type: 'PLAIN'
+ project-pattern: 'dahdi-linux'
+ branches:
+ - branch-compare-type: 'ANT'
+ branch-pattern: '**'
+ skip-vote:
+ successful: false
+ failed: false
+ unstable: false
+ notbuilt: false
+ silent: false
+ escape-quotes: false
+ server-name: gerrit.osmocom.org
+
+ builders:
+ - shell: |
+ rm -rf dahdi-linux
+ git init dahdi-linux
+ cd dahdi-linux
+ git config advice.detachedHead false
+ git fetch \
+ --depth=1 \
+ https://gerrit.osmocom.org/dahdi-linux \
+ "$GERRIT_REFSPEC"
+ git checkout FETCH_HEAD
+ cd ..
+
+ docker run \
+ --rm=true \
+ -e "KSRC=/linux" \
+ -v "$PWD/dahdi-linux:/build" \
+ -v "$PWD/linux:/linux" \
+ "$USER/debian-bookworm-build" \
+ timeout 10m su build -c /build/contrib/jenkins.sh
+
+ publishers:
+ - warnings:
+ console-log-parsers:
+ - 'GNU C Compiler 4 (gcc)'
+ resolve-relative-paths: true
+
+# vim: expandtab tabstop=2 shiftwidth=2
diff --git a/jobs/gerrit-verifications-parameters.yaml.inc b/jobs/gerrit-verifications-parameters.yaml.inc
new file mode 100644
index 0000000..32df1b6
--- /dev/null
+++ b/jobs/gerrit-verifications-parameters.yaml.inc
@@ -0,0 +1,53 @@
+# This file gets included as parameters element of various jobs in the
+# gerrit-verifications pipeline. All of these jobs have the same parameters
+# and can be called from one run_job() function, without creating log entries
+# about undefined parameters (OS#6261). See gerrit-verifications.yml for more
+# information.
+
+- string:
+ name: BRANCH_CI
+ description: |
+ osmo-ci.git branch
+ default: 'master'
+- string:
+ name: COMMENT_TYPE
+ description: set by gerrit verification pipeline job
+- string:
+ name: DISTRO
+ description: set by gerrit verification pipeline job
+- string:
+ name: GERRIT_BRANCH
+ description: set by gerrit verification pipeline job
+- string:
+ name: GERRIT_CHANGE_NUMBER
+ description: set by gerrit verification pipeline job
+- string:
+ name: GERRIT_HOST
+ description: set by gerrit verification pipeline job
+- string:
+ name: GERRIT_PATCHSET_NUMBER
+ description: set by gerrit verification pipeline job
+- string:
+ name: GERRIT_PATCHSET_REVISION
+ description: set by gerrit verification pipeline job
+- string:
+ name: GERRIT_PATCHSET_UPLOADER_NAME
+ description: set by gerrit verification pipeline job
+- string:
+ name: GERRIT_PORT
+ description: set by gerrit verification pipeline job
+- string:
+ name: GERRIT_PROJECT
+ description: set by gerrit verification pipeline job
+- string:
+ name: GERRIT_REFSPEC
+ description: set by gerrit verification pipeline job
+- string:
+ name: GERRIT_REPO_URL
+ description: set by gerrit verification pipeline job
+- string:
+ name: PIPELINE_BUILD_URL
+ description: set by gerrit verification pipeline job
+- string:
+ name: PROJECT_NAME
+ description: set by gerrit verification pipeline job
diff --git a/jobs/gerrit-verifications.yml b/jobs/gerrit-verifications.yml
index 77865b7..3be106d 100644
--- a/jobs/gerrit-verifications.yml
+++ b/jobs/gerrit-verifications.yml
@@ -1,14 +1,39 @@
# This file holds all gerrit build verifications https://jenkins.osmocom.org/jenkins/view/Jenkins-Gerrit/.
-# One cane simply add a gerrit job by adding project's repository to repos list.
+# One can simply add a gerrit job by adding project's repository to repos list.
+#
+# Overview of jobs involved in the pipeline:
+# https://osmocom.org/projects/osmocom-servers/wiki/Jenkins_gerrit-verifications_Pipeline
+#
+# How it works:
+# * Two jobs get added for each repository:
+# gerrit-{repos}
+# gerrit-{repos}-build
+# * The gerrit-{repos} job is a pipeline that runs the gerrit-{repos}-build job
+# and other jobs (linting, building debian packages, ...). The purpose of
+# this job is to reduce the amount of comments (and resulting mails) in
+# gerrit coming from the jenkins build verification. If each job ran
+# separately without this pipeline job, they would each generate mails.
+# The gerrit-{repos} job gets triggered from gerrit whenever a new patch to
+# test is available. It passes the GERRIT_BRANCH and GERRIT_REFSPEC variables
+# from gerrit to the jobs called in the pipeline.
+# * The gerrit-{repos}-build job runs contrib/jenkins.sh in docker in a matrix.
+# By default the matrix results in only one job, however for some projects we
+# use the matrix to build for multiple CPU architectures (x86_64, arm) or use
+# it to build multiple configurations of the same project (see osmo-bts).
+#
+# NOTE: after updating the job with Jenkins Job Builder as usual, check if a
+# new pipeline script was generated and approve it here:
+# https://jenkins.osmocom.org/jenkins/scriptApproval/
+# This used to be necessary when changing the pipeline script, adding new
+# projects etc. But it seems to get auto-approved now.
- project:
name: gerrit
# following default values can be overridden by each repo
disabled: false
concurrent: true
- node: osmocom-gerrit-debian9
# axes related defaults
- slave_axis: !!python/tuple [osmocom-gerrit-debian9]
+ slave_axis: !!python/tuple [osmocom-gerrit]
a1_name: a1
a1: !!python/tuple [default]
a2_name: a2
@@ -24,11 +49,8 @@
# Documentation for variables (keep in sync!):
# https://osmocom.org/projects/osmocom-servers/wiki/Jenkins_build_verification_jobs
docker_run: |
- ARTIFACT_STORE="$HOME/jenkins_build_artifact_store"
- mkdir -p "$ARTIFACT_STORE"
docker run --rm=true \
--cap-add SYS_PTRACE \
- -e ARTIFACT_STORE=/artifact_store \
-e ASCIIDOC_WARNINGS_CHECK="1" \
-e HOME=/build \
-e JOB_NAME="$JOB_NAME" \
@@ -39,33 +61,72 @@
-e WITH_MANUALS="1" \
-w /build -i -u build \
-v "$PWD:/build" \
- -v "$ARTIFACT_STORE:/artifact_store" \
- docker_img: '$USER/debian-stretch-jenkins'
- docker_img_erlang: '$USER/debian-buster-erlang'
+ docker_run_ccache: |
+ CCACHE_DIR="$HOME/ccache/gerrit-verifications"
+ mkdir -p "$CCACHE_DIR"
+ chown osmocom-build:osmocom-build "$CCACHE_DIR"
+
+ docker run --rm=true \
+ --cap-add SYS_PTRACE \
+ -e ASCIIDOC_WARNINGS_CHECK="1" \
+ -e HOME=/build \
+ -e JOB_NAME="$JOB_NAME" \
+ -e MAKE=make \
+ -e OSMOPY_DEBUG_TCP_SOCKETS="1" \
+ -e OSMO_GSM_MANUALS_DIR="/opt/osmo-gsm-manuals" \
+ -e PARALLEL_MAKE="$PARALLEL_MAKE" \
+ -e WITH_MANUALS="1" \
+ -w /build -i -u build \
+ -v "$PWD:/build" \
+ \
+ -e CCACHE_DIR="/ccache" \
+ -e PATH="/usr/lib/ccache:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
+ -v "$CCACHE_DIR:/ccache" \
+ docker_img: '$USER/debian-bookworm-build'
+ docker_img_erlang: '$USER/debian-bookworm-erlang'
timeout_cmd: '/usr/bin/timeout 30m'
- cmd: '{docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins.sh'
+ cmd: '{docker_run_ccache} {docker_img} {timeout_cmd} /build/contrib/jenkins.sh'
gerrit_url: 'ssh://jenkins@gerrit.osmocom.org:29418'
repos_url: '{gerrit_url}/{repos}'
gerrit_project: '{repos}'
+ # Which jobs to run in the pipeline
+ pipeline_build: true
+ pipeline_lint: true
+ pipeline_binpkgs: "debian:10 debian:12"
# in alphabetical order
repos:
- - asn1c
- - cellmgr-ng
+ - asn1c:
+ pipeline_binpkgs: ""
+ pipeline_lint: false
+
+ - cellmgr-ng:
+ pipeline_binpkgs: "debian:10 debian:12"
+
+ - docker-playground:
+ pipeline_build: false
+ pipeline_binpkgs: ""
+
- libasn1c
- libgtpnl
- libosmo-abis
+ - libosmo-gprs
- libosmo-netif
+ - libosmo-pfcp
- libosmo-sccp
- libosmocore:
+ slave_axis: !!python/tuple [io_uring] # OS#6405
a1_name: arch
a1: !!python/tuple [arm-none-eabi, amd64]
combination_filter: '!(arch=="arm-none-eabi" && label=="FreeBSD_amd64")'
- cmd: '{docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins_arch.sh "$arch"'
+ cmd: '{docker_run_ccache} {docker_img} {timeout_cmd} /build/contrib/jenkins_arch.sh "$arch"'
- libsmpp34
- - libtelnet
+
+ - libtelnet:
+ pipeline_binpkgs: ""
+
- libusrp
- openbsc:
@@ -76,11 +137,12 @@
a3_name: IU
a3: !!python/tuple [--disable-iu]
cmd: |
- {docker_run} \
+ {docker_run_ccache} \
-e IU="$IU" \
-e SMPP="$SMPP" \
-e MGCP="$MGCP" \
{docker_img} {timeout_cmd} /build/contrib/jenkins.sh
+ pipeline_binpkgs: ""
- openggsn:
a1_name: GTP
@@ -89,9 +151,10 @@
cmd: '{timeout_cmd} ./contrib/jenkins.sh'
- osmo-bsc
+ - osmo-bsc-nat
- osmo-bts:
- slave_axis: !!python/tuple [osmocom-master-debian9, rpi4-raspbian10]
+ slave_axis: !!python/tuple [osmocom-gerrit, rpi4-raspbian11]
a1_name: FIRMWARE_VERSION
a1: !!python/tuple [master, femtobts_v2.7, superfemto_v2.4, superfemto_v3.0.1pre, superfemto_v3.1, superfemto_v5.1, v2017.01, litecell15, oc2g, oc2g-next]
a2_name: BTS_MODEL
@@ -99,32 +162,60 @@
a3_name: WITH_MANUALS
a3: !!python/tuple ["1", "0"]
combination_filter: >
- (FIRMWARE_VERSION == "master" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (FIRMWARE_VERSION == "master" && BTS_MODEL == "trx" && WITH_MANUALS == "0" && label == "rpi4-raspbian10") ||
- (FIRMWARE_VERSION == "femtobts_v2.7" && BTS_MODEL == "sysmo" && WITH_MANUALS == "1" && label == "osmocom-master-debian9") ||
- (FIRMWARE_VERSION == "superfemto_v2.4" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (FIRMWARE_VERSION == "superfemto_v3.0.1pre" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (FIRMWARE_VERSION == "superfemto_v3.1" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (FIRMWARE_VERSION == "superfemto_v5.1" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (FIRMWARE_VERSION == "v2017.01" && BTS_MODEL == "lc15" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (FIRMWARE_VERSION == "litecell15" && BTS_MODEL == "lc15" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (FIRMWARE_VERSION == "oc2g" && BTS_MODEL == "oc2g" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (FIRMWARE_VERSION == "oc2g-next" && BTS_MODEL == "oc2g" && WITH_MANUALS == "0" && label == "osmocom-master-debian9")
+ (FIRMWARE_VERSION == "master" && WITH_MANUALS == "0" && label == "osmocom-gerrit") ||
+ (FIRMWARE_VERSION == "master" && BTS_MODEL == "trx" && WITH_MANUALS == "0" && label == "rpi4-raspbian11") ||
+ (FIRMWARE_VERSION == "femtobts_v2.7" && BTS_MODEL == "sysmo" && WITH_MANUALS == "1" && label == "osmocom-gerrit") ||
+ (FIRMWARE_VERSION == "superfemto_v2.4" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-gerrit") ||
+ (FIRMWARE_VERSION == "superfemto_v3.0.1pre" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-gerrit") ||
+ (FIRMWARE_VERSION == "superfemto_v3.1" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-gerrit") ||
+ (FIRMWARE_VERSION == "superfemto_v5.1" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-gerrit") ||
+ (FIRMWARE_VERSION == "v2017.01" && BTS_MODEL == "lc15" && WITH_MANUALS == "0" && label == "osmocom-gerrit") ||
+ (FIRMWARE_VERSION == "litecell15" && BTS_MODEL == "lc15" && WITH_MANUALS == "0" && label == "osmocom-gerrit") ||
+ (FIRMWARE_VERSION == "oc2g" && BTS_MODEL == "oc2g" && WITH_MANUALS == "0" && label == "osmocom-gerrit") ||
+ (FIRMWARE_VERSION == "oc2g-next" && BTS_MODEL == "oc2g" && WITH_MANUALS == "0" && label == "osmocom-gerrit")
concurrent: false
cmd: |
- {docker_run} \
+ DOCKER_IMG="{docker_img}"
+ case "$(arch)" in
+ arm*) DOCKER_IMG="$DOCKER_IMG-arm" ;;
+ esac
+ {docker_run_ccache} \
-e FIRMWARE_VERSION="$FIRMWARE_VERSION" \
-e WITH_MANUALS="$WITH_MANUALS" \
- {docker_img} {timeout_cmd} /build/contrib/jenkins_bts_model.sh "$BTS_MODEL"
+ "$DOCKER_IMG" {timeout_cmd} /build/contrib/jenkins_bts_model.sh "$BTS_MODEL"
- osmo_dia2gsup:
repos_url: '{gerrit_url}/erlang/{repos}'
gerrit_project: 'erlang/{repos}'
cmd: '{docker_run} {docker_img_erlang} {timeout_cmd} /build/contrib/jenkins.sh'
+ pipeline_binpkgs: "debian:11 debian:12"
+
+ - osmo-epdg:
+ repos_url: '{gerrit_url}/erlang/{repos}'
+ gerrit_project: 'erlang/{repos}'
+ cmd: '{docker_run} {docker_img_erlang} {timeout_cmd} /build/contrib/jenkins.sh'
+ pipeline_binpkgs: "debian:12"
- osmo-e1-hardware:
- docker_img: 'registry.osmocom.org/$USER/fpga-build'
- cmd: 'docker pull {docker_img} && {docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins.sh'
+ a2_name: JOB_TYPE
+ a2: !!python/tuple ["firmware", "gateware", "manuals", "software"]
+ cmd: |
+ DOCKER_IMG="{docker_img}"
+ case "$JOB_TYPE" in
+ firmware|gateware|manuals)
+ DOCKER_IMG="registry.osmocom.org/$USER/fpga-build"
+ docker pull "$DOCKER_IMG"
+ ;;
+ esac
+ {docker_run} \
+ -e JOB_TYPE="$JOB_TYPE" \
+ "$DOCKER_IMG" \
+ {timeout_cmd} /build/contrib/jenkins.sh
+ pipeline_binpkgs: ""
+
+ - osmo-ci:
+ pipeline_build: false
+ pipeline_binpkgs: ""
- osmo-ggsn:
a1_name: GTP
@@ -135,17 +226,19 @@
(GTP == "--enable-gtp-linux" && WITH_MANUALS == "0") ||
(GTP == "--disable-gtp-linux" && WITH_MANUALS == "1")
cmd: |
- {docker_run} \
+ {docker_run_ccache} \
-e GTP="$GTP" \
-e WITH_MANUALS="$WITH_MANUALS" \
{docker_img} {timeout_cmd} /build/contrib/jenkins.sh
- - osmo-gsm-manuals
+ - osmo-gsm-manuals:
+ pipeline_binpkgs: "debian:10 debian:12"
- osmo_gsup:
repos_url: '{gerrit_url}/erlang/{repos}'
gerrit_project: 'erlang/{repos}'
- cmd: '{docker_run} {docker_img_erlang} {timeout_cmd} /build/contrib/jenkins.sh'
+ cmd: '{docker_run_ccache} {docker_img_erlang} {timeout_cmd} /build/contrib/jenkins.sh'
+ pipeline_binpkgs: ""
- osmo-hlr
- osmo-iuh
@@ -157,10 +250,10 @@
a2_name: WITH_MANUALS
a2: !!python/tuple ["1", "0"]
combination_filter: >
- (IU == "--enable-iu" && WITH_MANUALS == "0") ||
- (IU == "--disable-iu" && WITH_MANUALS == "1")
+ (IU == "--enable-iu" && WITH_MANUALS == "1") ||
+ (IU == "--disable-iu" && WITH_MANUALS == "0")
cmd: |
- {docker_run} \
+ {docker_run_ccache} \
-e IU="$IU" \
-e WITH_MANUALS="$WITH_MANUALS" \
{docker_img} {timeout_cmd} /build/contrib/jenkins.sh
@@ -168,7 +261,7 @@
- osmo-pcap
- osmo-pcu:
- slave_axis: !!python/tuple [osmocom-master-debian9, rpi4-raspbian10]
+ slave_axis: !!python/tuple [osmocom-gerrit, rpi4-raspbian11]
a1_name: FIRMWARE_VERSION
a1: !!python/tuple [master, v2017.01, litecell15, oc2g]
a2_name: with_vty
@@ -178,24 +271,29 @@
a4_name: WITH_MANUALS
a4: !!python/tuple ["1", "0"]
combination_filter: >
- (with_vty == "True" && with_dsp == "sysmo" && FIRMWARE_VERSION=="master" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (with_vty == "True" && with_dsp == "lc15" && FIRMWARE_VERSION=="master" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (with_vty == "True" && with_dsp == "lc15" && FIRMWARE_VERSION=="v2017.01" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (with_vty == "True" && with_dsp == "lc15" && FIRMWARE_VERSION=="litecell15" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (with_vty == "True" && with_dsp == "oc2g" && FIRMWARE_VERSION=="oc2g" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (with_vty == "False" && with_dsp == "none" && FIRMWARE_VERSION=="master" && WITH_MANUALS == "1" && label == "osmocom-master-debian9") ||
- (with_vty == "False" && with_dsp == "none" && FIRMWARE_VERSION=="master" && WITH_MANUALS == "0" && label == "rpi4-raspbian10")
+ (with_vty == "True" && with_dsp == "sysmo" && FIRMWARE_VERSION=="master" && WITH_MANUALS == "0" && label == "osmocom-gerrit") ||
+ (with_vty == "True" && with_dsp == "lc15" && FIRMWARE_VERSION=="master" && WITH_MANUALS == "0" && label == "osmocom-gerrit") ||
+ (with_vty == "True" && with_dsp == "lc15" && FIRMWARE_VERSION=="v2017.01" && WITH_MANUALS == "0" && label == "osmocom-gerrit") ||
+ (with_vty == "True" && with_dsp == "lc15" && FIRMWARE_VERSION=="litecell15" && WITH_MANUALS == "0" && label == "osmocom-gerrit") ||
+ (with_vty == "True" && with_dsp == "oc2g" && FIRMWARE_VERSION=="oc2g" && WITH_MANUALS == "0" && label == "osmocom-gerrit") ||
+ (with_vty == "False" && with_dsp == "none" && FIRMWARE_VERSION=="master" && WITH_MANUALS == "1" && label == "osmocom-gerrit") ||
+ (with_vty == "False" && with_dsp == "none" && FIRMWARE_VERSION=="master" && WITH_MANUALS == "0" && label == "rpi4-raspbian11")
cmd: |
- {docker_run} \
+ DOCKER_IMG="{docker_img}"
+ case "$(arch)" in
+ arm*) DOCKER_IMG="$DOCKER_IMG-arm" ;;
+ esac
+ {docker_run_ccache} \
-e FIRMWARE_VERSION="$FIRMWARE_VERSION" \
-e with_vty="$with_vty" \
-e with_dsp="$with_dsp" \
-e WITH_MANUALS="$WITH_MANUALS" \
- {docker_img} {timeout_cmd} /build/contrib/jenkins.sh
+ "$DOCKER_IMG" {timeout_cmd} /build/contrib/jenkins.sh
- osmo-python-tests:
repos_url: '{gerrit_url}/python/{repos}'
gerrit_project: 'python/{repos}'
+ pipeline_binpkgs: "debian:10 debian:12"
- osmo-sgsn:
a1_name: IU
@@ -203,10 +301,10 @@
a2_name: WITH_MANUALS
a2: !!python/tuple ["1", "0"]
combination_filter: >
- (IU == "--enable-iu" && WITH_MANUALS == "0") ||
- (IU == "--disable-iu" && WITH_MANUALS == "1")
+ (IU == "--enable-iu" && WITH_MANUALS == "1") ||
+ (IU == "--disable-iu" && WITH_MANUALS == "0")
cmd: |
- {docker_run} \
+ {docker_run_ccache} \
-e IU="$IU" \
-e WITH_MANUALS="$WITH_MANUALS" \
{docker_img} {timeout_cmd} /build/contrib/jenkins.sh
@@ -215,74 +313,343 @@
repos_url: '{gerrit_url}/erlang/{repos}'
gerrit_project: 'erlang/{repos}'
cmd: '{docker_run} {docker_img_erlang} {timeout_cmd} /build/contrib/jenkins.sh'
+ pipeline_binpkgs: ""
- osmo-sip-connector
- osmo-trx:
- slave_axis: !!python/tuple [osmocom-master-debian9,rpi4-raspbian10]
+ slave_axis: !!python/tuple [osmocom-gerrit,rpi4-raspbian11]
a1_name: INSTR
- a1: !!python/tuple [--with-sse, --without-sse --with-neon, --with-neon-vfpv4]
+ a1: !!python/tuple [--with-sse, --without-sse, --with-neon, --with-neon-vfpv4]
a2_name: WITH_MANUALS
a2: !!python/tuple ["1", "0"]
combination_filter: >
- (INSTR == "--without-sse" && WITH_MANUALS == "1" && label == "osmocom-master-debian9") ||
- (INSTR == "--with-sse" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (INSTR == "--with-neon" && WITH_MANUALS == "0" && label == "rpi4-raspbian10") ||
- (INSTR == "--with-neon-vfpv4" && WITH_MANUALS == "0" && label == "rpi4-raspbian10")
+ (INSTR == "--without-sse" && WITH_MANUALS == "1" && label == "osmocom-gerrit") ||
+ (INSTR == "--with-sse" && WITH_MANUALS == "0" && label == "osmocom-gerrit") ||
+ (INSTR == "--with-neon" && WITH_MANUALS == "0" && label == "rpi4-raspbian11") ||
+ (INSTR == "--with-neon-vfpv4" && WITH_MANUALS == "0" && label == "rpi4-raspbian11")
cmd: |
- {docker_run} \
+ DOCKER_IMG="{docker_img}"
+ case "$(arch)" in
+ arm*) DOCKER_IMG="$DOCKER_IMG-arm" ;;
+ esac
+ {docker_run_ccache} \
-e INSTR="$INSTR" \
-e WITH_MANUALS="$WITH_MANUALS" \
- {docker_img} {timeout_cmd} /build/contrib/jenkins.sh
+ "$DOCKER_IMG" {timeout_cmd} /build/contrib/jenkins.sh
+
+ - osmocom-bb:
+ pipeline_binpkgs: "debian:12"
+
+ - osmo-tetra:
+ pipeline_binpkgs: ""
+
+ - osmo-sysmon:
+ pipeline_binpkgs: "debian:10 debian:12"
- - osmocom-bb
- - osmo-tetra
- - osmo-sysmon
- osmo-remsim
- simtrace2
- - osmo-asf4-dfu
- - osmo-ccid-firmware
+
+ - osmo-opencm3-projects:
+ cmd: '{docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins.sh'
+ pipeline_binpkgs: ""
+
+ - osmo-asf4-dfu:
+ pipeline_binpkgs: ""
+
+ - osmo-ccid-firmware:
+ pipeline_binpkgs: ""
+
- osmo-e1d
- osmo-cbc
- - osmo-e1-recorder
- - gapk
+
+ - osmo-e1-recorder:
+ pipeline_binpkgs: ""
+
+ - gapk:
+ pipeline_binpkgs: "debian:12"
+
- osmo-uecups
- - osmo-el2tpd
+
+ - osmo-el2tpd:
+ pipeline_binpkgs: ""
+
- osmo-smlc
- osmo-gbproxy
- osmo-hnodeb
+ - osmo-hnbgw:
+ a1_name: PFCP
+ a1: !!python/tuple ["1", "0"]
+ a2_name: WITH_MANUALS
+ a2: !!python/tuple ["1", "0"]
+ combination_filter: >
+ (PFCP == "1" && WITH_MANUALS == "1") ||
+ (PFCP == "0" && WITH_MANUALS == "0")
+ cmd: |
+ {docker_run_ccache} \
+ -e PFCP="$PFCP" \
+ -e WITH_MANUALS="$WITH_MANUALS" \
+ {docker_img} {timeout_cmd} /build/contrib/jenkins.sh
+
+ - osmo-upf:
+ pipeline_binpkgs: "debian:11 debian:12"
+
- pysim:
- slave_axis: !!python/tuple [simtester]
concurrent: false
- cmd: '{timeout_cmd} ./contrib/jenkins.sh'
+ slave_axis: !!python/tuple [simtester,osmocom-gerrit]
+ a2_name: JOB_TYPE
+ a2: !!python/tuple ["test", "distcheck", "pylint", "docs"]
+ combination_filter: >
+ (JOB_TYPE == "test" && label == "simtester") ||
+ (JOB_TYPE == "distcheck" && label == "osmocom-gerrit") ||
+ (JOB_TYPE == "pylint" && label == "osmocom-gerrit") ||
+ (JOB_TYPE == "docs" && label == "osmocom-gerrit")
+ cmd: |
+ case "$JOB_TYPE" in
+ "test")
+ {timeout_cmd} ./contrib/jenkins.sh
+ ;;
+ *)
+ {docker_run} \
+ -e JOB_TYPE="$JOB_TYPE" \
+ {docker_img} \
+ {timeout_cmd} /build/contrib/jenkins.sh
+ ;;
+ esac
+ pipeline_binpkgs: ""
- osmo-ttcn3-hacks:
- repos_url: 'https://gerrit.osmocom.org/{repos}'
slave_axis: !!python/tuple [ttcn3]
- concurrent: false
cmd: |
- set -e
- make deps; make clean; make compile
+ docker run \
+ --rm \
+ -v "$PWD:/build" \
+ "registry.osmocom.org/osmocom-build/debian-bookworm-titan" \
+ {timeout_cmd} \
+ sh -e -x -c "
+ useradd --uid=1000 build;
+ cd /build;
+ su build -c 'make deps';
+ su build -c 'make clean';
+ su build -c 'make compile';
+ "
+ pipeline_binpkgs: ""
+
+ - dahdi-tools:
+ pipeline_lint: false
+ pipeline_binpkgs: ""
jobs:
- 'gerrit-{repos}'
+ - 'gerrit-{repos}-build'
- job-template:
name: 'gerrit-{repos}'
+ project-type: pipeline
+ concurrent: true
+ parameters:
+ - bool:
+ name: PIPELINE_BUILD
+ description: Enable the build job (runs contrib/jenkins.sh)
+ default: '{obj:pipeline_build}'
+ - bool:
+ name: PIPELINE_LINT
+ description: Enable the lint job
+ default: '{obj:pipeline_lint}'
+ - string:
+ name: REPO_URL
+ description: Clone URL, to be passed to jobs started in the pipeline
+ default: '{obj:repos_url}'
+ - string:
+ name: PIPELINE_BINPKGS
+ description: |
+ Enable the binpkgs job (build deb/rpm packages) for the given space
+ separated distributions. Empty list disables the binpkgs job. The
+ distributions must have binary packages in the osmocom:master
+ repository.
+ default: '{obj:pipeline_binpkgs}'
+ dsl: |
+ def run_job(display_name, job_name, comment_type=null, distro=null) {{
+ try {{
+ echo "pipeline_${{display_name}}: running job ${{job_name}}"
+
+ // IMPORTANT: Keep gerrit-verifications-parameters.yaml.inc in sync
+ // with the list of parameters here, and make sure all jobs passed
+ // to run_job() include the .yaml.inc file. (OS#6261)
+ build job: job_name, parameters: [
+ string(name: "COMMENT_TYPE", value: comment_type),
+ string(name: "DISTRO", value: distro),
+ string(name: "GERRIT_BRANCH", value: "${{env.GERRIT_BRANCH}}"),
+ string(name: "GERRIT_CHANGE_NUMBER", value: "${{env.GERRIT_CHANGE_NUMBER}}"),
+ string(name: "GERRIT_HOST", value: "${{env.GERRIT_HOST}}"),
+ string(name: "GERRIT_PATCHSET_NUMBER", value: "${{env.GERRIT_PATCHSET_NUMBER}}"),
+ string(name: "GERRIT_PATCHSET_REVISION", value: "${{env.GERRIT_PATCHSET_REVISION}}"),
+ string(name: "GERRIT_PATCHSET_UPLOADER_NAME", value: "${{env.GERRIT_PATCHSET_UPLOADER_NAME}}"),
+ string(name: "GERRIT_PORT", value: "${{env.GERRIT_PORT}}"),
+ string(name: "GERRIT_PROJECT", value: "${{env.GERRIT_PROJECT}}"),
+ string(name: "GERRIT_REFSPEC", value: "${{env.GERRIT_REFSPEC}}"),
+ string(name: "GERRIT_REPO_URL", value: "${{env.REPO_URL}}"),
+ string(name: "PIPELINE_BUILD_URL", value: "${{env.BUILD_URL}}"),
+ string(name: "PROJECT_NAME", value: "{repos}"),
+ ]
+ echo "pipeline_${{display_name}}: SUCCESS"
+ }} catch (Exception e) {{
+ echo "pipeline_${{display_name}}: FAILED"
+ }}
+ }}
+
+ pipeline {{
+ agent none
+ stages {{
+ stage("Verification") {{
+ parallel {{
+
+ // Run the comment job to add the pipeline link to gerrit
+ stage("Start Comment") {{
+ steps {{
+ script {{
+ run_job("comment_start", "gerrit-verifications-comment", "start")
+ }}
+ }}
+ }}
+
+ stage("Build") {{
+ when {{
+ expression {{ params.PIPELINE_BUILD }}
+ }}
+ steps {{
+ script {{
+ run_job("build", "gerrit-{repos}-build")
+ }}
+ }}
+ }}
+
+ stage("Lint") {{
+ when {{
+ expression {{ params.PIPELINE_LINT }}
+ }}
+ steps {{
+ script {{
+ run_job("lint", "gerrit-lint")
+ }}
+ }}
+ }}
+
+ // Generating binpkgs stages is not possible if we want to run
+ // them in parallel and not have another job that just waits
+ // until they complete.
+
+ stage("binpkgs-0") {{
+ when {{
+ expression {{ params.PIPELINE_BINPKGS.split().size() > 0 }}
+ }}
+ steps {{
+ script {{
+ distro = params.PIPELINE_BINPKGS.split()[0]
+ run_job(distro, "gerrit-binpkgs", null, distro)
+ }}
+ }}
+ }}
+
+ stage("binpkgs-1") {{
+ when {{
+ expression {{ params.PIPELINE_BINPKGS.split().size() > 1 }}
+ }}
+ steps {{
+ script {{
+ distro = params.PIPELINE_BINPKGS.split()[1]
+ run_job(distro, "gerrit-binpkgs", null, distro)
+ }}
+ }}
+ }}
+
+ stage("binpkgs-2") {{
+ when {{
+ expression {{ params.PIPELINE_BINPKGS.split().size() > 2 }}
+ }}
+ steps {{
+ script {{
+ distro = params.PIPELINE_BINPKGS.split()[2]
+ run_job(distro, "gerrit-binpkgs", null, distro)
+ }}
+ }}
+ }}
+
+ }} // end of parallel
+ }} // end of Verification stage
+
+ // Run the comment job to get successful/failed links and add a
+ // comment + vote to gerrit
+ stage("Result Comment") {{
+ steps {{
+ script {{
+ run_job("comment_result", "gerrit-verifications-comment", "result")
+ }}
+ }}
+ }}
+
+ }}
+ }}
+ disabled: '{obj:disabled}'
+ retry-count: 3 # scm checkout
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 1000
+ description: |
+ Pipeline of CI for patches sent to
+ <a href="https://gerrit.osmocom.org/#/q/status:open+project:{repos}">{repos}</a>
+ <br/><br/>
+ <b>Auto-generated using Jenkins Job Builder. DO NOT EDIT MANUALLY!</b>
+
+ scm:
+ - git:
+ url: '{obj:repos_url}'
+ credentials-id: d5eda5e9-b59d-44ba-88d2-43473cb6e42d
+ branches:
+ - $GERRIT_BRANCH
+ refspec: $GERRIT_REFSPEC
+ name:
+ choosing-strategy: gerrit
+ wipe-workspace: false
+ skip-tag: true
+ submodule:
+ recursive: true
+
+ triggers:
+ - gerrit:
+ trigger-on:
+ - patchset-created-event:
+ exclude-drafts: true
+ exclude-no-code-change: true
+ projects:
+ - project-compare-type: 'PLAIN'
+ project-pattern: '{obj:gerrit_project}'
+ branches:
+ - branch-compare-type: 'ANT'
+ branch-pattern: '**'
+ skip-vote:
+ successful: false
+ failed: false
+ unstable: false
+ notbuilt: false
+ silent: true # comment + vote is done in gerrit-pipeline-result.yml
+ escape-quotes: false
+ server-name: gerrit.osmocom.org
+
+- job-template:
+ name: 'gerrit-{repos}-build'
project-type: matrix
- node: '{node}'
concurrent: '{obj:concurrent}'
disabled: '{obj:disabled}'
retry-count: 3 # scm checkout
properties:
- build-discarder:
days-to-keep: 30
- num-to-keep: 120
- artifact-days-to-keep: -1
- artifact-num-to-keep: -1
+ num-to-keep: 1000
description: |
- Pending gerrit code reviews of
+ Build job of CI for patches sent to
<a href="https://gerrit.osmocom.org/#/q/status:open+project:{repos}">{repos}</a>
<br/><br/>
<b>Auto-generated using Jenkins Job Builder. DO NOT EDIT MANUALLY!</b>
@@ -327,28 +694,8 @@
submodule:
recursive: true
- triggers:
- - gerrit:
- trigger-on:
- - patchset-created-event:
- exclude-drafts: true
- exclude-no-code-change: true
- projects:
- - project-compare-type: 'PLAIN'
- project-pattern: '{obj:gerrit_project}'
- branches:
- - branch-compare-type: 'ANT'
- branch-pattern: '**'
- skip-vote:
- successful: false
- failed: false
- unstable: false
- notbuilt: false
- silent: false
- escape-quotes: false
- no-name-and-email: false
- trigger-for-unreviewed-patches: true
- server-name: gerrit.osmocom.org
+ parameters:
+ !include: gerrit-verifications-parameters.yaml.inc
builders:
- shell: '{obj:cmd}'
diff --git a/jobs/master-builds-dahdi.yml b/jobs/master-builds-dahdi.yml
new file mode 100644
index 0000000..8f6d070
--- /dev/null
+++ b/jobs/master-builds-dahdi.yml
@@ -0,0 +1,82 @@
+- project:
+ name: master-builds-dahdi-linux-stable
+ type: stable
+ repo: 'https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git'
+ linux-ver:
+ - "linux-4.19.y"
+ - "linux-5.10.y"
+ - "linux-6.1.y"
+ jobs:
+ - 'master-dahdi-linux-{type}-{linux-ver}'
+
+- project:
+ name: master-builds-dahdi-linux-torvalds
+ type: torvalds
+ repo: 'https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git'
+ linux-ver:
+ - "master"
+ jobs:
+ - 'master-dahdi-linux-{type}-{linux-ver}'
+
+- job-template:
+ name: 'master-dahdi-linux-{type}-{linux-ver}'
+ project-type: freestyle
+ node: osmocom-master
+ concurrent: true
+ retry-count: 0 # scm checkout
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 120
+ description: |
+ Automatic jenkins test job of
+ <a href="https://gerrit.osmocom.org/#/q/status:open+project:dahdi-linux">dahdi-linux</a>
+ built against linux {linux-ver}
+ (<a href="https://osmocom.org/issues/5407">OS#5407</a>)
+ <br/><br/>
+ <b>Auto-generated using Jenkins Job Builder. DO NOT EDIT MANUALLY!</b>
+
+ scm:
+ # Let jenkins clone the big linux repository and keep it between builds
+ # (wipe-workspace is false). Clone dahdi-linux with code from gerrit in
+ # the shell part below.
+ - git:
+ basedir: 'linux'
+ url: '{repo}'
+ branches:
+ - '{linux-ver}'
+ refspec: '{linux-ver}'
+ shallow-clone: true
+ wipe-workspace: false
+ clean:
+ before: true
+
+ triggers:
+ - timed: "H 23 * * *"
+
+ builders:
+ - shell: |
+ rm -rf dahdi-linux
+ git clone \
+ --depth=1 \
+ https://gerrit.osmocom.org/dahdi-linux
+ git -C dahdi-linux log --oneline
+
+ docker run \
+ --rm=true \
+ -e "KSRC=/linux" \
+ -v "$PWD/dahdi-linux:/build" \
+ -v "$PWD/linux:/linux" \
+ "$USER/debian-bookworm-build" \
+ timeout 10m su build -c /build/contrib/jenkins.sh
+
+ publishers:
+ - warnings:
+ console-log-parsers:
+ - 'GNU C Compiler 4 (gcc)'
+ resolve-relative-paths: true
+ - email:
+ recipients: 'jenkins-notifications@lists.osmocom.org laforge@gnumonks.org'
+ send-to-individuals: true
+
+# vim: expandtab tabstop=2 shiftwidth=2
diff --git a/jobs/master-builds.yml b/jobs/master-builds.yml
index 65b641b..66043e5 100644
--- a/jobs/master-builds.yml
+++ b/jobs/master-builds.yml
@@ -5,9 +5,8 @@
# following default values can be overridden by each repo
disabled: false
concurrent: true
- node: 'osmocom-master-debian9'
# axes related defaults
- slave_axis: !!python/tuple [osmocom-master-debian9]
+ slave_axis: !!python/tuple [osmocom-master]
a1_name: a1
a1: !!python/tuple [default]
a2_name: a2
@@ -23,34 +22,68 @@
# Documentation for variables (keep in sync!):
# https://osmocom.org/projects/osmocom-servers/wiki/Jenkins_build_verification_jobs
docker_run: |
- ARTIFACT_STORE="$HOME/jenkins_build_artifact_store"
- mkdir -p "$ARTIFACT_STORE"
docker run --rm=true \
--cap-add SYS_PTRACE \
- -e ARTIFACT_STORE=/artifact_store \
-e ASCIIDOC_WARNINGS_CHECK="1" \
-e HOME=/build \
+ -e IS_MASTER_BUILD=1 \
-e JOB_NAME="$JOB_NAME" \
-e MAKE=make \
-e OSMOPY_DEBUG_TCP_SOCKETS="1" \
-e OSMO_GSM_MANUALS_DIR="/opt/osmo-gsm-manuals" \
-e PARALLEL_MAKE="$PARALLEL_MAKE" \
-e PUBLISH="1" \
+ -e SSH_AUTH_SOCK=/ssh-agent \
-e WITH_MANUALS="1" \
-w /build -i -u build \
+ -v $(readlink -f $SSH_AUTH_SOCK):/ssh-agent \
-v "$PWD:/build" \
- -v "$ARTIFACT_STORE:/artifact_store" \
-v "$HOME/.ssh:/home/build/.ssh:ro" \
- docker_img: '$USER/debian-stretch-jenkins'
- cmd: '{docker_run} {docker_img} /build/contrib/jenkins.sh'
- wrappers: []
+ docker_run_ccache: |
+ CCACHE_DIR="$HOME/ccache/master-builds"
+ mkdir -p "$CCACHE_DIR"
+ chown osmocom-build:osmocom-build "$CCACHE_DIR"
+
+ docker run --rm=true \
+ --cap-add SYS_PTRACE \
+ -e ASCIIDOC_WARNINGS_CHECK="1" \
+ -e HOME=/build \
+ -e IS_MASTER_BUILD=1 \
+ -e JOB_NAME="$JOB_NAME" \
+ -e MAKE=make \
+ -e OSMOPY_DEBUG_TCP_SOCKETS="1" \
+ -e OSMO_GSM_MANUALS_DIR="/opt/osmo-gsm-manuals" \
+ -e PARALLEL_MAKE="$PARALLEL_MAKE" \
+ -e PUBLISH="1" \
+ -e SSH_AUTH_SOCK=/ssh-agent \
+ -e WITH_MANUALS="1" \
+ -w /build -i -u build \
+ -v $(readlink -f $SSH_AUTH_SOCK):/ssh-agent \
+ -v "$PWD:/build" \
+ -v "$HOME/.ssh:/home/build/.ssh:ro" \
+ \
+ -e CCACHE_DIR="/ccache" \
+ -e PATH="/usr/lib/ccache:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
+ -v "$CCACHE_DIR:/ccache" \
+ docker_img: '$USER/debian-bookworm-build'
+ docker_img_erlang: '$USER/debian-bookworm-erlang'
+ timeout_cmd: '/usr/bin/timeout 30m'
+ cmd: '{docker_run_ccache} {docker_img} {timeout_cmd} /build/contrib/jenkins.sh'
trigger:
email: jenkins-notifications@lists.osmocom.org laforge@gnumonks.org
- git_base_url: git://git.osmocom.org
+ gerrit_url: 'https://gerrit.osmocom.org'
+ gitea_url: 'https://gitea.osmocom.org'
+ repos_url: '{gerrit_url}/{repos}'
+ branches:
+ - master
+ ssh:
+ - docs_ftp.osmocom.org
# in alphabetical order
repos:
- - asn1c
+ - asn1c:
+ branches:
+ - osmo-iuh/master
- libasn1c:
# Full triggers would be:
@@ -60,7 +93,7 @@
trigger: master-osmo-iuh
- libgtpnl:
- trigger: master-osmo-ggsn, openggsn
+ trigger: master-osmo-ggsn, openggsn, master-osmo-upf
- libosmo-abis:
# Full triggers would be:
@@ -73,21 +106,32 @@
master-libosmo-netif, master-osmo-bts
- libosmo-dsp:
+ repos_url: '{gitea_url}/sdr/{repos}'
cmd: |
- {docker_run} \
- {docker_img} sh -ex -c \
+ {docker_run_ccache} \
+ {docker_img} {timeout_cmd} sh -ex -c \
'autoreconf -fi; \
./configure; \
$MAKE $PARALLEL_MAKE; \
$MAKE distcheck'
trigger: master-osmo-gmr
- email: gerrit-log@lists.osmocom.org laforge@gnumonks.org 246tnt@gmail.com
+ email: jenkins-notifications@lists.osmocom.org laforge@gnumonks.org 246tnt@gmail.com
+
+ - libosmo-gprs:
+ # Full triggers would be:
+ # trigger: >
+ # master-osmo-pcu
+ # Optimized:
+ # (see comment at libosmocore)
+ trigger: >
+ master-osmo-pcu
- libosmo-netif:
# Full triggers would be:
# trigger: >
# master-libosmo-sccp, master-openbsc, mastere-osmo-hlr, master-osmo-iuh,
- # master-osmo-mgw, master-osmo-msc, master-osmo-sgsn
+ # master-osmo-mgw, master-osmo-msc, master-osmo-sgsn, master-osmo-hnodeb,
+ # master-osmo-hnbgw
# Optimized:
# (see comment at libosmocore)
trigger: >
@@ -96,32 +140,44 @@
master-osmo-mgw,
master-osmo-sgsn
+ - libosmo-pfcp:
+ # Full triggers would be:
+ # trigger: >
+ # master-osmo-upf, master-osmo-hnbgw
+ # Optimized:
+ # (see comment at libosmocore)
+ trigger: >
+ master-osmo-upf,
+ master-osmo-hnbgw
+
- libosmo-sccp:
# Full triggers would be:
# trigger: >
- # master-openbsc, master-osmo-bsc, master-osmo-iuh, master-osmo-msc, master-osmo-sgsn
+ # master-openbsc, master-osmo-bsc, master-osmo-iuh, master-osmo-msc,
+ # master-osmo-sgsn, master-osmo-hnodeb, master-osmo-hnbgw
# Optimized:
# (see comment at libosmocore)
trigger: >
master-openbsc,
master-osmo-bsc,
+ master-osmo-bsc-nat,
master-osmo-iuh,
master-osmo-msc,
- master-osmo-sgsn,
- master-osmo-hnodeb
+ master-osmo-sgsn
- libosmocore:
+ slave_axis: !!python/tuple [io_uring] # OS#6405
a1_name: arch
a1: !!python/tuple [arm-none-eabi, amd64]
combination_filter: '!(arch=="arm-none-eabi" && label=="FreeBSD_amd64")'
- cmd: '{docker_run} {docker_img} /build/contrib/jenkins_arch.sh "$arch"'
+ cmd: '{docker_run_ccache} {docker_img} {timeout_cmd} /build/contrib/jenkins_arch.sh "$arch"'
# Full triggers would be:
# trigger: >
# master-libosmo-abis, master-libosmo-netif, master-libosmo-sccp, master-openbsc,
# master-osmo-bsc, master-osmo-bts, master-osmo-ggsn, master-osmo-gmr, master-osmo-hlr,
# master-osmo-iuh, master-osmo-mgw, master-osmo-msc, master-osmo-pcap, master-osmo-pcu,
- # master-osmo-sgsn, master-osmo-sip-connector, master-osmo-tetra,
- # master-osmocom-bb, SIMtrace, xgoldmon
+ # master-osmo-sgsn, master-osmo-sip-connector, master-osmo-tetra, master-osmo-hnodeb
+ # master-osmocom-bb, master-simtrace, master-xgoldmon
# Optimized:
# libosmocore
# -> master-libosmo-abis
@@ -129,31 +185,38 @@
# -> master-libosmo-sccp
# -> master-openbsc
# -> master-osmo-bsc
+ # -> master-osmo-bsc-nat
# -> master-osmo-iuh
# -> master-osmo-msc
# -> master-osmo-sgsn
+ # -> master-osmo-hnodeb
# -> master-osmo-msc
# -> master-osmo-sgsn
# -> master-osmo-hlr
# -> master-osmo-mgw
# -> master-osmo-sgsn
# -> master-osmo-bts
+ # -> master-libosmo-gprs
+ # -> master-osmo-pcu
+ # -> master-libosmo-pfcp
+ # -> master-osmo-upf
+ # -> master-osmo-hnbgw
# -> master-osmo-ggsn
# -> master-osmo-sgsn
# [...]
trigger: >
master-libosmo-abis,
+ master-libosmo-pfcp,
master-osmo-gbproxy,
master-osmo-ggsn,
master-osmo-gmr,
master-osmo-pcap,
- master-osmo-pcu,
master-osmo-sip-connector,
master-osmo-tetra,
master-osmo-trx,
master-osmocom-bb,
- SIMtrace,
- xgoldmon
+ master-simtrace,
+ master-xgoldmon
- libsmpp34:
trigger: master-osmo-msc, master-openbsc
@@ -170,16 +233,17 @@
a3_name: IU
a3: !!python/tuple [--disable-iu]
cmd: |
- {docker_run} \
+ {docker_run_ccache} \
-e IU="$IU" \
-e SMPP="$SMPP" \
-e MGCP="$MGCP" \
- {docker_img} /build/contrib/jenkins.sh
+ {docker_img} {timeout_cmd} /build/contrib/jenkins.sh
- osmo-bsc
+ - osmo-bsc-nat
- osmo-bts:
- slave_axis: !!python/tuple [osmocom-master-debian9, rpi4-raspbian10]
+ slave_axis: !!python/tuple [osmocom-master, rpi4-raspbian11]
a1_name: FIRMWARE_VERSION
a1: !!python/tuple [master, femtobts_v2.7, superfemto_v2.4, superfemto_v3.0.1pre, superfemto_v3.1, superfemto_v5.1, v2017.01, origin/nrw/litecell15, origin/nrw/oc2g]
a2_name: BTS_MODEL
@@ -187,33 +251,72 @@
a3_name: WITH_MANUALS
a3: !!python/tuple ["1", "0"]
combination_filter: >
- (FIRMWARE_VERSION == "master" && WITH_MANUALS == "1" && label == "osmocom-master-debian9") ||
- (FIRMWARE_VERSION == "master" && BTS_MODEL == "trx" && WITH_MANUALS == "0" && label == "rpi4-raspbian10") ||
- (FIRMWARE_VERSION == "femtobts_v2.7" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (FIRMWARE_VERSION == "superfemto_v2.4" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (FIRMWARE_VERSION == "superfemto_v3.0.1pre" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (FIRMWARE_VERSION == "superfemto_v3.1" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (FIRMWARE_VERSION == "superfemto_v5.1" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (FIRMWARE_VERSION == "v2017.01" && BTS_MODEL == "lc15" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (FIRMWARE_VERSION == "origin/nrw/litecell15" && BTS_MODEL == "lc15" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (FIRMWARE_VERSION == "origin/nrw/oc2g" && BTS_MODEL == "oc2g" && WITH_MANUALS == "0" && label == "osmocom-master-debian9")
+ (FIRMWARE_VERSION == "master" && WITH_MANUALS == "1" && label == "osmocom-master") ||
+ (FIRMWARE_VERSION == "master" && BTS_MODEL == "trx" && WITH_MANUALS == "0" && label == "rpi4-raspbian11") ||
+ (FIRMWARE_VERSION == "femtobts_v2.7" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-master") ||
+ (FIRMWARE_VERSION == "superfemto_v2.4" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-master") ||
+ (FIRMWARE_VERSION == "superfemto_v3.0.1pre" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-master") ||
+ (FIRMWARE_VERSION == "superfemto_v3.1" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-master") ||
+ (FIRMWARE_VERSION == "superfemto_v5.1" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-master") ||
+ (FIRMWARE_VERSION == "v2017.01" && BTS_MODEL == "lc15" && WITH_MANUALS == "0" && label == "osmocom-master") ||
+ (FIRMWARE_VERSION == "origin/nrw/litecell15" && BTS_MODEL == "lc15" && WITH_MANUALS == "0" && label == "osmocom-master") ||
+ (FIRMWARE_VERSION == "origin/nrw/oc2g" && BTS_MODEL == "oc2g" && WITH_MANUALS == "0" && label == "osmocom-master")
concurrent: false
cmd: |
- {docker_run} \
+ DOCKER_IMG="{docker_img}"
+ case "$(arch)" in
+ arm*) DOCKER_IMG="$DOCKER_IMG-arm" ;;
+ esac
+ {docker_run_ccache} \
-e FIRMWARE_VERSION="$FIRMWARE_VERSION" \
-e WITH_MANUALS="$WITH_MANUALS" \
- {docker_img} /build/contrib/jenkins_bts_model.sh "$BTS_MODEL"
+ "$DOCKER_IMG" {timeout_cmd} /build/contrib/jenkins_bts_model.sh "$BTS_MODEL"
+
+ - osmo_dia2gsup:
+ repos_url: '{gerrit_url}/erlang/{repos}'
+ cmd: '{docker_run} {docker_img_erlang} {timeout_cmd} /build/contrib/jenkins.sh'
+
+ - osmo-epdg:
+ repos_url: '{gerrit_url}/erlang/{repos}'
+ cmd: '{docker_run} {docker_img_erlang} {timeout_cmd} /build/contrib/jenkins.sh'
- osmo-e1-hardware:
+ a2_name: JOB_TYPE
+ a2: !!python/tuple ["firmware", "gateware", "manuals", "software"]
+ cmd: |
+ DOCKER_IMG="{docker_img}"
+ case "$JOB_TYPE" in
+ firmware|gateware)
+ DOCKER_IMG="registry.osmocom.org/$USER/fpga-build"
+ docker pull "$DOCKER_IMG"
+ ;;
+ esac
+ {docker_run} \
+ -e JOB_TYPE="$JOB_TYPE" \
+ "$DOCKER_IMG" \
+ {timeout_cmd} /build/contrib/jenkins.sh --publish
+ ssh:
+ - binaries_ftp.osmocom.org
+ - docs_ftp.osmocom.org
+ - ice40-usbtrace:
+ repos_url: '{gitea_url}/electronics/{repos}'
docker_img: 'registry.osmocom.org/$USER/fpga-build'
- cmd: 'docker pull {docker_img} && {docker_run} {docker_img} /build/contrib/jenkins.sh --publish'
+ cmd: 'docker pull {docker_img} && {docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins.sh --publish'
+ ssh:
+ - binaries_ftp.osmocom.org
+ - docs_ftp.osmocom.org
- no2bootloader:
- git_base_url: https://github.com/no2fpga
+ repos_url: 'https://github.com/no2fpga/{repos}'
docker_img: 'registry.osmocom.org/$USER/fpga-build'
- cmd: 'docker pull {docker_img} && {docker_run} {docker_img} /build/contrib/jenkins.sh --publish'
- email: gerrit-log@lists.osmocom.org laforge@gnumonks.org kredon@sysmocom.de 246tnt@gmail.com
+ cmd: 'docker pull {docker_img} && {docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins.sh --publish'
+ email: jenkins-notifications@lists.osmocom.org laforge@gnumonks.org kredon@sysmocom.de 246tnt@gmail.com
+ ssh:
+ - binaries_ftp.osmocom.org
+ - docs_ftp.osmocom.org
+
+ - osmo-fl2k:
+ repos_url: '{gitea_url}/sdr/{repos}'
- - osmo-fl2k
- osmo-ggsn:
a1_name: GTP
a1: !!python/tuple [--enable-gtp-linux,--disable-gtp-linux]
@@ -223,31 +326,43 @@
(GTP == "--enable-gtp-linux" && WITH_MANUALS == "0") ||
(GTP == "--disable-gtp-linux" && WITH_MANUALS == "1")
cmd: |
- {docker_run} \
+ {docker_run_ccache} \
-e GTP="$GTP" \
-e WITH_MANUALS="$WITH_MANUALS" \
- {docker_img} /build/contrib/jenkins.sh
+ {docker_img} {timeout_cmd} /build/contrib/jenkins.sh
trigger: master-osmo-sgsn
+ - osmo-upf
+
- osmo-gmr:
- email: gerrit-log@lists.osmocom.org 246tnt@gmail.com
+ repos_url: '{gitea_url}/satellite/{repos}'
+ email: jenkins-notifications@lists.osmocom.org 246tnt@gmail.com
- - osmo-gsm-manuals:
- node: 'osmocom-master-debian9'
+ - osmo-gsm-manuals
+
+ - osmo_gsup:
+ repos_url: '{gerrit_url}/erlang/{repos}'
+ cmd: '{docker_run_ccache} {docker_img_erlang} {timeout_cmd} /build/contrib/jenkins.sh'
- osmo-gsm-tester:
- cmd: '{docker_run} {docker_img} /build/contrib/jenkins-build-manuals.sh'
+ cmd: '{docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins-build-manuals.sh'
- osmo-hlr
- osmo-iuh:
# Full triggers would be:
- # trigger: master-osmo-msc, master-osmo-sgsn
+ # trigger: master-osmo-msc, master-osmo-sgsn, master-osmo-hnodeb, master-osmo-hnbgw
# Optimized:
# (see comment at libosmocore)
trigger: >
master-osmo-msc,
- master-osmo-sgsn
+ master-osmo-sgsn,
+ master-osmo-hnodeb,
+ master-osmo-hnbgw
+
+ - osmo-ir77:
+ repos_url: '{gitea_url}/satellite/{repos}'
+ email: jenkins-notifications@lists.osmocom.org 246tnt@gmail.com
- osmo-mgw:
trigger: master-osmo-msc, master-osmo-bsc
@@ -258,18 +373,18 @@
a2_name: WITH_MANUALS
a2: !!python/tuple ["1", "0"]
combination_filter: >
- (IU == "--enable-iu" && WITH_MANUALS == "0") ||
- (IU == "--disable-iu" && WITH_MANUALS == "1")
+ (IU == "--enable-iu" && WITH_MANUALS == "1") ||
+ (IU == "--disable-iu" && WITH_MANUALS == "0")
cmd: |
- {docker_run} \
+ {docker_run_ccache} \
-e IU="$IU" \
-e WITH_MANUALS="$WITH_MANUALS" \
- {docker_img} /build/contrib/jenkins.sh
+ {docker_img} {timeout_cmd} /build/contrib/jenkins.sh
- osmo-pcap
- osmo-pcu:
- slave_axis: !!python/tuple [osmocom-master-debian9, rpi4-raspbian10]
+ slave_axis: !!python/tuple [osmocom-master, rpi4-raspbian11]
a1_name: FIRMWARE_VERSION
a1: !!python/tuple [master, v2017.01, origin/nrw/litecell15]
a2_name: with_vty
@@ -279,19 +394,26 @@
a4_name: WITH_MANUALS
a4: !!python/tuple ["1", "0"]
combination_filter: >
- (with_vty == "True" && with_dsp == "sysmo" && FIRMWARE_VERSION=="master" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (with_vty == "True" && with_dsp == "lc15" && FIRMWARE_VERSION=="master" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (with_vty == "True" && with_dsp == "lc15" && FIRMWARE_VERSION=="v2017.01" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (with_vty == "True" && with_dsp == "lc15" && FIRMWARE_VERSION=="origin/nrw/litecell15" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (with_vty == "False" && with_dsp == "none" && FIRMWARE_VERSION=="master" && WITH_MANUALS == "1" && label == "osmocom-master-debian9") ||
- (with_vty == "False" && with_dsp == "none" && FIRMWARE_VERSION=="master" && WITH_MANUALS == "0" && label == "rpi4-raspbian10")
+ (with_vty == "True" && with_dsp == "sysmo" && FIRMWARE_VERSION=="master" && WITH_MANUALS == "0" && label == "osmocom-master") ||
+ (with_vty == "True" && with_dsp == "lc15" && FIRMWARE_VERSION=="master" && WITH_MANUALS == "0" && label == "osmocom-master") ||
+ (with_vty == "True" && with_dsp == "lc15" && FIRMWARE_VERSION=="v2017.01" && WITH_MANUALS == "0" && label == "osmocom-master") ||
+ (with_vty == "True" && with_dsp == "lc15" && FIRMWARE_VERSION=="origin/nrw/litecell15" && WITH_MANUALS == "0" && label == "osmocom-master") ||
+ (with_vty == "False" && with_dsp == "none" && FIRMWARE_VERSION=="master" && WITH_MANUALS == "1" && label == "osmocom-master") ||
+ (with_vty == "False" && with_dsp == "none" && FIRMWARE_VERSION=="master" && WITH_MANUALS == "0" && label == "rpi4-raspbian11")
cmd: |
- {docker_run} \
+ DOCKER_IMG="{docker_img}"
+ case "$(arch)" in
+ arm*) DOCKER_IMG="$DOCKER_IMG-arm" ;;
+ esac
+ {docker_run_ccache} \
-e FIRMWARE_VERSION="$FIRMWARE_VERSION" \
-e with_vty="$with_vty" \
-e with_dsp="$with_dsp" \
-e WITH_MANUALS="$WITH_MANUALS" \
- {docker_img} /build/contrib/jenkins.sh
+ "$DOCKER_IMG" {timeout_cmd} /build/contrib/jenkins.sh
+
+ - osmo-python-tests:
+ repos_url: '{gerrit_url}/python/{repos}'
- osmo-sgsn:
a1_name: IU
@@ -299,49 +421,72 @@
a2_name: WITH_MANUALS
a2: !!python/tuple ["1", "0"]
combination_filter: >
- (IU == "--enable-iu" && WITH_MANUALS == "0") ||
- (IU == "--disable-iu" && WITH_MANUALS == "1")
+ (IU == "--enable-iu" && WITH_MANUALS == "1") ||
+ (IU == "--disable-iu" && WITH_MANUALS == "0")
cmd: |
- {docker_run} \
+ {docker_run_ccache} \
-e IU="$IU" \
-e WITH_MANUALS="$WITH_MANUALS" \
- {docker_img} /build/contrib/jenkins.sh
+ {docker_img} {timeout_cmd} /build/contrib/jenkins.sh
- osmo-sip-connector:
- email: gerrit-log@lists.osmocom.org laforge@gnumonks.org holger@freyther.de
+ email: jenkins-notifications@lists.osmocom.org laforge@gnumonks.org holger@freyther.de
- osmo-tetra:
- email: gerrit-log@lists.osmocom.org laforge@gnumonks.org 246tnt@gmail.com
+ email: jenkins-notifications@lists.osmocom.org laforge@gnumonks.org 246tnt@gmail.com
- osmo-trx:
- slave_axis: !!python/tuple [osmocom-master-debian9,rpi4-raspbian10]
+ slave_axis: !!python/tuple [osmocom-master,rpi4-raspbian11]
a1_name: INSTR
a1: !!python/tuple [--with-sse, --without-sse, --with-neon, --with-neon-vfpv4]
a2_name: WITH_MANUALS
a2: !!python/tuple ["1", "0"]
combination_filter: >
- (INSTR == "--without-sse" && WITH_MANUALS == "1" && label == "osmocom-master-debian9") ||
- (INSTR == "--with-sse" && WITH_MANUALS == "0" && label == "osmocom-master-debian9") ||
- (INSTR == "--with-neon" && WITH_MANUALS == "0" && label == "rpi4-raspbian10") ||
- (INSTR == "--with-neon-vfpv4" && WITH_MANUALS == "0" && label == "rpi4-raspbian10")
+ (INSTR == "--without-sse" && WITH_MANUALS == "1" && label == "osmocom-master") ||
+ (INSTR == "--with-sse" && WITH_MANUALS == "0" && label == "osmocom-master") ||
+ (INSTR == "--with-neon" && WITH_MANUALS == "0" && label == "rpi4-raspbian11") ||
+ (INSTR == "--with-neon-vfpv4" && WITH_MANUALS == "0" && label == "rpi4-raspbian11")
cmd: |
- {docker_run} \
+ DOCKER_IMG="{docker_img}"
+ case "$(arch)" in
+ arm*) DOCKER_IMG="$DOCKER_IMG-arm" ;;
+ esac
+ {docker_run_ccache} \
-e INSTR="$INSTR" \
-e WITH_MANUALS="$WITH_MANUALS" \
- {docker_img} /build/contrib/jenkins.sh
+ "$DOCKER_IMG" {timeout_cmd} /build/contrib/jenkins.sh
- osmocom-bb
- osmo-sysmon
- osmo-remsim
- osmo-asf4-dfu:
- cmd: '{docker_run} {docker_img} /build/contrib/jenkins.sh --publish'
- email: gerrit-log@lists.osmocom.org laforge@gnumonks.org ewild@sysmocom.de
+ cmd: '{docker_run_ccache} {docker_img} {timeout_cmd} /build/contrib/jenkins.sh --publish'
+ email: jenkins-notifications@lists.osmocom.org laforge@gnumonks.org ewild@sysmocom.de
+ ssh:
+ - binaries_ftp.osmocom.org
+ - docs_ftp.osmocom.org
- osmo-ccid-firmware:
- cmd: '{docker_run} {docker_img} /build/contrib/jenkins.sh --publish'
- email: gerrit-log@lists.osmocom.org laforge@gnumonks.org ewild@sysmocom.de
+ cmd: '{docker_run_ccache} {docker_img} {timeout_cmd} /build/contrib/jenkins.sh --publish'
+ email: jenkins-notifications@lists.osmocom.org laforge@gnumonks.org ewild@sysmocom.de
+ ssh:
+ - binaries_ftp.osmocom.org
+ - docs_ftp.osmocom.org
+ - simtrace:
+ repos_url: '{gitea_url}/sim-card/{repos}'
+ email: jenkins-notifications@lists.osmocom.org laforge@gnumonks.org kredon@sysmocom.de
- simtrace2:
- cmd: '{docker_run} {docker_img} /build/contrib/jenkins.sh --publish'
- email: gerrit-log@lists.osmocom.org laforge@gnumonks.org kredon@sysmocom.de
+ cmd: '{docker_run_ccache} {docker_img} {timeout_cmd} /build/contrib/jenkins.sh --publish'
+ email: jenkins-notifications@lists.osmocom.org laforge@gnumonks.org kredon@sysmocom.de
+ ssh:
+ - binaries_ftp.osmocom.org
+ - docs_ftp.osmocom.org
+ - osmo-opencm3-projects:
+ repos_url: '{gitea_url}/electronics/{repos}'
+ cmd: '{docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins.sh --publish'
+ email: jenkins-notifications@lists.osmocom.org laforge@gnumonks.org kredon@sysmocom.de
+ ssh:
+ - binaries_ftp.osmocom.org
+ - docs_ftp.osmocom.org
- osmo-e1d
- osmo-cbc
@@ -353,14 +498,60 @@
- osmo-gbproxy
- osmo-hnodeb
+ - osmo-hnbgw:
+ a1_name: PFCP
+ a1: !!python/tuple ["1", "0"]
+ a2_name: WITH_MANUALS
+ a2: !!python/tuple ["1", "0"]
+ combination_filter: >
+ (PFCP == "1" && WITH_MANUALS == "1") ||
+ (PFCP == "0" && WITH_MANUALS == "0")
+ cmd: |
+ {docker_run_ccache} \
+ -e PFCP="$PFCP" \
+ -e WITH_MANUALS="$WITH_MANUALS" \
+ {docker_img} {timeout_cmd} /build/contrib/jenkins.sh
+
- rtl-sdr:
+ repos_url: '{gitea_url}/sdr/{repos}'
email: laforge@gnumonks.org steve@steve-m.de
+ - pysim:
+ concurrent: false
+ slave_axis: !!python/tuple [simtester,osmocom-master]
+ a2_name: JOB_TYPE
+ a2: !!python/tuple ["test", "pylint", "docs"]
+ combination_filter: >
+ (JOB_TYPE == "test" && label == "simtester") ||
+ (JOB_TYPE == "pylint" && label == "osmocom-master") ||
+ (JOB_TYPE == "docs" && label == "osmocom-master")
+ cmd: |
+ case "$JOB_TYPE" in
+ "test")
+ {timeout_cmd} ./contrib/jenkins.sh
+ ;;
+ *)
+ {docker_run} \
+ -e JOB_TYPE="$JOB_TYPE" \
+ {docker_img} \
+ {timeout_cmd} /build/contrib/jenkins.sh
+ ;;
+ esac
+
- sysmo-usim-tool:
concurrent: false
- cmd: cd ./tests; ./run-tests
+ cmd: cd ./tests; {timeout_cmd} ./run-tests
slave_axis: !!python/tuple [simtester]
- git_base_url: https://git.sysmocom.de/sysmocom
+ repos_url: 'https://gitea.sysmocom.de/sysmocom/{repos}'
+
+ - dahdi-tools
+
+ - xgoldmon:
+ repos_url: 'https://github.com/2b-as/{repos}'
+ cmd: |
+ rm -rf osmo-ci
+ git clone --depth=1 https://gerrit.osmocom.org/osmo-ci -b master
+ {docker_run_ccache} {docker_img} {timeout_cmd} /build/osmo-ci/scripts/xgoldmon-jenkins.sh
jobs:
- 'master-{repos}'
@@ -375,11 +566,9 @@
- build-discarder:
days-to-keep: 30
num-to-keep: 120
- artifact-days-to-keep: -1
- artifact-num-to-keep: -1
description: |
Automatic jenkins test job of
- <a href="{git_base_url}/{repos}">{repos}</a>
+ <a href="{repos_url}">{repos}</a>
<br/><br/>
<b>Auto-generated using Jenkins Job Builder. DO NOT EDIT MANUALLY!</b>
@@ -387,7 +576,6 @@
sequential: '{obj:sequential}'
combination-filter: |
{obj:combination_filter}
- node: '{obj:node}'
axes:
- axis:
type: slave
@@ -412,9 +600,8 @@
scm:
- git:
- url: "{git_base_url}/{repos}"
- branches:
- - 'origin/master'
+ url: '{obj:repos_url}'
+ branches: '{obj:branches}'
refspec:
name:
wipe-workspace: false
@@ -432,15 +619,22 @@
- shell: '{obj:cmd}'
publishers:
+ - archive:
+ allow-empty: true
+ artifacts: '**/core, **/testsuite.log, **/workspace.tar.xz'
- warnings:
console-log-parsers:
- 'GNU C Compiler 4 (gcc)'
resolve-relative-paths: true
- trigger:
project: '{obj:trigger}'
+ - trigger:
+ project: 'Osmocom_OBS_master'
- email:
recipients: '{obj:email}'
send-to-individuals: true
- wrappers: '{obj:wrappers}'
+ wrappers:
+ - ssh-agent-credentials:
+ users: '{obj:ssh}'
# vim: expandtab tabstop=2 shiftwidth=2
diff --git a/jobs/octsim_osmo-ccid-firmware.yml b/jobs/octsim_osmo-ccid-firmware.yml
new file mode 100644
index 0000000..7db1039
--- /dev/null
+++ b/jobs/octsim_osmo-ccid-firmware.yml
@@ -0,0 +1,53 @@
+- job:
+ axes:
+ - axis:
+ name: label
+ type: slave
+ values:
+ - octsimtest
+ block-downstream: false
+ block-upstream: false
+ builders:
+ - shell: |
+ cd ./tests
+ mkdir -p sysmo-octsim/dl
+ ./run-tests
+ concurrent: false
+ description: octsim osmo-ccid-firmware CI - flash every build to octsim and see
+ if it still works
+ disabled: false
+ execution-strategy:
+ sequential: false
+ name: octsim_osmo-ccid-firmware
+ project-type: matrix
+ scm:
+ - git:
+ url: https://gerrit.osmocom.org/osmo-ccid-firmware
+ git-config-name: 'Jenkins Builder'
+ git-config-email: 'jenkins@osmocom.org'
+ skip-tag: true
+ branches:
+ - 'origin/master'
+ properties:
+ - build-blocker:
+ block-level: GLOBAL
+ blocking-jobs: []
+ queue-scanning: DISABLED
+ use-build-blocker: false
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 120
+ - rebuild:
+ auto-rebuild: false
+ rebuild-disabled: false
+ publishers:
+ - email:
+ notify-every-unstable-build: true
+ recipients: jsteiger@sysmocom.de
+ send-to-individuals: true
+ retry-count: '3'
+ triggers:
+ - reverse:
+ jobs: master-osmo-ccid-firmware,
+ - timed: H 22 * * *
+ wrappers: []
diff --git a/jobs/osmo-gsm-manuals-trigger.yml b/jobs/osmo-gsm-manuals-trigger.yml
index d83b9fa..29882e3 100644
--- a/jobs/osmo-gsm-manuals-trigger.yml
+++ b/jobs/osmo-gsm-manuals-trigger.yml
@@ -15,10 +15,13 @@
Roll out osmo-gsm-manuals to relevant build environments.<br>
<br>
Related: <a href="https://osmocom.org/issues/4912">OS#4912</a>
- node: osmocom-master-debian9
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
scm:
- git:
- url: git://git.osmocom.org/osmo-gsm-manuals
+ url: https://gerrit.osmocom.org/osmo-gsm-manuals
git-config-name: 'Jenkins Builder'
git-config-email: 'jenkins@osmocom.org'
skip-tag: true
diff --git a/jobs/osmo-gsm-tester-builder.yml b/jobs/osmo-gsm-tester-builder.yml
index 16f9419..cd50cb8 100644
--- a/jobs/osmo-gsm-tester-builder.yml
+++ b/jobs/osmo-gsm-tester-builder.yml
@@ -11,7 +11,7 @@
- osmo-ggsn
- osmo-hlr
- osmo-mgw
- - osmo-iuh
+ - osmo-hnbgw
- osmo-msc
- osmo-pcu
- osmo-sgsn
@@ -92,6 +92,8 @@
- add_param_build_branch:
name: OSMO_GSM_TESTER_BUILD_osmo_iuh
- add_param_build_branch:
+ name: OSMO_GSM_TESTER_BUILD_osmo_hnbgw
+ - add_param_build_branch:
name: OSMO_GSM_TESTER_BUILD_osmo_msc
- add_param_build_branch:
name: OSMO_GSM_TESTER_BUILD_osmo_hlr
@@ -114,7 +116,7 @@
name: osmo-gsm-tester-repo
scm:
- git:
- url: git://git.osmocom.org/osmo-gsm-tester
+ url: https://gerrit.osmocom.org/osmo-gsm-tester
branches:
- ${OSMO_GSM_TESTER_BRANCH}
wipe-workspace: false
diff --git a/jobs/osmo-gsm-tester-runner.yml b/jobs/osmo-gsm-tester-runner.yml
index a505629..4909fe7 100644
--- a/jobs/osmo-gsm-tester-runner.yml
+++ b/jobs/osmo-gsm-tester-runner.yml
@@ -32,7 +32,7 @@
name: osmo-gsm-tester-repo
scm:
- git:
- url: git://git.osmocom.org/osmo-gsm-tester
+ url: https://gerrit.osmocom.org/osmo-gsm-tester
branches:
- ${OSMO_GSM_TESTER_BRANCH}
wipe-workspace: false
@@ -63,7 +63,7 @@
- copy_artifact:
repo: osmo-gsm-tester_build-osmo-mgw
- copy_artifact:
- repo: osmo-gsm-tester_build-osmo-iuh
+ repo: osmo-gsm-tester_build-osmo-hnbgw
- copy_artifact:
repo: osmo-gsm-tester_build-osmo-msc
- copy_artifact:
@@ -111,6 +111,7 @@
- job-template:
name: 'osmo-gsm-tester_run-{stage}'
+ disabled: true
defaults: runner
node: 'osmo-gsm-tester-{stage}'
triggers:
@@ -171,8 +172,8 @@
branch-pattern: '**'
builders:
- shell: |
- # make sure no bin artifacts from a previous run remain
- rm -f *.tgz *.md5
+ # make sure no test results/bin artifacts from a previous run remain
+ rm -rf trial-* *.tgz *.md5
- copy_artifact_all
- shell: !include-raw: osmo-gsm-tester_run-gerrit.sh
publishers:
@@ -186,6 +187,7 @@
# ttcn3 job
- job:
name: 'osmo-gsm-tester_ttcn3'
+ disabled: true
defaults: runner
scm:
- 'osmo-gsm-tester-repo'
@@ -193,8 +195,8 @@
- timed: "H 23 * * *"
builders:
- shell: |
- # make sure no bin artifacts from a previous run remain
- rm -f *.tgz *.md5
+ # make sure no test results/bin artifacts from a previous run remain
+ rm -rf trial-* *.tgz *.md5
- copy_artifact_all
- copy_artifact:
repo: osmo-gsm-tester_build-osmocom-bb
@@ -211,7 +213,7 @@
- job:
name: 'osmo-gsm-tester_virtual'
defaults: runner
- node: 'osmocom-master-debian9'
+ node: 'osmocom-master'
scm:
- 'osmo-gsm-tester-repo'
triggers:
@@ -219,11 +221,46 @@
builders:
- shell: |
# make sure no bin artifacts from a previous run remain
- rm -f *.tgz *.md5
+ rm -rf *.tgz *.md5
- copy_artifact_all
- copy_artifact:
repo: osmo-gsm-tester_build-osmocom-bb
- - shell: >
+ - shell: |
+ # Set a trap to fix workspace permissions / kill the docker container on exit
+ clean_up() {
+ docker kill "osmo-gsm-tester-virtual" || true
+ docker run --rm \
+ -v "$WORKSPACE":/workspace \
+ debian:bullseye \
+ chmod -R a+rwX /workspace/
+ }
+ clean_up_trap() {
+ set +x
+ echo
+ echo "### Clean up ###"
+ echo
+ set -x
+
+ trap - EXIT INT TERM 0
+ clean_up
+ }
+ trap clean_up_trap EXIT INT TERM 0
+
+ # Make sure no test results from a previous run remain
+ clean_up
+ rm -rf trial-*
+
+ # Build the docker image
+ rm -rf docker-playground
+ git clone \
+ --depth=1 \
+ --branch="$DOCKER_PLAYGROUND_BRANCH" \
+ https://gerrit.osmocom.org/docker-playground \
+ docker-playground
+ git -C docker-playground log --oneline
+ make -C docker-playground/debian-buster-jenkins # OS#6126
+ make -C docker-playground/osmo-gsm-tester
+
unlink osmo-gsm-tester/sysmocom/resources.conf || true
ln -s resources.conf.virtual osmo-gsm-tester/sysmocom/resources.conf
@@ -241,6 +278,7 @@
-w /build -i \
-v "$PWD:/build" \
-v "$HOME/.ssh:/home/build/.ssh:ro" \
+ --name=osmo-gsm-tester-virtual \
--cap-add=sys_nice \
$USER/osmo-gsm-tester \
/bin/bash -c 'LANG="en_US.utf8" LC_ALL="en_US.UTF-8" LC_LANG="en_US.UTF-8" PATH="$PWD/osmo-gsm-tester/src:${PATH}" ./osmo-gsm-tester/contrib/jenkins-run.sh'
@@ -251,12 +289,19 @@
- junit:
results: 'trial-*/last_run/trial-*.xml'
allow-empty-results: true
+ - email:
+ notify-every-unstable-build: true
+ recipients: jenkins-notifications@lists.osmocom.org
parameters:
- string:
name: "OSMO_GSM_TESTER_OPTS"
default: "-s netreg_mass"
- description: "pass additional command line options to osmo-gsm-tester.py, e.g. to select specific suites:scenarios. Default: leave empty."
+ description: "pass additional command line options to osmo-gsm-tester.py, e.g. to select specific suites:scenarios"
- string:
name: "OSMO_GSM_TESTER_BRANCH"
- default: "origin/master"
+ default: "master"
description: "Which branch/sha should be used for testing"
+ - string:
+ name: "DOCKER_PLAYGROUND_BRANCH"
+ default: "master"
+ description: "Which branch/sha should be used for building the osmo-gsm-tester docker container"
diff --git a/jobs/osmo-gsm-tester-trigger.yml b/jobs/osmo-gsm-tester-trigger.yml
index 07edcf4..216dabc 100644
--- a/jobs/osmo-gsm-tester-trigger.yml
+++ b/jobs/osmo-gsm-tester-trigger.yml
@@ -8,6 +8,10 @@
- job:
name: 'osmo-gsm-tester_build-trigger-all'
description: 'Manually kick off all osmo-gsm-tester_build-osmo-* builds at once.'
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
project-type: freestyle
node: osmo-gsm-tester-build
publishers:
diff --git a/jobs/osmocom-api.yml b/jobs/osmocom-api.yml
new file mode 100644
index 0000000..312496b
--- /dev/null
+++ b/jobs/osmocom-api.yml
@@ -0,0 +1,46 @@
+- job:
+ concurrent: false
+ description: This job builds the doxygen API documentation for latest libosmo*
+ and uploads it to <a href="https://downloads.osmocom.org/api/latest/"
+ >https://downloads.osmocom.org/api/latest/</a>
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
+ name: Osmocom_API
+ node: osmocom-master
+ project-type: freestyle
+ publishers:
+ - email:
+ notify-every-unstable-build: true
+ recipients: jenkins-notifications@lists.osmocom.org
+ triggers:
+ - timed: "H 00 * * *"
+ wrappers:
+ - ssh-agent-credentials:
+ users:
+ - api_ftp.osmocom.org
+ parameters:
+ - string:
+ name: BRANCH
+ description: osmo-ci.git branch
+ default: 'master'
+ scm:
+ - git:
+ branches:
+ - '$BRANCH'
+ url: https://gerrit.osmocom.org/osmo-ci
+ git-config-name: 'Jenkins Builder'
+ git-config-email: 'jenkins@osmocom.org'
+ wipe-workspace: true
+ builders:
+ - shell: |
+ docker run \
+ --rm=true \
+ -e SSH_AUTH_SOCK=/ssh-agent \
+ -u build \
+ -v "$PWD:/build" \
+ -v $(readlink -f $SSH_AUTH_SOCK):/ssh-agent \
+ -w /build \
+ "$USER/debian-bookworm-build" \
+ scripts/osmocom-api-doxygen.sh
diff --git a/jobs/osmocom-build-tags-against-master.yml b/jobs/osmocom-build-tags-against-master.yml
index 1520ec2..e33c9a3 100644
--- a/jobs/osmocom-build-tags-against-master.yml
+++ b/jobs/osmocom-build-tags-against-master.yml
@@ -11,7 +11,11 @@
description: |
Build old releases of programs/libraries against "master of the day" libosmo*
(<a href="https://osmocom.org/issues/3765">OS#3765</a>)
- node: osmocom-master-debian9
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
+ node: osmocom-master
parameters:
- string:
name: BRANCH
@@ -20,16 +24,23 @@
default: '*/master'
builders:
- shell: |
- PARALLEL_MAKE=-j8 scripts/osmocom-build-old-tags-against-master.sh
+ docker run \
+ --rm=true \
+ -e PARALLEL_MAKE="-j8" \
+ -u build \
+ -v "$PWD:/build" \
+ -w /build \
+ "$USER/debian-bookworm-build" \
+ scripts/osmocom-build-old-tags-against-master.sh
scm:
- git:
branches:
- '$BRANCH'
- url: git://git.osmocom.org/osmo-ci
+ url: https://gerrit.osmocom.org/osmo-ci
git-config-name: 'Jenkins Builder'
git-config-email: 'jenkins@osmocom.org'
triggers:
- - timed: "@midnight"
+ - timed: "H 01 * * *"
publishers:
- archive:
allow-empty: false
diff --git a/jobs/osmocom-depcheck.yml b/jobs/osmocom-depcheck.yml
index 1e5533d..5e20f95 100644
--- a/jobs/osmocom-depcheck.yml
+++ b/jobs/osmocom-depcheck.yml
@@ -12,7 +12,11 @@
Verifies that Osmocom programs really build with the dependency
versions they claim to support in configure.ac.
(Generated by job-builder)
- node: osmocom-master-debian9
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
+ node: osmocom-master
parameters:
- string:
name: PROJECTS
@@ -26,7 +30,7 @@
name: GIT_URL_PREFIX
description: |
Where to clone the sources from
- default: 'git://git.osmocom.org/'
+ default: 'https://gerrit.osmocom.org/'
- bool:
name: BUILD
description: |
@@ -65,7 +69,7 @@
- git:
branches:
- '$BRANCH'
- url: git://git.osmocom.org/osmo-ci
+ url: https://gerrit.osmocom.org/osmo-ci
git-config-name: 'Jenkins Builder'
git-config-email: 'jenkins@osmocom.org'
diff --git a/jobs/osmocom-list-commits.yml b/jobs/osmocom-list-commits.yml
index f2651f4..d367499 100644
--- a/jobs/osmocom-list-commits.yml
+++ b/jobs/osmocom-list-commits.yml
@@ -11,7 +11,11 @@
description: |
Generate and upload a list of Osmocom git repositories and their latest tag and commits.
<b>The latest list is <a href="https://jenkins.osmocom.org/jenkins/job/Osmocom-list-commits/lastSuccessfulBuild/artifact/commits.txt/*view*/">here</a>.</b>
- node: osmocom-master-debian9
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
+ node: osmocom-master
parameters:
- string:
name: BRANCH
@@ -26,11 +30,11 @@
- git:
branches:
- '$BRANCH'
- url: git://git.osmocom.org/osmo-ci
+ url: https://gerrit.osmocom.org/osmo-ci
git-config-name: 'Jenkins Builder'
git-config-email: 'jenkins@osmocom.org'
triggers:
- - timed: "@midnight"
+ - timed: "H 01 * * *"
publishers:
- archive:
allow-empty: false
diff --git a/jobs/osmocom-obs-check-builders.yml b/jobs/osmocom-obs-check-builders.yml
new file mode 100644
index 0000000..01fdba5
--- /dev/null
+++ b/jobs/osmocom-obs-check-builders.yml
@@ -0,0 +1,32 @@
+---
+- project:
+ name: Osmocom_OBS_check_builders
+ jobs:
+ - Osmocom_OBS_check_builders
+
+- job-template:
+ name: 'Osmocom_OBS_check_builders'
+ project-type: freestyle
+ defaults: global
+ description: |
+ Verify that the expected amount of builders are connected to
+ obs.osmocom.org.
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
+ builders:
+ - shell: |
+ scripts/obs/check_builders.sh
+ scm:
+ - git:
+ branches:
+ - 'origin/master'
+ url: https://gerrit.osmocom.org/osmo-ci
+ triggers:
+ - timed: "@hourly"
+ node: obs
+ publishers:
+ - email:
+ notify-every-unstable-build: true
+ recipients: 'jenkins-notifications@lists.osmocom.org'
diff --git a/jobs/osmocom-obs-check-new-distros.yml b/jobs/osmocom-obs-check-new-distros.yml
new file mode 100644
index 0000000..760479a
--- /dev/null
+++ b/jobs/osmocom-obs-check-new-distros.yml
@@ -0,0 +1,39 @@
+---
+- job:
+ name: 'Osmocom_OBS_check_new_distros'
+ project-type: freestyle
+ concurrent: false
+ defaults: global
+ description: |
+ Check whether new distributions (Debian, Raspyberry Pi OS, Ubuntu) are
+ available in openSUSE's OBS that we need to add to the Osmocom OBS.
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
+ node: obs
+ parameters:
+ - string:
+ name: CI_BRANCH
+ description: osmo-ci.git branch
+ default: 'master'
+ scm:
+ - git:
+ branches:
+ - '$CI_BRANCH'
+ url: https://gerrit.osmocom.org/osmo-ci
+ git-config-name: 'Jenkins Builder'
+ git-config-email: 'jenkins@osmocom.org'
+ builders:
+ - shell: |
+ export PYTHONUNBUFFERED=1
+ ./scripts/obs/check_new_distros.py --docker
+ triggers:
+ - timed: "H 19 * * *"
+ wrappers:
+ - ansicolor:
+ colormap: xterm
+ publishers:
+ - email:
+ notify-every-unstable-build: true
+ recipients: 'jenkins-notifications@lists.osmocom.org'
diff --git a/jobs/osmocom-obs-ewild.yml b/jobs/osmocom-obs-ewild.yml
new file mode 100644
index 0000000..111ec09
--- /dev/null
+++ b/jobs/osmocom-obs-ewild.yml
@@ -0,0 +1,72 @@
+---
+- project:
+ name: Osmocom_OBS_ewild
+ jobs:
+ - Osmocom_OBS_ewild_{branch_suffix}
+ branch_suffix:
+ - mssdr
+
+- job-template:
+ name: 'Osmocom_OBS_ewild_{branch_suffix}'
+ project-type: freestyle
+ concurrent: false
+ defaults: global
+ description: |
+ Iterate over Osmocom projects, and update projects from the
+ "Hoernchen/{branch_suffix}" branch in OBS. If a package exists in OBS
+ for a project, but it doesn't have the git branch anymore then delete the
+ package from OBS.
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
+ node: obs
+ parameters:
+ - string:
+ name: CI_BRANCH
+ description: osmo-ci.git branch
+ default: 'master'
+ builders:
+ - shell: |
+ export PYTHONUNBUFFERED=1
+ BRANCH="Hoernchen/{branch_suffix}"
+ PROJ="home:ewild:{branch_suffix}"
+ VERSION_APPEND="~ewild.{branch_suffix}"
+ PACKAGE="$GERRIT_PROJECT"
+
+ # When manually triggered, run on all Osmocom packages
+ if [ -z "$PACKAGE" ]; then
+ PACKAGE="ALL_OSMOCOM_PACKAGES"
+ fi
+
+ ./scripts/obs/update_obs_project.py \
+ --apiurl https://obs.osmocom.org \
+ --conflict-pkgname "osmocom-nightly" \
+ --delete \
+ --docker \
+ --feed "master" \
+ --git-branch "$BRANCH" \
+ --version-append "$VERSION_APPEND" \
+ "$PROJ" \
+ "$PACKAGE"
+ scm:
+ - git:
+ branches:
+ - '$CI_BRANCH'
+ url: https://gerrit.osmocom.org/osmo-ci
+ git-config-name: 'Jenkins Builder'
+ git-config-email: 'jenkins@osmocom.org'
+ triggers:
+ - gerrit:
+ trigger-on:
+ - ref-updated-event
+ projects:
+ - project-compare-type: 'ANT'
+ project-pattern: '**'
+ branches:
+ - branch-pattern: 'Hoernchen/{branch_suffix}'
+ server-name: gerrit.osmocom.org
+ publishers:
+ - email:
+ notify-every-unstable-build: true
+ recipients: 'ewild@sysmocom.de'
diff --git a/jobs/osmocom-obs-nhofmeyr.yml b/jobs/osmocom-obs-nhofmeyr.yml
new file mode 100644
index 0000000..427dcd2
--- /dev/null
+++ b/jobs/osmocom-obs-nhofmeyr.yml
@@ -0,0 +1,73 @@
+---
+- project:
+ name: Osmocom_OBS_nhofmeyr
+ jobs:
+ - Osmocom_OBS_nhofmeyr_{branch_suffix}
+ branch_suffix:
+ - dev
+
+- job-template:
+ name: 'Osmocom_OBS_nhofmeyr_{branch_suffix}'
+ project-type: freestyle
+ concurrent: false
+ defaults: global
+ description: |
+ Iterate over Osmocom projects, and update projects from the
+ "neels/{branch_suffix}" branch in OBS. If a package exists in OBS
+ for a project, but it doesn't have the git branch anymore then delete the
+ package from OBS. (created from osmo-ci.git/jobs/osmocom-obs-nhofmeyr.yml
+ which was originally a copy of osmocom-obs-rhizomatica.yml)
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
+ node: obs
+ parameters:
+ - string:
+ name: CI_BRANCH
+ description: osmo-ci.git branch
+ default: 'master'
+ builders:
+ - shell: |
+ export PYTHONUNBUFFERED=1
+ BRANCH="neels/{branch_suffix}"
+ PROJ="home:nhofmeyr:{branch_suffix}"
+ VERSION_APPEND="~nhofmeyr.{branch_suffix}"
+ PACKAGE="$GERRIT_PROJECT"
+
+ # When manually triggered, run on all Osmocom packages
+ if [ -z "$PACKAGE" ]; then
+ PACKAGE="ALL_OSMOCOM_PACKAGES"
+ fi
+
+ ./scripts/obs/update_obs_project.py \
+ --apiurl https://obs.osmocom.org \
+ --conflict-pkgname "osmocom-latest" \
+ --delete \
+ --docker \
+ --feed "master" \
+ --git-branch "$BRANCH" \
+ --version-append "$VERSION_APPEND" \
+ "$PROJ" \
+ "$PACKAGE"
+ scm:
+ - git:
+ branches:
+ - '$CI_BRANCH'
+ url: https://gerrit.osmocom.org/osmo-ci
+ git-config-name: 'Jenkins Builder'
+ git-config-email: 'jenkins@osmocom.org'
+ triggers:
+ - gerrit:
+ trigger-on:
+ - ref-updated-event
+ projects:
+ - project-compare-type: 'ANT'
+ project-pattern: '**'
+ branches:
+ - branch-pattern: 'neels/{branch_suffix}'
+ server-name: gerrit.osmocom.org
+ publishers:
+ - email:
+ notify-every-unstable-build: true
+ recipients: 'nhofmeyr@sysmocom.de'
diff --git a/jobs/osmocom-obs-rhizomatica.yml b/jobs/osmocom-obs-rhizomatica.yml
new file mode 100644
index 0000000..92ab465
--- /dev/null
+++ b/jobs/osmocom-obs-rhizomatica.yml
@@ -0,0 +1,73 @@
+---
+- project:
+ name: Osmocom_OBS_rhizomatica
+ jobs:
+ - Osmocom_OBS_rhizomatica_{branch_suffix}
+ branch_suffix:
+ - testing
+ - production
+
+- job-template:
+ name: 'Osmocom_OBS_rhizomatica_{branch_suffix}'
+ project-type: freestyle
+ concurrent: false
+ defaults: global
+ description: |
+ Iterate over Osmocom projects, and update projects from the
+ "rhizomatica/{branch_suffix}" branch in OBS. If a package exists in OBS
+ for a project, but it doesn't have the git branch anymore then delete the
+ package from OBS (<a href="https://osmocom.org/issues/5981">OS#5981</a>).
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
+ node: obs
+ parameters:
+ - string:
+ name: CI_BRANCH
+ description: osmo-ci.git branch
+ default: 'master'
+ builders:
+ - shell: |
+ export PYTHONUNBUFFERED=1
+ BRANCH="rhizomatica/{branch_suffix}"
+ PROJ="rhizomatica:{branch_suffix}"
+ VERSION_APPEND="~rhizomatica.{branch_suffix}"
+ PACKAGE="$GERRIT_PROJECT"
+
+ # When manually triggered, run on all Osmocom packages
+ if [ -z "$PACKAGE" ]; then
+ PACKAGE="ALL_OSMOCOM_PACKAGES"
+ fi
+
+ ./scripts/obs/update_obs_project.py \
+ --apiurl https://obs.osmocom.org \
+ --conflict-pkgname "osmocom-latest" \
+ --delete \
+ --docker \
+ --feed "master" \
+ --git-branch "$BRANCH" \
+ --version-append "$VERSION_APPEND" \
+ "$PROJ" \
+ "$PACKAGE"
+ scm:
+ - git:
+ branches:
+ - '$CI_BRANCH'
+ url: https://gerrit.osmocom.org/osmo-ci
+ git-config-name: 'Jenkins Builder'
+ git-config-email: 'jenkins@osmocom.org'
+ triggers:
+ - gerrit:
+ trigger-on:
+ - ref-updated-event
+ projects:
+ - project-compare-type: 'ANT'
+ project-pattern: '**'
+ branches:
+ - branch-pattern: 'rhizomatica/{branch_suffix}'
+ server-name: gerrit.osmocom.org
+ publishers:
+ - email:
+ notify-every-unstable-build: true
+ recipients: 'jenkins-notifications@lists.osmocom.org'
diff --git a/jobs/osmocom-obs-sync.yml b/jobs/osmocom-obs-sync.yml
new file mode 100644
index 0000000..9193edc
--- /dev/null
+++ b/jobs/osmocom-obs-sync.yml
@@ -0,0 +1,54 @@
+---
+- job:
+ name: 'Osmocom_OBS_sync'
+ project-type: freestyle
+ concurrent: false
+ defaults: global
+ description: |
+ Sync base distributions (Debian:12 etc.) from openSUSE's OBS to our OBS
+ (<a href="https://osmocom.org/issues/6165">OS#6165</a>).
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
+ node: obs
+ parameters:
+ - string:
+ name: CI_BRANCH
+ description: osmo-ci.git branch
+ default: 'master'
+ scm:
+ - git:
+ branches:
+ - '$CI_BRANCH'
+ url: https://gerrit.osmocom.org/osmo-ci
+ git-config-name: 'Jenkins Builder'
+ git-config-email: 'jenkins@osmocom.org'
+ builders:
+ - shell: |
+ export PYTHONUNBUFFERED=1
+ export DISTROS="
+ AlmaLinux:8
+ Debian:10
+ Debian:11
+ Debian:12
+ Debian:Next
+ Raspbian:10
+ Raspbian:11
+ Raspbian:12
+ Ubuntu:20.04
+ Ubuntu:22.04
+ Ubuntu:23.10
+ "
+ ./scripts/obs/sync_obs_projects.py \
+ --docker \
+ $DISTROS
+ triggers:
+ - timed: "H 18 * * *"
+ wrappers:
+ - ansicolor:
+ colormap: xterm
+ publishers:
+ - email:
+ notify-every-unstable-build: true
+ recipients: 'jenkins-notifications@lists.osmocom.org'
diff --git a/jobs/osmocom-obs-wireshark.yml b/jobs/osmocom-obs-wireshark.yml
new file mode 100644
index 0000000..213a1f3
--- /dev/null
+++ b/jobs/osmocom-obs-wireshark.yml
@@ -0,0 +1,42 @@
+---
+- project:
+ name: Osmocom_OBS
+ jobs:
+ - Osmocom_OBS_wireshark
+
+- job:
+ name: 'Osmocom_OBS_wireshark'
+ project-type: freestyle
+ defaults: global
+ description: |
+ See <a href="https://osmocom.org/issues/2537">OS#2537</a>
+ for more information.
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
+ node: obs
+ parameters:
+ - string:
+ name: BRANCH
+ description: osmo-ci.git branch
+ default: 'master'
+ builders:
+ - shell: |
+ export PYTHONUNBUFFERED=1
+ ./scripts/obs/update_obs_wireshark.sh osmocom:wireshark
+ scm:
+ - git:
+ branches:
+ - '$BRANCH'
+ url: https://gerrit.osmocom.org/osmo-ci
+ git-config-name: 'Jenkins Builder'
+ git-config-email: 'jenkins@osmocom.org'
+ wipe-workspace: true
+ triggers:
+ - timed: "H 19 * * *"
+
+ publishers:
+ - email:
+ notify-every-unstable-build: true
+ recipients: 'jenkins-notifications@lists.osmocom.org'
diff --git a/jobs/osmocom-obs.yml b/jobs/osmocom-obs.yml
index 9bdadeb..eb77f4d 100644
--- a/jobs/osmocom-obs.yml
+++ b/jobs/osmocom-obs.yml
@@ -4,21 +4,27 @@
jobs:
- Osmocom_OBS_{type}
type:
- - nightly
- - latest
- - next
+ - master:
+ conflict_version: ""
+ - nightly:
+ # For nightly we don't provide ABI compatibility, make sure packages
+ # from different build dates are not mixed by accident
+ conflict_version: "$(date +%Y%m%d%H%M)"
+ - latest:
+ conflict_version: ""
- job-template:
name: 'Osmocom_OBS_{type}'
project-type: freestyle
+ concurrent: false
defaults: global
description: |
- <ul>
- <li><a href="https://build.opensuse.org/project/show/network:osmocom:{type}">osmocom:{type} home page</a>
- <li><a href="https://build.opensuse.org/project/monitor/network:osmocom:{type}">osmocom:{type} build monitor</a>
- <li><a href="https://osmocom.org/projects/cellular-infrastructure/wiki/Binary_Packages">Wiki: binary packages</a>
- </ul>
- (Generated by job-builder)
+ See <a href="https://osmocom.org/projects/cellular-infrastructure/wiki/Binary_Packages">Wiki: binary packages</a>
+ for more information.
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
node: obs
parameters:
- string:
@@ -27,17 +33,24 @@
default: 'refs/remotes/origin/master'
builders:
- shell: |
- export PROJ=network:osmocom:{type}
- ./scripts/osmocom-{type}-packages.sh
+ export PYTHONUNBUFFERED=1
+
+ ./scripts/obs/update_obs_project.py \
+ --apiurl https://obs.osmocom.org \
+ --conflict-version {conflict_version} \
+ --docker \
+ --feed {type} \
+ --meta \
+ osmocom:{type}
scm:
- git:
branches:
- '$BRANCH'
- url: git://git.osmocom.org/osmo-ci
+ url: https://gerrit.osmocom.org/osmo-ci
git-config-name: 'Jenkins Builder'
git-config-email: 'jenkins@osmocom.org'
triggers:
- - timed: "H 00 * * *" # run before repo-install-test.yml
+ - timed: "H 20 * * *"
publishers:
- email:
diff --git a/jobs/osmocom-release-manuals.yml b/jobs/osmocom-release-manuals.yml
new file mode 100644
index 0000000..c9d04ae
--- /dev/null
+++ b/jobs/osmocom-release-manuals.yml
@@ -0,0 +1,45 @@
+---
+- project:
+ name: Osmocom-release-manuals
+ jobs:
+ - Osmocom-release-manuals
+
+- job:
+ name: 'Osmocom-release-manuals'
+ node: osmocom-master
+ project-type: freestyle
+ defaults: global
+ description: |
+ Build manuals for all new Osmocom releases
+ (<a href="https://osmocom.org/issues/5902">OS#5902</a>).
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
+ parameters:
+ - string:
+ name: BRANCH_CI
+ description: |
+ osmo-ci.git branch
+ default: 'master'
+ builders:
+ - shell: |
+ scripts/manuals/publish-manuals-for-tags.sh
+ scm:
+ - git:
+ branches:
+ - '$BRANCH_CI'
+ url: https://gerrit.osmocom.org/osmo-ci
+ git-config-name: 'Jenkins Builder'
+ git-config-email: 'jenkins@osmocom.org'
+ wipe-workspace: true
+ triggers:
+ - timed: "H 04 * * *"
+ publishers:
+ - email:
+ notify-every-unstable-build: true
+ recipients: 'jenkins-notifications@lists.osmocom.org'
+ wrappers:
+ - ssh-agent-credentials:
+ users:
+ - docs_ftp.osmocom.org
diff --git a/jobs/osmocom-release-tarballs.yml b/jobs/osmocom-release-tarballs.yml
index 930fb42..4b3c8dc 100644
--- a/jobs/osmocom-release-tarballs.yml
+++ b/jobs/osmocom-release-tarballs.yml
@@ -6,60 +6,36 @@
- job-template:
name: 'Osmocom-release-tarballs'
+ node: osmocom-master
project-type: freestyle
defaults: global
description: |
For each relevant Osmocom repository and its tags, build release tarballs (<a href="https://osmocom.org/issues/3870">OS#3870</a>).
<br><br>
Publishes to: <a href="https://ftp.osmocom.org/releases/">https://ftp.osmocom.org/releases</a>
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
parameters:
- string:
name: BRANCH_OSMO_CI
description: |
- osmo-ci.git branch where the osmocom-release-tarballs.sh gets pulled from
- default: 'master'
- - string:
- name: BRANCH_DOCKER_PLAYGROUND
- description: |
- docker-playground.git branch where the Dockerfile gets pulled from
+ osmo-ci.git branch with scripts/tarballs/publish-tarballs-for-tags.sh
default: 'master'
builders:
- shell: |
- export OSMO_BRANCH_DOCKER_PLAYGROUND="$BRANCH_DOCKER_PLAYGROUND"
- . scripts/common.sh
- docker_images_require \
- "debian-stretch-obs-latest" \
- "debian-stretch-build-dist"
-
- docker run \
- --rm=true \
- -v "$PWD:/osmo-ci" \
- -w /osmo-ci \
- -e KEEP_TEMP="$KEEP_TEMP" \
- "$USER/debian-stretch-build-dist" /osmo-ci/scripts/osmocom-release-tarballs.sh
-
- cat > "$WORKSPACE/known_hosts" <<EOF
- [ftp.osmocom.org]:48 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDDgQ9HntlpWNmh953a2Gc8NysKE4orOatVT1wQkyzhARnfYUerRuwyNr1GqMyBKdSI9amYVBXJIOUFcpV81niA7zQRUs66bpIMkE9/rHxBd81SkorEPOIS84W4vm3SZtuNqa+fADcqe88Hcb0ZdTzjKILuwi19gzrQyME2knHY71EOETe9Yow5RD2hTIpB5ecNxI0LUKDq+Ii8HfBvndPBIr0BWYDugckQ3Bocf+yn/tn2/GZieFEyFpBGF/MnLbAAfUKIdeyFRX7ufaiWWz5yKAfEhtziqdAGZaXNaLG6gkpy3EixOAy6ZXuTAk3b3Y0FUmDjhOHllbPmTOcKMry9
- [ftp.osmocom.org]:48 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPdWn1kEousXuKsZ+qJEZTt/NSeASxCrUfNDW3LWtH+d8Ust7ZuKp/vuyG+5pe5pwpPOgFu7TjN+0lVjYJVXH54=
- [ftp.osmocom.org]:48 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK8iivY70EiR5NiGChV39gRLjNpC8lvu1ZdHtdMw2zuX
- EOF
-
- SSH_COMMAND="ssh -o 'UserKnownHostsFile=$WORKSPACE/known_hosts' -p 48"
- cd _release_tarballs
- rsync -avz --delete -e "$SSH_COMMAND" . releases@ftp.osmocom.org:web-files/
+ PUBLISH=1 scripts/tarballs/publish-tarballs-for-tags.sh
scm:
- git:
branches:
- '$BRANCH_OSMO_CI'
- url: git://git.osmocom.org/osmo-ci
+ url: https://gerrit.osmocom.org/osmo-ci
git-config-name: 'Jenkins Builder'
git-config-email: 'jenkins@osmocom.org'
- # Don't wipe workspace to keep generated release tarballs
- wipe-workspace: false
+ wipe-workspace: true
triggers:
- - timed: "@midnight"
- # Limit to one specific builder (so we don't regenerate all tarballs when switching to another builder)
- node: build2-deb9build-ansible
+ - timed: "H 05 * * *"
publishers:
- email:
notify-every-unstable-build: true
@@ -67,6 +43,6 @@
wrappers:
- ssh-agent-credentials:
users:
- - releases
+ - releases_ftp.osmocom.org
# vim: expandtab tabstop=2 shiftwidth=2
diff --git a/jobs/registry-rebuild-upload-fpga-build.yml b/jobs/registry-rebuild-upload-fpga-build.yml
index 9338bce..1846e49 100644
--- a/jobs/registry-rebuild-upload-fpga-build.yml
+++ b/jobs/registry-rebuild-upload-fpga-build.yml
@@ -8,10 +8,14 @@
name: registry-rebuild-upload-fpga-build
description: |
Rebuild + push fpga-build docker images in our private registry.osmocom.org
- node: osmocom-master-debian9
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
+ node: osmocom-master
scm:
- git:
- url: git://git.osmocom.org/docker-playground
+ url: https://gerrit.osmocom.org/docker-playground
skip-tag: true
branches:
- 'origin/master'
@@ -25,7 +29,7 @@
docker push $REGISTRY/$USER/fpga-build
cd ..
triggers:
- - timed: "@weekly"
+ - timed: "H 02 * * H"
publishers:
- email:
diff --git a/jobs/registry-rebuild-upload-titan.yml b/jobs/registry-rebuild-upload-titan.yml
index 7e50aed..a535920 100644
--- a/jobs/registry-rebuild-upload-titan.yml
+++ b/jobs/registry-rebuild-upload-titan.yml
@@ -1,34 +1,69 @@
-
----
- project:
name: registry-rebuild-upload-titan
jobs:
- registry-rebuild-upload-titan
+ - registry-rebuild-upload-ttcn3-bts-tests
+
+- scm:
+ name: osmocom-docker-playground
+ scm:
+ - git:
+ url: https://gerrit.osmocom.org/docker-playground
+ skip-tag: true
+ branches:
+ - 'origin/master'
+
- job:
name: registry-rebuild-upload-titan
description: |
- Rebuild + push ttcn3 base images (debian-stretch-titan) in our private
+ Rebuild + push ttcn3 base images (debian-bookworm-titan) in our private
registry.osmocom.org
<br><br>
Related:<br>
<ul>
<li> <a href="https://osmocom.org/issues/5017">OS#5017</a>
</ul>
- node: osmocom-master-debian9
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
+ node: osmocom-master
scm:
- - git:
- url: git://git.osmocom.org/docker-playground
- skip-tag: true
- branches:
- - 'origin/master'
-
+ - 'osmocom-docker-playground'
builders:
- shell: |
REGISTRY="registry.osmocom.org"
- make -C debian-stretch-titan REGISTRY_HOST=$REGISTRY
- docker push "$REGISTRY/$USER/debian-stretch-titan"
+ make -C debian-bookworm-titan REGISTRY_HOST=$REGISTRY
+ docker push "$REGISTRY/$USER/debian-bookworm-titan"
+ publishers:
+ - email:
+ notify-every-unstable-build: true
+ recipients: 'jenkins-notifications@lists.osmocom.org'
+- job:
+ name: registry-rebuild-upload-ttcn3-bts-tests
+ description: |
+ Rebuild + push ttcn3-bts-test image in our private registry.osmocom.org.
+ This docker container is used by osmo-gsm-tester main unit to run TTCN3
+ BTS_Tests testsuite against real hardware.
+ <br><br>
+ Related:<br>
+ <ul>
+ <li>SYS#5765
+ </ul>
+ node: osmocom-master
+ triggers:
+ - reverse:
+ jobs:
+ - registry-rebuild-upload-titan
+ scm:
+ - 'osmocom-docker-playground'
+ builders:
+ - shell: |
+ REGISTRY="registry.osmocom.org"
+ make -C ttcn3-bts-test REGISTRY_HOST=$REGISTRY
+ docker push "$REGISTRY/$USER/ttcn3-bts-test"
publishers:
- email:
notify-every-unstable-build: true
diff --git a/jobs/registry-triggers.yml b/jobs/registry-triggers.yml
index 0763496..da7d402 100644
--- a/jobs/registry-triggers.yml
+++ b/jobs/registry-triggers.yml
@@ -12,10 +12,13 @@
name: 'registry-trigger-{repos}'
description: |
Update registry.osmocom.org after changes in {repos}.
- node: osmocom-master-debian9
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
scm:
- git:
- url: git://git.osmocom.org/{repos}
+ url: https://gerrit.osmocom.org/{repos}
branches:
- 'origin/master'
trigger:
@@ -24,7 +27,7 @@
- pollscm:
cron: "H/5 * * * *"
ignore-post-commit-hooks: false
- - timed: "H H * * *"
+ - timed: "H 02 * * *"
publishers:
- trigger:
project: '{obj:trigger}'
diff --git a/jobs/registry-update-base-images.yml b/jobs/registry-update-base-images.yml
index 48e8b59..b51ae95 100644
--- a/jobs/registry-update-base-images.yml
+++ b/jobs/registry-update-base-images.yml
@@ -12,11 +12,22 @@
<ul>
<li> <a href="https://osmocom.org/issues/4839">OS#4839</a>
</ul>
- node: osmocom-master-debian9
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
+ node: osmocom-master
builders:
- shell: |
REGISTRY="registry.osmocom.org"
- IMAGES="debian:stretch debian:buster debian:jessie debian:sid ubuntu:zesty centos:centos8"
+ IMAGES="
+ debian:buster
+ debian:bullseye
+ debian:bookworm
+ debian:sid
+
+ centos:centos7
+ "
for i in $IMAGES; do
src=$i
dst=${REGISTRY}/$i
@@ -27,7 +38,7 @@
docker push $dst
done
triggers:
- - timed: "@daily"
+ - timed: "H 00 * * *"
publishers:
- email:
diff --git a/jobs/repo-install-test.yml b/jobs/repo-install-test.yml
index ad48f1b..0891e24 100644
--- a/jobs/repo-install-test.yml
+++ b/jobs/repo-install-test.yml
@@ -4,12 +4,12 @@
jobs:
- Osmocom-repo-install-{distro}
distro:
- - centos8:
- feeds: !!python/tuple [next, nightly]
- - debian9:
- feeds: !!python/tuple [nightly, latest]
- debian10:
feeds: !!python/tuple [nightly, latest]
+ - debian11:
+ feeds: !!python/tuple [nightly, latest]
+ - debian12:
+ feeds: !!python/tuple [nightly, latest]
- job-template:
name: Osmocom-repo-install-{distro}
@@ -25,42 +25,40 @@
<li> <a href="https://osmocom.org/issues/3176">OS#3176</a>: install all Osmocom debian packages
<li> <a href="https://osmocom.org/issues/3369">OS#3369</a>: check if systemd services start properly
<li> <a href="https://osmocom.org/issues/4563">OS#4563</a>: extend for centos
+ <li> <a href="https://osmocom.org/issues/5365">OS#5365</a>: run in qemu
</ul>
- node: osmocom-master-debian9
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
parameters:
- string:
name: BRANCH_CI
description: |
osmo-ci.git branch
default: 'master'
- - string:
- name: BRANCH_DP
- description: |
- docker-playground.git branch
- default: 'master'
axes:
- axis:
type: slave
name: label
- values: !!python/tuple [repo-install-test]
+ values: !!python/tuple [qemu]
- axis:
type: user-defined
name: feed
values: '{obj:feeds}'
builders:
- shell: |
- export OSMO_BRANCH_DOCKER_PLAYGROUND="$BRANCH_DP"
export FEED="$feed"
scripts/repo-install-test.sh "{distro}"
scm:
- git:
branches:
- '$BRANCH_CI'
- url: git://git.osmocom.org/osmo-ci
+ url: https://gerrit.osmocom.org/osmo-ci
git-config-name: 'Jenkins Builder'
git-config-email: 'jenkins@osmocom.org'
triggers:
- - timed: "H 10 * * *" # run after osmocom-obs.yml
+ - timed: "H 06 * * *"
publishers:
- email:
diff --git a/jobs/ttcn3-testsuites-kernel-git.yml b/jobs/ttcn3-testsuites-kernel-git.yml
index 1bc07d5..6ce8fe8 100644
--- a/jobs/ttcn3-testsuites-kernel-git.yml
+++ b/jobs/ttcn3-testsuites-kernel-git.yml
@@ -8,13 +8,11 @@
disabled: false
description: ""
node: ttcn3
- blocking: "^{job-name}.*"
wipe_workspace: false
jobs:
- "{job-name}"
job-name:
- ttcn3-ggsn-test-kernel-git:
- blocking: "^(ttcn3|TTCN3-.*)-ggsn-test.*"
# Cache the linux source tree in the workspace of one node
node: ttcn3-ggsn-test-kernel-git
description: |
@@ -61,7 +59,7 @@
description: |
Version of the Osmocom stack to test against the kernel. Either
<code>latest</code> for the latest release version, or
- <code>master</code> for HEAD of git.osmocom.org.
+ <code>master</code> for HEAD of Osmocom git repositories.
default: 'latest'
- string:
name: BRANCH
@@ -96,7 +94,7 @@
- git:
branches:
- '$BRANCH'
- url: git://git.osmocom.org/docker-playground
+ url: https://gerrit.osmocom.org/docker-playground
git-config-name: 'Jenkins Builder'
git-config-email: 'jenkins@osmocom.org'
wipe-workspace: "{obj:wipe_workspace}"
@@ -105,8 +103,8 @@
results: '**/junit-xml-*.log'
allow-empty-results: false
- email:
- notify-every-unstable-build: true
- recipients: laforge@gnumonks.org osmith@sysmocom.de
+ notify-every-unstable-build: false
+ recipients: '{obj:email}'
send-to-individuals: false
- archive:
allow-empty: false
@@ -116,14 +114,12 @@
fingerprint: false
only-if-success: false
properties:
- - build-blocker:
- use-build-blocker: true
- blocking-jobs:
- - "{obj:blocking}"
- build-discarder:
days-to-keep: 30
num-to-keep: 120
- artifact-days-to-keep: -1
- artifact-num-to-keep: -1
+ wrappers:
+ - ansicolor:
+ colormap: xterm
+ email: jenkins-notifications@lists.osmocom.org
# vim: expandtab tabstop=2 shiftwidth=2
diff --git a/jobs/ttcn3-testsuites.yml b/jobs/ttcn3-testsuites.yml
index e3e9f3e..8a17e04 100644
--- a/jobs/ttcn3-testsuites.yml
+++ b/jobs/ttcn3-testsuites.yml
@@ -5,7 +5,7 @@
disabled: false
description: |
Run the a testsuite from
- <a href="https://git.osmocom.org/docker-playground/">
+ <a href="https://gitea.osmocom.org/osmocom/docker-playground">
docker-playground.git</a>.<br>
If this job ends in <code>-latest</code>, the job runs the latest
stable versions of the Osmocom stack. Otherwise the nightly builds.<br>
@@ -13,36 +13,37 @@
TTCN3 Testsuite</a> for more information.
timer: "@midnight"
node: ttcn3
- # Block *-latest for all master jobs (without suffix)
- blocking: "^{job-name}.*"
wipe_workspace: true
jobs:
- "{job-name}"
# Folders from docker-playground.git
+ # timer: 10 per hour from 04:00 - 18:00, see jobs/README.adoc
job-name:
- # debian-stretch nightly - order by time
+ # debian nightly - order by time
- nplab-m3ua-test:
- blocking: "^(nplab|NPLAB-.*)-m3ua-test.*"
- timer: 00 03 * * *
+ timer: H 04 * * *
- nplab-sua-test:
- blocking: "^(nplab|NPLAB-.*)-sua-test.*"
- timer: 15 03 * * *
+ timer: H 04 * * *
- ttcn3-ggsn-test:
- blocking: "^(ttcn3|TTCN3-.*)-ggsn-test.*"
- timer: 00 04 * * *
+ timer: H 04 * * *
- ttcn3-ggsn-test-kernel:
- blocking: "^(ttcn3|TTCN3-.*)-ggsn-test.*"
- timer: 10 04 * * *
+ timer: H 04 * * *
description: |
Test osmo-ggsn master with the Debian kernel.
+ - ttcn3-ggsn-test-kernel-pablo-gtp:
+ timer: H 04 * * *
+ description: |
+ Test osmo-ggsn master with the Linux kernel built from main of
+ <a href="https://git.kernel.org/pub/scm/linux/kernel/git/pablo/gtp.git/">pablo/gtp.git</a>.
+ <br>
+ This is a temporary job, to be removed once the IPv6 related GTP
+ patches are upstreamed (OS#1952).
- ttcn3-ggsn-test-kernel-latest:
- blocking: "^(ttcn3|TTCN3-.*)-ggsn-test.*"
- timer: 20 04 * * *
+ timer: H 04 * * *
description: |
Test osmo-ggsn latest with the Debian kernel.
- ttcn3-ggsn-test-kernel-latest-net-next:
- blocking: "^(ttcn3|TTCN3-.*)-ggsn-test.*"
- timer: 25 04 * * *
+ timer: H 04 * * *
# Cache the linux source tree in the workspace of one node
wipe_workspace: false
node: ttcn3-ggsn-test-kernel-git
@@ -51,8 +52,7 @@
<a href="https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git/">
net-next.git</a>.
- ttcn3-ggsn-test-kernel-latest-torvalds:
- blocking: "^(ttcn3|TTCN3-.*)-ggsn-test.*"
- timer: 30 04 * * *
+ timer: H 04 * * *
# Cache the linux source tree in the workspace of one node
wipe_workspace: false
node: ttcn3-ggsn-test-kernel-git
@@ -60,275 +60,180 @@
Test osmo-ggsn latest with the Linux kernel built from master of
<a href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/">
torvalds/linux.git</a>.
+ - ttcn3-ggsn-test-ogs:
+ timer: H 04 * * *
+ description: |
+ Test open5gs-smfd + open5gs-upfd main.
+ email: jenkins-notifications@lists.osmocom.org acetcom@gmail.com
- ttcn3-msc-test:
- blocking: "^(ttcn3|TTCN3-.*)-msc-test.*"
- timer: 40 04 * * *
+ timer: H 04 * * *
- ttcn3-pcu-test:
- blocking: "^(ttcn3|TTCN3-.*)-pcu-test.*"
- timer: 00 05 * * *
+ timer: H 04 * * *
- ttcn3-pcu-test-sns:
- blocking: "^(ttcn3|TTCN3-.*)-pcu-test.*"
- timer: 15 05 * * *
+ timer: H 05 * * *
- ttcn3-sgsn-test:
- blocking: "^(ttcn3|TTCN3-.*)-sgsn-test.*"
- timer: 30 05 * * *
+ timer: H 05 * * *
- ttcn3-bsc-test:
- blocking: "^(ttcn3|TTCN3-.*)-bsc-test.*"
- timer: 00 06 * * *
+ timer: H 05 * * *
- ttcn3-mgw-test:
- blocking: "^(ttcn3|TTCN3-.*)-mgw-test.*"
- timer: 30 06 * * *
+ timer: H 05 * * *
- ttcn3-hlr-test:
- blocking: "^(ttcn3|TTCN3-.*)-hlr-test.*"
- timer: 45 06 * * *
+ timer: H 05 * * *
- ttcn3-bts-test:
- blocking: "^(ttcn3|TTCN3-.*)-bts-test.*"
- timer: 00 07 * * *
+ timer: H 05 * * *
- ttcn3-sip-test:
- blocking: "^(ttcn3|TTCN3-.*)-sip-test.*"
- timer: 30 07 * * *
+ timer: H 05 * * *
- ttcn3-bscnat-test:
- blocking: "^(ttcn3|TTCN3-.*)-bscnat-test.*"
- timer: 00 08 * * *
+ timer: H 05 * * *
disabled: true
- ttcn3-bsc-test-sccplite:
- blocking: "^(ttcn3|TTCN3-.*)-bsc-test.*"
- timer: 30 08 * * *
+ timer: H 05 * * *
- ttcn3-remsim-test:
- blocking: "^(ttcn3|TTCN3-.*)-remsim-test.*"
- timer: 00 09 * * *
+ timer: H 05 * * *
- ttcn3-stp-test:
- blocking: "^(ttcn3|TTCN3-.*)-stp-test.*"
- timer: 30 09 * * *
+ timer: H 06 * * *
- ttcn3-sccp-test:
- blocking: "^(ttcn3|TTCN3-.*)-sccp-test.*"
- timer: 00 10 * * *
+ timer: H 06 * * *
- ttcn3-smlc-test:
- blocking: "^(ttcn3|TTCN3-.*)-smlc-test.*"
- timer: 10 10 * * *
+ timer: H 06 * * *
- ttcn3-gbproxy-test:
- blocking: "^(ttcn3|TTCN3-.*)-gbproxy-test.*"
- timer: 20 10 * * *
+ timer: H 06 * * *
- ttcn3-gbproxy-test-fr:
- blocking: "^(ttcn3|TTCN3-.*)-gbproxy-test-fr.*"
- timer: 30 10 * * *
+ timer: H 06 * * *
node: hdlc
- ttcn3-cbc-test:
- blocking: "^(ttcn3|TTCN3-.*)-cbc-test.*"
- timer: 40 10 * * *
+ timer: H 06 * * *
- ttcn3-ns-test:
- blocking: "^(ttcn3|TTCN3-.*)-ns-test.*"
- timer: 00 11 * * *
+ timer: H 06 * * *
- ttcn3-ns-test-sns:
- blocking: "^(ttcn3|TTCN3-.*)-ns-test-sns.*"
- timer: 10 11 * * *
+ timer: H 06 * * *
- ttcn3-ns-test-fr:
- blocking: "^(ttcn3|TTCN3-.*)-ns-test-fr.*"
- timer: 20 11 * * *
+ timer: H 06 * * *
node: hdlc
- ttcn3-ns-test-sgsn-sns:
- blocking: "^(ttcn3|TTCN3-.*)-ns-test-sgsn-sns.*"
- timer: 30 11 * * *
+ timer: H 06 * * *
- ttcn3-pcap-client-test:
- blocking: "^(ttcn3|TTCN3-.*)-pcap-client-test.*"
- timer: 40 11 * * *
+ timer: H 07 * * *
- ttcn3-bsc-test-vamos:
- blocking: "^(ttcn3|TTCN3-.*)-bsc-test.*"
- timer: 00 12 * * *
+ timer: H 07 * * *
+ - ttcn3-hnodeb-test:
+ timer: H 07 * * *
+ - ttcn3-upf-test:
+ timer: H 07 * * *
+ - ttcn3-hnbgw-test:
+ timer: H 07 * * *
+ - ttcn3-fr-test:
+ timer: H 07 * * *
+ node: hdlc
+ - ttcn3-pgw-test:
+ timer: H 07 * * *
+ description: |
+ Test open5gs-smfd + open5gs-upfd PGW main.
+ email: jenkins-notifications@lists.osmocom.org acetcom@gmail.com
+ - ttcn3-mme-test-ogs:
+ timer: H 07 * * *
+ description: |
+ Test open5gs-mmed
+ email: jenkins-notifications@lists.osmocom.org acetcom@gmail.com
+ - ttcn3-hss-test-ogs:
+ timer: H 07 * * *
+ description: |
+ Test open5gs-hssd
+ email: jenkins-notifications@lists.osmocom.org acetcom@gmail.com
+ - ttcn3-dia2gsup-test:
+ timer: H 07 * * *
+ - ttcn3-epdg-test:
+ timer: H 07 * * *
+ # osmo-epdg requires OTP and kernel >= debian12:
+ node: build4-deb12build-ansible
- # debian-stretch latest stable
+ # debian latest stable
- nplab-m3ua-test-latest:
- blocking: "^(nplab|NPLAB-.*)-m3ua-test.*"
- timer: 00 03 * * *
+ timer: H 07 * * *
- nplab-sua-test-latest:
- blocking: "^(nplab|NPLAB-.*)-sua-test.*"
- timer: 15 03 * * *
+ timer: H 08 * * *
- ttcn3-ggsn-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-ggsn-test.*"
- timer: 00 04 * * *
+ timer: H 08 * * *
- ttcn3-msc-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-msc-test.*"
- timer: 30 04 * * *
+ timer: H 08 * * *
- ttcn3-pcu-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-pcu-test.*"
- timer: 00 05 * * *
+ timer: H 08 * * *
- ttcn3-pcu-test-sns-latest:
- blocking: "^(ttcn3|TTCN3-.*)-pcu-test.*"
- timer: 15 05 * * *
+ timer: H 08 * * *
- ttcn3-sgsn-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-sgsn-test.*"
- timer: 30 05 * * *
+ timer: H 08 * * *
- ttcn3-bsc-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-bsc-test.*"
- timer: 00 06 * * *
+ timer: H 08 * * *
- ttcn3-mgw-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-mgw-test.*"
- timer: 30 06 * * *
+ timer: H 08 * * *
- ttcn3-hlr-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-hlr-test.*"
- timer: 45 06 * * *
+ timer: H 08 * * *
- ttcn3-bts-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-bts-test.*"
- timer: 00 07 * * *
+ timer: H 08 * * *
- ttcn3-sip-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-sip-test.*"
- timer: 30 07 * * *
+ timer: H 09 * * *
- ttcn3-bscnat-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-bscnat-test.*"
- timer: 00 08 * * *
+ timer: H 09 * * *
disabled: true
- ttcn3-bsc-test-sccplite-latest:
- blocking: "^(ttcn3|TTCN3-.*)-bsc-test.*"
- timer: 30 08 * * *
+ timer: H 09 * * *
- ttcn3-remsim-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-remsim-test.*"
- timer: 00 09 * * *
+ timer: H 09 * * *
- ttcn3-stp-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-stp-test.*"
- timer: 30 09 * * *
- #- ttcn3-sccp-test-latest:
- # blocking: "^(ttcn3|TTCN3-.*)-sccp-test.*"
- # timer: 00 10 * * *
+ timer: H 09 * * *
+ - ttcn3-sccp-test-latest:
+ timer: H 09 * * *
- ttcn3-smlc-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-smlc-test.*"
- timer: 10 10 * * *
- # disabled until we have a stable release that supports IP-SNS in SGSN role
- #- ttcn3-gbproxy-test-latest:
- #blocking: "^(ttcn3|TTCN3-.*)-gbproxy-test.*"
- #timer: 20 10 * * *
+ timer: H 09 * * *
+ - ttcn3-gbproxy-test-latest:
+ timer: H 09 * * *
+ - ttcn3-gbproxy-test-fr-latest:
+ timer: H 09 * * *
+ node: hdlc
- ttcn3-cbc-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-cbc-test.*"
- timer: 40 10 * * *
+ timer: H 09 * * *
- ttcn3-pcap-client-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-pcap-client-test.*"
- timer: 40 11 * * *
-
+ timer: H 10 * * *
+ - ttcn3-hnodeb-test-latest:
+ timer: H 10 * * *
+ - ttcn3-upf-test-latest:
+ timer: H 10 * * *
+ disabled: true
+ - ttcn3-hnbgw-test-latest:
+ timer: H 10 * * *
- # Jobs below use uppercase TTCN3-$DISTRO, so they get listed separately from the ttcn3-* (debian-stretch) jobs
+ # Jobs below use uppercase TTCN3-$DISTRO, so they get listed separately from the ttcn3-* (debian) jobs
- # centos8 nightly
- - TTCN3-centos-ggsn-test:
- blocking: "^(ttcn3|TTCN3-.*)-ggsn-test.*"
- timer: 00 04 * * *
- - TTCN3-centos-msc-test:
- blocking: "^(ttcn3|TTCN3-.*)-msc-test.*"
- timer: 30 04 * * *
- - TTCN3-centos-pcu-test:
- blocking: "^(ttcn3|TTCN3-.*)-pcu-test.*"
- timer: 00 05 * * *
- - TTCN3-centos-pcu-test-sns:
- blocking: "^(ttcn3|TTCN3-.*)-pcu-test.*"
- timer: 15 05 * * *
- - TTCN3-centos-sgsn-test:
- blocking: "^(ttcn3|TTCN3-.*)-sgsn-test.*"
- timer: 30 05 * * *
- - TTCN3-centos-bsc-test:
- blocking: "^(ttcn3|TTCN3-.*)-bsc-test.*"
- timer: 00 06 * * *
- - TTCN3-centos-mgw-test:
- blocking: "^(ttcn3|TTCN3-.*)-mgw-test.*"
- timer: 30 06 * * *
- - TTCN3-centos-hlr-test:
- blocking: "^(ttcn3|TTCN3-.*)-hlr-test.*"
- timer: 45 06 * * *
- - TTCN3-centos-bts-test:
- blocking: "^(ttcn3|TTCN3-.*)-bts-test.*"
- timer: 00 07 * * *
- - TTCN3-centos-sip-test:
- blocking: "^(ttcn3|TTCN3-.*)-sip-test.*"
- timer: 30 07 * * *
- - TTCN3-centos-bsc-test-sccplite:
- blocking: "^(ttcn3|TTCN3-.*)-bsc-test.*"
- timer: 30 08 * * *
- - TTCN3-centos-remsim-test:
- blocking: "^(ttcn3|TTCN3-.*)-remsim-test.*"
- timer: 00 09 * * *
- - TTCN3-centos-stp-test:
- blocking: "^(ttcn3|TTCN3-.*)-stp-test.*"
- timer: 30 09 * * *
- - TTCN3-centos-gbproxy-test:
- blocking: "^(ttcn3|TTCN3-.*)-gbproxy-test.*"
- timer: 20 10 * * *
- - TTCN3-centos-cbc-test:
- blocking: "^(ttcn3|TTCN3-.*)-cbc-test.*"
- timer: 40 10 * * *
+ # centos7 nightly
+ - TTCN3-centos7-pcap-client-test:
+ timer: H 10 * * *
- # centos8 latest stable
- - TTCN3-centos-ggsn-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-ggsn-test.*"
- timer: 00 04 * * *
- - TTCN3-centos-msc-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-msc-test.*"
- timer: 30 04 * * *
- - TTCN3-centos-pcu-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-pcu-test.*"
- timer: 00 05 * * *
- - TTCN3-centos-pcu-test-sns-latest:
- blocking: "^(ttcn3|TTCN3-.*)-pcu-test.*"
- timer: 15 05 * * *
- - TTCN3-centos-sgsn-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-sgsn-test.*"
- timer: 30 05 * * *
- - TTCN3-centos-bsc-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-bsc-test.*"
- timer: 00 06 * * *
- - TTCN3-centos-mgw-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-mgw-test.*"
- timer: 30 06 * * *
- - TTCN3-centos-hlr-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-hlr-test.*"
- timer: 45 06 * * *
- - TTCN3-centos-bts-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-bts-test.*"
- timer: 00 07 * * *
- - TTCN3-centos-sip-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-sip-test.*"
- timer: 30 07 * * *
- - TTCN3-centos-bsc-test-sccplite-latest:
- blocking: "^(ttcn3|TTCN3-.*)-bsc-test.*"
- timer: 30 08 * * *
- # No osmo-remsim packages for centos8
- #- TTCN3-centos-remsim-test-latest:
- # blocking: "^(ttcn3|TTCN3-.*)-remsim-test.*"
- # timer: 00 09 * * *
- - TTCN3-centos-stp-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-stp-test.*"
- timer: 30 09 * * *
- - TTCN3-centos-smlc-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-smlc-test.*"
- timer: 10 10 * * *
- #- TTCN3-centos-sccp-test-latest:
- # blocking: "^(ttcn3|TTCN3-.*)-sccp-test.*"
- # timer: 00 10 * * *
- #- TTCN3-centos-gbproxy-test-latest:
- #blocking: "^(ttcn3|TTCN3-.*)-gbproxy-test.*"
- #timer: 20 10 * * *
- - TTCN3-centos-cbc-test-latest:
- blocking: "^(ttcn3|TTCN3-.*)-cbc-test.*"
- timer: 40 10 * * *
+ # centos7 latest
+ - TTCN3-centos7-pcap-client-test-latest:
+ timer: H 10 * * *
- # centos8 2021q1 (SYS#5370)
- - TTCN3-centos-pcu-test-2021q1:
- blocking: "^(ttcn3|TTCN3-.*)-pcu-test.*"
- timer: 00 11 * * *
- - TTCN3-centos-pcu-test-sns-2021q1:
- blocking: "^(ttcn3|TTCN3-.*)-pcu-test.*"
- timer: 15 11 * * *
- - TTCN3-centos-bsc-test-2021q1:
- blocking: "^(ttcn3|TTCN3-.*)-bsc-test.*"
- timer: 30 11 * * *
- - TTCN3-centos-mgw-test-2021q1:
- blocking: "^(ttcn3|TTCN3-.*)-mgw-test.*"
- timer: 45 11 * * *
- - TTCN3-centos-bts-test-2021q1:
- blocking: "^(ttcn3|TTCN3-.*)-bts-test.*"
- timer: 00 12 * * *
- - TTCN3-centos-smlc-test-2021q1:
- blocking: "^(ttcn3|TTCN3-.*)-smlc-test.*"
- timer: 15 12 * * *
- - TTCN3-centos-gbproxy-test-2021q1:
- blocking: "^(ttcn3|TTCN3-.*)-gbproxy-test.*"
- timer: 30 12 * * *
+ # With LIBOSMO_IO_BACKEND=IO_URING (OS#6357)
+ - ttcn3-bsc-test-io_uring:
+ timer: H 11 * * *
+ node: io_uring # OS#6405
+ - ttcn3-msc-test-io_uring:
+ timer: H 11 * * *
+ node: io_uring # OS#6405
+ - ttcn3-hnbgw-test-io_uring:
+ timer: H 11 * * *
+ node: io_uring # OS#6405
+ - ttcn3-gbproxy-test-io_uring:
+ timer: H 11 * * *
+ node: io_uring # OS#6405
+ - ttcn3-sgsn-test-io_uring:
+ timer: H 11 * * *
+ node: io_uring # OS#6405
+ - ttcn3-stp-test-io_uring:
+ timer: H 11 * * *
+ node: io_uring # OS#6405
+ - ttcn3-mgw-test-io_uring:
+ timer: H 11 * * *
+ node: io_uring # OS#6405
- job-template:
name: '{job-name}'
@@ -357,6 +262,15 @@
# Run against debian's kernel
export KERNEL_TEST=1
;;
+ ttcn3-ggsn-test-kernel-pablo-gtp)
+ DIR="ttcn3-ggsn-test"
+ export IMAGE_SUFFIX="master"
+ export KERNEL_TEST=1
+ export KERNEL_BUILD=1
+ export KERNEL_URL="https://git.kernel.org/pub/scm/linux/kernel/git/pablo/gtp.git"
+ export KERNEL_REMOTE_NAME="pablo-gtp"
+ export KERNEL_BRANCH="main"
+ ;;
ttcn3-ggsn-test-kernel-latest)
DIR="ttcn3-ggsn-test"
export IMAGE_SUFFIX="latest"
@@ -380,21 +294,17 @@
export KERNEL_REMOTE_NAME="torvalds"
export KERNEL_BRANCH="master"
;;
- TTCN3-centos-*-latest)
- DIR="$(echo "{job-name}" | sed s/^TTCN3-centos\-/ttcn3-/ | sed s/\-latest$//)"
- export IMAGE_SUFFIX="latest-centos8"
+ TTCN3-centos7-*-latest)
+ DIR="$(echo "{job-name}" | sed s/^TTCN3-centos7\-/ttcn3-/ | sed s/\-latest$//)"
+ export IMAGE_SUFFIX="latest-centos7"
;;
- TTCN3-centos-*-20*q*)
- export OSMOCOM_REPO_TESTSUITE_MIRROR="http://download.opensuse.org"
- export OSMOCOM_REPO_MIRROR="https://downloads.osmocom.org"
- export OSMOCOM_REPO_PATH="osmo-maintained"
- export OSMOCOM_REPO_VERSION="$(echo "{job-name}" | grep -o -P -- "20\d\dq.*$")" # e.g. "2021q1"
- export IMAGE_SUFFIX="$OSMOCOM_REPO_VERSION-centos8"
- DIR="$(echo "{job-name}" | sed s/^TTCN3-centos\-/ttcn3-/ | sed s/\-$OSMOCOM_REPO_VERSION$//)"
+ TTCN3-centos7-*)
+ DIR="$(echo "{job-name}" | sed s/^TTCN3-centos7\-/ttcn3-/)"
+ export IMAGE_SUFFIX="master-centos7"
;;
- TTCN3-centos-*)
- DIR="$(echo "{job-name}" | sed s/^TTCN3-centos\-/ttcn3-/)"
- export IMAGE_SUFFIX="master-centos8"
+ *-io_uring)
+ DIR="$(echo "{job-name}" | sed s/\-io_uring$//)"
+ export DOCKER_ARGS="-e LIBOSMO_IO_BACKEND=IO_URING"
;;
*-latest)
DIR="$(echo "{job-name}" | sed s/\-latest$//)"
@@ -412,7 +322,7 @@
- git:
branches:
- '$BRANCH'
- url: git://git.osmocom.org/docker-playground
+ url: https://gerrit.osmocom.org/docker-playground
git-config-name: 'Jenkins Builder'
git-config-email: 'jenkins@osmocom.org'
wipe-workspace: "{obj:wipe_workspace}"
@@ -423,8 +333,8 @@
results: '**/junit-xml-*.log'
allow-empty-results: false
- email:
- notify-every-unstable-build: true
- recipients: laforge@gnumonks.org osmith@sysmocom.de
+ notify-every-unstable-build: false
+ recipients: '{obj:email}'
send-to-individuals: false
- archive:
allow-empty: false
@@ -434,14 +344,12 @@
fingerprint: false
only-if-success: false
properties:
- - build-blocker:
- use-build-blocker: true
- blocking-jobs:
- - "{obj:blocking}"
- build-discarder:
days-to-keep: 30
num-to-keep: 120
- artifact-days-to-keep: -1
- artifact-num-to-keep: -1
+ wrappers:
+ - ansicolor:
+ colormap: xterm
+ email: jenkins-notifications@lists.osmocom.org
# vim: expandtab tabstop=2 shiftwidth=2
diff --git a/jobs/update-osmo-ci-on-slaves.yml b/jobs/update-osmo-ci-on-slaves.yml
index 060bba2..c826175 100644
--- a/jobs/update-osmo-ci-on-slaves.yml
+++ b/jobs/update-osmo-ci-on-slaves.yml
@@ -6,42 +6,60 @@
type: slave
name: label
values:
- - simtester
- - host2-deb9build-ansible
- - build2-deb9build-ansible
- build2-deb10build-ansible
- - rpi4-raspbian10build-ansible-1
- - rpi4-raspbian10build-ansible-2
- - rpi4-raspbian10build-ansible-3
- - gtp0-deb9build
- - gtp0-deb10build32
+ - build2-deb11build-ansible
+ - build4-deb12build-ansible
- gtp0-deb10fr
+ - host2-deb11build-ansible
+ - rpi4-raspbian11build-ansible-1
+ - rpi4-raspbian11build-ansible-2
+ - rpi4-raspbian11build-ansible-3
+ - simtester
properties:
- build-discarder:
days-to-keep: 30
num-to-keep: 120
+ parameters:
+ - string:
+ name: OSMO_BRANCH_CI
+ description: |
+ osmo-ci.git branch
+ default: 'master'
+ - string:
+ # Used in scripts/common.sh:docker_images_require()
+ name: OSMO_BRANCH_DOCKER_PLAYGROUND
+ description: |
+ docker-playground.git branch
+ default: 'master'
scm:
- git:
- url: git://git.osmocom.org/osmo-ci
+ url: https://gerrit.osmocom.org/osmo-ci
git-config-name: 'Jenkins Builder'
git-config-email: 'jenkins@osmocom.org'
skip-tag: true
branches:
- - 'origin/master'
+ - '$OSMO_BRANCH_CI'
+ wipe-workspace: true
triggers:
- pollscm:
cron: "H/5 * * * *"
ignore-post-commit-hooks: false
+ - timed: "H H * * *"
builders:
- shell: './contrib/jenkins.sh'
+
+ wrappers:
+ - ansicolor:
+ colormap: xterm
+
description: |
<b>Auto-generated using Jenkins Job Builder. DO NOT EDIT MANUALLY!</b>
publishers:
- email:
- recipients: intern@lists.sysmocom.de
+ recipients: jenkins-notifications@lists.osmocom.org
# Trigger from docker-playground.git too
- job:
@@ -49,10 +67,9 @@
description: |
Trigger <a href="https://jenkins.osmocom.org/jenkins/job/update-osmo-ci-on-slaves/">update-osmo-ci-on-slaves</a>
on changes in docker-playground.git, so it can rebuild docker containers relevant for the jenkins slaves.
- node: osmocom-master-debian9
scm:
- git:
- url: git://git.osmocom.org/docker-playground
+ url: https://gerrit.osmocom.org/docker-playground
git-config-name: 'Jenkins Builder'
git-config-email: 'jenkins@osmocom.org'
skip-tag: true
diff --git a/jobs/update-osmo-python-on-slaves.yml b/jobs/update-osmo-python-on-slaves.yml
index 598d867..0842fba 100644
--- a/jobs/update-osmo-python-on-slaves.yml
+++ b/jobs/update-osmo-python-on-slaves.yml
@@ -6,12 +6,14 @@
type: slave
name: label
values:
- - host2-deb9build-ansible
- - build2-deb9build-ansible
- build2-deb10build-ansible
- - rpi4-raspbian10build-ansible-1
- - rpi4-raspbian10build-ansible-2
- - rpi4-raspbian10build-ansible-3
+ - build2-deb11build-ansible
+ - build3-deb11build-ansible
+ - build4-deb12build-ansible
+ - host2-deb11build-ansible
+ - rpi4-raspbian11build-ansible-1
+ - rpi4-raspbian11build-ansible-2
+ - rpi4-raspbian11build-ansible-3
properties:
- build-discarder:
days-to-keep: 30
@@ -40,6 +42,6 @@
project: update-osmo-ci-on-slaves
threshold: FAILURE
- email:
- recipients: intern@lists.sysmocom.de
+ recipients: jenkins-notifications@lists.osmocom.org
# vim: expandtab tabstop=2 shiftwidth=2
diff --git a/lint/checkpatch/checkpatch.pl b/lint/checkpatch/checkpatch.pl
index 06e6e36..eada2a9 100755
--- a/lint/checkpatch/checkpatch.pl
+++ b/lint/checkpatch/checkpatch.pl
@@ -2813,6 +2813,7 @@ sub process {
foreach (@exclude) {
if ($realfile =~ m@$_@) {
$skipme = 1;
+ last; # break out of the loop
}
}
if ($skipme) {
@@ -3436,7 +3437,7 @@ sub process {
my $msg_level = \&ERROR;
$msg_level = \&CHK if ($file);
&{$msg_level}("FSF_MAILING_ADDRESS",
- "Do not include the paragraph about writing to the Free Software Foundation's mailing address from the sample GPL notice. The FSF has changed addresses in the past, and may do so again. Linux already includes a copy of the GPL.\n" . $herevet)
+ "Do not include the paragraph about writing to the Free Software Foundation's mailing address from the sample GPL notice. The FSF has changed addresses in the past, and may do so again. This Osmocom project should already include a copy of the (A)GPL.\n" . $herevet)
}
# check for Kconfig help text having a real description
@@ -5076,11 +5077,16 @@ sub process {
$opv eq '*U' || $opv eq '-U' ||
$opv eq '&U') { # Osmocom specific: &&U removed
if ($ctx !~ /[WEBC]x./ && $ca !~ /(?:\)|!|~|\*|-|\&|\||\+\+|\-\-|\{)$/) {
- if (ERROR("SPACING",
- "space required before that '$op' $at\n" . $hereptr)) {
- if ($n != $last_after + 2) {
- $good = $fix_elements[$n] . " " . ltrim($fix_elements[$n + 1]);
- $line_fixed = 1;
+ # Osmocom specific: inside struct osmo_tdef we write non-spec timers as
+ # T=-1234. Do not complain about having no space before the minus
+ # character.
+ if ($opline !~ /\.T=/) {
+ if (ERROR("SPACING",
+ "space required before that '$op' $at\n" . $hereptr)) {
+ if ($n != $last_after + 2) {
+ $good = $fix_elements[$n] . " " . ltrim($fix_elements[$n + 1]);
+ $line_fixed = 1;
+ }
}
}
}
@@ -5150,7 +5156,7 @@ sub process {
}
} elsif ($ctx =~ /Wx[^WCE]|[^WCE]xW/) {
if (ERROR("SPACING",
- "need consistent spacing around '$op' $at\n" . $hereptr)) {
+ "need consistent spacing around '$op' $at (or typedef missing in osmo-ci/lint/checkpatch/typedefs_osmo.txt?)\n" . $hereptr)) {
$good = rtrim($fix_elements[$n]) . " " . trim($fix_elements[$n + 1]) . " ";
if (defined $fix_elements[$n + 2]) {
$fix_elements[$n + 2] =~ s/^\s+//;
@@ -6007,7 +6013,7 @@ sub process {
$sum_allowed += $_;
}
if ($sum_allowed == 0) {
- WARN("BRACES",
+ WARN("BRACES_NOT_NECESSARY", # Osmocom specific: renamed from BRACES
"braces {} are not necessary for any arm of this statement\n" . $herectx);
} elsif ($sum_allowed != $allow &&
$seen != $allow) {
@@ -6063,7 +6069,7 @@ sub process {
my $cnt = statement_rawlines($block);
my $herectx = get_stat_here($linenr, $cnt, $here);
- WARN("BRACES",
+ WARN("BRACES_NOT_NECESSARY", # Osmocom specific: renamed from BRACES
"braces {} are not necessary for single statement blocks\n" . $herectx);
}
}
diff --git a/lint/checkpatch/checkpatch_json.py b/lint/checkpatch/checkpatch_json.py
new file mode 100755
index 0000000..81946ca
--- /dev/null
+++ b/lint/checkpatch/checkpatch_json.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-only
+
+"""
+This utilty generate json output to post comment in gerrit.
+
+INPUT: output of checkpatch.pl.
+OUTPUT: json format output that can be used to post comment in gerrit
+"""
+import os
+import sys
+import json
+
+data = {}
+data['comments'] = []
+list_temp = {}
+
+def update_struct( file_path, msg_output, line_number):
+ if file_path not in list_temp:
+ list_temp[file_path] = []
+ list_temp[file_path].append({
+ "robot_id" : "checkpatch",
+ "robot_run_id" : sys.argv[3],
+ "url" : sys.argv[4],
+ "line" : line_number,
+ "message" : msg_output,}
+ )
+
+def parse_file(input_file):
+ fp = open (input_file, "r")
+ for line in fp:
+ if line.startswith("ERROR:"):
+ msg_output = line.split("ERROR:")[1].strip()
+ elif line.startswith("WARNING:"):
+ msg_output = line.split("WARNING:")[1].strip()
+ elif ": FILE:" in line:
+ temp = line.split("FILE:")
+ file_path = temp[1].split(":")[0]
+ line_number = temp[1].split(":")[1]
+ update_struct( file_path.strip(), msg_output, str(line_number) )
+ else:
+ continue
+ fp.close()
+
+def main():
+ if (len(sys.argv) < 5) or (sys.argv[1] == "-h"):
+ print("HELP:")
+ print(sys.argv[0] + " <input file> <output file in json> <job-id> <job-url>")
+ sys.exit()
+
+ print(sys.argv[1])
+ parse_file(sys.argv[1])
+ data['robot_comments'] = list_temp
+ print(json.dumps(data))
+ out_file = open( sys.argv[2] , "w")
+ json.dump(data, out_file, sort_keys=True, indent=4)
+ out_file.close()
+
+if __name__ == "__main__":
+ main()
diff --git a/lint/checkpatch/checkpatch_osmo.sh b/lint/checkpatch/checkpatch_osmo.sh
index 91e9abe..f56464a 100755
--- a/lint/checkpatch/checkpatch_osmo.sh
+++ b/lint/checkpatch/checkpatch_osmo.sh
@@ -1,12 +1,34 @@
#!/bin/sh -e
SCRIPT_DIR="$(dirname "$(realpath "$0")")"
-PROJECT="$(basename "$(git rev-parse --show-toplevel)")"
+PROJECT_DIR="$(git rev-parse --show-toplevel)"
+PROJECT="$(basename "$PROJECT_DIR")"
exclude_paths_common() {
- # Stdout and stderr of regression tests
- echo '--exclude \.(ok|err)$'
+ # Test output
+ echo '--exclude \.(ok|err|ctrl|vty)$'
# Generated from commit log, which may contain spelling errors (OS#5232)
echo '--exclude ^debian/changelog$'
+ # Patch files
+ echo '--exclude \.patch$'
+ # License files (OS#6397)
+ echo '--exclude ^(LICENSE|COPYING)'
+ # Symlinks (no newline at end of file)
+ find -type l -printf '--exclude ^%P$\n'
+}
+
+exclude_paths_common_asn1c() {
+ local pattern="^ \* Generated by asn1c-"
+ local files="$(find -name '*.[hc]' -printf '%P\n')"
+ local i
+
+ if [ -z "$files" ]; then
+ return
+ fi
+
+ for i in $(grep -l "$pattern" $files); do
+ # Example: --exclude ^include/osmocom/sabp/SABP_Data-Coding-Scheme.h$
+ echo '--exclude ^'$i'$'
+ done
}
exclude_paths_project() {
@@ -40,24 +62,33 @@ exclude_paths_project() {
# * EMBEDDED_FUNCTION_NAME: often __func__ isn't used, arguably not much benefit in changing this when touching code
# * EXECUTE_PERMISSIONS: not followed, files need to be executable: git-version-gen, some in debian/
# * FILE_PATH_CHANGES: we don't use a MAINTAINERS file
-# * FUNCTION_WITHOUT_ARGS: not followed: warns about func() instead of func(void)
# * GLOBAL_INITIALISERS: we initialise globals to NULL for talloc ctx (e.g. *tall_lapd_ctx = NULL)
# * IF_0: used intentionally
# * INITIALISED_STATIC: we use this, see also http://lkml.iu.edu/hypermail/linux/kernel/0808.1/2235.html
# * LINE_CONTINUATIONS: false positives
# * LINE_SPACING: we don't always put a blank line after declarations
# * LONG_LINE*: should be 120 chars, but exceptions are done often so don't fail here
+# * MACRO_WITH_FLOW_CONTROL: not followed
+# * MISSING_SPACE: warns about breaking strings at space characters, not useful for long strings of hex chars
# * PREFER_DEFINED_ATTRIBUTE_MACRO: macros like __packed not defined in libosmocore
# * PREFER_FALLTHROUGH: pseudo keyword macro "fallthrough" is not defined in libosmocore
# * REPEATED_WORD: false positives in doxygen descriptions (e.g. '\param[in] data Data passed through...')
# * SPDX_LICENSE_TAG: we don't place it on line 1
# * SPLIT_STRING: we do split long messages over multiple lines
# * STRING_FRAGMENTS: sometimes used intentionally to improve readability
+# * SYMBOLIC_PERMS: recommends using octal permissions over symbolic ones, not followed
+# * TRACING_LOGGING: recommends to use kernel's internal ftrace instead of printf("%s()\n", __func__)
+# * TRAILING_STATEMENTS: not followed, e.g. 'while (osmo_select_main_ctx(1) > 0);' is put in one line
+# * UNNECESSARY_BREAK: not followed (see https://gerrit.osmocom.org/c/libosmo-netif/+/26429)
# * UNNECESSARY_INT: not followed (see https://gerrit.osmocom.org/c/libosmocore/+/25345)
# * UNSPECIFIED_INT: not followed (doesn't seem useful for us)
+# * VOLATILE: using volatile makes sense in embedded projects so this warning is not useful for us
+
+cd "$PROJECT_DIR"
$SCRIPT_DIR/checkpatch.pl \
$(exclude_paths_common) \
+ $(exclude_paths_common_asn1c) \
$(exclude_paths_project) \
--ignore ASSIGN_IN_IF \
--ignore AVOID_EXTERNS \
@@ -69,7 +100,6 @@ $SCRIPT_DIR/checkpatch.pl \
--ignore EMBEDDED_FUNCTION_NAME \
--ignore EXECUTE_PERMISSIONS \
--ignore FILE_PATH_CHANGES \
- --ignore FUNCTION_WITHOUT_ARGS \
--ignore GLOBAL_INITIALISERS \
--ignore IF_0 \
--ignore INITIALISED_STATIC \
@@ -78,15 +108,23 @@ $SCRIPT_DIR/checkpatch.pl \
--ignore LONG_LINE \
--ignore LONG_LINE_COMMENT \
--ignore LONG_LINE_STRING \
+ --ignore MACRO_WITH_FLOW_CONTROL \
+ --ignore MISSING_SPACE \
--ignore PREFER_DEFINED_ATTRIBUTE_MACRO \
--ignore PREFER_FALLTHROUGH \
--ignore REPEATED_WORD \
--ignore SPDX_LICENSE_TAG \
--ignore SPLIT_STRING \
--ignore STRING_FRAGMENTS \
+ --ignore SYMBOLIC_PERMS \
+ --ignore TRACING_LOGGING \
+ --ignore TRAILING_STATEMENTS \
+ --ignore UNNECESSARY_BREAK \
--ignore UNNECESSARY_INT \
--ignore UNSPECIFIED_INT \
+ --ignore VOLATILE \
--max-line-length 120 \
+ --typedefsfile "$SCRIPT_DIR/typedefs_osmo.txt" \
--no-signoff \
--no-tree \
"$@"
diff --git a/lint/checkpatch/spelling.txt b/lint/checkpatch/spelling.txt
index 7b6a012..f60a02f 100644
--- a/lint/checkpatch/spelling.txt
+++ b/lint/checkpatch/spelling.txt
@@ -7,6 +7,9 @@
# The format of each line is:
# mistake||correction
#
+# Osmocom specific:
+s.m.f.c.||s.f.m.c.
+#
abandonning||abandoning
abigious||ambiguous
abitrary||arbitrary
@@ -50,7 +53,6 @@ acient||ancient
acitions||actions
acitve||active
acknowldegement||acknowledgment
-acknowledgement||acknowledgment
ackowledge||acknowledge
ackowledged||acknowledged
acording||according
diff --git a/lint/checkpatch/typedefs_osmo.txt b/lint/checkpatch/typedefs_osmo.txt
new file mode 100644
index 0000000..204ba0a
--- /dev/null
+++ b/lint/checkpatch/typedefs_osmo.txt
@@ -0,0 +1,5 @@
+A_SEQUENCE_OF
+A_SET_OF
+l1if_compl_cb
+ranap_message
+osmo_ss7_asp_rx_unknown_cb
diff --git a/lint/docker_run_rm.sh b/lint/docker_run_rm.sh
new file mode 100755
index 0000000..3d6648e
--- /dev/null
+++ b/lint/docker_run_rm.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+# Verify that "docker run" has a "--rm" in the same line or next line, so we
+# don't fill up space on jenkins nodes with never deleted containers:
+# https://osmocom.org/projects/osmocom-servers/wiki/Docker_cache_clean_up
+
+RET=0
+
+for i in $(git grep -l '^[^#]*docker run'); do
+ if [ -z "$(grep -A1 "docker run" "$i" | grep -- "--rm")" ]; then
+ echo "ERROR: missing --rm after 'docker run' (same line or next line):"
+ grep --color=always -H -n -A1 "docker run" "$i"
+ echo
+ RET=1
+ fi
+done
+
+exit $RET
diff --git a/lint/lint_diff.sh b/lint/lint_diff.sh
index f8daab7..662bdd3 100755
--- a/lint/lint_diff.sh
+++ b/lint/lint_diff.sh
@@ -19,20 +19,60 @@ if [ -z "$COMMIT" ]; then
fi
fi
+ERROR=0
+
+echo "Running docker_run_rm.sh on the whole tree..."
+echo
+if ! "$SCRIPT_DIR"/docker_run_rm.sh; then
+ ERROR=1
+fi
+
echo "Running checkpatch on 'git diff $COMMIT'..."
echo
-if git diff -U0 "$COMMIT" | "$SCRIPT_DIR/checkpatch/checkpatch_osmo.sh" - \
+if ! git diff -U0 "$COMMIT" | "$SCRIPT_DIR/checkpatch/checkpatch_osmo.sh" - \
--color=always \
--mailback \
--show-types \
--showfile \
--terse
then
- exit 0
+ ERROR=1
fi
-echo
-echo "Please fix the linting errors above. More information:"
-echo "https://osmocom.org/projects/cellular-infrastructure/wiki/Linting"
-echo
-exit 1
+
+if [ "$ERROR" = 1 ]; then
+ echo
+ echo "Please fix the linting errors above. More information:"
+ echo "https://osmocom.org/projects/cellular-infrastructure/wiki/Linting"
+ echo
+
+ if [ -n "$JENKINS_HOME" ]; then
+ echo "Leaving review comments in gerrit..."
+ set -x
+
+ # Run again, but in the proper format for checkpatch_json.py
+ # and store the output in a file
+ git diff -U0 "$COMMIT" | "$SCRIPT_DIR/checkpatch/checkpatch_osmo.sh" \
+ > ../checkpatch_output || true
+ cd ..
+ # Convert to gerrit review format
+ "$SCRIPT_DIR/checkpatch/checkpatch_json.py" \
+ checkpatch_output \
+ gerrit_report.json \
+ "$BUILD_TAG" \
+ "$BUILD_URL"
+ # Apply as review in gerrit
+ ssh \
+ -p "$GERRIT_PORT" \
+ -l jenkins \
+ "$GERRIT_HOST" \
+ gerrit \
+ review \
+ --project "$GERRIT_PROJECT" \
+ "$GERRIT_CHANGE_NUMBER,$GERRIT_PATCHSET_NUMBER" \
+ --json \
+ < gerrit_report.json
+ fi
+
+ exit 1
+fi
diff --git a/qemu-kvm/ggsn-kernel-gtp/postinst.sh b/qemu-kvm/ggsn-kernel-gtp/postinst.sh
index 4cbe64c..27df200 100755
--- a/qemu-kvm/ggsn-kernel-gtp/postinst.sh
+++ b/qemu-kvm/ggsn-kernel-gtp/postinst.sh
@@ -48,7 +48,7 @@ apt-get clean
# add osmocom:nightly feed + install libosmocore-dev
apt-key add /tmp/postinst/Release.key
-echo "deb http://download.opensuse.org/repositories/network:/osmocom:/nightly/Debian_9.0/ ./" > /etc/apt/sources.list.d/osmocom-nightly.list
+echo "deb http://downloads.osmocom.org/packages/osmocom:/nightly/Debian_9.0/ ./" > /etc/apt/sources.list.d/osmocom-nightly.list
apt-get update
apt-get install -y --no-install-recommends \
libosmocore-dev
@@ -65,10 +65,10 @@ DEBIAN_FRONTEND=noninteractive apt-get purge -y nano laptop-detect tasksel dicti
# check out sources we need from their respective repositories
cd /usr/local/src
-git clone git://git.netfilter.org/libmnl
+git clone https://git.netfilter.org/libmnl
(cd libmnl && autoreconf -fi && ./configure && make && make install)
-git clone git://git.osmocom.org/libgtpnl
+git clone https://gerrit.osmocom.org/libgtpnl
(cd libgtpnl && autoreconf -fi && ./configure && make && make install)
-git clone git://git.osmocom.org/osmo-ggsn
+git clone https://gerrit.osmocom.org/osmo-ggsn
(cd osmo-ggsn && autoreconf -fi && ./configure --enable-gtp-linux && make && make install)
ldconfig
diff --git a/qemu-kvm/ggsn-kernel-gtp/postinst/Release.key b/qemu-kvm/ggsn-kernel-gtp/postinst/Release.key
index a737316..ecca084 100644
--- a/qemu-kvm/ggsn-kernel-gtp/postinst/Release.key
+++ b/qemu-kvm/ggsn-kernel-gtp/postinst/Release.key
@@ -1,20 +1,26 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: GnuPG v1.4.5 (GNU/Linux)
-mQENBFJBt/wBCADAht3d/ilNuyzaXYw/QwTRvmjyoDvfXw+H/3Fvk1zlDZoiKPPc
-a1wCVBINUZl7vYM2OXqbJwYa++JP2Q48xKSvC6thbRc/YLievkbcvTemf7IaREfl
-CTjoYpoqXHa9kHMw1aALDm8CNU88jZmnV7v9L6hKkbYDxie+jpoj7D6B9JlxgNJ4
-5dQyRNsFGVcIl4Vplt1HyGc5Q5nQI/VgS2rlF/IOXmhRQBc4LEDdU8R2IKnkU4ee
-S7TWanAigGAQhxGuCkS39/CWzc1DhLhjlNhBl/+RTPejkqJtAy00ZLps3+RqUN1Y
-CU/Fsr7aRlYVGqQ/BlptwV0XQ2VVYJX2oEBBABEBAAG0MG5ldHdvcmsgT0JTIFBy
-b2plY3QgPG5ldHdvcmtAYnVpbGQub3BlbnN1c2Uub3JnPokBPAQTAQIAJgUCXm/4
-pgIbAwUJEEzwqgYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEGLrGgkXKA3f/1AH
-/A7WVSpfM4wV/DMqZPTsSjChB4JyDotxpV7qHZzBC5aaP2dINZyi9PayIwZWbvCY
-VKvt+Fw8oCGC9F9mdh10Xe+ElHeVNSihzABPuu1RkRkb1nvkymScy0yxydodYOBi
-K4WQ+BhpijXWmYvOekIwbS5Hi9BHpfgK4TinK0xsvh1bVLeQJ8YjrnNFIAR2CnBa
-X7Y72Up/kKL08DdQzuS+mKrJtAQlGMtIsukWC2ajYQMkNwm8Gvhpn8za113dCkBW
-XAFnlQqQobKwC7b19QgEtJI/YpGSrRc6WaZxPyAjscbWQlFEAB900sVj4BWT55ig
-7O2uSdsCVhTuU7T0ztwsgvmIRgQTEQIABgUCUkG3/AAKCRA7MBG3a51lIzhdAJ9v
-d6XPffMZRcCGgDEY5OaTn/MsCQCgrXbeZpFJgnirSrc8rRonvzYFiF4=
-=/Tek
+mQENBGKzE1QBCADFcM3ZzggvgxNRNNqDGWf5xIDAiK5qzFLdGes7L6F9VCHdaPy0
+RAOB5bFb/Q1tSDFNEBLtaauXKz+4iGL6qMVjZcyjzpB5w4jKN+kkrFRhjDNUv/SH
+BX6d+P7v5WBGSNArNgA8D1BGzckp5a99EZ0okMJFEqIcN40PD6OGugpq5XnVV5Nk
+e93fLa2Cu8vhFBcVn6CuHeEhsmuMf6NLbQRNfNNCEEUYaZn7beMYtpZ7t1djsKx5
+1xGm50OzI22FLu8lELQ9d7qMVGRG3WHYawX9BDteRybiyqxfwUHm1haWazRJtlGt
+UWyzvwAb80BK1J2Nu5fbAa3w5CoEPAbUuCyrABEBAAG0JW9zbW9jb20gT0JTIFBy
+b2plY3QgPG9zbW9jb21Ab3Ntb2NvbT6JAVQEEwEIAD4WIQRrKp83ktFetw1Oao+G
+pzC2U3JZcwUCYrMV4wIbAwUJBB6yjwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAK
+CRCGpzC2U3JZc4FRCACQQkKIrnvQ7n2u7GSmyVZa3I+oLoFXSGqaGyey5TW/nrMm
+vFDKU3qliHiuNSmUY35SnAhXUsvqOYppxVRoO1MLrqUvzMOnIWqkJpf8mtjGUnsW
+jyVeto7Rsjs75y2i1Hk+e7ljb/V65J3NlfrfEYWbqR9AKd53ReNXTdrQ0J05A38N
+GdI4Ld/2lNISAwaBmGhqdeKsLHpQw/JERU1TApVJR1whFiIwDF1rOCg9GPnNKIk7
+yRZdK267XzztrainX/cbPILyzUZEDhYs6wQuyACyQ1YUxZIxrwVfk7PMNay8CrLH
+z42B73Ne5IAj8+op/3iJafFONLm7YXiDUFN+QDYAiQEzBBMBCAAdFiEExoiYhHND
+S7aVYlnqa51NyAUyjdsFAmKzE1UACgkQa51NyAUyjdvuZgf+OXmr//i7u7Gg7eWB
+7e0qUsyCId9lXS8J437x3K6ciJfD7/6RSy8TFW5Nglm/uSkbyq582I8t+SoOirMD
+E6cg9U/5+h5s46bAf+Kd2XS/6tLGeNLM18i4el8CP06NpFzDrsKu76uYFpyRiiHD
+otBdtgxeLJ83LugGfZslF+/5cigJkAJMhAdVvGO8h85R6fba8ZSOKtMKkaQRfi76
+nhyOrJPlLuS+DLEnHwdkOFgtKnxHdjM97K+Tx0gisb6uwaWroXfSLnhP8RTLLZZy
+Z+noU1Hw3c+mn4c/NYbcC/uwHYHKRzuf9gHnQ3dGgv0Z5sbeLRVo92hjGj7Ftlyd
+4hmKBg==
+=HxK4
-----END PGP PUBLIC KEY BLOCK-----
diff --git a/qemu-kvm/ggsn-kernel-gtp/postinst/authorized_keys b/qemu-kvm/ggsn-kernel-gtp/postinst/authorized_keys
index 817a537..6a7cc8a 100644
--- a/qemu-kvm/ggsn-kernel-gtp/postinst/authorized_keys
+++ b/qemu-kvm/ggsn-kernel-gtp/postinst/authorized_keys
@@ -1,2 +1 @@
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCmaAZWIdZZjFZHq7Xm5OCUz/Cm46haad+pm2iZ2uSQCIXubjN8HGz1XujBMH1DHhSDK+L0JOBEtxHaY4JR7F177V5R8o50VNwmtTrTaNBPS8WPyVBDuHjjDVSZA+d6QT3Y+3X/FhfoFKmGc2a8MZRmE8yDGW484y7MTZ4XBHqy5vWnnywxWCZm7kVqNccM4oQ4tPWlMTxk6+tgWC4wvk3axsUVUm0ak/WKuZLoX+jBWlsWXR4uwaOEqblpNMv98CxjLgpjEBTvZxgrgY35eAm11S9lmvxFSjLIZA1V/2huAA/9MSNvI4dX4ZQPHfLCsMqFjhkIY0hvFK2uX8hlxHhOR6GkOq3n3MdcKCMI+JAISCP6g++DdQO5wzfOLeSdpTETNpIxeFkjQLS+HnuAcgjRK6O7czI+xYfCiBcaoslsTIYi33Pg7gPuDjwAyVdKqQPVdYPluR8W5+FT1lFtf1jwIB3jgt+v9q+r6d4LfkOCAU17yxG6b40P+FqK9O7X+iUkva/lbghdAw7DimczDIuEMpVVVL3/WhmCIMrfJPG/RtMdoczcAw4q4FxBretZR1Ke4AomUvUklFwfMtLqijxBJN2Owpy7jtqggs+v6cUX54fKh7/SeVrgyIgeUtgJp6lqaxlMao8SoR59YoqLqZueYOp9QxYujA70AGoMMwxsEw== laforge@nataraja
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDAX61B6rs76JfKmtqqFoR6o9KAP1/WVz9/COtjNrBDc6IfqDy46ZTlKwud4Nm3O3QF5cbdxx/hnOvep4SsLTArYuD8EcBtMoKhh8+c6Y30wR9l3kUWen4JEUJjFOCX/jr2SG4tikF/0S2zUJmILyFMw53pc1KSOmEewiWI8Z6U15ntLWEljWg+BWPs25kgQ5qeA3TO9ygNPFUrEQDGJlzLVubENwZOweTMll89bFaD+6jbjBG4ViPsSlzSvan9CThrTCrVCSmTObfOz3cEci0Wvgh1mBuOyfo5GyXQ7V8tHTF9AU5lGLybeyiXOki3f+bf9da8rnHuswR6ap8VsXrra2QFr40ro7ACCc7TC5Sa62c2Y8pC2Buuv3WcZ1x5PG7gRbU3iZqR0r7HeQGP/lVD0Z9uSV+9dSvg4uhsSS59XzRoI4gElHBiSGdxm0EKB+WI/0y+G9z10qyZggDPHqD67BY1Qpi5uQxQrFiOc5eadC0ucjjA2o4KqEid2cHI4qu8YLBns5X+GE1BB9o/ix5eSYFTBju0FlgzLaYEmo6zIURpKKvKfPEgpU/ZYWRX1SaUlohXaJxB60k87eyHNDCFwyIFDklZ8LB0WBvTJGqJaDmRcR9Idz4dP8UWCHD0hiShSLKW5sOFzkJflFRmaTWmBV8W/mwc9lVJrYuaSd7S4w== laforge@sunbeam.rsa
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF+89UJw1295JWouFrLnIVbxsX0i5X46J16olfb5gqyL laforge@nataraja
diff --git a/scripts/common-obs-conflict.sh b/scripts/common-obs-conflict.sh
deleted file mode 100644
index 98c8b70..0000000
--- a/scripts/common-obs-conflict.sh
+++ /dev/null
@@ -1,146 +0,0 @@
-#!/bin/sh
-# Create conflicting dummy packages in OBS (opensuse build service), so users can't mix packages
-# built from different branches by accident
-
-OSMO_OBS_CONFLICT_PKGVER="${OSMO_OBS_CONFLICT_PKGVER:-1.0.0}"
-
-# Create the conflicting package for debian
-#
-# $1: name of dummy package (e.g. "osmocom-nightly")
-# $2-*: name of conflicting packages (e.g. "osmocom-latest")
-#
-# Generates the following directory structure:
-# debian
-# ├── changelog
-# ├── compat
-# ├── control
-# ├── copyright
-# ├── rules
-# └── source
-# └── format
-osmo_obs_prepare_conflict_deb() {
- local pkgname="$1"
- shift
- local oldpwd="$PWD"
-
- mkdir -p "debian/source"
- cd "debian"
-
- # Fill control
- cat << EOF > control
-Source: ${pkgname}
-Section: unknown
-Priority: optional
-Maintainer: Oliver Smith <osmith@sysmocom.de>
-Build-Depends: debhelper (>= 9)
-Standards-Version: 3.9.8
-
-Package: ${pkgname}
-Depends: \${misc:Depends}
-Architecture: any
-EOF
- printf "Conflicts: " >> control
- first=1
- for i in "$@"; do
- if [ "$first" -eq 1 ]; then
- first=0
- else
- printf ", " >> control
- fi
- printf "%s" "$i" >> control
- done
- printf "\n" >> control
- cat << EOF >> control
-Description: Dummy package, which conflicts with: $@
-EOF
-
- # Fill changelog
- cat << EOF > changelog
-${pkgname} (${OSMO_OBS_CONFLICT_PKGVER}) unstable; urgency=medium
-
- * Dummy package, which conflicts with: $@
-
- -- Oliver Smith <osmith@sysmocom.de> Thu, 13 Jun 2019 12:50:19 +0200
-EOF
-
- # Fill rules
- cat << EOF > rules
-#!/usr/bin/make -f
-%:
- dh \$@
-EOF
-
- # Finish up debian dir
- chmod +x rules
- echo "9" > compat
- echo "3.0 (native)" > source/format
- touch copyright
-
- cd "$oldpwd"
-}
-
-# Create the conflicting package for rpm (e.g. contrib/osmocom-nightly.spec.in). The remaining
-# placeholders are replaced in osmo_obs_add_rpm_spec().
-#
-# $1: name of dummy package (e.g. "osmocom-nightly")
-# $2-*: name of conflicting packages (e.g. "osmocom-latest")
-osmo_obs_prepare_conflict_rpm() {
- local pkgname="$1"
- shift
- local spec_in="contrib/$pkgname.spec.in"
-
- mkdir -p contrib
-
- cat << EOF > "$spec_in"
-Name: $pkgname
-Version: @VERSION@
-Release: 0
-Summary: Dummy package, which conflicts with: $@
-License: AGPL-3.0-or-later
-Group: Hardware/Mobile
-Source: @SOURCE@
-EOF
-
- for i in "$@"; do
- echo "Conflicts: $i" >> "$spec_in"
- done
-
- cat << EOF >> "$spec_in"
-%description
-Dummy package, which conflicts with: $@
-%files
-EOF
-
-}
-
-# Print names of packages that the conflict package from the current feed
-# (e.g. osmocom-nightly) should conflict with (e.g. osmocom-latest,
-# osmocom-next, osmocom-2021q1)
-osmo_obs_prepare_conflict_args() {
- for i in $FEEDS_ALL; do
- if [ "$i" != "$FEED" ]; then
- echo "osmocom-$i"
- fi
- done
-}
-
-# Create conflicting packages, based on global $FEED and $FEEDS_ALL vars
-osmo_obs_prepare_conflict() {
- local pkgname="osmocom-$FEED"
- local conflict_args="$(osmo_obs_prepare_conflict_args)"
- local oldpwd="$PWD"
-
- mkdir -p "$pkgname"
- cd "$pkgname"
-
- osmo_obs_prepare_conflict_deb "$pkgname" $conflict_args
- osmo_obs_prepare_conflict_rpm "$pkgname" $conflict_args
-
- # Put in git repository
- git init .
- git add -A
- git commit -m "auto-commit: $pkgname dummy package" || true
- git tag -f "$OSMO_OBS_CONFLICT_PKGVER"
-
- cd "$oldpwd"
-}
diff --git a/scripts/common-obs.sh b/scripts/common-obs.sh
deleted file mode 100644
index 1a10eb6..0000000
--- a/scripts/common-obs.sh
+++ /dev/null
@@ -1,313 +0,0 @@
-#!/bin/sh
-# Various common code used in the OBS (opensuse build service) related osmo-ci shell scripts
-. "$(dirname "$0")/common-obs-conflict.sh"
-
-FEEDS_ALL="
- 2021q1
- latest
- next
- nightly
-"
-
-osmo_cmd_require \
- dch \
- dh \
- dpkg-buildpackage \
- gbp \
- git \
- meson \
- mktemp \
- osc \
- patch \
- sed
-
-if [ -z "$PROJ" ]; then
- echo "PROJ environment variable is not set"
- exit 1
-fi
-
-# Add dependency to all (sub)packages in debian/control and commit the change.
-# $1: path to debian/control file
-# $2: package name (e.g. "libosmocore")
-# $3: dependency package name (e.g. "osmocom-nightly")
-# $4: dependency package version (optional, e.g. "1.0.0.202101151122")
-osmo_obs_add_depend_deb() {
- local d_control="$1"
- local pkgname="$2"
- local depend="$3"
- local dependver="$4"
-
- if [ "$pkgname" = "$depend" ]; then
- echo "NOTE: skipping dependency on itself: $depend"
- return
- fi
-
- if [ -n "$dependver" ]; then
- depend="$depend (= $dependver)"
- fi
-
- # Note: adding the comma at the end should be fine. If there is a Depends: line, it is most likely not empty. It
- # should at least have ${misc:Depends} according to lintian.
- sed "s/^Depends: /Depends: $depend, /g" -i "$d_control"
-
- git -C "$(dirname "$d_control")" commit -m "auto-commit: debian: depend on $depend" .
-}
-
-# Add dependency to all (sub)packages in rpm spec file
-# $1: path to rpm spec file
-# $2: package name (e.g. "libosmocore")
-# $3: dependency package name (e.g. "osmocom-nightly")
-# $4: dependency package version (optional, e.g. "1.0.0.202101151122")
-osmo_obs_add_depend_rpm() {
- local spec="$1"
- local pkgname="$2"
- local depend="$3"
- local dependver="$4"
-
- if [ "$pkgname" = "$depend" ]; then
- echo "NOTE: skipping dependency on itself: $depend"
- return
- fi
-
- if [ -n "$dependver" ]; then
- depend="$depend = $dependver"
- fi
-
- ( while IFS= read -r line; do
- echo "$line"
-
- case "$line" in
- # Main package
- "Name:"*)
- echo "Requires: $depend"
- ;;
- # Subpackages
- "%package"*)
- echo "Requires: $depend"
- ;;
- # Build recipe
- "%build"*)
- if [ -n "$dependver" ]; then
- cat << EOF
-# HACK: don't let rpmlint abort the build when it finds that a library depends
-# on a package with a specific version. The path used here is listed in:
-# https://build.opensuse.org/package/view_file/devel:openSUSE:Factory:rpmlint/rpmlint-mini/rpmlint-mini.config?expand=1
-# Instead of writing to the SOURCES dir, we could upload osmocom-rpmlintrc as
-# additional source for each package. But that's way more effort, not worth it.
-echo "setBadness('shlib-fixed-dependency', 0)" \\
- > "%{_sourcedir}/osmocom-rpmlintrc"
-
-EOF
- fi
- ;;
- esac
- done < "$spec" ) > "$spec.new"
-
- mv "$spec.new" "$spec"
-}
-
-# Copy a project's rpm spec.in file to the osc package dir, set the version/source, depend on the conflicting dummy
-# package and 'osc add' it
-# $1: oscdir (path to checked out OSC package)
-# $2: repodir (path to git repository)
-# $3: package name (e.g. "libosmocore")
-# $4: dependency package name (e.g. "osmocom-nightly")
-# $5: dependency package version (optional, e.g. "1.0.0.202101151122")
-osmo_obs_add_rpm_spec() {
- local oscdir="$1"
- local repodir="$2"
- local name="$3"
- local depend="$4"
- local dependver="$5"
- local spec_in="$(find "$repodir" -name "$name.spec.in")"
- local spec="$oscdir/$name.spec"
- local tarball
- local version
- local epoch
-
- if [ -z "$spec_in" ]; then
- echo "WARNING: RPM spec missing: $name.spec.in"
- return
- fi
-
- cp "$spec_in" "$spec"
-
- osmo_obs_add_depend_rpm "$spec" "$name" "$depend" "$dependver"
-
- # Set version and epoch from "Version: [EPOCH:]VERSION" in .dsc
- version="$(grep "^Version: " "$oscdir"/*.dsc | cut -d: -f2- | xargs)"
- case $version in
- *:*)
- epoch=$(echo "$version" | cut -d : -f 1)
- version=$(echo "$version" | cut -d : -f 2)
- ;;
- esac
- if [ -n "$epoch" ]; then
- sed -i "s/^Version:.*/Version: $version\nEpoch: $epoch/g" "$spec"
- else
- sed -i "s/^Version:.*/Version: $version/g" "$spec"
- fi
-
- # Set source file
- tarball="$(cd "$oscdir" && ls -1 "${name}_"*".tar."*)"
- sed -i "s/^Source:.*/Source: $tarball/g" "$spec"
-
- osc add "$spec"
-}
-
-# Get the path to a distribution specific patch, either from osmo-ci.git or from the project repository.
-# $PWD must be the project repository dir.
-# $1: distribution name (e.g. "debian8")
-# $2: project repository (e.g. "osmo-trx", "limesuite")
-osmo_obs_distro_specific_patch() {
- local distro="$1"
- local repo="$2"
- local ret
-
- ret="$OSMO_CI_DIR/obs-patches/$repo/build-for-$distro.patch"
- if [ -f "$ret" ]; then
- echo "$ret"
- return
- fi
-
- ret="debian/patches/build-for-$distro.patch"
- if [ -f "$ret" ]; then
- echo "$ret"
- return
- fi
-}
-
-# Check if checkout or build of a given package should be skipped, based on the
-# PACKAGES environment variable.
-# $1: package name (e.g. "libosmocore")
-osmo_obs_skip_pkg() {
- local pkgname="$1"
-
- if [ -z "$PACKAGES" ]; then
- # Don't skip
- return 1
- fi
-
- for i in "osmocom-$FEED" $PACKAGES; do
- if [ "$i" = "$pkgname" ]; then
- return 1
- fi
- done
-
- # Skip
- return 0
-}
-
-# Copy an already checked out repository dir and apply a distribution specific patch.
-# $PWD must be where all repositories are checked out in subdirs.
-# $1: distribution name (e.g. "debian8")
-# $2: project repository (e.g. "osmo-trx", "limesuite")
-osmo_obs_checkout_copy() {
- local distro="$1"
- local repo="$2"
- local patch
-
- if osmo_obs_skip_pkg "$repo"; then
- return
- fi
-
- echo
- echo "====> Checking out $repo-$distro"
-
- # Verify distro name for consistency
- local distros="
- debian8
- debian10
- "
- local found=0
- local distro_i
- for distro_i in $distros; do
- if [ "$distro_i" = "$distro" ]; then
- found=1
- break
- fi
- done
- if [ "$found" -eq 0 ]; then
- echo "ERROR: invalid distro name: $distro, should be one of: $distros"
- exit 1
- fi
-
- # Copy
- if [ -d "$repo-$distro" ]; then
- rm -rf "$repo-$distro"
- fi
- cp -a "$repo" "$repo-$distro"
- cd "$repo-$distro"
-
- # Commit patch
- patch="$(osmo_obs_distro_specific_patch "$distro" "$repo")"
- if [ -z "$patch" ]; then
- echo "ERROR: no patch found for distro=$distro, repo=$repo"
- exit 1
- fi
- patch -p1 < "$patch"
- git commit -m "auto-commit: apply $patch" debian/
- cd ..
-}
-
-# Run git-version-gen inside Osmocom repositories, so the .tarball-version
-# becomes part of the source repository. Usually this would be done with
-# "make dist", but we use git-buildpackage instead.
-osmo_obs_git_version_gen() {
- if [ -x ./git-version-gen ]; then
- ./git-version-gen . > .tarball-version 2>/dev/null
- fi
-}
-
-# Return a version based on the latest tag and commit (e.g. "1.5.1.93.47cc")
-# or fall back to the last debian version (e.g. "2.2.6"). Run
-# osmo_obs_git_version_gen before. $PWD must be inside a git repository.
-osmo_obs_get_commit_version() {
- local version=""
-
- if [ -e ".tarball-version" ]; then
- version=$(cat .tarball-version)
- # debian doesn't allow '-' in version.
- version=$(echo "$version" | sed 's/-/./g' )
- fi
-
- # deb version
- deb_version=$(head -1 debian/changelog | cut -d ' ' -f 2 | sed 's,(,,' | sed 's,),,')
- if [ -z "$version" ] || [ "$version" = "UNKNOWN" ]; then
- version="$deb_version"
- else
- # add epoch from debian/changelog
- case $deb_version in
- *:*)
- epoch=$(echo "$deb_version" | cut -d : -f 1)
- version=$epoch:$version
- ;;
- esac
- fi
-
- echo -n "$version"
-}
-
-# Verify that $FEED is in $FEEDS and $FEEDS_ALL
-osmo_obs_verify_feed() {
- local i
- local j
-
- for i in $FEEDS; do
- if [ "$i" != "$FEED" ]; then
- continue
- fi
-
- for j in $FEEDS_ALL; do
- if [ "$j" = "$i" ]; then
- return
- fi
- done
-
- echo "feed found in FEEDS but not FEEDS_ALL: $FEED"
- exit 1
- done
-
- echo "unsupported feed: $FEED"
- exit 1
-}
diff --git a/scripts/common.sh b/scripts/common.sh
index 85725f6..1081859 100644
--- a/scripts/common.sh
+++ b/scripts/common.sh
@@ -1,34 +1,48 @@
#!/bin/sh
# Various functions and variables used in multiple osmo-ci shell scripts
OSMO_CI_DIR="$(realpath "$(dirname "$0")/..")"
-OSMO_GIT_URL="https://git.osmocom.org"
+OSMO_GIT_URL_GITEA="https://gitea.osmocom.org"
OSMO_GIT_URL_GERRIT="https://gerrit.osmocom.org"
# Osmocom repositories of which we want to build release tarballs automatically, and list the current versions at
# https://jenkins.osmocom.org/jenkins/job/Osmocom-list-commits/lastSuccessfulBuild/artifact/commits.txt
+# List is ordered alphabetically.
OSMO_RELEASE_REPOS="
libasn1c
libosmo-abis
- libosmocore
+ libosmo-gprs
libosmo-netif
+ libosmo-pfcp
libosmo-sccp
+ libosmocore
libsmpp34
libusrp
osmo-bsc
osmo-bts
+ osmo-cbc
+ osmo-e1d
+ osmo-gbproxy
osmo-ggsn
+ osmo-gsm-manuals
osmo-hlr
+ osmo-hnodeb
+ osmo-hnbgw
osmo-iuh
osmo-mgw
osmo-msc
+ osmo-pcap
osmo-pcu
+ osmo-remsim
osmo-sgsn
osmo-sip-connector
+ osmo-smlc
osmo-sysmon
osmo-trx
+ osmo-uecups
+ osmo-upf
osmocom-bb
+ simtrace2
"
-#TODO: Add above once we have releases: osmo-hnodeb, osmo-gbproxy
OSMO_BRANCH_DOCKER_PLAYGROUND="${OSMO_BRANCH_DOCKER_PLAYGROUND:-master}"
@@ -38,7 +52,9 @@ OSMO_BRANCH_DOCKER_PLAYGROUND="${OSMO_BRANCH_DOCKER_PLAYGROUND:-master}"
osmo_git_head_commit() {
# git output:
# f90496f577e78944ce8db1aa5b900477c1e479b0 HEAD
- ret="$(git ls-remote "$OSMO_GIT_URL/$1" HEAD)"
+ local url ret
+ url="$(osmo_git_clone_url "$1")"
+ ret="$(git ls-remote "$url" HEAD)"
ret="$(echo "$ret" | awk '{print $1}')"
echo "$ret"
}
@@ -58,17 +74,32 @@ osmo_git_last_commits_tags() {
# ee618ecbedec82dfd240334bc87d0d1c806477b0 refs/tags/debian/0.9.13-0_jrsantos.1
# a3fdd24af099b449c9856422eb099fb45a5595df refs/tags/debian/0.9.13-0_jrsantos.1^{}
# ...
- ret="$(git ls-remote --tags "$OSMO_GIT_URL/$1")"
- ret="$(echo "$ret" | grep 'refs/tags/[0-9.]*$' || true)"
+ local project="$1"
+ local amount="$2"
+ local default_str="$3"
+ local url ret pattern
+
+ case "$project" in
+ gapk|osmo-fl2k|rtl-sdr)
+ pattern='refs/tags/v[0-9.]*$'
+ ;;
+ *)
+ pattern='refs/tags/[0-9.]*$'
+ ;;
+ esac
+
+ url="$(osmo_git_clone_url "$project")"
+ ret="$(git ls-remote --tags "$url")"
+ ret="$(echo "$ret" | grep "$pattern" || true)"
ret="$(echo "$ret" | sort -V -t/ -k3)"
- if [ "$2" != "all" ]; then
- ret="$(echo "$ret" | tail -n "$2")"
+ if [ "$amount" != "all" ]; then
+ ret="$(echo "$ret" | tail -n "$amount")"
fi
if [ -n "$ret" ]; then
echo "$ret"
else
- echo "$3"
+ echo "$default_str"
fi
}
@@ -92,23 +123,27 @@ osmo_git_last_tags() {
echo "$ret" | cut -d/ -f 3
}
-# Pass all arguments to "git clone", but write the current date and time before the clone and on failure.
-# This helps analyzing errors with git.osmocom.org (OS#4083).
-osmo_git_clone_date() {
- date "+%Y-%m-%d %H:%M:%S"
- if ! git clone "$@"; then
- date "+%Y-%m-%d %H:%M:%S"
- exit 1
- fi
-}
-
-# Print git clone URL for an Osmocom git repository. Prefer the gerrit clone URL, because cloning from the regular URL
-# sometimes results in the "garbage at end of loose object" error (OS#4083).
+# Echo git clone URL for an Osmocom git repository. For projects developed on
+# gerrit, use the gerrit URL to avoid the mirror sync delay, for other
+# repositories use the gitea URL.
+# https://osmocom.org/projects/cellular-infrastructure/wiki/Git_infrastructure
# $1: Osmocom project (e.g. "osmo-hlr")
osmo_git_clone_url() {
case "$1" in
- rtl-sdr|osmo-fl2k|libosmo-dsp|libgtpnl|libasn1c|libusrp|libsmpp34)
- echo "$OSMO_GIT_URL"/"$1"
+ libgtpnl|libasn1c|libsmpp34)
+ echo "$OSMO_GIT_URL_GITEA"/cellular-infrastructure/"$1"
+ ;;
+ rtl-sdr|osmo-fl2k|libosmo-dsp|libusrp)
+ echo "$OSMO_GIT_URL_GITEA"/sdr/"$1"
+ ;;
+ osmo-gmr)
+ echo "$OSMO_GIT_URL_GITEA"/satellite/"$1"
+ ;;
+ osmo-isdntap)
+ echo "$OSMO_GIT_URL_GITEA"/retronetworking/"$1"
+ ;;
+ osmo_dia2gsup)
+ echo "$OSMO_GIT_URL_GERRIT"/erlang/"$1"
;;
*)
echo "$OSMO_GIT_URL_GERRIT"/"$1"
@@ -142,7 +177,7 @@ docker_images_require() {
if [ -d "_docker_playground" ]; then
git -C _docker_playground fetch
else
- git clone https://git.osmocom.org/docker-playground/ _docker_playground
+ git clone https://gerrit.osmocom.org/docker-playground/ _docker_playground
fi
cd _docker_playground
diff --git a/scripts/docker-cleanup.sh b/scripts/docker-cleanup.sh
index e7c7d1b..982ccea 100755
--- a/scripts/docker-cleanup.sh
+++ b/scripts/docker-cleanup.sh
@@ -1,9 +1,46 @@
-#!/bin/sh
+#!/bin/sh -x
+# https://osmocom.org/projects/osmocom-servers/wiki/Docker_cache_clean_up
-# delete all but the latest images
-IMAGES=`docker image ls | grep \^osmocom-build | grep -v latest | awk -F ' ' '{print $1":"$2}'`
-for f in $IMAGES; do
- docker image rm $f
-done
+kill_docker_containers_running_longer_than_24h() {
+ docker ps
+ set +x
-docker image prune -f
+ local date_24h_ago="$(date "+%s" -d"24 hours ago")"
+ docker ps --format "{{.ID}}|{{.Names}}|{{.CreatedAt}}" | while read -r line; do
+ local id="$(echo "$line" | cut -d '|' -f 1)"
+ local name="$(echo "$line" | cut -d '|' -f 2)"
+ local created_at="$(echo "$line" | cut -d '|' -f 3 | cut -d ' ' -f 1-3)"
+ local date_created_at="$(date "+%s" -d "$created_at")"
+
+ if [ "$date_created_at" -gt "$date_24h_ago" ]; then
+ echo "$name: not running for >24h"
+ continue
+ fi
+
+ case "$name" in
+ jenkins-*|*ttcn3*|osmo-gsm-tester*) ;;
+ *)
+ echo "$name: does not match name pattern"
+ continue
+ ;;
+ esac
+
+ echo "$name ($id): has been running for >24h, killing"
+ docker kill "$id"
+ done
+
+ set -x
+ docker ps
+}
+
+kill_docker_containers_running_longer_than_24h
+
+# delete all containers where we forgot to use --rm with docker run,
+# older than 24 hours
+docker container prune --filter "until=24h" -f
+
+# remove unused networks older than 24 hours
+docker network prune --filter "until=24h" -f
+
+# remove docker buildkit cache
+docker builder prune --all --filter "until=24h" -f
diff --git a/scripts/jenkins-gerrit/comment_generate.py b/scripts/jenkins-gerrit/comment_generate.py
new file mode 100755
index 0000000..e127c4c
--- /dev/null
+++ b/scripts/jenkins-gerrit/comment_generate.py
@@ -0,0 +1,260 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import argparse
+import io
+import json
+import re
+import urllib.request
+
+jenkins_url = "https://jenkins.osmocom.org"
+re_start_build = re.compile("Starting building: gerrit-[a-zA-Z-_0-9]* #[0-9]*")
+re_result = re.compile("^pipeline_([a-zA-Z-_0-9:]*): (SUCCESS|FAILED)$")
+re_job_type = re.compile("JOB_TYPE=([a-zA-Z-_0-9]*),")
+re_distro = re.compile("Building binary packages for distro: '([a-zA-Z0-9:].*)'")
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(
+ description="Prepare a comment to be submitted to gerrit. Depending on"
+ " the comment type, (start) either a link to the pipeline,"
+ " or (result) a summary of failed / successful builds from"
+ " the pipeline we run for patches submitted to gerrit.")
+ parser.add_argument("build_url",
+ help="$BUILD_URL of the pipeline job, e.g."
+ " https://jenkins.osmocom.org/jenkins/job/gerrit-osmo-bsc-nat/17/")
+ parser.add_argument("-o", "--output", help="output json file")
+ parser.add_argument("-t", "--type", help="comment type",
+ choices=["start", "result"], required=True)
+ parser.add_argument("-n", "--notify-on-success", action="store_true",
+ help="always indicate in json that the owner should be"
+ " notified via mail, not only on failure")
+ return parser.parse_args()
+
+
+def stage_binpkgs_from_url(job_url):
+ """ Multiple gerrit-binpkgs jobs may be started to build binary packages
+ for multiple distributions. It is not clear from the job name / URL of
+ a job for which distro it is building, so read it from the log output.
+ :returns: a distro like "debian:12" """
+ global re_distro
+
+ url = f"{job_url}/consoleText"
+ with urllib.request.urlopen(url) as response:
+ content = response.read().decode("utf-8")
+ match = re_distro.search(content)
+ assert match, f"couldn't find distro name in log: {url}"
+ return match.group(1)
+
+
+def stage_from_job_name(job_name, job_url):
+ if job_name == "gerrit-verifications-comment":
+ # The job that runs this script. Don't include it in the summary.
+ return None
+ if job_name == "gerrit-lint":
+ return "lint"
+ if job_name == "gerrit-binpkgs":
+ return stage_binpkgs_from_url(job_url)
+ if job_name == "gerrit-pipeline-endianness":
+ return "endianness"
+ if job_name.endswith("-build"):
+ return "build"
+ assert False, f"couldn't figure out stage from job_name: {job_name}"
+
+
+def parse_pipeline(build_url):
+ """ Parse started jobs and result from the pipeline log.
+ :returns: a dict that looks like:
+ {"build": {"name": "gerrit-osmo-bsc-nat-build", id=7,
+ "passed": True, "url": "https://..."},
+ "lint": {...},
+ "deb": {...},
+ "rpm: {...}} """
+ global re_start_build
+ global re_result
+ global jenkins_url
+ ret = {}
+
+ url = f"{build_url}/consoleText"
+ with urllib.request.urlopen(url) as response:
+ for line in io.TextIOWrapper(response, encoding='utf-8'):
+ # Parse start build lines
+ for match in re_start_build.findall(line):
+ job_name = match.split(" ")[2]
+ job_id = int(match.split(" ")[3].replace("#", ""))
+ job_url = f"{jenkins_url}/jenkins/job/{job_name}/{job_id}"
+ stage = stage_from_job_name(job_name, job_url)
+ if stage:
+ ret[stage] = {"url": job_url, "name": job_name, "id": job_id}
+
+ # Parse result lines
+ match = re_result.match(line)
+ if match:
+ stage = match.group(1)
+ if stage.startswith("comment_"):
+ # Jobs that run this script, not relevant for summary
+ continue
+ assert stage in ret, f"found result for stage {stage}, but" \
+ " didn't find where it was started. The" \
+ " re_start_build regex probably needs to be adjusted" \
+ " to match the related gerrit-*-build job.\n\n" \
+ f"ret: {ret}"
+ ret[stage]["passed"] = (match.group(2) == "SUCCESS")
+
+ return ret
+
+
+def parse_build_matrix(job):
+ """ Parse started jobs and result from the matrix of the build job. Usually
+ it is only one job, but for some projects we build for multiple arches
+ (x86_64, arm) or build multiple times with different configure flags.
+ :param job: "build" dict from parse_pipeline()
+ :returns: a list of jobs in the matrix, looks like:
+ [{"passed": True, "url": "https://..."}, ...]
+ """
+ global jenkins_url
+
+ ret = []
+ url = f"{job['url']}/consoleFull"
+ with urllib.request.urlopen(url) as response:
+ for line in io.TextIOWrapper(response, encoding='utf-8'):
+ if " completed with result " in line:
+ url = line.split("<a href='", 1)[1].split("'", 1)[0]
+ url = f"{jenkins_url}{url}{job['id']}"
+ result = line.split(" completed with result ")[1].rstrip()
+ passed = result == "SUCCESS"
+ ret += [{"passed": passed, "url": url}]
+ return ret
+
+
+def jobs_for_summary(pipeline, build_matrix):
+ """ Sort the jobs from pipeline and build matrix into passed/failed lists.
+ :returns: a dict that looks like:
+ {"passed": [{"stage": "build", "url": "https://..."}, ...],
+ "failed": [...]} """
+ ret = {"passed": [], "failed": []}
+
+ # Build errors are most interesting, display them first
+ for job in build_matrix:
+ category = "passed" if job["passed"] else "failed"
+ ret[category] += [{"stage": "build", "url": job["url"]}]
+
+ # Hide the build matrix job (we show the jobs started by it instead), as
+ # long as there is at least one failed started job when the matrix failed
+ matrix_failed = "build" in pipeline and not pipeline["build"]["passed"]
+ show_build_matrix_job = matrix_failed and not ret["failed"]
+
+ # Add jobs from the pipeline
+ for stage, job in pipeline.items():
+ if stage == "build" and not show_build_matrix_job:
+ continue
+ category = "passed" if job["passed"] else "failed"
+ ret[category] += [{"stage": stage, "url": job["url"]}]
+
+ return ret
+
+
+def get_job_short_name(job):
+ """ :returns: a short job name, usually the stage (lint, deb, rpm, build).
+ Or in case of build a more useful name like the JOB_TYPE part
+ of the URL if it is found. For osmo-e1-hardware it could be
+ one of: manuals, gateware, firmware, software """
+ global re_job_type
+ stage = job["stage"]
+
+ if stage == "build":
+ match = re_job_type.search(job["url"])
+ if match:
+ return match.group(1)
+
+ return stage
+
+
+def get_jobs_list_str(jobs):
+ lines = []
+ for job in jobs:
+ lines += [f"* [{get_job_short_name(job)}] {job['url']}/consoleFull\n"]
+ return "".join(sorted(lines))
+
+
+def get_comment_result(build_url, notify_on_success):
+ """ Generate a summary of failed and successful builds for gerrit.
+ :returns: a dict that is expected by gerrit's set-review api, e.g.
+ {"tag": "jenkins",
+ "message": "...",
+ "labels": {"Code-Review": -1},
+ "notify": "OWNER"} """
+ summary = ""
+ pipeline = parse_pipeline(build_url)
+
+ build_matrix = []
+ if "build" in pipeline:
+ build_matrix = parse_build_matrix(pipeline["build"])
+
+ jobs = jobs_for_summary(pipeline, build_matrix)
+
+ if jobs["failed"]:
+ summary += f"{len(jobs['failed'])} failed:\n"
+ summary += get_jobs_list_str(jobs["failed"])
+ summary += "\n"
+
+ summary += f"{len(jobs['passed'])} passed:\n"
+ summary += get_jobs_list_str(jobs["passed"])
+
+ if "build" in pipeline and "deb" in pipeline and "rpm" in pipeline and \
+ not pipeline["build"]["passed"] and pipeline["deb"]["passed"] \
+ and pipeline["rpm"]["passed"]:
+ summary += "\n"
+ summary += "The build job(s) failed, but deb/rpm jobs passed.\n"
+ summary += "We don't enable external/vty tests when building\n"
+ summary += "packages, so maybe those failed. Check the logs.\n"
+
+ if "lint" in pipeline and not pipeline["lint"]["passed"]:
+ summary += "\n"
+ summary += "Please fix the linting errors. More information:\n"
+ summary += "https://osmocom.org/projects/cellular-infrastructure/wiki/Linting\n"
+
+ summary += "\n"
+ if jobs["failed"]:
+ summary += "Build Failed\n"
+ summary += "\n"
+ summary += f"Find the Retrigger button here:\n{build_url}\n"
+ vote = -1
+ notify = "OWNER"
+ else:
+ summary += "Build Successful\n"
+ vote = 1
+ notify = "OWNER" if notify_on_success else "NONE"
+
+ # Reference:
+ # https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#set-review
+ # https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#review-input
+ return {"tag": "jenkins",
+ "message": summary,
+ "labels": {"Verified": vote},
+ "notify": notify}
+
+
+def get_comment_start(build_url):
+ return {"tag": "jenkins",
+ "message": f"Build Started\n{build_url}consoleFull",
+ "notify": "NONE"}
+
+
+def main():
+ args = parse_args()
+ if args.type == "result":
+ comment = get_comment_result(args.build_url, args.notify_on_success)
+ else:
+ comment = get_comment_start(args.build_url)
+
+ print()
+ print(comment["message"])
+ print(f"notify: {comment['notify']}")
+
+ if args.output:
+ with open(args.output, "w") as handle:
+ json.dump(comment, handle, indent=4)
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/jenkins-gerrit/comment_send.sh b/scripts/jenkins-gerrit/comment_send.sh
new file mode 100755
index 0000000..24fec27
--- /dev/null
+++ b/scripts/jenkins-gerrit/comment_send.sh
@@ -0,0 +1,33 @@
+#!/bin/sh -e
+
+# By default, a mail notification will only be sent if the gerrit verification
+# failed. Add yourself here to also receive notifications on successs.
+notify_on_success_users="
+ pespin
+"
+
+arg_notify=""
+for i in $notify_on_success_users; do
+ if [ "$GERRIT_PATCHSET_UPLOADER_NAME" = "$i" ]; then
+ arg_notify="--notify-on-success"
+ break
+ fi
+done
+
+set -x
+
+./comment_generate.py "$PIPELINE_BUILD_URL" \
+ -o gerrit_report.json \
+ -t "$COMMENT_TYPE" \
+ $arg_notify
+
+ssh \
+ -p "$GERRIT_PORT" \
+ -l jenkins \
+ "$GERRIT_HOST" \
+ gerrit \
+ review \
+ --project "$GERRIT_PROJECT" \
+ "$GERRIT_CHANGE_NUMBER,$GERRIT_PATCHSET_NUMBER" \
+ --json \
+ < gerrit_report.json
diff --git a/scripts/manuals/0001-build-custom-dblatex.sty-remove-DRAFT.patch b/scripts/manuals/0001-build-custom-dblatex.sty-remove-DRAFT.patch
new file mode 100644
index 0000000..2addeb4
--- /dev/null
+++ b/scripts/manuals/0001-build-custom-dblatex.sty-remove-DRAFT.patch
@@ -0,0 +1,32 @@
+From 92a645d74fde8b86ab0a8f6e3d089a0f1edc34d0 Mon Sep 17 00:00:00 2001
+From: Oliver Smith <osmith@sysmocom.de>
+Date: Wed, 8 Mar 2023 17:48:17 +0100
+Subject: [PATCH] build/custom-dblatex.sty: remove DRAFT
+
+This patch will be used when generating manuals for the tagged
+releases, so they don't have the draft watermark.
+
+Related: OS#5902
+Change-Id: Id9a27b01412b0e615638f20fa5e36f5b114a15d8
+---
+ build/custom-dblatex.sty | 4 ----
+ 1 file changed, 4 deletions(-)
+
+diff --git a/build/custom-dblatex.sty b/build/custom-dblatex.sty
+index 16931ad..0d53bb1 100644
+--- a/build/custom-dblatex.sty
++++ b/build/custom-dblatex.sty
+@@ -20,10 +20,6 @@
+ \usepackage{alltt}
+ \usepackage{upquote}
+
+-% "DRAFT" on first page
+-\definecolor{LtGrey}{rgb}{0.875,0.875,0.875}
+-\AddToShipoutPicture*{ \AtTextCenter{ \makebox(0,0)[c]{\resizebox{\textwidth}{!}{ \rotatebox{45}{\textsf{\textbf{\color{LtGrey}DRAFT}}}}} } }
+-
+ \def\Company{sysmocom - s.f.m.c. GmbH}
+
+ \def\DBKcover{
+--
+2.34.1
+
diff --git a/scripts/manuals/publish-manuals-for-tags.sh b/scripts/manuals/publish-manuals-for-tags.sh
new file mode 100755
index 0000000..3f95ce7
--- /dev/null
+++ b/scripts/manuals/publish-manuals-for-tags.sh
@@ -0,0 +1,434 @@
+#!/bin/sh -e
+. "$(dirname "$0")/../common.sh"
+OSMO_CI_DIR="$(realpath $(dirname "$0")/../..)"
+TEMP="$OSMO_CI_DIR/_temp_manuals"
+WEB_PATH="/downloads/home/docs/web-files"
+SSH_COMMAND="ssh -o UserKnownHostsFile=$TEMP/src/osmo-gsm-manuals/build/known_hosts -p 48"
+DOCKER_IMAGE="$USER/debian-bookworm-build"
+LOG_PREFIX="::"
+
+# Releases that were made before shared osmo-gsm-manuals, or where build fails
+# for other reasons.
+TAGS_IGNORE="
+ openbsc:0.9.0,
+ openbsc:0.9.1,
+ openbsc:0.9.2,
+ openbsc:0.9.3,
+ openbsc:0.9.4,
+ openbsc:0.9.5,
+ openbsc:0.9.6,
+ openbsc:0.9.8,
+ openbsc:0.9.9,
+ openbsc:0.9.10,
+ openbsc:0.9.11,
+ openbsc:0.9.12,
+ openbsc:0.9.13,
+ openbsc:0.9.14,
+ openbsc:0.9.15,
+ openbsc:0.9.16,
+ openbsc:0.10.0,
+ openbsc:0.10.1,
+ openbsc:0.11.0,
+ openbsc:0.12.0,
+ openbsc:0.13.0,
+ openbsc:0.14.0,
+ openbsc:0.15.0,
+ openbsc:1.0.0,
+ openbsc:1.1.0,
+
+ osmo-bsc:1.0.1,
+ osmo-bsc:1.1.0,
+ osmo-bsc:1.1.1,
+ osmo-bsc:1.1.2,
+ osmo-bsc:1.2.0,
+ osmo-bsc:1.2.1,
+ osmo-bsc:1.2.2,
+ osmo-bsc:1.3.0,
+ osmo-bsc:1.4.0,
+ osmo-bsc:1.4.1,
+ osmo-bsc:1.5.0,
+ osmo-bsc:1.6.0,
+
+ osmo-bts:0.0.1,
+ osmo-bts:0.1.0,
+ osmo-bts:0.2.0,
+ osmo-bts:0.3.0,
+ osmo-bts:0.4.0,
+ osmo-bts:0.6.0,
+ osmo-bts:0.7.0,
+ osmo-bts:0.8.0,
+ osmo-bts:0.8.1,
+ osmo-bts:1.0.0,
+ osmo-bts:1.0.1,
+ osmo-bts:1.1.0,
+ osmo-bts:1.2.0,
+
+ osmo-e1d:0.0.1,
+ osmo-e1d:0.1.0,
+ osmo-e1d:0.1.1,
+ osmo-e1d:0.2.0,
+ osmo-e1d:0.2.1,
+ osmo-e1d:0.2.2,
+ osmo-e1d:0.3.0,
+ osmo-e1d:0.4.0,
+
+ osmo-ggsn:1.0.0,
+ osmo-ggsn:1.1.0,
+ osmo-ggsn:1.2.0,
+ osmo-ggsn:1.2.1,
+ osmo-ggsn:1.2.2,
+
+ osmo-gsm-tester:0.1,
+
+ osmo-hlr:0.0.1,
+ osmo-hlr:0.1.0,
+ osmo-hlr:0.2.0,
+ osmo-hlr:0.2.1,
+
+ osmo-mgw:1.0.1,
+ osmo-mgw:1.0.2,
+ osmo-mgw:1.1.0,
+ osmo-mgw:1.2.0,
+ osmo-mgw:1.2.1,
+ osmo-mgw:1.3.0,
+ osmo-mgw:1.4.0,
+
+ osmo-msc:1.0.1,
+ osmo-msc:1.1.0,
+ osmo-msc:1.1.1,
+ osmo-msc:1.1.2,
+ osmo-msc:1.2.0,
+ osmo-msc:1.3.0,
+ osmo-msc:1.3.1,
+ osmo-msc:1.4.0,
+ osmo-msc:1.5.0,
+ osmo-msc:1.6.0,
+ osmo-msc:1.6.1,
+ osmo-msc:1.6.2,
+ osmo-msc:1.6.3,
+
+ osmo-pcap:0.0.1,
+ osmo-pcap:0.0.2,
+ osmo-pcap:0.0.3,
+ osmo-pcap:0.0.4,
+ osmo-pcap:0.0.5,
+ osmo-pcap:0.0.6,
+ osmo-pcap:0.0.7,
+ osmo-pcap:0.0.8,
+ osmo-pcap:0.0.9,
+ osmo-pcap:0.0.10,
+ osmo-pcap:0.0.11,
+ osmo-pcap:0.1.0,
+ osmo-pcap:0.1.1,
+ osmo-pcap:0.1.2,
+ osmo-pcap:0.1.3,
+
+ osmo-pcu:0.1,
+ osmo-pcu:0.1.0,
+ osmo-pcu:0.2,
+ osmo-pcu:0.2.0,
+ osmo-pcu:0.4.0,
+ osmo-pcu:0.5.0,
+ osmo-pcu:0.5.1,
+
+ osmo-remsim:0.0,
+ osmo-remsim:0.1.0,
+ osmo-remsim:0.2.0,
+ osmo-remsim:0.2.1,
+
+ osmo-sgsn:0.10.0,
+ osmo-sgsn:0.10.1,
+ osmo-sgsn:0.11.0,
+ osmo-sgsn:0.12.0,
+ osmo-sgsn:0.13.0,
+ osmo-sgsn:0.14.0,
+ osmo-sgsn:0.15.0,
+ osmo-sgsn:0.9.0,
+ osmo-sgsn:0.9.1,
+ osmo-sgsn:0.9.10,
+ osmo-sgsn:0.9.11,
+ osmo-sgsn:0.9.12,
+ osmo-sgsn:0.9.13,
+ osmo-sgsn:0.9.13+deb1,
+ osmo-sgsn:0.9.14,
+ osmo-sgsn:0.9.14-onwaves1,
+ osmo-sgsn:0.9.15,
+ osmo-sgsn:0.9.16,
+ osmo-sgsn:0.9.2,
+ osmo-sgsn:0.9.3,
+ osmo-sgsn:0.9.4,
+ osmo-sgsn:0.9.5,
+ osmo-sgsn:0.9.6,
+ osmo-sgsn:0.9.8,
+ osmo-sgsn:0.9.9,
+ osmo-sgsn:1.0.1,
+ osmo-sgsn:1.1.0,
+ osmo-sgsn:1.10.0,
+ osmo-sgsn:1.2.0,
+ osmo-sgsn:1.3.0,
+ osmo-sgsn:1.4.0,
+ osmo-sgsn:1.4.1,
+ osmo-sgsn:1.5.0,
+ osmo-sgsn:1.6.0,
+ osmo-sgsn:1.6.1,
+
+ osmo-sip-connector:0.0.1,
+ osmo-sip-connector:1.1.0,
+ osmo-sip-connector:1.1.1,
+
+ libosmo-sccp:0.0.1,
+ libosmo-sccp:0.0.2,
+ libosmo-sccp:0.0.3,
+ libosmo-sccp:0.0.4,
+ libosmo-sccp:0.0.5,
+ libosmo-sccp:0.0.5.1,
+ libosmo-sccp:0.0.6,
+ libosmo-sccp:0.0.6.1,
+ libosmo-sccp:0.0.6.2,
+ libosmo-sccp:0.0.6.3,
+ libosmo-sccp:0.10.0,
+ libosmo-sccp:0.7.0,
+ libosmo-sccp:0.8.0,
+ libosmo-sccp:0.8.1,
+ libosmo-sccp:0.9.0,
+ libosmo-sccp:1.0.0,
+ libosmo-sccp:1.1.0,
+ libosmo-sccp:1.2.0,
+
+ osmo-trx:0.2.0,
+ osmo-trx:0.3.0,
+ osmo-trx:0.4.0,
+
+ pysim:1.0,
+"
+
+mkdir -p \
+ "$TEMP" \
+ "$TEMP/src"
+
+check_ssh_auth_sock() {
+ if [ -z "$SSH_AUTH_SOCK" ]; then
+ echo "ERROR: SSH_AUTH_SOCK is not set"
+ exit 1
+ fi
+}
+
+# Additional configure options to use, so manuals include all VTY commands
+# $1: repo name
+get_configure_opts_from_repo_name() {
+ case "$1" in
+ osmo-hnbgw)
+ echo "--enable-pfcp"
+ ;;
+ osmo-msc|osmo-sgsn)
+ echo "--enable-iu"
+ ;;
+ esac
+}
+
+# $1: docs dir
+get_repo_name_from_docs_dir() {
+ case "$1" in
+ osmo-stp)
+ echo "libosmo-sccp"
+ ;;
+ *)
+ echo "$1"
+ ;;
+ esac
+}
+
+# $1: repo name
+get_docs_dir_from_repo_name() {
+ case "$1" in
+ libosmo-sccp)
+ echo "osmo-stp"
+ ;;
+ *)
+ echo "$1"
+ ;;
+ esac
+}
+
+
+# $1: path on server, e.g. "/docs/osmo-bsc"
+get_server_ls() {
+ local dir="$1"
+ local out="$TEMP/ls$(echo "$dir" | tr / _)"
+
+ echo "$LOG_PREFIX Listing files on server: $dir"
+
+ if [ -e "$out" ]; then
+ echo "Skipped, file exists: $out"
+ return
+ fi
+
+ dir="$(echo "$dir" | sed "s.^/docs.$WEB_PATH.")"
+ $SSH_COMMAND docs@ftp.osmocom.org "ls -1 $dir" >"$out"
+}
+
+# $1: repository
+get_git_tags() {
+ local repo="$1"
+ local out="$TEMP/git_tags_$repo"
+
+ echo "$LOG_PREFIX Getting git tags"
+
+ if [ -e "$out" ]; then
+ echo "Skipped, file exists: $out"
+ return
+ fi
+
+ osmo_git_last_tags "$repo" "all" >"$out"
+}
+
+# $1: docs dir
+# $2: tag
+manuals_exist() {
+ local docs_dir="$1"
+ local tag="$2"
+
+ grep -q "^$tag$" "$TEMP"/ls_docs_"$docs_dir"
+}
+
+# $1: repository
+# $2: tag
+is_tag_ignored() {
+ local repo="$1"
+ local tag="$2"
+
+ case "$TAGS_IGNORE" in
+ *"$repo:$tag,"*)
+ return 0
+ ;;
+ esac
+
+ return 1
+}
+
+# $1: repository
+# $2: tag
+clone_repo() {
+ local repo="$1"
+ local tag="$2"
+ local gitdir="$TEMP/src/$repo"
+
+ if ! [ -d "$gitdir" ]; then
+ local url="$(osmo_git_clone_url "$repo")"
+ echo "$LOG_PREFIX Cloning $url"
+ git -C "$TEMP/src" clone "$url" "$repo"
+ fi
+
+ echo "$LOG_PREFIX Checkout $tag"
+ cd "$gitdir"
+ git reset --hard HEAD
+ git checkout "$tag"
+ git submodule update --init
+ git clean -dxf
+
+ # Fix depends on packages that don't exist anymore
+ sed -i 's/dh-systemd \(.*\),//g' debian/control
+ sed -i 's/python-minimal,//g' debian/control
+}
+
+# $1: repository
+# $2: tag
+build_publish_manuals() {
+ local repo="$1"
+ local tag="$2"
+ local configure_opts="--enable-manuals $(get_configure_opts_from_repo_name "$repo")"
+ echo "$LOG_PREFIX Building manuals"
+
+ if ! docker run \
+ --rm \
+ -e "BUILD_RELEASE=1" \
+ -e "DEBIAN_FRONTEND=noninteractive" \
+ -e "OSMO_GSM_MANUALS_DIR=/opt/osmo-gsm-manuals" \
+ -e "OSMO_REPOSITORY=$(get_docs_dir_from_repo_name "$repo")" \
+ -e "PUBLISH_REF=$tag" \
+ -e "SSH_AUTH_SOCK=/ssh-agent" \
+ -v "$OSMO_CI_DIR/scripts/manuals:/manuals" \
+ -v "$TEMP/src/$repo/:/build" \
+ -v $(readlink -f $SSH_AUTH_SOCK):/ssh-agent \
+ "$DOCKER_IMAGE" \
+ sh -ex -c "
+ apt-get update
+
+ # The docker image has the nightly repository
+ # configured, in which packages can't be installed from
+ # different build dates. Upgrade osmocom-nightly first
+ # to prevent errors in apt-get build-dep below.
+ apt-get -y upgrade osmocom-nightly
+
+ # Install dependencies
+ case $repo in
+ *)
+ apt-get -y build-dep /build
+ ;;
+ esac
+
+ # Remove DRAFT in osmo-gsm-manuals
+ cd /opt/osmo-gsm-manuals/
+ patch -p1 < /manuals/0001-build-custom-dblatex.sty-remove-DRAFT.patch
+
+ # Build manuals
+ cd /build
+ case $repo in
+ openbsc)
+ for dir in manuals/*/; do
+ su build -c \"make -C \$dir\"
+ done
+ ;;
+ *)
+ su build -c \"autoreconf -fi\"
+ su build -c \"./configure $configure_opts\"
+ su build -c \"make -j$(nproc)\"
+ ;;
+ esac
+
+
+ # Publish manuals
+ case $repo in
+ openbsc)
+ for dir in manuals/*/; do
+ su build -c \"make -C \$dir publish\"
+ done
+ ;;
+ *)
+ su build -c \"make -C doc/manuals publish\"
+ ;;
+ esac
+ "; then
+ echo "$LOG_PREFIX Building manuals failed!"
+ exit 1
+ fi
+}
+
+check_ssh_auth_sock
+
+# Get the UserKnownHostsFile for $SSH_COMMAND
+clone_repo osmo-gsm-manuals master
+
+get_server_ls "/docs"
+
+for docs_dir in $(cat "$TEMP"/ls_docs); do
+ repo="$(get_repo_name_from_docs_dir "$docs_dir")"
+ LOG_PREFIX=":: ($repo)"
+ get_server_ls "/docs/$docs_dir"
+ get_git_tags "$repo"
+
+ echo "$LOG_PREFIX Building missing manuals"
+ for tag in $(cat "$TEMP"/git_tags_"$repo"); do
+ LOG_PREFIX=":: ($repo, $tag)"
+ if manuals_exist "$docs_dir" "$tag"; then
+ echo "$LOG_PREFIX: skipping, manuals exist"
+ continue
+ elif is_tag_ignored "$repo" "$tag"; then
+ echo "$LOG_PREFIX: skipping, tag is ignored"
+ continue
+ fi
+
+ clone_repo "$repo" "$tag"
+ build_publish_manuals "$repo" "$tag"
+ done
+done
diff --git a/scripts/obs/README b/scripts/obs/README
new file mode 100644
index 0000000..cf8e201
--- /dev/null
+++ b/scripts/obs/README
@@ -0,0 +1,141 @@
+Overview
+========
+This directory contains the following scripts, related to building Osmocom
+projects in OBS (Open Build Service) and building binary packages. Here is an
+overview of the scripts, run them with -h to get a more detailed description.
+
+* build_srcpkg.py: build one source package for an Osmocom project
+* update_obs_project.py: generate source packages and upload them to OBS
+* build_binpkg.py: build rpm/deb packages for one Osmocom project
+* sync_obs_projects.py: sync projects from another instance (OS#6165)
+
+Docker
+------
+The above scripts have a -d parameter, that allows running them inside docker.
+This is how we run it in jenkins to avoid installing dependencies on the host.
+
+Note that when building binary packages, libosmocore socket tests and possibly
+others will fail unless docker has ipv6 configured:
+https://osmocom.org/issues/4700#note-3
+
+Usage Example: Submitting source packages to Osmocom's OBS build server
+=======================================================================
+
+I want to test changes to the packaging of osmo-hnbgw.
+They are committed on a private branch osmo-hnbgw.git:neels/pkg.
+I want to test this in my OBS "Home Project" called home:nhofmeyr:test.
+
+Here are the steps of what I do:
+
+
+OBS home project
+----------------
+
+Sign up / sign in to obs.osmocom.org and create the "test" project under the
+"Home Project" link (right next to the "Logout" link on the OBS web interface),
+so that https://obs.osmocom.org/project/show/home:nhofmeyr:test exists.
+
+Make sure I can list the project using the 'osc' tool.
+That requires an osc config file. Easiest is to let osc create one:
+
+ $ osc -A https://obs.osmocom.org list home:nhofmeyr:test
+ Username: nhofmeyr
+ Password: ************
+ Select credentials manager: 4
+
+Verify that it worked:
+
+ $ osc list home:nhofmeyr:test
+ libosmo-pfcp
+ osmo-hnbgw
+
+FYI, the config file (oscrc) will look like this:
+
+ [general]
+ apiurl = https://obs.osmocom.org
+
+ [https://obs.osmocom.org]
+ user=nhofmeyr
+ pass=***********
+ credentials_mgr_class=osc.credentials.PlaintextConfigFileCredentialsManager
+
+
+Publish patches in private branch
+---------------------------------
+
+(Optional: bypassing gerrit.osmocom.org explained in next section below.)
+
+Push my private branch to Osmocom's git repository at gerrit.osmocom.org -- not
+submit for review, just push a private branch.
+
+Why is that? The obs scripts here potentially maim a git tree, so it uses a
+separate git clone, which is cloned from gerrit.osmocom.org. When my private
+branch is pushed there, I can trivially use it.
+
+ cd ~/osmo-dev/src/osmo-hnbgw
+ git push --set-upstream origin neels/pkg
+
+
+Optional: fetch from local git repos
+------------------------------------
+
+Instead of using gerrit.osmocom.org, I can change the config of a locally
+cached repository, so that the branch is fetched from my local working copy.
+That is useful if I want to avoid pushing my branch upstream.
+
+Create initial git clone in _cache/:
+
+ cd ~/osmo-dev/src/osmo-ci/scripts/obs/
+ ./build_srcpkg.py osmo-hnbgw
+
+Set the 'origin' of the './_cache/osmo-hnbgw' git clone to my local working
+copy in '~/osmo-dev/src/osmo-hnbgw':
+
+ git -C _cache/osmo-hnbgw remote set-url origin '~/osmo-dev/src/osmo-hnbgw'
+
+After this, no need to push to Osmocom's git, i just commit patches on my local
+branch in my git clone in '~/osmo-dev/src/osmo-hnbgw'. The obs script will
+fetch my local branch, known as 'origin/neels/pkg' in its cached git tree.
+
+Verify:
+
+ $ git -C _cache/osmo-hnbgw remote -v
+ origin ~/osmo-dev/src/osmo-hnbgw (fetch)
+ origin ~/osmo-dev/src/osmo-hnbgw (push)
+
+
+Build and upload source package to OBS
+--------------------------------------
+
+ cd ~/osmo-dev/src/osmo-ci/scripts/obs/
+ ./update_obs_project.py -b neels/pkg home:nhofmeyr:test osmo-hnbgw
+
+The -b option chooses a custom branch to build.
+
+
+See results
+-----------
+
+I can now see my hnbgw package listed:
+
+ $ osc list home:nhofmeyr:test
+ libosmo-pfcp
+ osmo-hnbgw
+
+I could query things via the osc tool:
+
+ $ osc results home:nhofmeyr:test osmo-hnbgw
+ neels_test2 x86_64 unresolvable
+ neels_test x86_64 failed
+
+Or point my web browser at
+https://obs.osmocom.org/project/show/home:nhofmeyr:test
+
+
+Repeat
+------
+
+The dev cycle of rebuilding a change is:
+
+* Commit changes on private branch (and push to Osmocom's git server if necessary),
+* Re-run './update_obs_project.py -g -b origin/...' as above.
diff --git a/scripts/obs/build_binpkg.py b/scripts/obs/build_binpkg.py
new file mode 100755
index 0000000..ae2fe53
--- /dev/null
+++ b/scripts/obs/build_binpkg.py
@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import argparse
+import fnmatch
+import lib
+import multiprocessing
+import os
+import sys
+import lib.binpkg_deb
+import lib.config
+import lib.docker
+import lib.git
+import lib.metapkg
+import lib.srcpkg
+
+
+def arg_type_docker_distro(arg):
+ for pattern in lib.config.docker_distro_other:
+ if fnmatch.fnmatch(arg, pattern):
+ return arg
+ raise ValueError
+
+
+def main():
+ distro_default = lib.config.docker_distro_default
+ jobs_default = multiprocessing.cpu_count() + 1
+
+ parser = argparse.ArgumentParser(
+ description="Build a deb or rpm package as it would be done on"
+ " obs.osmocom.org. Use after building a source package"
+ " with build_srcpkg.py."
+ f" Output dir: {lib.config.path_temp}/binpkgs")
+ parser.add_argument("-d", "--docker", type=arg_type_docker_distro,
+ const=distro_default, nargs="?", metavar="DISTRO",
+ help="build the package in docker for a specific"
+ f" distro (default: {distro_default}, other:"
+ f" almalinux:8, debian:10, ubuntu:22.04 etc.)")
+ parser.add_argument("-f", "--feed", dest="docker_feed", default="master",
+ choices=["master", "nightly", "latest"],
+ help="the OBS feed to configure inside docker, against"
+ " which the package will get built (use nightly"
+ " if master doesn't get built for DISTRO)")
+ parser.add_argument("-j", "--jobs", type=int, default=jobs_default,
+ help=f"parallel running jobs (default: {jobs_default})")
+ parser.add_argument("-r", "--run-shell-on-error", action="store_true",
+ help="run an interactive shell if the build fails")
+ parser.add_argument("-v", "--verbose", action="store_true",
+ help="always print shell commands and their output,"
+ " instead of only printing them on error")
+ parser.add_argument("package",
+ help="package name, e.g. libosmocore")
+ args = parser.parse_args()
+
+ lib.set_args(args)
+
+ srcdir = f"{lib.config.path_temp}/srcpkgs/{args.package}"
+ if not os.path.exists(srcdir):
+ print(f"ERROR: {args.package}: no srcpkg found, run build_srcpkg.py"
+ " first!")
+ sys.exit(1)
+
+ bindir = f"{lib.config.path_temp}/binpkgs"
+ lib.run_cmd(["rm", "-rf", bindir])
+ os.makedirs(bindir)
+
+ distro = args.docker if args.docker else distro_default
+
+ env = {"JOBS": str(args.jobs),
+ "PACKAGE": args.package,
+ "BUILDUSER": os.environ["USER"],
+ "PACKAGEFORMAT": "deb"}
+
+ docker_args = []
+ if args.run_shell_on_error:
+ env["RUN_SHELL_ON_ERROR"] = "1"
+ docker_args += ["-i", "-t"]
+
+ # Add capability needed for building without network
+ docker_args += ["--cap-add=NET_ADMIN"]
+
+ script_path = "data/build.sh"
+
+ if not distro.startswith("debian:") and not distro.startswith("ubuntu:"):
+ env["PACKAGEFORMAT"] = "rpm"
+
+ if args.docker:
+ image_type = "build_binpkg"
+
+ # Optimization: use docker container with osmo-gsm-manuals-dev already
+ # installed if it is in build depends
+ if env["PACKAGEFORMAT"] == "deb" \
+ and lib.srcpkg.requires_osmo_gsm_manuals_dev(args.package):
+ image_type += "_manuals"
+
+ env["BUILDUSER"] = "user"
+ lib.docker.run_in_docker_and_exit(script_path,
+ image_type=image_type,
+ distro=distro,
+ pass_argv=False, env=env,
+ docker_args=docker_args)
+ else:
+ lib.run_cmd(["sudo", "-E", script_path], env=env,
+ cwd=lib.config.path_top)
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/obs/build_srcpkg.py b/scripts/obs/build_srcpkg.py
new file mode 100755
index 0000000..a5feeea
--- /dev/null
+++ b/scripts/obs/build_srcpkg.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import argparse
+import sys
+import lib
+import lib.config
+import lib.docker
+import lib.git
+import lib.metapkg
+import lib.srcpkg
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ description="Clone the git repository and build the debian source"
+ " package as well as an rpm .spec file. This is the same"
+ " code that runs to generate source packages which we"
+ " upload to https://obs.osmocom.org."
+ f" Output dir: {lib.config.path_temp}/srcpkgs")
+ lib.add_shared_arguments(parser)
+ parser.add_argument("-g", "--gerrit-id", type=int, default=0,
+ help="clone particular revision from gerrit using given ID")
+ parser.add_argument("package", nargs="?",
+ help="package name, e.g. libosmocore or open5gs")
+ args = parser.parse_args()
+
+ if not args.meta and not args.package:
+ print("ERROR: specify -m and/or a package. See -h for help.")
+ sys.exit(1)
+
+ lib.set_args(args)
+
+ if args.docker:
+ lib.docker.run_in_docker_and_exit("build_srcpkg.py")
+
+ if not args.ignore_req:
+ lib.check_required_programs()
+
+ if args.package:
+ args.package = lib.set_proper_package_name(args.package)
+ lib.remove_temp()
+
+ if args.meta:
+ lib.metapkg.build()
+
+ if args.package:
+ lib.srcpkg.build(args.package, args.gerrit_id)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/obs/check_builders.sh b/scripts/obs/check_builders.sh
new file mode 100755
index 0000000..65a0d76
--- /dev/null
+++ b/scripts/obs/check_builders.sh
@@ -0,0 +1,19 @@
+#!/bin/sh -ex
+min=10
+max=500
+wget -q https://obs.osmocom.org -O index.html
+
+set +x
+for i in $(seq $min $max); do
+ if grep -q " of $i build hosts" index.html; then
+ echo
+ echo "Check successful, $i builders are connected to OBS"
+ echo
+ exit 0
+ fi
+done
+
+echo
+echo "ERROR: expected at least $min builders to be connected to OBS!"
+echo
+exit 1
diff --git a/scripts/obs/check_new_distros.py b/scripts/obs/check_new_distros.py
new file mode 100755
index 0000000..962d547
--- /dev/null
+++ b/scripts/obs/check_new_distros.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2023 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import argparse
+import lib.docker
+import lib.osc
+import sys
+
+projects_opensuse = None
+projects_osmocom = None
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description="Check for new distribution"
+ " projects on the openSUSE OBS, that we want to configure in the"
+ " Osmocom OBS as soon as they are available")
+ parser.add_argument("-d", "--docker",
+ help="run in docker to avoid installing required pkgs",
+ action="store_true")
+ parser.add_argument("-v", "--verbose", action="store_true",
+ help="always print shell commands and their output,"
+ " instead of only printing them on error")
+
+ advanced = parser.add_argument_group("advanced options")
+ advanced.add_argument("-A", "--apiurl", help="source OBS API URL"
+ " (default: https://api.opensuse.org)",
+ default="https://api.opensuse.org")
+ advanced.add_argument("-p", "--prefix", default="openSUSE.org-mirror",
+ help="destination OBS prefix"
+ " (default: openSUSE.org-mirror)")
+ advanced.add_argument("-t", "--to-apiurl", help="destination OBS API URL"
+ " (default: https://obs.osmocom.org)",
+ default="https://obs.osmocom.org")
+
+ args = parser.parse_args()
+ lib.set_args(args)
+
+ lib.osc.check_oscrc()
+
+ if args.docker:
+ lib.docker.run_in_docker_and_exit("check_new_distros.py", add_oscrc=True)
+
+
+def find_highest_distro_project(distro):
+ highest = None
+ for project in projects_opensuse:
+ if not project.startswith(f"{distro}:"):
+ continue
+
+ num = project[len(distro) + 1:]
+ if not lib.config.check_new_distros_version_regex.match(num):
+ if lib.args.verbose:
+ print(f"ignoring {distro}:{num} (doesn't match version regex)")
+ continue
+
+ if not highest or float(num) > float(highest):
+ highest = num
+
+ if highest:
+ return f"{distro}:{highest}"
+
+ return None
+
+
+def check_distro(distro):
+ highest = find_highest_distro_project(distro)
+ if not highest:
+ print(f"ERROR: {distro}: not found in {lib.args.apiurl}")
+ return False
+
+ # check if it is in the osmo obs
+ exp = f"{lib.args.prefix}:{highest}"
+ if exp in projects_osmocom:
+ print(f"{exp}: OK")
+ return True
+
+ print()
+ print(f"ERROR: {exp} not found")
+ print()
+ print("Follow this guide to add it to the Osmocom OBS:")
+ print("https://osmocom.org/projects/cellular-infrastructure/wiki/Add_a_new_distribution_to_OBS")
+ print()
+
+ return False
+
+
+def main():
+ global projects_opensuse
+ global projects_osmocom
+
+ parse_args()
+ ret = 0
+
+ # Get list of projects from Osmocom OBS
+ lib.osc.set_apiurl(lib.args.to_apiurl)
+ projects_osmocom = lib.osc.get_projects()
+
+ # Get list of projects from openSUSE OBS
+ lib.osc.set_apiurl(lib.args.apiurl)
+ projects_opensuse = lib.osc.get_projects()
+
+ # Check for missing distros in Osmocom OBS
+ for distro in lib.config.check_new_distros:
+ if not check_distro(distro):
+ ret = 1
+
+ sys.exit(ret)
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/obs/data/Release.key b/scripts/obs/data/Release.key
new file mode 100644
index 0000000..ecca084
--- /dev/null
+++ b/scripts/obs/data/Release.key
@@ -0,0 +1,26 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1.4.5 (GNU/Linux)
+
+mQENBGKzE1QBCADFcM3ZzggvgxNRNNqDGWf5xIDAiK5qzFLdGes7L6F9VCHdaPy0
+RAOB5bFb/Q1tSDFNEBLtaauXKz+4iGL6qMVjZcyjzpB5w4jKN+kkrFRhjDNUv/SH
+BX6d+P7v5WBGSNArNgA8D1BGzckp5a99EZ0okMJFEqIcN40PD6OGugpq5XnVV5Nk
+e93fLa2Cu8vhFBcVn6CuHeEhsmuMf6NLbQRNfNNCEEUYaZn7beMYtpZ7t1djsKx5
+1xGm50OzI22FLu8lELQ9d7qMVGRG3WHYawX9BDteRybiyqxfwUHm1haWazRJtlGt
+UWyzvwAb80BK1J2Nu5fbAa3w5CoEPAbUuCyrABEBAAG0JW9zbW9jb20gT0JTIFBy
+b2plY3QgPG9zbW9jb21Ab3Ntb2NvbT6JAVQEEwEIAD4WIQRrKp83ktFetw1Oao+G
+pzC2U3JZcwUCYrMV4wIbAwUJBB6yjwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAK
+CRCGpzC2U3JZc4FRCACQQkKIrnvQ7n2u7GSmyVZa3I+oLoFXSGqaGyey5TW/nrMm
+vFDKU3qliHiuNSmUY35SnAhXUsvqOYppxVRoO1MLrqUvzMOnIWqkJpf8mtjGUnsW
+jyVeto7Rsjs75y2i1Hk+e7ljb/V65J3NlfrfEYWbqR9AKd53ReNXTdrQ0J05A38N
+GdI4Ld/2lNISAwaBmGhqdeKsLHpQw/JERU1TApVJR1whFiIwDF1rOCg9GPnNKIk7
+yRZdK267XzztrainX/cbPILyzUZEDhYs6wQuyACyQ1YUxZIxrwVfk7PMNay8CrLH
+z42B73Ne5IAj8+op/3iJafFONLm7YXiDUFN+QDYAiQEzBBMBCAAdFiEExoiYhHND
+S7aVYlnqa51NyAUyjdsFAmKzE1UACgkQa51NyAUyjdvuZgf+OXmr//i7u7Gg7eWB
+7e0qUsyCId9lXS8J437x3K6ciJfD7/6RSy8TFW5Nglm/uSkbyq582I8t+SoOirMD
+E6cg9U/5+h5s46bAf+Kd2XS/6tLGeNLM18i4el8CP06NpFzDrsKu76uYFpyRiiHD
+otBdtgxeLJ83LugGfZslF+/5cigJkAJMhAdVvGO8h85R6fba8ZSOKtMKkaQRfi76
+nhyOrJPlLuS+DLEnHwdkOFgtKnxHdjM97K+Tx0gisb6uwaWroXfSLnhP8RTLLZZy
+Z+noU1Hw3c+mn4c/NYbcC/uwHYHKRzuf9gHnQ3dGgv0Z5sbeLRVo92hjGj7Ftlyd
+4hmKBg==
+=HxK4
+-----END PGP PUBLIC KEY BLOCK-----
diff --git a/scripts/obs/data/build.sh b/scripts/obs/data/build.sh
new file mode 100755
index 0000000..7dfbda7
--- /dev/null
+++ b/scripts/obs/data/build.sh
@@ -0,0 +1,15 @@
+#!/bin/sh -e
+
+if ! data/build_"$PACKAGEFORMAT".sh; then
+ echo
+ echo "ERROR: build failed!"
+ echo
+ if [ -n "$RUN_SHELL_ON_ERROR" ]; then
+ bash
+ fi
+ exit 1
+fi
+
+echo
+echo "Build successful!"
+echo
diff --git a/scripts/obs/data/build_binpkg.Dockerfile b/scripts/obs/data/build_binpkg.Dockerfile
new file mode 100644
index 0000000..55bfe0c
--- /dev/null
+++ b/scripts/obs/data/build_binpkg.Dockerfile
@@ -0,0 +1,90 @@
+ARG DISTRO_FROM
+FROM ${DISTRO_FROM}
+ARG DISTRO
+ARG FEED
+ARG UID
+
+COPY Release.key /tmp/Release.key
+
+RUN useradd --uid=${UID} -m user
+
+# Only install build-essential here, and what's needed to add the Osmocom
+# repository. Everything else must be defined as dependency in the package
+# build recipe. For rpm-based distributions, there is no build-essential or
+# similar package. Instead add relevant packages from prjconf, e.g.:
+# https://build.opensuse.org/projects/CentOS:CentOS-8/prjconf
+# For debian, make sure we don't have man pages as otherwise it takes some time
+# to regenerate the manuals database when installing build dependencies.
+# SYS#5818: using almalinux:8 instead of centos:8
+RUN case "$DISTRO" in \
+ debian*|ubuntu*) \
+ echo "path-exclude=/usr/share/man/*" \
+ > /etc/dpkg/dpkg.cfg.d/exclude-man-pages && \
+ rm -rf /usr/share/man/ && \
+ apt-get update && \
+ apt-get install -y --no-install-recommends \
+ build-essential \
+ ca-certificates \
+ fakeroot \
+ git \
+ gnupg2 \
+ iproute2 \
+ && \
+ apt-get clean \
+ ;; \
+ almalinux*) \
+ dnf -y install \
+ autoconf \
+ automake \
+ binutils \
+ dnf-utils \
+ gcc \
+ gcc-c++ \
+ glibc-devel \
+ iproute \
+ libtool \
+ make \
+ redhat-rpm-config \
+ rpm-build \
+ rpmdevtools \
+ wget && \
+ yum config-manager --set-enabled powertools && \
+ su user -c rpmdev-setuptree \
+ ;; \
+ esac
+
+# Add master repository, where packages immediately get updated after merging
+# patches to master.
+# sed: first letter uppercase (testing -> Testing)
+RUN set -x; \
+ VERSION="$(echo "$DISTRO" | cut -d : -f 2 | sed 's/./\u&/')"; \
+ case "$DISTRO" in \
+ debian:*) \
+ apt-key add /tmp/Release.key && \
+ rm /tmp/Release.key && \
+ echo "deb https://downloads.osmocom.org/packages/osmocom:/$FEED/Debian_$VERSION/ ./" \
+ > /etc/apt/sources.list.d/osmocom-$FEED.list \
+ ;; \
+ ubuntu:*) \
+ apt-key add /tmp/Release.key && \
+ rm /tmp/Release.key && \
+ echo "deb https://downloads.osmocom.org/packages/osmocom:/$FEED/xUbuntu_$VERSION/ ./" \
+ > /etc/apt/sources.list.d/osmocom-$FEED.list \
+ ;; \
+ almalinux:*) \
+ { echo "[network_osmocom_$FEED]"; \
+ echo "name=osmocom:$FEED"; \
+ echo "type=rpm-md"; \
+ echo "baseurl=https://downloads.osmocom.org/packages/osmocom:/$FEED/CentOS_$VERSION/"; \
+ echo "gpgcheck=1"; \
+ echo "gpgkey=https://downloads.osmocom.org/packages/osmocom:/$FEED/CentOS_$VERSION/repodata/repomd.xml.key"; \
+ echo "enabled=1"; \
+ } > /etc/yum.repos.d/network:osmocom:$FEED.repo \
+ ;; \
+ *) \
+ echo "can't install repo for $DISTRO" && \
+ exit 1 \
+ ;; \
+ esac
+
+WORKDIR /obs/
diff --git a/scripts/obs/data/build_binpkg_manuals.Dockerfile b/scripts/obs/data/build_binpkg_manuals.Dockerfile
new file mode 100644
index 0000000..c2b1211
--- /dev/null
+++ b/scripts/obs/data/build_binpkg_manuals.Dockerfile
@@ -0,0 +1,20 @@
+# Optimization: installing osmo-gsm-manuals-dev and its many, many dependencies
+# takes quite a long time - sometimes longer than building the package itself
+# (related: OS#4132). Instead of doing this every time before starting a build,
+# here is a second docker container that already has it installed. This gets
+# used by build_binpkg.py in case the package to build depends on
+# osmo-gsm-manuals-dev and the build is done for Debian. Note that right now we
+# don't build the manuals for rpm-based distributions.
+ARG DISTRO_FROM
+FROM ${DISTRO_FROM}
+ARG DISTRO
+
+RUN case "$DISTRO" in \
+ debian*|ubuntu*) \
+ apt-get update && \
+ apt-get install -y --no-install-recommends \
+ osmo-gsm-manuals-dev \
+ && \
+ apt-get clean \
+ ;; \
+ esac
diff --git a/scripts/obs/data/build_deb.sh b/scripts/obs/data/build_deb.sh
new file mode 100755
index 0000000..931919d
--- /dev/null
+++ b/scripts/obs/data/build_deb.sh
@@ -0,0 +1,25 @@
+#!/bin/sh -ex
+
+apt_get="apt-get"
+if [ -n "$INSIDE_DOCKER" ]; then
+ export DEBIAN_FRONTEND=noninteractive
+ apt_get="apt-get -y"
+fi
+
+su "$BUILDUSER" -c "tar -C _temp/binpkgs -xvf _temp/srcpkgs/$PACKAGE/*.tar.*"
+cd _temp/binpkgs/*
+
+$apt_get update
+$apt_get build-dep .
+
+if [ -n "$INSIDE_DOCKER" ]; then
+ ip link set eth0 down
+fi
+
+su "$BUILDUSER" -c "dpkg-buildpackage -us -uc -j$JOBS"
+
+# Show contents
+cd ..
+for i in *.deb; do
+ dpkg -c "$i"
+done
diff --git a/scripts/obs/data/build_rpm.sh b/scripts/obs/data/build_rpm.sh
new file mode 100755
index 0000000..a73d164
--- /dev/null
+++ b/scripts/obs/data/build_rpm.sh
@@ -0,0 +1,44 @@
+#!/bin/sh -ex
+
+if ! [ -d /home/$BUILDUSER/rpmbuild/SOURCES ]; then
+ set +x
+ echo "ERROR: rpmdev-setuptree did not run"
+ echo "If this is an rpm based system and you want to build the package"
+ echo "here, run rpmdev-setuptree. Otherwise consider building the"
+ echo "package in docker (-d)."
+ exit 1
+fi
+
+yum_builddep="yum-builddep"
+if [ -n "$INSIDE_DOCKER" ]; then
+ yum_builddep="yum-builddep -y"
+fi
+
+spec="$(basename "$(find _temp/srcpkgs/"$PACKAGE" -name '*.spec')")"
+
+su "$BUILDUSER" -c "cp _temp/srcpkgs/$PACKAGE/$spec ~/rpmbuild/SPECS"
+su "$BUILDUSER" -c "cp _temp/srcpkgs/$PACKAGE/*.tar.* ~/rpmbuild/SOURCES"
+su "$BUILDUSER" -c "cp _temp/srcpkgs/$PACKAGE/rpmlintrc ~/rpmbuild/SOURCES"
+su "$BUILDUSER" -c "cp /obs/data/rpmmacros ~/.rpmmacros"
+
+# Force refresh of package index data (OS#6038)
+dnf makecache --refresh
+
+$yum_builddep "/home/$BUILDUSER/rpmbuild/SPECS/$spec"
+
+if [ -n "$INSIDE_DOCKER" ]; then
+ ip link set eth0 down
+fi
+
+su "$BUILDUSER" -c "rpmbuild -bb ~/rpmbuild/SPECS/$spec"
+
+# Make built rpms available outside of docker
+if [ -n "$INSIDE_DOCKER" ]; then
+ su "$BUILDUSER" -c "mv ~/rpmbuild/RPMS/*/*.rpm _temp/binpkgs/"
+fi
+
+# Show contents
+cd _temp/binpkgs
+for i in *.rpm; do
+ rpm -qlp "$i"
+done
diff --git a/scripts/obs/data/build_srcpkg.Dockerfile b/scripts/obs/data/build_srcpkg.Dockerfile
new file mode 100644
index 0000000..80488ea
--- /dev/null
+++ b/scripts/obs/data/build_srcpkg.Dockerfile
@@ -0,0 +1,31 @@
+# Change distro in lib/config.py:docker_distro_default
+ARG DISTRO_FROM
+FROM ${DISTRO_FROM}
+ARG UID
+
+RUN apt-get update && \
+ apt-get upgrade -y && \
+ apt-get install -y --no-install-recommends \
+ ca-certificates \
+ colordiff \
+ debhelper \
+ dh-python \
+ dpkg-dev \
+ fakeroot \
+ git \
+ git-review \
+ gnupg2 \
+ libxml2-utils \
+ lsb-release \
+ meson \
+ osc \
+ python3-packaging \
+ python3-setuptools \
+ quilt \
+ rebar3 \
+ sed \
+ && \
+ apt-get clean
+
+RUN useradd --uid=${UID} -m user
+USER user
diff --git a/scripts/obs/data/rpmlintrc b/scripts/obs/data/rpmlintrc
new file mode 100644
index 0000000..aa3903c
--- /dev/null
+++ b/scripts/obs/data/rpmlintrc
@@ -0,0 +1,5 @@
+# Don't abort the build when finding a library that depends on a package with
+# a specific version. This is intentional for nightly builds, we don't want
+# libraries from different build dates to be mixed as they might have ABI
+# incompatibilities.
+setBadness('shlib-fixed-dependency', 0)
diff --git a/scripts/obs/data/rpmmacros b/scripts/obs/data/rpmmacros
new file mode 100644
index 0000000..5ac26c7
--- /dev/null
+++ b/scripts/obs/data/rpmmacros
@@ -0,0 +1,5 @@
+# Default values for macros from OBS project config:
+# https://build.opensuse.org/projects/CentOS:CentOS-8/prjconf
+
+%ext_info .gz
+%ext_man .gz
diff --git a/scripts/obs/lib/__init__.py b/scripts/obs/lib/__init__.py
new file mode 100644
index 0000000..5292dc5
--- /dev/null
+++ b/scripts/obs/lib/__init__.py
@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import importlib
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import inspect
+import lib.config
+
+# Argparse result
+args = None
+
+# Print output of commands as they run, not only on error
+cmds_verbose = False
+
+
+def add_shared_arguments(parser):
+ """ Arguments shared between build_srcpkg.py and update_obs_project.py. """
+ parser.add_argument("-f", "--feed",
+ help="package feed (default: nightly). The feed"
+ " determines the git revision to be built:"
+ " 'nightly' and 'master' build 'origin/master',"
+ " 'latest' builds the last signed tag,"
+ " other feeds build their respective branch.",
+ metavar="FEED", default="nightly",
+ choices=lib.config.feeds)
+ parser.add_argument("-a", "--allow-unknown-package", action="store_true",
+ help="don't complain if the name of the package is not"
+ " stored in lib/config.py")
+ parser.add_argument("-b", "--git-branch", help="instead of using a branch"
+ " based on the feed, checkout this git branch",
+ metavar="BRANCH", default=None)
+ parser.add_argument("-d", "--docker",
+ help="run in docker to avoid installing required pkgs",
+ action="store_true")
+ parser.add_argument("-s", "--git-skip-fetch",
+ help="do not fetch already cloned git repositories",
+ action="store_false", dest="git_fetch")
+ parser.add_argument("-S", "--git-skip-checkout",
+ help="do not checkout and reset to a branch/tag",
+ action="store_false", dest="git_checkout")
+ parser.add_argument("-m", "--meta", action="store_true",
+ help="build a meta package (e.g. osmocom-nightly)")
+ parser.add_argument("-i", "--ignore-req", action="store_true",
+ help="skip required programs check")
+ parser.add_argument("-c", "--conflict-version", nargs="?",
+ help="Of the generated source packages, all Osmocom"
+ " packages (not e.g. open5gs, see lib/config.py"
+ " for full list) depend on a meta-package such as"
+ " osmocom-nightly, osmocom-latest, osmocom-2021q1"
+ " etc. These meta-packages conflict with each"
+ " other to ensure that one does not mix e.g."
+ " latest and nightly packages by accident."
+ " With this -c argument, it is possible to let"
+ " these packages depend on a meta-package of a"
+ " specific version. This is used for nightly and"
+ " 20YYqX packages to ensure they are not mixed"
+ " from different build dates (ABI compatibility"
+ " is only on latest).")
+ parser.add_argument("-p", "--conflict-pkgname", nargs="?",
+ help="name of the meta-package to depend on (default:"
+ " osmocom-$feed)")
+ parser.add_argument("-M", "--no-meta", action="store_true",
+ help="Don't depend on the meta package (helpful when"
+ " building one-off packages for development)")
+ parser.add_argument("-v", "--verbose", action="store_true",
+ help="always print shell commands and their output,"
+ " instead of only printing them on error")
+ parser.add_argument("-e", "--version-append",
+ help="add a string at the end of the version, e.g."
+ " '~osmocom' for the wireshark package to"
+ " indicate that it is the version from our repo")
+
+
+def set_cmds_verbose(new_val):
+ global cmds_verbose
+ cmds_verbose = new_val
+
+
+def set_args(new_val):
+ global args
+ args = new_val
+ set_cmds_verbose(args.verbose)
+
+
+def check_required_programs():
+ ok = True
+
+ for program in lib.config.required_programs:
+ if not shutil.which(program):
+ print(f"ERROR: missing program: {program}")
+ ok = False
+
+ for module in lib.config.required_python_modules:
+ if not importlib.find_loader(module):
+ print(f"ERROR: missing python3 module: {module}")
+ ok = False
+
+ if not ok:
+ print("Either install them or use the -d argument to run in docker")
+ sys.exit(1)
+
+
+def set_proper_package_name(package):
+ if package in lib.config.projects_osmocom:
+ return package
+ if package in lib.config.projects_other:
+ return package
+
+ # Add prefix to Osmocom package if missing
+ for package_cfg in lib.config.projects_osmocom:
+ if os.path.basename(package_cfg) == package:
+ return package_cfg
+
+ if lib.args.allow_unknown_package:
+ return package
+
+ print(f"ERROR: unknown package: {package}")
+ print("See projects_osmocom and projects_other in obs/lib/config.py")
+ sys.exit(1)
+
+
+def exit_error_cmd(completed, error_msg):
+ """ :param completed: return from run_cmd() below """
+ global cmds_verbose
+
+ print()
+ print(f"ERROR: {error_msg}")
+ print()
+ print(f"*** command ***\n{completed.args}\n")
+ print(f"*** returncode ***\n{completed.returncode}\n")
+
+ if not cmds_verbose:
+ print(f"*** output ***\n{completed.output}")
+
+ print("*** python trace ***")
+ raise RuntimeError("shell command related error, find details right above"
+ " this python trace")
+
+
+def run_cmd(cmd, check=True, *args, **kwargs):
+ """ Like subprocess.run, but has check=True and text=True by default and
+ allows capturing the output while displaying it at the same time. By
+ default the output is hidden unless there's an error, with -v the
+ output gets written to stdout.
+ :returns: subprocess.CompletedProcess instance, but with combined
+ stdout + stderr written to ret.output
+ :param check: stop with error if exit code is not 0 """
+ global cmds_verbose
+
+ caller = inspect.stack()[2][3]
+ if cmds_verbose:
+ print(f"+ {caller}(): {cmd}")
+
+ with tempfile.TemporaryFile(encoding="utf8", mode="w+") as output_buf:
+ p = subprocess.Popen(cmd, stdin=subprocess.DEVNULL,
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+ text=True, bufsize=1, *args, **kwargs)
+
+ while True:
+ out = p.stdout.read(1)
+ if out == "" and p.poll() is not None:
+ break
+ if out != "":
+ output_buf.write(out)
+ if cmds_verbose:
+ sys.stdout.write(out)
+ sys.stdout.flush()
+
+ output_buf.seek(0)
+ setattr(p, "output", output_buf.read())
+
+ if p.returncode == 0 or not check:
+ return p
+
+ exit_error_cmd(p, "command failed unexpectedly")
+
+
+def remove_temp():
+ run_cmd(["rm", "-rf", lib.config.path_temp])
+
+
+def remove_cache_extra_files():
+ """ dpkg-buildpackage outputs all files to the top dir of the package
+ dir, so it will always put them in _cache when building e.g. the debian
+ source package of _cache/libosmocore. Clear all extra files from _cache
+ that don't belog to the git repositories which we actually want to
+ cache. """
+ run_cmd(["find", lib.config.path_cache, "-maxdepth", "1", "-type", "f",
+ "-delete"])
+
+
+def get_output_path(project):
+ return f"{lib.config.path_temp}/srcpkgs/{os.path.basename(project)}"
diff --git a/scripts/obs/lib/binpkg_deb.py b/scripts/obs/lib/binpkg_deb.py
new file mode 100644
index 0000000..b26623c
--- /dev/null
+++ b/scripts/obs/lib/binpkg_deb.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import os
+import glob
+import lib.config
+
+
+def extract_source(srcdir, bindir):
+ tarball = glob.glob(f"{srcdir}/*.tar.*")[0]
+
+ print(f"extracting {os.path.basename(tarball)}")
+ lib.run_cmd(["tar", "-xf", tarball], cwd=bindir)
+
+ return glob.glob(f"{bindir}/*/")[0]
+
+
+def build(srcdir, jobs):
+ bindir = f"{lib.config.path_temp}/binpkg"
+ extractdir = extract_source(srcdir, bindir)
+
+ lib.set_cmds_verbose(True)
+
+ # install deps
+ lib.run_cmd(["apt-get", "-y", "build-dep", "."], cwd=extractdir)
+
+ print("running dpkg-buildpackage")
+ lib.run_cmd(["dpkg-buildpackage", "-us", "-uc", f"-j{jobs}"],
+ cwd=extractdir)
diff --git a/scripts/obs/lib/binpkg_rpm.py b/scripts/obs/lib/binpkg_rpm.py
new file mode 100644
index 0000000..b26623c
--- /dev/null
+++ b/scripts/obs/lib/binpkg_rpm.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import os
+import glob
+import lib.config
+
+
+def extract_source(srcdir, bindir):
+ tarball = glob.glob(f"{srcdir}/*.tar.*")[0]
+
+ print(f"extracting {os.path.basename(tarball)}")
+ lib.run_cmd(["tar", "-xf", tarball], cwd=bindir)
+
+ return glob.glob(f"{bindir}/*/")[0]
+
+
+def build(srcdir, jobs):
+ bindir = f"{lib.config.path_temp}/binpkg"
+ extractdir = extract_source(srcdir, bindir)
+
+ lib.set_cmds_verbose(True)
+
+ # install deps
+ lib.run_cmd(["apt-get", "-y", "build-dep", "."], cwd=extractdir)
+
+ print("running dpkg-buildpackage")
+ lib.run_cmd(["dpkg-buildpackage", "-us", "-uc", f"-j{jobs}"],
+ cwd=extractdir)
diff --git a/scripts/obs/lib/config.py b/scripts/obs/lib/config.py
new file mode 100644
index 0000000..b4e23a0
--- /dev/null
+++ b/scripts/obs/lib/config.py
@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import os
+import re
+
+# Lists are ordered alphabetically.
+
+path_top = os.path.normpath(f"{os.path.realpath(__file__)}/../..")
+path_cache = f"{path_top}/_cache"
+path_temp = f"{path_top}/_temp"
+
+# Keep in sync with packages installed in data/Dockerfile
+required_programs = [
+ "dh",
+ "dh_python3",
+ "dpkg-buildpackage",
+ "fakeroot",
+ "find",
+ "git",
+ "git-review",
+ "meson",
+ "osc",
+ "rebar3",
+ "sed",
+]
+
+required_python_modules = [
+ "packaging",
+ "setuptools",
+]
+
+feeds = [
+ "2022q1",
+ "2022q2",
+ "2023q1",
+ "latest",
+ "master",
+ "nightly",
+]
+
+# Osmocom projects: generated source packages will depend on a meta package,
+# such as osmocom-nightly, osmocom-latest or osmocom-2022q1. This meta package
+# prevents that packages from different feeds are mixed by accident.
+# NOTE: Before adding new projects, add them to jobs/gerrit-verifications.yml
+# and ensure the rpm and deb packages build successfully in jenkins.
+projects_osmocom = [
+ "erlang/osmo_dia2gsup",
+ "erlang/osmo-epdg",
+ "gapk",
+ "libasn1c",
+ "libgtpnl",
+ "libosmo-abis",
+ "libosmo-dsp",
+ "libosmo-gprs",
+ "libosmo-netif",
+ "libosmo-pfcp",
+ "libosmo-sccp",
+ "libosmocore",
+ "libsmpp34",
+ "libusrp",
+ "osmo-bsc",
+ "osmo-bsc-nat",
+ "osmo-bts",
+ "osmo-cbc",
+ "osmo-e1d",
+ "osmo-fl2k",
+ "osmo-gbproxy",
+ "osmo-ggsn",
+ "osmo-gsm-manuals",
+ "osmo-hlr",
+ "osmo-hnbgw",
+ "osmo-hnodeb",
+ "osmo-iuh",
+ "osmo-mgw",
+ "osmo-msc",
+ "osmo-pcap",
+ "osmo-pcu",
+ "osmo-remsim",
+ "osmo-sgsn",
+ "osmo-sip-connector",
+ "osmo-smlc",
+ "osmo-sysmon",
+ "osmo-trx",
+ "osmo-uecups",
+ "osmo-upf",
+ "osmocom-bb",
+ "python/osmo-python-tests",
+ "rtl-sdr",
+ "simtrace2",
+]
+projects_other = [
+ "limesuite",
+ "neocon",
+ "open5gs",
+]
+
+git_url_default = "https://gerrit.osmocom.org" # /project gets appended
+git_url_other = {
+ "libosmo-dsp": "https://gitea.osmocom.org/sdr/libosmo-dsp",
+ "limesuite": "https://github.com/myriadrf/LimeSuite",
+ "neocon": "https://github.com/laf0rge/neocon",
+ "open5gs": "https://github.com/open5gs/open5gs",
+ "osmo-fl2k": "https://gitea.osmocom.org/sdr/osmo-fl2k",
+ "rtl-sdr": "https://gitea.osmocom.org/sdr/rtl-sdr",
+}
+
+git_branch_default = "master"
+git_branch_other = {
+ "open5gs": "main",
+}
+
+git_latest_tag_pattern_default = "^[0-9]*\\.[0-9]*\\.[0-9]*$"
+git_latest_tag_pattern_other = {
+ "limesuite": "^v[0-9]*\\.[0-9]*\\.[0-9]*$",
+ "open5gs": "^v[0-9]*\\.[0-9]*\\.[0-9]*$",
+ "osmo-fl2k": "^v[0-9]*\\.[0-9]*\\.[0-9]*$",
+ "rtl-sdr": "^v[0-9]*\\.[0-9]*\\.[0-9]*$",
+ "wireshark": "^v[0-9]*\\.[0-9]*\\.[0-9a-z]*$",
+}
+
+docker_distro_default = "debian:12"
+docker_distro_other = [
+ "almalinux:*", # instead of centos (SYS#5818)
+ "debian:*",
+ "ubuntu:*",
+]
+
+#
+# Options related to sync from build.opensuse.org (OS#6165)
+#
+
+sync_remove_paths = [
+ # This path has a kernel-obs-build package that other OBS instances use to
+ # build armv7l/hl packages, but we don't need it
+ "OBS:DefaultKernel",
+]
+
+sync_set_maintainers = [
+ "osmocom-jenkins",
+]
+
+# Distributions for which we want to make sure we add the latest release as
+# soon as it is available in openSUSE's OBS
+# https://osmocom.org/projects/cellular-infrastructure/wiki/Linux_Distributions
+check_new_distros = [
+ "Debian",
+ "Raspbian",
+ "Ubuntu",
+]
+
+check_new_distros_version_regex = re.compile(r'[0-9.]+$')
diff --git a/scripts/obs/lib/debian.py b/scripts/obs/lib/debian.py
new file mode 100644
index 0000000..6703f8b
--- /dev/null
+++ b/scripts/obs/lib/debian.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import datetime
+import os
+import shlex
+import lib
+import lib.git
+
+# Imports that may not be available during startup, ignore it here and rely on
+# lib.check_required_programs() checking this later on (possibly after the
+# script executed itself in docker if using --docker).
+try:
+ import packaging.version
+except ImportError:
+ pass
+
+def control_add_depend(project, pkgname, version):
+ """ :param pkgname: of the meta-package to depend on (e.g. osmocom-nightly)
+ :param version: of the meta-pkgname to depend on or None """
+ repo_path = lib.git.get_repo_path(project)
+
+ if version:
+ depend = f"{pkgname} (= {version})"
+ else:
+ depend = pkgname
+
+ cmd = ["sed", f"s/^Depends: /Depends: {depend}, /", "-i", "debian/control"]
+ lib.run_cmd(cmd, cwd=repo_path)
+
+
+def changelog_add_entry(project, version):
+ """ :param version: for the new changelog entry """
+ feed = lib.args.feed
+ repo_path = lib.git.get_repo_path(project)
+ changelog_path = f"{repo_path}/debian/changelog"
+
+ changelog_old = open(changelog_path).read()
+
+ # Package name might be different from project name, read it from changelog
+ pkgname = changelog_old.split(" ", 1)[0]
+ assert pkgname
+
+ # Debian doesn't allow '-' in version
+ version = version.replace("-", ".")
+
+ # Debian changelog requires this specific date format
+ date = datetime.datetime.now(datetime.timezone.utc)
+ date_str = date.strftime("%a, %d %b %Y %H:%M:%S %z")
+
+ # Add new changelog entry
+ with open(changelog_path, "w") as f:
+ f.write(f"{pkgname} ({version}) unstable; urgency=medium\n")
+ f.write("\n")
+ f.write(" * Automatically generated changelog entry for building the"
+ f" Osmocom {feed} feed\n")
+ f.write("\n")
+ f.write(f" -- Osmocom OBS scripts <info@osmocom.org> {date_str}\n")
+ f.write("\n")
+ f.write(changelog_old)
+
+
+def fix_source_format(project):
+ """ Always use format "3.0 (native)" (e.g. limesuite has "3.0 (quilt)")."""
+ repo_path = lib.git.get_repo_path(project)
+ format_path = f"{repo_path}/debian/source/format"
+ if not os.path.exists(format_path):
+ return
+
+ expected = "3.0 (native)"
+ current = open(format_path, "r").read().rstrip()
+
+ if current == expected:
+ return
+
+ print(f"{project}: fixing debian/source/format ({current} => {expected})")
+ open(format_path, "w").write(f"{expected}\n")
+
+
+def get_last_version_from_changelog(project):
+ repo_path = lib.git.get_repo_path(project)
+ changelog_path = f"{repo_path}/debian/changelog"
+
+ assert os.path.exists(changelog_path), f"{project}: missing debian/changelog"
+
+ changelog = open(changelog_path).read()
+ assert changelog, f"{project}: debian/changelog is empty"
+
+ ret = changelog.split("(", 1)[1].split(")", 1)[0]
+ assert ret, f"{project}: couldn't find last version in debian/changelog"
+
+ return ret
+
+
+def changelog_add_entry_if_needed(project, version):
+ """ Adjust the changelog if the version in the changelog is different from
+ the given version. """
+ version_changelog = get_last_version_from_changelog(project)
+
+ # Don't use a lower number (OS#6173)
+ try:
+ if packaging.version.parse(version_changelog.split("-")[0]) > \
+ packaging.version.parse(version.split("-")[0]):
+ print(f"{project}: WARNING: version from changelog"
+ f" ({version_changelog}) is higher than version based on git tag"
+ f" ({version}), using version from changelog (git tag not pushed"
+ " yet?)")
+ return
+ except packaging.version.InvalidVersion:
+ # packaging.version.parse can parse the version numbers used in Osmocom
+ # projects (where we need the above check), but not e.g. some versions
+ # from wireshark. Don't abort here if that is the case.
+ pass
+
+ if version_changelog == version:
+ return
+
+ print(f"{project}: adding debian/changelog entry ({version_changelog} =>"
+ f" {version})")
+ changelog_add_entry(project, version)
+
+
+def build_source_package(project):
+ fix_source_format(project)
+ print(f"{project}: building debian source package")
+ lib.run_cmd(["dpkg-buildpackage", "-S", "-uc", "-us", "-d"],
+ cwd=lib.git.get_repo_path(project))
+
+
+def move_files_to_output(project):
+ path_output = lib.get_output_path(project)
+ lib.run_cmd(f"mv *.tar* *.dsc {shlex.quote(path_output)}", shell=True,
+ cwd=lib.config.path_cache)
diff --git a/scripts/obs/lib/docker.py b/scripts/obs/lib/docker.py
new file mode 100644
index 0000000..779099b
--- /dev/null
+++ b/scripts/obs/lib/docker.py
@@ -0,0 +1,119 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import os
+import shutil
+import subprocess
+import sys
+import lib
+import lib.config
+
+
+def get_image_name(distro, image_type):
+ ret = f"{distro}-osmocom-obs-{image_type}"
+ ret = ret.replace(":","-").replace("_","-")
+ return ret
+
+
+def get_distro_from(distro, image_type):
+ # Manuals: depend on regular image (data/build_binpkg_manuals.Dockerfile)
+ if image_type.endswith("_manuals"):
+ return get_image_name(distro, image_type.replace("_manuals", ""))
+
+ return distro
+
+
+def build_image(distro, image_type):
+ image_name = get_image_name(distro, image_type)
+ distro_from = get_distro_from(distro, image_type)
+
+ print(f"docker: building image {image_name}")
+
+ # Set the feed of packages to be configured inside the docker container
+ # (master, nightly, latest). This can be set with build_binpkg.py --feed,
+ # to reproduce a build error that happens with a distro that is only in
+ # nightly but not in the master feed (all ubuntu versions as of writing).
+ build_arg_feed = []
+ if getattr(lib.args, "docker_feed", None):
+ build_arg_feed = ["--build-arg", f"FEED={lib.args.docker_feed}"]
+
+ lib.run_cmd(["docker", "build",
+ "--build-arg", f"DISTRO={distro}",
+ "--build-arg", f"DISTRO_FROM={distro_from}",
+ "--build-arg", f"UID={os.getuid()}"] +
+ build_arg_feed +
+ ["-t", image_name,
+ "-f", f"{lib.config.path_top}/data/{image_type}.Dockerfile",
+ f"{lib.config.path_top}/data"])
+
+
+def get_oscrc():
+ ret = os.path.expanduser("~/.oscrc")
+ if "OSCRC" in os.environ:
+ ret = os.environ["OSCRC"]
+
+ if os.path.exists(ret):
+ return os.path.realpath(ret)
+
+ print("ERROR: couldn't find ~/.oscrc. Put it there or set OSCRC.")
+ sys.exit(1)
+
+
+def run_in_docker_and_exit(script_path, add_oscrc=False,
+ image_type="build_srcpkg", distro=None,
+ pass_argv=True, env={}, docker_args=[]):
+ """
+ :param script_path: what to run inside docker, relative to scripts/obs/
+ :param add_oscrc: put user's oscrc in docker (contains obs credentials!)
+ :param image_type: which Dockerfile to use (data/{image_type}.Dockerfile)
+ :param distro: which Linux distribution to use, e.g. "debian:11"
+ :param pass_argv: pass arguments from sys.argv to the script
+ :param env: dict of environment variables
+ :param docker_args: extra arguments to pass to docker
+ """
+ if "INSIDE_DOCKER" in os.environ:
+ return
+
+ if not shutil.which("docker"):
+ print("ERROR: docker is not installed")
+ sys.exit(1)
+
+ if not distro:
+ distro = lib.config.docker_distro_default
+ image_name = get_image_name(distro, image_type)
+
+ oscrc = None
+ if add_oscrc:
+ oscrc = get_oscrc()
+
+ # Unless the docker image is up-to-date, building will take a few
+ # minutes or so, therefore print the output. No need to restore
+ # set_cmds_verbose, as we use subprocess.run() below and exit afterwards.
+ lib.set_cmds_verbose(True)
+
+ # Manuals: build regular image first (data/build_binpkg_manuals.Dockerfile)
+ if image_type.endswith("_manuals"):
+ build_image(distro, image_type.replace("_manuals",""))
+
+ build_image(distro, image_type)
+
+ cmd = ["docker", "run",
+ "--rm",
+ "-e", "INSIDE_DOCKER=1",
+ "-e", "PYTHONUNBUFFERED=1",
+ "-v", f"{lib.config.path_top}:/obs"]
+
+ for env_key, env_val in env.items():
+ cmd += ["-e", f"{env_key}={env_val}"]
+
+ if oscrc:
+ cmd += ["-v", f"{oscrc}:/home/user/.oscrc"]
+
+ cmd += docker_args
+ cmd += [image_name, f"/obs/{script_path}"]
+
+ if pass_argv:
+ cmd += sys.argv[1:]
+
+ print(f"docker: running: {script_path} inside docker")
+ ret = subprocess.run(cmd)
+ sys.exit(ret.returncode)
diff --git a/scripts/obs/lib/git.py b/scripts/obs/lib/git.py
new file mode 100644
index 0000000..0194f77
--- /dev/null
+++ b/scripts/obs/lib/git.py
@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import os
+import re
+import lib.config
+
+
+def get_repo_path(project):
+ return f"{lib.config.path_cache}/{os.path.basename(project)}"
+
+
+def get_repo_url(project):
+ if project in lib.config.git_url_other:
+ return lib.config.git_url_other[project]
+ return f"{lib.config.git_url_default}/{project}"
+
+
+def get_latest_tag_pattern(project):
+ if project in lib.config.git_latest_tag_pattern_other:
+ return lib.config.git_latest_tag_pattern_other[project]
+ return lib.config.git_latest_tag_pattern_default
+
+
+def clone(project):
+ fetch = lib.args.git_fetch
+ repo_path = get_repo_path(project)
+ url = get_repo_url(project)
+
+ if os.path.exists(repo_path):
+ if fetch:
+ print(f"{project}: 'git fetch'")
+ lib.run_cmd(["git", "fetch"], cwd=repo_path)
+ else:
+ print(f"{project}: using cached {url} (not cloning, not fetching)")
+ return
+
+ print(f"{project}: cloning {url}")
+ os.makedirs(lib.config.path_cache, exist_ok=True)
+ lib.run_cmd(["git", "clone", url, repo_path])
+
+ lib.run_cmd(["git", "config", "user.name", "Osmocom OBS scripts"],
+ cwd=repo_path)
+ lib.run_cmd(["git", "config", "user.email", "info@osmocom.org"],
+ cwd=repo_path)
+
+
+def clean(project):
+ repo_path = get_repo_path(project)
+ print(f"{project}: 'git clean -ffxd'")
+ lib.run_cmd(["git", "clean", "-ffxd"], cwd=repo_path)
+
+
+def checkout(project, branch):
+ repo_path = get_repo_path(project)
+ if not lib.args.git_checkout:
+ ref = lib.run_cmd(["git", "log", "--pretty=oneline", "--abbrev-commit",
+ "-1"], cwd=repo_path).output.rstrip()
+ print(f"{project}: skipping git checkout, current commit: {ref}")
+ return
+ print(f"{project}: 'git checkout -f {branch}'")
+ lib.run_cmd(["git", "checkout", "-f", branch], cwd=repo_path)
+ print(f"{project}: 'git reset --hard {branch}'")
+ lib.run_cmd(["git", "reset", "--hard", branch], cwd=repo_path)
+ print(f"{project}: 'git submodule update --init'")
+ lib.run_cmd(["git", "submodule", "update", "--init"], cwd=repo_path)
+
+
+def checkout_from_review(project, gerrit_id):
+ """ checkout a given gerrit ID """
+ repo_path = get_repo_path(project)
+ lib.run_cmd(["git", "review", "-s"], cwd=repo_path)
+ lib.run_cmd(["git", "review", "-d", str(gerrit_id)], cwd=repo_path)
+
+
+def get_default_branch(project):
+ if project in lib.config.git_branch_other:
+ return lib.config.git_branch_other[project]
+ return lib.config.git_branch_default
+
+
+def checkout_default_branch(project):
+ branch = get_default_branch(project)
+ checkout(project, f"origin/{branch}")
+
+
+def get_head(project):
+ repo_path = get_repo_path(project)
+ ret = lib.run_cmd(["git", "rev-parse", "HEAD"], cwd=repo_path)
+ return ret.output.rstrip()
+
+
+def get_head_remote(project, branch, branch_missing_ok=True):
+ if not branch:
+ branch = get_default_branch(project)
+ repo_url = get_repo_url(project)
+
+ print(f"{project}: getting head from git remote for {branch}")
+ ls_remote = lib.run_cmd(["git", "ls-remote", repo_url, f"heads/{branch}"])
+
+ ret = ls_remote.output.split("\t")[0]
+
+ # If the branch is missing from the remote, git ls-remote exits with 0 and
+ # the output is empty
+ if not ret:
+ if branch_missing_ok:
+ print(f"{project}: branch not found: {branch}")
+ return None
+ lib.exit_error_cmd(ls_remote, "failed to find head commit for"
+ f" {project} in output")
+
+ return ret
+
+
+def get_latest_tag(project):
+ pattern_str = get_latest_tag_pattern(project)
+ pattern = re.compile(pattern_str)
+ repo_path = get_repo_path(project)
+
+ git_tag_ret = lib.run_cmd(["git", "tag", "-l", "--sort=-v:refname"],
+ cwd=repo_path)
+
+ for line in git_tag_ret.output.split('\n'):
+ line = line.strip('\r')
+ if pattern.match(line):
+ return line
+
+ lib.exit_error_cmd(git_tag_ret, f"couldn't find latest tag for {project},"
+ f" regex used on output: {pattern_str}")
+
+
+def get_latest_tag_remote(project):
+ pattern_str = get_latest_tag_pattern(project)
+ pattern = re.compile(pattern_str)
+
+ print(f"{project}: getting latest tag from git remote")
+ ls_remote = lib.run_cmd(["git", "ls-remote", "--tags", "--sort=-v:refname",
+ get_repo_url(project)])
+ for line in ls_remote.output.split('\n'):
+ # Tags are listed twice, skip the ones with ^{} at the end
+ if "^{}" in line:
+ continue
+
+ if "refs/tags/" not in line:
+ continue
+
+ line = line.rstrip().split("refs/tags/")[1]
+ if pattern.match(line):
+ return line
+
+ # No tag found probably means the repository was just created and doesn't
+ # have a release tag yet
+ return None
+
+
+def checkout_latest_tag(project):
+ checkout(project, get_latest_tag(project))
diff --git a/scripts/obs/lib/metapkg.py b/scripts/obs/lib/metapkg.py
new file mode 100644
index 0000000..38ed930
--- /dev/null
+++ b/scripts/obs/lib/metapkg.py
@@ -0,0 +1,108 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import os
+import lib
+import lib.config
+import lib.debian
+import lib.rpm_spec
+
+
+def get_conflicts():
+ ret = []
+ for f in lib.config.feeds:
+ if f == lib.args.feed:
+ continue
+ ret += [f"osmocom-{f}"]
+ return ret
+
+
+def prepare_source_dir():
+ path = f"{lib.config.path_cache}/osmocom-{lib.args.feed}"
+
+ if os.path.exists(path):
+ lib.run_cmd(["rm", "-rf", path])
+
+ os.makedirs(f"{path}/debian")
+ os.makedirs(f"{path}/contrib")
+
+
+def generate_debian_pkg(version):
+ feed = lib.args.feed
+ path = f"{lib.config.path_cache}/osmocom-{feed}"
+ conflicts = get_conflicts()
+
+ with open(f"{path}/debian/control", "w") as f:
+ f.write(f"Source: osmocom-{feed}\n")
+ f.write("Section: unknown\n")
+ f.write("Priority: optional\n")
+ f.write("Maintainer: Osmocom OBS scripts <info@osmocom.org>\n")
+ f.write("Build-Depends: debhelper (>= 10)\n")
+ f.write("Standards-Version: 3.9.8\n")
+ f.write("\n")
+ f.write(f"Package: osmocom-{feed}\n")
+ f.write("Depends: ${misc:Depends}\n")
+ f.write("Architecture: any\n")
+ f.write(f"Conflicts: {', '.join(conflicts)}\n")
+ f.write(f"Description: Dummy package, conflicts with {conflicts}\n")
+
+ with open(f"{path}/debian/changelog", "w") as f:
+ f.write(f"osmocom-{feed} ({version}) unstable; urgency=medium\n")
+ f.write("\n")
+ f.write(f" * Dummy package, which conflicts with: {conflicts}\n")
+ f.write("\n")
+ f.write(" -- Osmocom OBS scripts <info@osmocom.org> Tue, 25 Jul 2022"
+ " 15:48:00 +0200\n")
+
+ with open(f"{path}/debian/rules", "w") as f:
+ f.write("#!/usr/bin/make -f\n")
+ f.write("%:\n")
+ f.write("\tdh $@\n")
+
+ lib.run_cmd(["chmod", "+x", f"{path}/debian/rules"])
+
+ with open(f"{path}/debian/compat", "w") as f:
+ f.write("10\n")
+
+
+def generate_rpm_spec(version):
+ feed = lib.args.feed
+ print(f"osmocom-{feed}: generating rpm spec file")
+ path = (f"{lib.config.path_cache}/osmocom-{feed}/contrib/osmocom-{feed}"
+ ".spec.in")
+ conflicts = get_conflicts()
+
+ with open(path, "w") as f:
+ f.write(f"Name: osmocom-{feed}\n")
+ f.write(f"Version: {version}\n")
+ f.write(f"Summary: Dummy package, conflicts with: {conflicts}\n")
+ f.write("Release: 0\n")
+ f.write("License: AGPL-3.0-or-later\n")
+ f.write("Group: Hardware/Mobile\n")
+ for conflict in conflicts:
+ f.write(f"Conflicts: {conflict}\n")
+ f.write("%description\n")
+ f.write(f"Dummy package, which conflicts with: {conflicts}\n")
+ f.write("%files\n")
+
+
+def build():
+ feed = lib.args.feed
+ pkgname = lib.args.conflict_pkgname or f"osmocom-{feed}"
+ conflict_version = lib.args.conflict_version
+ version = conflict_version if conflict_version else "1.0.0"
+ print(f"{pkgname}: generating meta package {version}")
+
+ prepare_source_dir()
+ generate_debian_pkg(version)
+
+ os.makedirs(lib.get_output_path(pkgname))
+ lib.remove_cache_extra_files()
+
+ lib.debian.build_source_package(pkgname)
+ lib.debian.move_files_to_output(pkgname)
+
+ generate_rpm_spec(version)
+ lib.rpm_spec.copy_to_output(pkgname)
+
+ lib.remove_cache_extra_files()
+ return version
diff --git a/scripts/obs/lib/osc.py b/scripts/obs/lib/osc.py
new file mode 100644
index 0000000..ef2fc4b
--- /dev/null
+++ b/scripts/obs/lib/osc.py
@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+
+import glob
+import os
+import shlex
+import shutil
+import sys
+import lib
+import lib.config
+
+apiurl = None
+proj = None
+
+
+def check_oscrc():
+ configdir = os.environ.get("XDG_CONFIG_HOME", "~/.config")
+ paths = ["~/.oscrc", f"{configdir}/osc/oscrc"]
+ for path in paths:
+ if os.path.exists(os.path.expanduser(path)):
+ return
+
+ print("ERROR: oscrc does not exist, please create one as explained in the"
+ " README.")
+ sys.exit(1)
+
+
+def set_apiurl(url, obs_proj=None):
+ global apiurl
+ global proj
+
+ if obs_proj is not None:
+ if ":" not in obs_proj:
+ print(f"ERROR: this doesn't look like a valid OBS project: {obs_proj}")
+ sys.exit(1)
+ proj = obs_proj
+
+ apiurl = url
+
+
+def run_osc(cmd, *args, **kwargs):
+ global apiurl
+
+ # For some osc commands like 'osc add *' it makes sense to use a flat
+ # string and shell=True, hence support both list and string in this wrapper
+ if isinstance(cmd, str):
+ if apiurl:
+ cmd = f"osc -A {shlex.quote(apiurl)} {cmd}"
+ else:
+ cmd = f"osc {cmd}"
+ else:
+ if apiurl:
+ cmd = ["osc", "-A", apiurl] + cmd
+ else:
+ cmd = ["osc"] + cmd
+
+ return lib.run_cmd(cmd, *args, **kwargs)
+
+
+def get_remote_pkgs():
+ print(f"OBS: getting packages in {proj}")
+ ret = run_osc(["list", proj])
+ return ret.output.rstrip().split("\n")
+
+
+def get_package_version(package):
+ feed = lib.args.feed
+ print(f"{package}: getting OBS version")
+ ret = run_osc(["list", proj, os.path.basename(package)])
+
+ # Empty OBS package
+ if ret.output == '\n':
+ return "0"
+
+ # Extract the version from the file list
+ for line in ret.output.split('\n'):
+ line = line.rstrip()
+
+ if feed == "master" and package != "osmocom-master":
+ # Use commit_*.txt
+ if line.startswith("commit_") and line.endswith(".txt"):
+ return line.split("_")[1].split(".")[0]
+ else:
+ # Use *.dsc
+ if line.endswith(".dsc"):
+ return line.split("_")[-1][:-4]
+
+ lib.exit_error_cmd(ret, "failed to find package version on OBS by"
+ " extracting the version from the file list")
+
+
+def create_package(package):
+ print(f"{package}: creating new OBS package")
+
+ # cut off repository prefix like in "python/osmo-python-tests"
+ package = os.path.basename(package)
+
+ path_meta = f"{lib.config.path_temp}/_meta"
+ path_meta_obs = f"source/{proj}/{package}/_meta"
+
+ with open(path_meta, "w") as f:
+ f.write(f'<package name="{package}" project="{proj}">\n')
+ f.write(f'<title>{package}</title>\n')
+ f.write('<description></description>\n')
+ f.write('</package>\n')
+
+ run_osc(["api", "-X", "PUT", "-T", path_meta, path_meta_obs])
+
+ os.unlink(path_meta)
+
+
+def remove_temp_osc():
+ lib.run_cmd(["rm", "-rf", f"{lib.config.path_temp}/osc"])
+
+
+def update_package(package, version):
+ print(f"{package}: updating OBS package")
+
+ # cut off repository prefix like in "python/osmo-python-tests"
+ package = os.path.basename(package)
+
+ path_output = lib.get_output_path(package)
+ path_temp_osc = f"{lib.config.path_temp}/osc"
+ path_temp_osc_pkg = f"{path_temp_osc}/{proj}/{package}"
+
+ remove_temp_osc()
+ os.makedirs(path_temp_osc)
+
+ run_osc(["checkout", proj, package], cwd=path_temp_osc)
+
+ if glob.glob(f"{path_temp_osc_pkg}/*"):
+ run_osc("del *", shell=True, cwd=path_temp_osc_pkg)
+
+ lib.run_cmd(f"mv * {shlex.quote(path_temp_osc_pkg)}", shell=True,
+ cwd=path_output)
+
+ run_osc("add *", shell=True, cwd=path_temp_osc_pkg)
+ run_osc(["commit", "-m", f"upgrade to {version}"], cwd=path_temp_osc_pkg)
+
+ remove_temp_osc()
+
+
+def delete_package(package, commit_msg):
+ print(f"{package}: removing from OBS ({commit_msg})")
+ run_osc(["rdelete", "-m", commit_msg, proj, os.path.basename(package)])
+
+
+def get_prjconf(output_file):
+ print(f"{proj}: getting prjconf")
+ prjconf = lib.osc.run_osc(["meta", "prjconf", proj]).output
+ with open(output_file, "w") as h:
+ h.write(prjconf)
+
+
+def update_prjconf(prjconf_file, commit_msg):
+ print(f"{proj}: updating prjconf")
+ lib.osc.run_osc(["meta",
+ "prjconf",
+ "-F", prjconf_file,
+ "-m", commit_msg,
+ proj])
+
+
+def get_meta(output_file):
+ print(f"{proj}: getting meta")
+ meta = lib.osc.run_osc(["meta", "prj", proj]).output
+ with open(output_file, "w") as h:
+ h.write(meta)
+
+
+def update_meta(meta_file, commit_msg):
+ print(f"{proj}: updating meta")
+ lib.osc.run_osc(["meta",
+ "prj",
+ "-F", meta_file,
+ "-m", commit_msg,
+ proj])
+
+def get_projects():
+ print(f"OBS: getting list of projects ({apiurl})")
+ return lib.osc.run_osc(["ls"]).output.rstrip().split("\n")
diff --git a/scripts/obs/lib/rpm_spec.py b/scripts/obs/lib/rpm_spec.py
new file mode 100644
index 0000000..a7c4379
--- /dev/null
+++ b/scripts/obs/lib/rpm_spec.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import glob
+import os
+import shutil
+import lib
+import lib.git
+
+
+def get_spec_in_path(project):
+ repo_path = lib.git.get_repo_path(project)
+ ret = glob.glob(f"{repo_path}/**/contrib/*.spec.in", recursive=True)
+ assert len(ret) < 2
+ return ret[0] if ret else None
+
+
+def get_source_name(project):
+ """ Get the name to the .tar.xz file generated by dpkg-buildpackage, that
+ we also use in the rpm spec file. This requires
+ lib.debian.move_files_to_output() to run first."""
+ path_output = lib.get_output_path(project)
+ ret = glob.glob(f"{path_output}/*.tar.*")
+ assert len(ret) == 1
+ return os.path.basename(ret[0])
+
+
+def add_depend(project, pkgname, version):
+ """ Add a 'Requires: ...' line below the 'Name:' and '%package' lines of
+ the .spec.in file.
+ :param pkgname: of the meta-package to depend on (e.g. osmocom-nightly)
+ :param version: of the meta-pkgname to depend on or None """
+ path = get_spec_in_path(project)
+
+ req_line = f"Requires: {pkgname}"
+ if version:
+ req_line += f" = {version}"
+
+ # Main package
+ lib.run_cmd(["sed", "-e", "/^Name:/a\\", "-e", req_line, "-i", path])
+
+ # Subpackages
+ lib.run_cmd(["sed", "-e", "/^%package/a\\", "-e", req_line, "-i", path])
+
+
+def generate(project, version, epoch):
+ """ Update the version and source in the .spec.in file.
+ :param version: from get-version-gen script """
+ print(f"{project}: generating rpm spec file")
+ path = get_spec_in_path(project)
+ source = get_source_name(project)
+
+ # rpmbuild doesn't allow '-' in version
+ version = version.replace("-", ".")
+
+ # Version
+ lib.run_cmd(["sed", f"s/^Version:.*/Version: {version}/", "-i", path])
+ if epoch:
+ epoch_line = f"Epoch: {epoch}"
+ lib.run_cmd(["sed", "-e", "/^Version:/a\\", "-e", epoch_line, "-i",
+ path])
+
+ # Source: add the one generated by dpkg-buildpackage and another line with
+ # rpmlintrc
+ lib.run_cmd(["sed", f"s/^Source:.*/Source: {source}/", "-i", path])
+ rpmlint_line = "Source1: rpmlintrc"
+ lib.run_cmd(["sed", "-e", "/^Source:/a\\", "-e", rpmlint_line, "-i", path])
+
+ # rpmbuild assumes the directory inside the source tarball has the version
+ # at the end, but dpkg-buildpackage doesn't generate it that way. Set the
+ # directory with "%setup -n dirname".
+ lib.run_cmd(["sed", f"s/^%setup/%setup -n {os.path.basename(project)}/",
+ "-i", path])
+
+
+def copy_to_output(project):
+ # rpm spec
+ path_spec_in = get_spec_in_path(project)
+ path_output = lib.get_output_path(project)
+ name_spec = os.path.basename(path_spec_in)[:-3] # remove '.in'
+ shutil.copy(path_spec_in, f"{path_output}/{name_spec}")
+
+ # rpmlintrc
+ shutil.copy(f"{lib.config.path_top}/data/rpmlintrc", path_output)
diff --git a/scripts/obs/lib/srcpkg.py b/scripts/obs/lib/srcpkg.py
new file mode 100644
index 0000000..aef7732
--- /dev/null
+++ b/scripts/obs/lib/srcpkg.py
@@ -0,0 +1,233 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import glob
+import os
+import pathlib
+import sys
+import lib.config
+import lib.debian
+import lib.rpm_spec
+
+
+def checkout_for_feed(project):
+ """ checkout a commit, either latest tag or master or 20YY branch """
+ feed = lib.args.feed
+ branch = lib.args.git_branch
+ if branch:
+ lib.git.checkout(project, f"origin/{branch}")
+ elif feed == "latest":
+ lib.git.checkout_latest_tag(project)
+ elif feed in ["master", "nightly"]:
+ lib.git.checkout_default_branch(project)
+ else: # 2022q1 etc
+ lib.git.checkout(project, f"origin/{feed}")
+
+
+def get_git_version(project):
+ """ :returns: the string from git-version-gen, e.g. '1.7.0.10-76bdb' """
+ repo_path = lib.git.get_repo_path(project)
+
+ # Run git-version-gen if it is in the repository
+ script_path = f"{repo_path}/git-version-gen"
+ if os.path.exists(script_path):
+ ret = lib.run_cmd([script_path, "."], cwd=repo_path).output
+ if not ret:
+ lib.exit_error_cmd(ret, "empty output from git-version-gen")
+ return ret
+
+ # Generate a version string similar to git-version-gen, but run use git
+ # describe --tags, so it works with non-annotated tags as well (needed for
+ # e.g. limesuite's tags).
+ pattern = lib.git.get_latest_tag_pattern(project)
+ pattern = pattern.replace("^", "", 1)
+ pattern = pattern.replace("$", "", -1)
+ result = lib.run_cmd(["git", "describe",
+ "--abbrev=4",
+ "--tags",
+ f"--match={pattern}",
+ "HEAD"], cwd=repo_path, check=False)
+
+ if result.returncode == 128:
+ print(f"{project}: has no git tags, using 0.0.0 as version")
+ commit = lib.run_cmd(["git", "rev-parse", "HEAD"],
+ cwd=repo_path).output[0:4]
+ count = lib.run_cmd(["git", "rev-list", "--count", "HEAD"],
+ cwd=repo_path).output.rstrip()
+ return f"0.0.0.{count}-{commit}"
+
+ if result.returncode != 0:
+ lib.exit_error_cmd(result, "command failed unexpectedly")
+
+ ret = result.output.rstrip()
+
+ # Like git-version-gen:
+ # * Change the first '-' to '.'
+ # * Remove the 'g' in git describe's output string
+ # * Remove the leading 'v'
+ ret = ret.replace("-", ".", 1)
+ ret = ret.replace("-g", "-", 1)
+ if ret.startswith("v"):
+ ret = ret[1:]
+
+ return ret
+
+
+def get_version_for_feed(project):
+ if lib.args.feed == "latest":
+ # There's always a tag if we are here. If there was none, the build
+ # would have been skipped for latest.
+ ret = lib.git.get_latest_tag(project)
+ return ret[1:] if ret.startswith("v") else ret
+
+ ret = get_git_version(project)
+
+ # Try to get the last version from the debian/changelog if we can't get
+ # it with git-version-gen, like it was done in the previous OBS scripts
+ if ret == "UNKNOWN":
+ ret = lib.debian.get_last_version_from_changelog(project)
+ # cut off epoch, we retrieve it separately in get_epoch() below
+ if ":" in ret:
+ ret = ret.split(":")[1]
+
+ # Append the conflict_version to increase the version even if the commit
+ # did not change (OS#5135)
+ conflict_version = lib.args.conflict_version
+ if conflict_version:
+ ret = f"{ret}.{conflict_version}"
+
+ return ret
+
+
+def get_epoch(project):
+ """ The osmo-gbproxy used to have the same package version as osmo-sgsn
+ until 2021 where it was split into its own git repository. From then
+ on, osmo-gbproxy has a 0.*.* package version, which is smaller than
+ the previous 1.*.* from osmo-sgsn. We had to set the epoch to 1 for
+ osmo-gbproxy so package managers know these 0.*.* versions are higher
+ than the previous 1.*.* ones that are still found in e.g. debian 11.
+ The epoch is set in debian/changelog, retrieve it from there.
+ :returns: the epoch number if set, e.g. "1" or an empty string """
+ version_epoch = lib.debian.get_last_version_from_changelog(project)
+
+ if ":" in version_epoch:
+ return version_epoch.split(":")[0]
+
+ return ""
+
+
+def prepare_project_open5gs():
+ """ Download the subproject sources here, so the package can be built in
+ OBS without Internet access. """
+ lib.run_cmd(["meson", "subprojects", "download"],
+ cwd=lib.git.get_repo_path("open5gs"))
+
+
+def prepare_project_limesuite():
+ """ Fix bug in 23.10: https://github.com/myriadrf/LimeSuite/pull/386 """
+ lib.run_cmd(["mv", "-v",
+ "liblimesuite22.09-1.install",
+ "liblimesuite23.10-1.install"],
+ cwd=f"{lib.git.get_repo_path('limesuite')}/debian",
+ check=False)
+
+
+def run_generate_build_dep(project):
+ """ Run contrib/generate_build_dep.sh if it exists in the given project, to
+ to download sources for dependencies (see e.g. osmo_dia2gsup.git). """
+ repo_path = lib.git.get_repo_path(project)
+ script_path = "contrib/generate_build_dep.sh"
+
+ if os.path.exists(f"{repo_path}/{script_path}"):
+ print(f"{project}: running {script_path}")
+ lib.run_cmd(script_path, cwd=repo_path)
+
+
+def write_tarball_version(project, version):
+ repo_path = lib.git.get_repo_path(project)
+
+ with open(f"{repo_path}/.tarball-version", "w") as f:
+ f.write(f"{version}\n")
+
+
+def write_commit_txt(project):
+ """ Write the current git commit to commit_$commit.txt file, so it gets
+ uploaded to OBS along with the rest of the source package. This allows
+ figuring out if the source package is still up-to-date or not for the
+ master feed. """
+ output_path = lib.get_output_path(project)
+ commit = lib.git.get_head(project)
+
+ print(f"{project}: adding commit_{commit}.txt")
+ pathlib.Path(f"{output_path}/commit_{commit}.txt").touch()
+
+
+def build(project, gerrit_id=0):
+ conflict_version = lib.args.conflict_version
+ feed = lib.args.feed
+ version_append = lib.args.version_append
+
+ lib.git.clone(project)
+ lib.git.clean(project)
+ if gerrit_id > 0:
+ lib.git.checkout_from_review(project, gerrit_id)
+ else:
+ checkout_for_feed(project)
+
+ version = get_version_for_feed(project)
+ if version_append:
+ version += version_append
+ epoch = get_epoch(project)
+ version_epoch = f"{epoch}:{version}" if epoch else version
+
+ has_rpm_spec = lib.rpm_spec.get_spec_in_path(project) is not None
+
+ print(f"{project}: building source package {version_epoch}")
+ write_tarball_version(project, version_epoch)
+
+ if project in lib.config.projects_osmocom and not lib.args.no_meta:
+ metapkg = lib.args.conflict_pkgname or f"osmocom-{feed}"
+ lib.debian.control_add_depend(project, metapkg, conflict_version)
+ if has_rpm_spec:
+ lib.rpm_spec.add_depend(project, metapkg, conflict_version)
+
+ lib.debian.changelog_add_entry_if_needed(project, version_epoch)
+
+ os.makedirs(lib.get_output_path(project))
+ lib.remove_cache_extra_files()
+
+ project_specific_func = f"prepare_project_{os.path.basename(project)}"
+ if project_specific_func in globals():
+ print(f"{project}: running {project_specific_func}")
+ globals()[project_specific_func]()
+
+ if project in lib.config.projects_osmocom:
+ run_generate_build_dep(project)
+
+ lib.debian.build_source_package(project)
+ lib.debian.move_files_to_output(project)
+
+ if has_rpm_spec:
+ lib.rpm_spec.generate(project, version, epoch)
+ lib.rpm_spec.copy_to_output(project)
+
+ if feed == "master":
+ write_commit_txt(project)
+
+ lib.remove_cache_extra_files()
+ return version_epoch
+
+
+def requires_osmo_gsm_manuals_dev(project):
+ """ Check if an already built source package has osmo-gsm-manuals-dev in
+ Build-Depends of the .dsc file """
+ path_dsc = glob.glob(f"{lib.get_output_path(project)}/*.dsc")
+ assert len(path_dsc) == 1, f"failed to get dsc path for {project}"
+
+ with open(path_dsc[0], "r") as handle:
+ for line in handle.readlines():
+ if line.startswith("Build-Depends:") \
+ and "osmo-gsm-manuals-dev" in line:
+ return True
+
+ return False
diff --git a/scripts/obs/sync_obs_projects.py b/scripts/obs/sync_obs_projects.py
new file mode 100755
index 0000000..cc4635a
--- /dev/null
+++ b/scripts/obs/sync_obs_projects.py
@@ -0,0 +1,322 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2023 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import argparse
+import glob
+import hashlib
+import html
+import os
+import shlex
+import shutil
+import sys
+import xml.etree.ElementTree
+
+import lib
+import lib.docker
+import lib.osc
+
+temp_source_prjconf = f"{lib.config.path_temp}/sync_source_prjconf"
+temp_source_meta = f"{lib.config.path_temp}/sync_source_meta"
+temp_dest_old_meta = f"{lib.config.path_temp}/sync_dest_old_meta"
+temp_dest_old_prjconf = f"{lib.config.path_temp}/sync_dest_old_prjconf"
+temp_dest_new_meta = f"{lib.config.path_temp}/sync_dest_new_meta"
+temp_dest_new_prjconf = f"{lib.config.path_temp}/sync_dest_new_prjconf"
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description="Sync OBS projects (prjconf,"
+ " meta) from another instance (OS#6165)")
+ parser.add_argument("-d", "--docker",
+ help="run in docker to avoid installing required pkgs",
+ action="store_true")
+ parser.add_argument("-n", "--no-skip-up-to-date",
+ dest="skip_up_to_date", action="store_false",
+ help="always assume projects are outdated")
+ parser.add_argument("-v", "--verbose", action="store_true",
+ help="always print shell commands and their output,"
+ " instead of only printing them on error")
+ parser.add_argument("projects",
+ help="source OBS project, e.g. Debian:12",
+ nargs="+")
+
+ advanced = parser.add_argument_group("advanced options")
+ advanced.add_argument("-A", "--apiurl", help="source OBS API URL"
+ " (default: https://api.opensuse.org)",
+ default="https://api.opensuse.org")
+ advanced.add_argument("-p", "--prefix", default="openSUSE.org-mirror",
+ help="destination OBS prefix"
+ " (default: openSUSE.org-mirror)")
+ advanced.add_argument("-t", "--to-apiurl", help="destination OBS API URL"
+ " (default: https://obs.osmocom.org)",
+ default="https://obs.osmocom.org")
+ advanced.add_argument("-w", "--weburl", default="https://build.opensuse.org",
+ help="source OBS web URL (default:"
+ " https://build.opensuse.org)")
+
+ args = parser.parse_args()
+ lib.set_args(args)
+
+ lib.osc.check_oscrc()
+
+ if args.docker:
+ lib.docker.run_in_docker_and_exit("sync_obs_projects.py", add_oscrc=True)
+
+
+def check_required_programs():
+ required_programs = [
+ "colordiff",
+ "xmllint",
+ ]
+
+ ok = True
+ for program in required_programs:
+ if not shutil.which(program):
+ print(f"ERROR: missing program: {program}")
+ ok = False
+
+ if not ok:
+ print("Either install them or use the -d argument to run in docker")
+ sys.exit(1)
+
+
+def generate_prjconf_header(project):
+ """ This header gets prepended to the prjconf, before it gets written to
+ the destination OBS. This script uses it to determine whether the
+ project needs to be updated next time it runs. """
+ with open(temp_source_prjconf, "rb") as h:
+ source_prjconf = h.read()
+ with open(temp_source_meta, "rb") as h:
+ source_meta = h.read()
+
+ ret = "### This project gets synced from:\n"
+ ret += f"### {lib.args.weburl}/project/show/{project}\n"
+ ret += "### \n"
+ ret += "### Do not modify manually. See OS#6165.\n"
+ ret += "### \n"
+ ret += f"### Sync information:\n"
+ ret += f"### - source meta: {hashlib.md5(source_meta).hexdigest()}\n"
+ ret += f"### - source prjconf: {hashlib.md5(source_prjconf).hexdigest()}\n"
+ ret += "\n"
+
+ return ret
+
+
+def is_up_to_date(header, projects, project):
+ project_new = f"{lib.args.prefix}:{project}"
+
+ if project_new not in projects:
+ print(f"{project_new}: is outdated (not in destination OBS)")
+ return False
+
+ lib.osc.get_prjconf(temp_dest_old_prjconf)
+ with open(temp_dest_old_prjconf, "r") as h:
+ dest_prjconf = h.read()
+
+ if dest_prjconf.startswith(header):
+ if not lib.args.skip_up_to_date:
+ print(f"{project_new}: is up-to-date, but -n is set")
+ return False
+ print(f"{project_new}: is up-to-date")
+ return True
+
+ print(f"{project_new}: is outdated")
+ return False
+
+
+def get_relevant_arches(project):
+ if project.startswith("AlmaLinux:"):
+ return ["x86_64"]
+ if project.startswith("Raspbian:"):
+ return ["armv7l"]
+
+ return ["aarch64",
+ "armv7l",
+ "i586",
+ "x86_64"]
+
+
+def rewrite_meta(project):
+ project_new = f"{lib.args.prefix}:{project}"
+ print(f"{project}: rewriting meta for {project_new}")
+ tree = xml.etree.ElementTree.parse(temp_source_meta)
+ root = tree.getroot()
+ arches = get_relevant_arches(project)
+
+ # Update <project name="...">
+ assert root.get("name") == project
+ root.set("name", project_new)
+
+ for description in root.findall("description"):
+ href = f"{lib.args.weburl}/project/show/{project}"
+ description.text = ("This project gets synced from:"
+ f" <a href='{html.escape(href)}'>{project}</a>\n"
+ "Do not modify manually. See OS#6165.\n")
+
+ for repository in root.findall(".repository"):
+ repo_name = repository.get("name")
+ print(f" adjusting repository: {repo_name}")
+ for path in repository.findall(".path"):
+ # Update <path project="...">
+ path_project_old = path.get("project")
+ path_project_new = f"{lib.args.prefix}:{path_project_old}"
+ path.set("project", path_project_new)
+
+ # Remove unneeded paths
+ for path_check in lib.config.sync_remove_paths:
+ if path_project_old == path_check:
+ print(f" removing path: {path_project_old}")
+ repository.remove(path)
+ break
+
+ # Remove arches we don't build for
+ for arch in repository.findall(".arch"):
+ if arch.text not in arches:
+ print(f" removing arch: {arch.text}")
+ repository.remove(arch)
+ for download in repository.findall(".download"):
+ if download.get("arch") not in arches:
+ repository.remove(download)
+
+ # Debian: meta configs on build.opensuse.org reference PGP keys with an
+ # experimental feature that is not yet in the stable version of OBS
+ # (e.g. <pubkey>debian-archive-12</pubkey>):
+ # https://github.com/openSUSE/open-build-service/pull/14528
+ # Also we don't have such a pubkeydir set up on our OBS server. Assume
+ # ftp.de.debian.org is a trusted mirror, switch to HTTPS and skip the
+ # PGP verification by removing the pubkey blocks.
+ if project.startswith("Debian:"):
+ for download in repository.findall(".download"):
+ url = download.get("url")
+ print(f" changing url to https: {url}")
+ assert url.startswith("http://ftp.de.debian.org/debian"), \
+ f"unexpected mirror URL"
+ download.set("url", url.replace("http://", "https://"))
+ for pubkey in download.findall("pubkey"):
+ download.remove(pubkey)
+
+ # Remove original maintainers
+ for person in root.findall(".person"):
+ root.remove(person)
+
+ # Add new maintainers
+ for userid in lib.config.sync_set_maintainers:
+ print(f" set maintainer: {userid}")
+ person = xml.etree.ElementTree.Element("person")
+ person.set("userid", userid)
+ person.set("role", "maintainer")
+ # Insert into same position: after title and description
+ root.insert(2, person)
+
+ tree.write(temp_dest_new_meta)
+
+
+def rewrite_prjconf(project, header):
+ project_new = f"{lib.args.prefix}:{project}"
+ print(f"{project}: rewriting prjconf for {project_new}")
+
+ prjconf = ""
+ with open(temp_source_prjconf, "r") as f:
+ for line in f:
+ line = line.rstrip()
+
+ # Remove unneeded dependencies
+ if line == "VMInstall: kernel-obs-build":
+ print(f" commenting out: {line}")
+ line = f"# (commented out by sync) {line}"
+
+ prjconf += f"{line}\n"
+
+ # Extend the AlmaLinux prjconf to also set CentOS variables, as some of
+ # our prjconfigs and spec files rely on them
+ if project == "AlmaLinux:8":
+ print(f" appending CentOS 8 variables")
+ prjconf += "\n"
+ prjconf += "# CentOS 8 compat added by sync script\n"
+ prjconf += "%define centos_version 800\n"
+ prjconf += "%define centos_ver 8\n"
+ prjconf += "%define centos 8\n"
+ prjconf += "Macros:\n"
+ prjconf += "%centos_version 800\n"
+ prjconf += "%centos_ver 8\n"
+ prjconf += "%centos 8\n"
+ prjconf += ":Macros\n"
+
+ with open(temp_dest_new_prjconf, "w") as f:
+ f.write(header)
+ f.write(prjconf)
+
+
+def show_diff(projects, project):
+ project_new = f"{lib.args.prefix}:{project}"
+ if project_new not in projects:
+ return
+
+ # Show prjconf diff (old prjconf was retrieved in is_up_to_date())
+ diff = lib.run_cmd(["colordiff",
+ "-c3",
+ temp_dest_old_prjconf,
+ temp_dest_new_prjconf],
+ check=False)
+ if diff.returncode:
+ print(f"{project_new}: prjconf changes:")
+ print(diff.output, end="")
+ else:
+ print(f"{project_new}: prjconf is unchanged")
+
+ # Show meta diff
+ lib.osc.get_meta(temp_dest_old_meta)
+ for file in [temp_dest_old_meta, temp_dest_new_meta]:
+ lib.run_cmd(f"xmllint --format {shlex.quote(file)} > {shlex.quote(file)}.pretty",
+ shell=True)
+ diff = lib.run_cmd(["colordiff",
+ "-c3",
+ f"{temp_dest_old_meta}.pretty",
+ f"{temp_dest_new_meta}.pretty"],
+ check=False)
+ if diff.returncode:
+ print(f"{project_new}: meta changes:")
+ print(diff.output, end="")
+ else:
+ print(f"{project_new}: meta is unchanged")
+
+
+def main():
+ parse_args()
+ check_required_programs()
+
+ os.makedirs(lib.config.path_temp, exist_ok=True)
+
+ # Get destination OBS projects
+ lib.osc.set_apiurl(lib.args.to_apiurl, None)
+ dest_projects = lib.osc.get_projects()
+
+ for project in lib.args.projects:
+ # Talk to source OBS
+ lib.osc.set_apiurl(lib.args.apiurl, project)
+
+ # Get source prjconf, meta
+ lib.osc.get_prjconf(temp_source_prjconf)
+ lib.osc.get_meta(temp_source_meta)
+
+ # Talk to dest OBS
+ project_new = f"{lib.args.prefix}:{project}"
+ lib.osc.set_apiurl(lib.args.to_apiurl, project_new)
+
+ # Check if dest is up-to-date
+ header = generate_prjconf_header(project)
+ if is_up_to_date(header, dest_projects, project):
+ continue
+
+ # Rewrite configs and show diff
+ rewrite_prjconf(project, header)
+ rewrite_meta(project)
+ show_diff(dest_projects, project)
+
+ # Update dest prjconf & meta
+ commit_msg = f"sync with {lib.args.weburl}/project/show/{project}"
+ lib.osc.update_meta(temp_dest_new_meta, commit_msg)
+ lib.osc.update_prjconf(temp_dest_new_prjconf, commit_msg)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/obs/update_obs_project.py b/scripts/obs/update_obs_project.py
new file mode 100755
index 0000000..a456218
--- /dev/null
+++ b/scripts/obs/update_obs_project.py
@@ -0,0 +1,251 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import argparse
+import os
+import sys
+import traceback
+import lib
+import lib.config
+import lib.docker
+import lib.git
+import lib.metapkg
+import lib.osc
+import lib.srcpkg
+
+srcpkgs_built = {} # dict of pkgname: version
+srcpkgs_skipped = [] # list of pkgnames
+srcpkgs_deleted = [] # list of pkgnames
+srcpkgs_failed_build = [] # list of pkgnames
+srcpkgs_failed_upload = [] # list of pkgnames
+srcpkgs_updated = [] # list of pkgnames
+
+
+def parse_packages(packages_arg):
+ ret = []
+ if packages_arg:
+ for package in packages_arg:
+ if package == "ALL_OSMOCOM_PACKAGES":
+ ret += lib.config.projects_osmocom
+ else:
+ ret += [lib.set_proper_package_name(package)]
+ return ret
+
+ # Default to all
+ ret += lib.config.projects_osmocom
+ ret += lib.config.projects_other
+ return ret
+
+
+def build_srcpkg(package, is_meta_pkg):
+ global srcpkgs_built
+ global srcpkgs_failed_build
+
+ version = None
+
+ try:
+ if is_meta_pkg:
+ version = lib.metapkg.build()
+ else:
+ version = lib.srcpkg.build(package)
+ srcpkgs_built[package] = version
+ except Exception as ex:
+ traceback.print_exception(type(ex), ex, ex.__traceback__)
+ print()
+ print(f"{package}: build failed")
+ srcpkgs_failed_build += [package]
+
+
+def delete_srcpkg(package):
+ global srcpkgs_deleted
+ branch = lib.args.git_branch
+
+ lib.osc.delete_package(package, f"branch {branch} does not exist anymore")
+ srcpkgs_deleted += [package]
+
+
+def is_up_to_date(obs_version, git_latest_version):
+ if obs_version == git_latest_version:
+ return True
+
+ # e.g. open5gs has "v" infront of version in git tag
+ if f"v{obs_version}" == git_latest_version:
+ return True
+
+ return False
+
+
+def build_srcpkg_if_needed(pkgs_remote, package, is_meta_pkg):
+ global srcpkgs_skipped
+ feed = lib.args.feed
+ branch = lib.args.git_branch
+ delete = lib.args.delete
+
+ if feed in ["master", "latest"]:
+ """ Check if we can skip this package by comparing the OBS version with
+ the git remote. """
+ if is_meta_pkg:
+ conflict_version = lib.args.conflict_version
+ latest_version = conflict_version if conflict_version else "1.0.0"
+ else:
+ if feed == "master":
+ latest_version = lib.git.get_head_remote(package, branch,
+ branch_missing_ok=delete)
+ else:
+ latest_version = lib.git.get_latest_tag_remote(package)
+
+ if latest_version is None:
+ if delete and os.path.basename(package) in pkgs_remote:
+ delete_srcpkg(package)
+ return
+
+ print(f"{package}: skipping (no git tag/branch found)")
+ srcpkgs_skipped += [package]
+ return
+
+ if os.path.basename(package) not in pkgs_remote:
+ print(f"{package}: building source package (not in OBS)")
+ else:
+ obs_version = lib.osc.get_package_version(package)
+ if is_up_to_date(obs_version, latest_version):
+ if lib.args.skip_up_to_date:
+ print(f"{package}: skipping ({obs_version} is up-to-date)")
+ srcpkgs_skipped += [package]
+ return
+ else:
+ print(f"{package}: building source package"
+ f" ({obs_version} is up-to-date, but"
+ " --no-skip-up-to-date is set)")
+ else:
+ print(f"{package}: building source package (outdated:"
+ f" {latest_version} <=> {obs_version} in OBS)")
+ else:
+ print(f"{package}: building source package (feed is {feed})")
+
+ build_srcpkg(package, is_meta_pkg)
+
+
+def upload_srcpkg(pkgs_remote, package, version):
+ if os.path.basename(package) not in pkgs_remote:
+ lib.osc.create_package(package)
+ lib.osc.update_package(package, version)
+
+
+def build_srcpkgs(pkgs_remote, packages):
+ print()
+ print("### Building source packages ###")
+ print()
+
+ if lib.args.meta:
+ feed = lib.args.feed
+ build_srcpkg_if_needed(pkgs_remote, f"osmocom-{feed}", True)
+
+ for package in packages:
+ build_srcpkg_if_needed(pkgs_remote, package, False)
+
+
+def upload_srcpkgs(pkgs_remote):
+ global srcpkgs_built
+ global srcpkgs_failed_upload
+ global srcpkgs_updated
+
+ srcpkgs_failed_upload = []
+ srcpkgs_updated = []
+
+ if not srcpkgs_built:
+ return
+
+ print()
+ print("### Uploading built packages ###")
+ print()
+
+ for package, version in srcpkgs_built.items():
+ try:
+ upload_srcpkg(pkgs_remote, package, version)
+ srcpkgs_updated += [package]
+ except Exception as ex:
+ traceback.print_exception(type(ex), ex, ex.__traceback__)
+ print()
+ print(f"{package}: upload failed")
+ srcpkgs_failed_upload += [package]
+
+
+def exit_with_summary():
+ global srcpkgs_updated
+ global srcpkgs_skipped
+ global srcpkgs_failed_build
+ global srcpkgs_failed_upload
+
+ print()
+ print("### Summary ###")
+ print()
+ print(f"Updated: {len(srcpkgs_updated)}")
+ print(f"Skipped: {len(srcpkgs_skipped)}")
+ print(f"Failed (srcpkg build): {len(srcpkgs_failed_build)}")
+ print(f"Failed (srcpkg upload): {len(srcpkgs_failed_upload)}")
+ print(f"Deleted: {len(srcpkgs_deleted)}")
+
+ if not srcpkgs_failed_build and not srcpkgs_failed_upload:
+ sys.exit(0)
+
+ print()
+ print("List of failed packages:")
+ for package in srcpkgs_failed_build:
+ print(f"* {package} (srcpkg build)")
+ for package in srcpkgs_failed_upload:
+ print(f"* {package} (srcpkg upload)")
+
+ sys.exit(1)
+
+
+def validate_args(args):
+ # Only with feed=master we check the current commit of a branch on a remote
+ # git repository before trying to update/delete a package from OBS
+ if args.delete and args.feed != "master":
+ print("ERROR: --delete can only be used with --feed=master")
+ sys.exit(1)
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ description="Generate source packages and upload them to OBS.")
+ lib.add_shared_arguments(parser)
+ parser.add_argument("-A", "--apiurl", help="OBS API URL or .oscrc alias"
+ " (e.g. https://obs.osmocom.org)")
+ parser.add_argument("-n", "--no-skip-up-to-date",
+ dest="skip_up_to_date", action="store_false",
+ help="for latest feed, build and upload packages even"
+ " if the version did not change")
+ parser.add_argument("--delete", action="store_true",
+ help="remove packages from OBS if the git branch (-b)"
+ " does not exist anymore")
+ parser.add_argument("obs_project",
+ help="OBS project, e.g. home:osmith:nightly")
+ parser.add_argument("package", nargs="*",
+ help="package name, e.g. libosmocore or open5gs or"
+ " ALL_OSMOCOM_PACKAGES, default is all packages")
+ args = parser.parse_args()
+ validate_args(args)
+ lib.set_args(args)
+ packages = parse_packages(args.package)
+
+ if args.docker:
+ lib.docker.run_in_docker_and_exit("update_obs_project.py", True)
+
+ lib.osc.check_oscrc()
+ lib.osc.set_apiurl(args.apiurl, args.obs_project)
+
+ if not args.ignore_req:
+ lib.check_required_programs()
+
+ lib.remove_temp()
+
+ pkgs_remote = lib.osc.get_remote_pkgs()
+
+ build_srcpkgs(pkgs_remote, packages)
+ upload_srcpkgs(pkgs_remote)
+ exit_with_summary()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/obs/update_obs_wireshark.sh b/scripts/obs/update_obs_wireshark.sh
new file mode 100755
index 0000000..e422292
--- /dev/null
+++ b/scripts/obs/update_obs_wireshark.sh
@@ -0,0 +1,47 @@
+#!/bin/sh -e
+DIR="$(realpath "$(dirname "$0")")"
+PROJ="$1"
+
+BRANCHES="
+ osmith/deb-packaging
+ osmocom/qcdiag
+ laforge/rspro
+"
+
+prepare_git_repo() {
+ cd "$DIR"
+ if ! [ -d _cache/wireshark ]; then
+ mkdir -p _cache
+ git -C _cache clone https://gitlab.com/wireshark/wireshark.git
+ git -C _cache/wireshark remote add osmocom https://gitea.osmocom.org/osmocom/wireshark
+ fi
+
+ cd _cache/wireshark
+ git fetch --all
+ git clean -fdx
+ git checkout -f -B osmocom/all-in-one origin/master
+
+ for b in $BRANCHES; do
+ git merge --no-edit "osmocom/$b"
+ done
+}
+
+update_obs_project() {
+ cd "$DIR"
+ ./update_obs_project.py \
+ --apiurl https://obs.osmocom.org \
+ --docker \
+ --allow-unknown-package \
+ --git-skip-checkout \
+ --git-skip-fetch \
+ --version-append "~osmocom" \
+ "$PROJ" \
+ wireshark
+}
+
+set -x
+prepare_git_repo
+
+if [ -n "$PROJ" ]; then
+ update_obs_project
+fi
diff --git a/scripts/osmo-build.sh b/scripts/osmo-build.sh
deleted file mode 100644
index 39cbae6..0000000
--- a/scripts/osmo-build.sh
+++ /dev/null
@@ -1,230 +0,0 @@
-#!/bin/sh
-#
-# This script enables artifacts holding dependencies on a jenkins job level to
-# speed up builds. Basically, it holds logic to check whether the necessary artifact
-# is available. If so it fetches artifact, unpacks it and if cp/tar succeeded
-# it triggers the actual build.
-#
-# Otherwise it simply builds all dependencies from source by using osmo-build-dep.sh
-# and archives deps to the ARTIFACT_STORE afterwards. Revisions of locally built
-# dependencies are detrmined after dependencies are built to ensure catching new
-# changes in dep_n+1 meanwhile building dep_n.
-#
-# Furthermore, ARTIFACT_STORE environment variable has to be set on all jenkins slaves.
-# The JOB_NAME variable will be injected to each jenkins' job by jenkins itself.
-# When using script within a docker container one must inject jenkins' JOB_NAME variable
-# to the container and ensure that ARTIFACT_STORE is mounted to the container's
-# internal ARTIFACT_STORE.
-#
-# Artifacts will be stored as follows:
-#
-# $ARTIFACT_STORE/$JOB_NAME/<dep_1>.<branch_1>.<rev_1>_...
-# ..._<dep_n>.<tag_n>.tar.gz
-#
-# Note: each matrix-build has its own directory inside ARTIFACT_STORE.
-#
-# In order to make use of osmo-build.sh one needs to source it, e.g. from
-# ./contrib/jenkins.sh. Furthermore, jenkins should check out the git tree of
-# the project to be built in the workspace root. Following functions needs to be
-# declared within a build script that sources osmo-build.sh:
-#
-# - artifact_name()
-# - build_deps()
-# - build_project()
-#
-# This is an example for building "libosmo-netif" which depends on "libosmocore"
-# and "libosmo-abis".
-#
-# #!/bin/sh
-#
-# artifact_deps() {
-# # $1 will be one of folllowing functions:
-# # - artifact_name_by_local_repo()
-# # - artifact_name_by_remote_repo()
-# # osmo-build.sh takes care about which function to use
-#
-# x="$($1 libosmocore)"
-# x="${x}_$($1 libosmo-abis)"
-#
-# echo "${x}.tar.gz"
-# }
-#
-# build_deps() {
-# # all commands to build necessary dependencies
-# osmo-build-dep.sh libosmocore master ac_cv_path_DOXYGEN=false
-# "$deps"/libosmocore/contrib/verify_value_string_arrays_are_terminated.py $(find . -name "*.[hc]")
-# osmo-build-dep.sh libosmo-abis
-# }
-#
-# build_project() {
-# # Necessary commands to build the project, expecting all dependencies have
-# # been built or fetched. Commands within build_project() will be executed
-# # in jenkins' $WORKSPACE.
-#
-# autoreconf --install --force
-# ./configure --enable-sanitize
-# $MAKE $PARALLEL_MAKE
-# $MAKE distcheck || cat-testlogs.sh
-# }
-#
-##
-# # source osmo-build.sh to fire the build
-# . osmo-build.sh
-
-log() {
- set +x
- echo
- echo "[INFO] $1"
- echo
- set -x
-}
-
-# SOURCING SANITY
-log "source sanity check to ensure that sourcing script holds necessary functions"
-type artifact_deps
-type build_deps
-type build_project
-log "check whether necessary dependency build scripts are in PATH"
-type osmo-build-dep.sh
-type osmo-deps.sh
-
-# BUILD FUNCTIONS
-init_build() {
-
- if [ -z "$JOB_NAME" ]; then
- log "[ERROR] JOB_NAME variable is not set, running in Jenkins?"
- exit 1
- fi
-
- if [ -z "$ARTIFACT_STORE" ]; then
- log "[ERROR] ARTIFACT_STORE variable is not set on this build slave"
- exit 1
- fi
-
- base="$(pwd)"
- deps="$base/deps"
- inst="$deps/install"
- rm -rf "$deps" || true
-
- # obtain the project name from the git clone found in the workspace root
- project=$(git config --get --local remote.origin.url \
- | cut -d '/' -f4 | cut -d '.' -f1)
-
- # replace invalid char for dirs in $JOB_NAME (jenkins variable)
- # ( '/' separates job name and matrix-axis)
- job_name="$( echo "$JOB_NAME" | tr '/' '_')"
-
- export base deps inst project job_name
- export PKG_CONFIG_PATH="$inst/lib/pkgconfig:$PKG_CONFIG_PATH"
- export LD_LIBRARY_PATH="$inst/lib"
-
- log "$project build initialized"
-}
-
-build() {
-
- init_build
-
- artifact_name="$(artifact_name)"
-
- if [ -f "$ARTIFACT_STORE/$job_name/$artifact_name" ]; then
- fetch_artifact "$ARTIFACT_STORE/$job_name" "$artifact_name"
- else
- log "Compile $project dependencies from source."
- mkdir -p "$deps"
- rm -rf "$inst"
-
- build_deps
- archive_artifact
- fi
-
- log "building $project"
- build_project
-}
-
-# ARTIFACT FUNCTIONS
-artifact_name() {
- # in case deps is empty or does not exist we
- if [ -d "$deps" ]; then
- artifact_deps "branch_and_rev_of_local_repo"
- cd "$base"
- else
- artifact_deps "branch_and_rev_of_remote_repo"
- fi
-}
-
-branch_and_rev_of_local_repo() {
- cd "$deps/$1"
- rev="$(git rev-parse --short HEAD)"
- branch="$(git rev-parse --abbrev-ref HEAD)"
-
- # check whether it is a tag
- if [ "$branch" = "HEAD" ]; then
- tag="$(git describe --tags HEAD)"
- tag="$(echo "$tag" | tr '/' '_')"
- echo "$1.$tag"
- else
- branch="$( echo "$branch" | tr '/' '_')"
- echo "$1.$branch.$rev"
- fi
-}
-
-branch_and_rev_of_remote_repo() {
- if [ -z "${2+x}" ]; then branch="master"; else branch="$2"; fi
- branch="$( echo "$branch" | tr '/' '_')"
- rev="$(git ls-remote "git://git.osmocom.org/$1" "refs/heads/$branch")"
-
- # check whether branch is a tag
- if [ "$rev" = "" ]; then
- echo "$1.$branch"
- else
- rev="$(echo "$rev" | cut -c 1-7)"
- echo "$1.$branch.$rev"
- fi
-}
-
-archive_artifact() {
- log "Archiving artifact to artifactStore."
-
- cd "$base"
- artifact="$(artifact_name)"
- # temp_job_store is necessary to atomically move it to production.
- temp_job_store="$ARTIFACT_STORE/tmp/$job_name/"
- job_store="$ARTIFACT_STORE/$job_name/"
-
- if [ ! -f "$temp_job_store/$artifact" ]; then
- mkdir -p "$job_store" "$temp_job_store"
- # remove outdated artifact first to avoid temporarily
- # doubling of artifact storage consumption
- rm -f "$job_store/*"
- tar czf "$temp_job_store/$artifact" "deps"
- mv -n "$temp_job_store/$artifact" "$job_store/$artifact"
- rm -rf "$temp_job_store"
-
- log_artifact_hash "$job_store/$artifact"
- fi
-}
-
-fetch_artifact() {
- log "Fetching artifact from artifactStore."
-
- log_artifact_hash "$1/$2"
- cp "$1/$2" .
- log_artifact_hash "$2"
- tar xzf "$2"
-
- if [ $? -gt 0 ]; then
- log "Artifact could not be fetched, triggering build_deps()"
- build_deps
- else
- log "Artifact successfully fetched, triggering $project compilation"
- fi
-}
-
-# checksum is not used by script itself,
-# but might be handy in logs when debugging.
-log_artifact_hash() {
- log "name: $1 \n sha256: $(sha256sum "$1" | cut -d ' ' -f1)"
-}
-
-build
diff --git a/scripts/osmo-ci-docker-rebuild.sh b/scripts/osmo-ci-docker-rebuild.sh
index 73b3508..c9e2b63 100755
--- a/scripts/osmo-ci-docker-rebuild.sh
+++ b/scripts/osmo-ci-docker-rebuild.sh
@@ -2,12 +2,15 @@
set -e -x
cd "$(dirname "$0")/.."
. scripts/common.sh
-docker_images_require \
- "debian-stretch-jenkins" \
- "debian-buster-jenkins" \
- "debian-buster-erlang" \
-if [ "$(arch)" = "x86_64" ]; then
+case "$(arch)" in
+x86_64)
docker_images_require \
- "osmo-gsm-tester"
-fi
+ "debian-bookworm-build" \
+ "debian-bookworm-erlang"
+ ;;
+arm*)
+ docker_images_require \
+ "debian-bookworm-build-arm"
+ ;;
+esac
diff --git a/scripts/osmo-depcheck/config.py b/scripts/osmo-depcheck/config.py
index 13c1a95..94a1cb5 100644
--- a/scripts/osmo-depcheck/config.py
+++ b/scripts/osmo-depcheck/config.py
@@ -2,7 +2,7 @@
# Copyright 2018 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
# Where to clone sources from (with trailing slash)
-git_url_prefix = "git://git.osmocom.org/"
+git_url_prefix = "https://gerrit.osmocom.org/"
# Default projects to build when none are specified on the command line
projects = ("osmo-bts",
diff --git a/scripts/osmo-depcheck/dependencies.py b/scripts/osmo-depcheck/dependencies.py
index 9b5187d..d2b9927 100644
--- a/scripts/osmo-depcheck/dependencies.py
+++ b/scripts/osmo-depcheck/dependencies.py
@@ -14,7 +14,7 @@ def git_clone(workdir, prefix, cache_git_fetch, repository, version):
""" Clone a missing git repository and checkout a specific version tag.
:param workdir: path to where all data (git, build, install) is stored
- :param prefix: git url prefix (e.g. "git://git.osmocom.org/")
+ :param prefix: git url prefix (e.g. "https://gerrit.osmocom.org/")
:param cache_git_fetch: list of repositories that have already been
fetched in this run of osmo-depcheck
:param repository: Osmocom git repository name (e.g. "libosmo-abis")
@@ -56,7 +56,7 @@ def generate(workdir, prefix, cache_git_fetch, initial, rev):
repository, parsing the "configure.ac" file, and recursing.
:param workdir: path to where all data (git, build, install) is stored
- :param prefix: git url prefix (e.g. "git://git.osmocom.org/")
+ :param prefix: git url prefix (e.g. "https://gerrit.osmocom.org/")
:param cache_git_fetch: list of repositories that have already been
fetched in this run of osmo-depcheck
:param initial: the first program to look at (e.g. "osmo-bts")
diff --git a/scripts/osmo-deps.sh b/scripts/osmo-deps.sh
index 5a7dcc2..59ecce0 100755
--- a/scripts/osmo-deps.sh
+++ b/scripts/osmo-deps.sh
@@ -1,4 +1,5 @@
#!/bin/sh
+. "$(dirname "$0")/common.sh"
set -ex
project="$1"
branch="${2:-master}"
@@ -8,9 +9,8 @@ if [ "x$(git branch -a | grep -c "remotes/origin/$branch\$")" != "x0" ]; then
fi
-if ! test -d "$project";
-then
- git clone "git://git.osmocom.org/$project" "$project"
+if ! test -d "$project"; then
+ git clone "$(osmo_git_clone_url "$project")" "$project"
fi
cd "$project"
diff --git a/scripts/osmo-layer1-headers.sh b/scripts/osmo-layer1-headers.sh
index f23a888..b9c43c5 100755
--- a/scripts/osmo-layer1-headers.sh
+++ b/scripts/osmo-layer1-headers.sh
@@ -11,12 +11,12 @@ set -e -x
case "$1" in
sysmo)
- uri="https://git.sysmocom.de/sysmo-bts/layer1-api"
+ uri="https://gitea.sysmocom.de/sysmo-bts/layer1-api"
version_prefix=""
version="origin/master"
;;
oct)
- uri="git://git.osmocom.org/octphy-2g-headers"
+ uri="https://gitea.osmocom.org/cellular-infrastructure/octphy-2g-headers"
version_prefix=""
version="origin/master"
;;
diff --git a/scripts/osmocom-api-doxygen.sh b/scripts/osmocom-api-doxygen.sh
new file mode 100755
index 0000000..cfa5e61
--- /dev/null
+++ b/scripts/osmocom-api-doxygen.sh
@@ -0,0 +1,56 @@
+#!/bin/sh -ex
+SSH_CMD="ssh -o UserKnownHostsFile=/build/contrib/known_hosts -p 48"
+
+# Repositories for which doxygen documentation will be generated and
+# uploaded, also dependencies which need to be built
+repos_api="
+ libosmocore
+ libosmo-abis
+ libosmo-dsp
+ libosmo-netif
+ libosmo-sccp
+ osmo-gmr
+"
+
+# Source common.sh from osmo-ci.git for osmo_git_clone_url()
+. scripts/common.sh
+
+# Check early that SSH works
+$SSH_CMD api@ftp.osmocom.org -T -- true
+
+# Put git repos and install data in a subdir, so it isn't in the root
+# of the cloned osmo-ci.git repository
+mkdir _osmocom_api
+cd _osmocom_api
+
+# Prepare pkgconfig path
+export PKG_CONFIG_PATH=$PWD/install/lib/pkgconfig
+mkdir -p "$PKG_CONFIG_PATH"
+
+# Clone and build the repositories
+for i in $repos_api; do
+ git clone "$(osmo_git_clone_url "$i")"
+ cd "$i"
+ autoreconf -fi
+ ./configure \
+ --prefix=$PWD/../install \
+ --with-systemdsystemunitdir=no
+ make $PARALLEL_MAKE install
+ cd ..
+done
+
+# Upload all docs
+for i in $repos_api; do
+ if ! [ -d "$i"/doc ]; then
+ # e.g. libosmo-abis is built as dependency for others but doesn't
+ # have its own doxygen documentation as of writing
+ continue
+ fi
+
+ rsync \
+ -avz \
+ --delete \
+ -e "$SSH_CMD" \
+ ./"$i"/doc/ \
+ api@ftp.osmocom.org:web-files/latest/"$i"/
+done
diff --git a/scripts/osmocom-build-old-tags-against-master.sh b/scripts/osmocom-build-old-tags-against-master.sh
index 71d5a1f..5f0d04d 100755
--- a/scripts/osmocom-build-old-tags-against-master.sh
+++ b/scripts/osmocom-build-old-tags-against-master.sh
@@ -54,6 +54,8 @@ tags_to_ignore() {
echo "1.0.0" # testsuite
echo "1.3.1" # testsuite: libosmocore 6407c822ae0d45d047deb0dff095f028a90c1ecc
echo "1.3.2" # testsuite: libosmocore 6407c822ae0d45d047deb0dff095f028a90c1ecc
+ echo "1.4.0" # error: potential null pointer dereference, fixed in 1.4.2
+ echo "1.4.1" # error: potential null pointer dereference, fixed in 1.4.2
;;
osmo-bsc)
# Depends on libosmo-legacy-mgcp, but missing LIBOSMOLEGACYMGCP_CFLAGS so we can't build it with
@@ -66,6 +68,11 @@ tags_to_ignore() {
echo "1.2.0" # testsuite: libosmocore 6407c822ae0d45d047deb0dff095f028a90c1ecc
echo "1.2.1" # testsuite: libosmocore 6407c822ae0d45d047deb0dff095f028a90c1ecc
;;
+ osmo-hlr)
+ echo "1.4.0" # testsuite: libosmocore I446e54d0ddf4a18c46ee022b1249af73552e3ce1
+ echo "1.5.0" # testsuite: libosmocore I446e54d0ddf4a18c46ee022b1249af73552e3ce1
+ echo "1.6.0" # testsuite: libosmocore I446e54d0ddf4a18c46ee022b1249af73552e3ce1
+ ;;
osmo-mgw)
echo "1.3.0" # testsuite
echo "1.4.0" # testsuite
diff --git a/scripts/osmocom-latest-packages.sh b/scripts/osmocom-latest-packages.sh
deleted file mode 100755
index e218f1e..0000000
--- a/scripts/osmocom-latest-packages.sh
+++ /dev/null
@@ -1,240 +0,0 @@
-#!/bin/sh
-# Generate source packages and upload them to OBS, for the latest feed.
-# New packages are only uploaded if the source changed.
-# Environment variables:
-# * PROJ: the OBS namespace to upload to (e.g. "network:osmocom:latest")
-# * FEED:
-# * "latest": use latest tagged release (default)
-# * other: use last commit of branch of same name, exit with error if it does
-# not exist
-# * PACKAGES: set to a space-separated list of packages to skip all others
-. "$(dirname "$0")/common.sh"
-. "$(dirname "$0")/common-obs.sh"
-
-# Values for FEED env var. Adjust FEEDS_ALL in common-obs when changing.
-FEEDS="
- latest
-"
-
-set -e
-set -x
-
-DT=$(date +%Y%m%d%H%M)
-TOP=$(pwd)
-DEBSRCDIR="$TOP/debsrc"
-FEED="${FEED:-latest}"
-
-### OBS build
-prepare() {
- # start with a checkout of the project
- if [ -d $PROJ ]; then
- (cd $PROJ && osc up)
- else
- osc co $PROJ
- fi
- [ -d "$DEBSRCDIR" ] && rm -rf "$DEBSRCDIR"
- mkdir "$DEBSRCDIR"
-
- cd "$TOP"
- osmo_obs_prepare_conflict
-}
-
-get_last_tag() {
- project="$1"
- if [ "$project" = "limesuite" ] || [ "$project" = "open5gs" ]; then
- ver_regexp="^v[0-9]*.[0-9]*.[0-9]*$"
- else
- ver_regexp="^[0-9]*.[0-9]*.[0-9]*$"
- fi
- VER=$(git -C "${TOP}/${project}" tag -l --sort=v:refname | grep "$ver_regexp" | tail -n 1)
- echo "${VER}"
-}
-
-checkout() {
- project=$1
- url=$2
- gitbpargs=""
-
- if osmo_obs_skip_pkg "$project"; then
- return
- fi
-
- if [ -z "$url" ]; then
- url="$(osmo_git_clone_url "$project")"
- fi
-
- echo
- echo "====> Checking out $project"
- cd "$TOP"
- [ -d "$project" ] || osmo_git_clone_date "$url" "$project"
- cd "$project"
- git fetch
-
- if [ "$FEED" = "latest" ]; then
- VER=$(get_last_tag "$project")
- git checkout -f -B "$VER" "refs/tags/$VER"
- else
- git checkout -f -B "$FEED" "origin/$FEED"
- fi
-
- if [ "$project" = "open5gs" ]; then
- meson subprojects download freeDiameter
- fi
-}
-
-# Generate a source package and upload it to OBS
-# $1: package name (e.g. "libosmocore")
-# $2: arguments to pass to "gbp buildpackage"
-build() {
- project=$1
- gitbpargs="$2"
- output="$DEBSRCDIR/$project"
-
- if osmo_obs_skip_pkg "$project"; then
- return
- fi
-
- echo
- echo "====> Building $project"
- cd "$TOP/$project"
-
- osmo_obs_git_version_gen
-
- if [ "$FEED" = "latest" ]; then
- debian_branch=$(get_last_tag "$project")
- else
- debian_branch="$FEED"
- # Set new debian changelog version with commit appended. This version will
- # become part of resulting filenames, and will change if commits have been
- # added to the feed's branch.
- VER="$(osmo_obs_get_commit_version)"
- dch -b -v "$VER" -m "Snapshot build"
- git commit -m "$VER snapshot" debian/
- fi
-
- osmo_obs_add_depend_deb "./debian/control" "$project" "osmocom-$FEED"
-
- if [ "$project" = "open5gs" ]; then
- # we cannot control the output directory of the generated source :(
- dpkg-buildpackage -S -uc -us -d
- mkdir -p "$output"
- mv "../$name"*.tar* "../$name"*.dsc "$output"
- elif [ -x ./git-version-gen ]; then
- gbp buildpackage -S -uc -us -d --git-ignore-branch "--git-export-dir=$output" \
- "--git-debian-branch=$debian_branch" --git-ignore-new $gitbpargs \
- --git-postexport='cp $GBP_GIT_DIR/../.tarball-version $GBP_TMP_DIR/'
- else
- gbp buildpackage -S -uc -us -d --git-ignore-branch "--git-export-dir=$output" \
- "--git-debian-branch=$debian_branch" --git-ignore-new $gitbpargs
- fi
-
- if [ ! -d "$TOP/$PROJ/$project" ] ; then
- # creating a new package is different from using old ones
- mkdir "$TOP/$PROJ/$project"
- mv "$output/"*.dsc "$TOP/$PROJ/$project/"
- mv "$output/"*.tar* "$TOP/$PROJ/$project/"
- cd "$TOP/$PROJ"
- osc add "$project"
- else
- cd "$TOP/$PROJ/$project"
-
- # update OBS only if the filename doesn't match
- file=$(cd "$output/" ; ls ./*.dsc)
- if [ ! -e "$file" ] ; then
- osc rm ./* || true
- mv "$output/"*.dsc .
- mv "$output/"*.tar* .
- osc add ./*
- fi
- fi
-
- osmo_obs_add_rpm_spec "$TOP/$PROJ/$project" "$TOP/$project" "$project" "osmocom-$FEED"
-
- cd "$TOP"
-}
-
-# add those once they have tagged any versions that include the 'debian' sub-dir:
-#rtl-sdr
-#osmo-fl2k
-
-build_osmocom() {
- prepare
-
- # NOTE: when adding a repository that is not in gerrit, adjust osmo_git_clone_url()
- checkout limesuite https://github.com/myriadrf/LimeSuite
- checkout osmo-gsm-manuals
- checkout libosmocore
- checkout libosmo-sccp
- checkout libosmo-abis
- checkout libosmo-netif
- checkout libsmpp34
- checkout libasn1c
- checkout libgtpnl
- checkout libusrp
- checkout osmo-iuh
- checkout osmo-ggsn
- checkout osmo-sgsn
- checkout osmo-pcap
- checkout osmo-trx
- checkout osmo-sip-connector
- checkout osmo-bts
- checkout osmo-pcu
- checkout osmo-hlr
- checkout osmo-mgw
- checkout osmo-msc
- checkout osmo-bsc
- checkout simtrace2
- checkout libosmo-dsp
- checkout osmo-sysmon
- checkout osmo-remsim
- checkout open5gs https://github.com/open5gs/open5gs
- checkout neocon https://github.com/laf0rge/neocon
- checkout osmo-uecups
- checkout osmo-e1d
- checkout osmo-smlc
- checkout osmo-cbc
- checkout osmo-gbproxy
-
- cd "$TOP"
-
- build osmocom-$FEED
- build limesuite --git-upstream-tree="$(get_last_tag limesuite)"
- build osmo-gsm-manuals
- build libosmocore
- build libosmo-sccp
- build libosmo-abis
- build libosmo-netif
- build libsmpp34
- build libasn1c
- build libgtpnl
- build libusrp
- build osmo-iuh
- build osmo-ggsn
- build osmo-sgsn
- build osmo-pcap
- build osmo-trx
- build osmo-sip-connector
- build osmo-bts
- build osmo-pcu
- build osmo-hlr
- build osmo-mgw
- build osmo-msc
- build osmo-bsc
- build simtrace2
- build libosmo-dsp
- build osmo-sysmon
- build osmo-remsim
- build open5gs
- build neocon
- build osmo-uecups
- build osmo-e1d
- build osmo-smlc
- build osmo-cbc
- build osmo-gbproxy
-
- cd "$TOP/$PROJ"
- osc ci -m "$FEED versions of $DT" --noservice
-}
-
-osmo_obs_verify_feed
-build_osmocom
diff --git a/scripts/osmocom-list-commits.sh b/scripts/osmocom-list-commits.sh
index b584195..3fec196 100755
--- a/scripts/osmocom-list-commits.sh
+++ b/scripts/osmocom-list-commits.sh
@@ -15,10 +15,11 @@ for repo in $OSMO_RELEASE_REPOS; do
last_tag="$(osmo_git_last_tags "$repo" 1 "-")"
last_commit="$(osmo_git_last_commits "$repo" 1 "-")"
head_commit="$(osmo_git_head_commit "$repo")"
+ clone_url="$(osmo_git_clone_url "$repo")"
printf "$FORMAT_STR" \
"$repo.git" \
- "$OSMO_GIT_URL/$repo" \
+ "$clone_url" \
"$last_tag" \
"$last_commit" \
"$head_commit"
diff --git a/scripts/osmocom-next-packages.sh b/scripts/osmocom-next-packages.sh
deleted file mode 100755
index aa9ab90..0000000
--- a/scripts/osmocom-next-packages.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-# Generate source packages and upload them to OBS, for the next feed.
-. "$(dirname "$0")/common.sh"
-
-export FEED="next"
-$OSMO_CI_DIR/scripts/osmocom-nightly-packages.sh
diff --git a/scripts/osmocom-nightly-packages.sh b/scripts/osmocom-nightly-packages.sh
deleted file mode 100755
index 2bdc4f0..0000000
--- a/scripts/osmocom-nightly-packages.sh
+++ /dev/null
@@ -1,274 +0,0 @@
-#!/bin/bash
-# Generate source packages and upload them to OBS, for the nightly or next feed.
-# New packages are always uploaded, even if the source does not change. Only
-# packages of the same build date (DT) can be installed together.
-# Environment variables:
-# * PROJ: the OBS namespace to upload to (e.g. "network:osmocom:nightly")
-# * FEED: controls the source branch that is used:
-# * "nightly": use "master" branch (default)
-# * "next": use "next" branch if it exists, otherwise use "master" branch
-# * other (e.g. "2021q1"): use last commit of branch of same name, exit with
-# error if it does not exist
-# * PACKAGES: set to a space-separated list of packages to skip all others
-. "$(dirname "$0")/common.sh"
-. "$(dirname "$0")/common-obs.sh"
-
-# Values for FEED env var. Adjust FEEDS_ALL in common-obs when changing.
-FEEDS="
- 2021q1
- next
- nightly
-"
-
-set -e
-set -x
-
-DT=$(date +%Y%m%d%H%M)
-OSMO_OBS_CONFLICT_PKGVER="$OSMO_OBS_CONFLICT_PKGVER.$DT"
-TOP=$(pwd)/$(mktemp -d nightly-3g_XXXXXX)
-FEED="${FEED:-nightly}"
-
-### OBS build
-prepare() {
- # clean up the whole space
- mkdir -p "$REPO/osc/"
- cd "$REPO/osc"
- osc co "$PROJ"
-
- cd "$REPO"
- osmo_obs_prepare_conflict
-}
-
-get_last_tag() {
- project="$1"
- if [ "$project" = "limesuite" ]; then
- ver_regexp="^v[0-9]*.[0-9]*.[0-9]*$"
- else
- ver_regexp="^[0-9]*.[0-9]*.[0-9]*$"
- fi
- VER=$(git -C "${REPO}/${project}" tag -l --sort=v:refname | grep "$ver_regexp" | tail -n 1)
- echo "${VER}"
-}
-
-### common
-checkout() {
- local name=$1
- local url=$2
- local branch=$3
-
- if osmo_obs_skip_pkg "$name"; then
- return
- fi
-
- if [ -z "$url" ]; then
- url="$(osmo_git_clone_url "$name")"
- fi
-
- if [ -z "$branch" ]; then
- case "$FEED" in
- nightly|next) ;;
- *) branch="$FEED" ;; # e.g. 2021q1
- esac
- fi
-
- cd "$REPO"
-
- if [ -n "$branch" ] ; then
- osmo_git_clone_date "$url" -b "$branch"
- else
- osmo_git_clone_date "$url"
- fi
-
- if [ "$FEED" = "next" ] && git -C "$name" show-branch remotes/origin/next >/dev/null 2>&1; then
- git -C "$name" checkout next
- fi
-
- cd -
-}
-
-# Generate a source package and upload it to OBS
-# $1: package name (e.g. "libosmocore")
-# $2: update debian dir, unless set to "no_commit":
-# * add dependency on osmocom-$FEED package
-# * add new version to changelog (e.g. "1.5.1.96.c96d7.202104281354")
-# $3: arguments to pass to "gbp buildpackage"
-build() {
- local name=$1
- local changelog=$2
- local gitbpargs=$3
- local repodir=$REPO/$name
- local oscdir=$REPO/osc/$PROJ/$name
- local dependver="$OSMO_OBS_CONFLICT_PKGVER"
-
- if osmo_obs_skip_pkg "$name"; then
- return
- fi
-
- if [ -d "$oscdir" ] ; then
- # remove earlier version
- cd "$oscdir"
- osc rm -- * || true
- else
- # new package
- mkdir -p "$oscdir/"
- cd "$REPO/osc/$PROJ/"
- osc add "$name"
- fi
-
- cd "$repodir"
-
- if [ "$changelog" != "no_commit" ] ; then
- osmo_obs_git_version_gen
- # Add date to increase version even if commit did not change (OS#5135)
- VER="$(osmo_obs_get_commit_version).$DT"
- osmo_obs_add_depend_deb "./debian/control" "$name" "osmocom-$FEED" "$dependver"
- dch -b -v "$VER" -m "Snapshot build"
- git commit -m "$VER snapshot" debian/
- fi
-
- mkdir -p "$DATA/$name"
- if [ "$name" = "open5gs" ]; then
- # we cannot control the output directory of the generated source :(
- dpkg-buildpackage -S -uc -us -d
- mv "../$name"*.tar* "../$name"*.dsc "$DATA/$name/"
- elif [ -f .tarball-version ]; then
- gbp buildpackage -S -uc -us -d --git-ignore-branch "--git-export-dir=$DATA/$name" \
- --git-ignore-new $gitbpargs \
- --git-postexport='cp $GBP_GIT_DIR/../.tarball-version $GBP_TMP_DIR/'
- else
- gbp buildpackage -S -uc -us -d --git-ignore-branch "--git-export-dir=$DATA/$name" \
- --git-ignore-new $gitbpargs
- fi
-
- mv "$DATA/$name/"*.tar* "$DATA/$name/"*.dsc "$oscdir/"
-
- cd "$oscdir"
- osc add -- *.tar* *.dsc
- osmo_obs_add_rpm_spec "$oscdir" "$repodir" "$name" "osmocom-$FEED" "$dependver"
- osc ci -m "Snapshot $name $DT" --noservice
-}
-
-post() {
- cd "$REPO/osc/$PROJ/"
- osc status
-}
-
-checkout_limesuite() {
- if osmo_obs_skip_pkg "limesuite"; then
- return
- fi
-
- cd "$REPO"
- git clone https://github.com/myriadrf/LimeSuite limesuite
- TAG="$(get_last_tag limesuite)"
- cd limesuite
- git checkout "$TAG"
-}
-
-checkout_open5gs() {
- if osmo_obs_skip_pkg "open5gs"; then
- return
- fi
-
- cd "$REPO"
- git clone https://github.com/open5gs/open5gs
- cd open5gs
- meson subprojects download freeDiameter
-}
-
-build_osmocom() {
- DATA=$TOP/data
- REPO=$TOP/repo
-
- # rather than including a dangerous 'rm -rf *' here, lets delegate to the user:
- if [ -n "$(ls $TOP)" ]; then
- echo "ERROR: I need to run in an empty directory."
- exit 1
- fi
-
- prepare
-
- # NOTE: when adding a repository that is not in gerrit, adjust osmo_git_clone_url()
- checkout_limesuite
- checkout osmo-gsm-manuals
- checkout libosmocore
- checkout libosmo-sccp
- checkout libosmo-abis
- checkout libosmo-netif
- checkout libsmpp34
- checkout libasn1c
- checkout libgtpnl
- checkout libusrp
- checkout osmo-iuh
- checkout osmo-ggsn
- checkout osmo-sgsn
- checkout osmo-pcap
- checkout osmo-trx
- checkout osmo-sip-connector
- checkout osmo-bts
- checkout osmo-pcu
- checkout osmo-hlr
- checkout osmo-mgw
- checkout osmo-msc
- checkout osmo-bsc
- checkout python/osmo-python-tests
- checkout rtl-sdr
- checkout osmo-fl2k
- checkout simtrace2
- checkout libosmo-dsp
- checkout osmo-sysmon
- checkout osmo-remsim
- checkout_open5gs
- checkout neocon https://github.com/laf0rge/neocon
- checkout osmo-uecups
- checkout osmo-e1d
- checkout osmo-smlc
- checkout osmo-cbc
- checkout osmo-gbproxy
-
- cd "$REPO"
-
- build osmocom-$FEED no_commit
- build limesuite no_commit --git-upstream-tree="$(get_last_tag limesuite)"
- build osmo-gsm-manuals
- build libosmocore
- build libosmo-sccp
- build libosmo-abis
- build libosmo-netif
- build libsmpp34
- build libasn1c
- build libgtpnl
- build libusrp
- build osmo-iuh
- build osmo-ggsn
- build osmo-sgsn
- build osmo-pcap
- build osmo-trx
- build osmo-sip-connector
- build osmo-bts
- build osmo-pcu
- build osmo-hlr
- build osmo-mgw
- build osmo-msc
- build osmo-bsc
- build osmo-python-tests
- build rtl-sdr
- build osmo-fl2k
- build simtrace2
- build libosmo-dsp
- build osmo-sysmon
- build osmo-remsim
- build open5gs
- build neocon
- build osmo-uecups
- build osmo-e1d
- build osmo-smlc
- build osmo-cbc
- build osmo-gbproxy
- build osmo-hnodeb
-
- post
-}
-
-osmo_obs_verify_feed
-build_osmocom
diff --git a/scripts/osmocom-packages-docker.sh b/scripts/osmocom-packages-docker.sh
index a7a804b..0dda19a 100755
--- a/scripts/osmocom-packages-docker.sh
+++ b/scripts/osmocom-packages-docker.sh
@@ -25,7 +25,9 @@ nightly|next|latest)
esac
docker run \
+ --rm \
-e "FEED=$FEED" \
+ -e "OBS_SERVER=$OBS_SERVER" \
-e "OSMO_OBS_CONFLICT_PKGVER=$OSMO_OBS_CONFLICT_PKGVER" \
-e "PACKAGES=$PACKAGES" \
-e "PROJ=$PROJ" \
diff --git a/scripts/osmocom-release-tarballs.sh b/scripts/osmocom-release-tarballs.sh
deleted file mode 100755
index 9894a47..0000000
--- a/scripts/osmocom-release-tarballs.sh
+++ /dev/null
@@ -1,216 +0,0 @@
-#!/bin/sh -e
-# Iterate over all relevant Osmocom repositories and generate release tarballs for each of the repository tags. The tags
-# are queried from the git server without cloning the repositories first, so we can clone them only if we need to build
-# a missing tarball. All repositories are deleted afterwards to save space.
-#
-# Environment variables:
-# * KEEP_TEMP: do not delete cloned repositories (use for development)
-# * PARALLEL_MAKE: -jN argument for make (default: -j5).
-
-. "$(dirname "$0")/common.sh"
-cd "$(dirname "$0")"
-PARALLEL_MAKE="${PARALLEL_MAKE:--j5}"
-OUTPUT="$(cd ..; pwd)/_release_tarballs"
-TEMP="$(cd ..; pwd)/_temp"
-
-# Print all tags for which no release tarball should be built.
-# $1: Osmocom repository
-tags_to_ignore() {
- case "$1" in
- libosmocore)
- # configure.ac:144: error: required file 'src/gb/Makefile.in' not found
- echo "0.5.0"
- echo "0.5.1"
- ;;
- libsmpp34)
- # duplicate of 1.12.0
- echo "1.12"
- ;;
- osmo-bsc)
- # openbsc
- echo "1.0.1"
- # Requires libosmo-legacy-mgcp
- echo "1.1.0"
- echo "1.1.1"
- echo "1.1.2"
- echo "1.2.0"
- echo "1.2.1"
- echo "1.2.2"
- ;;
- osmo-bts)
- # gsm_data_shared.h:464:26: error: field 'power_params' has incomplete type
- echo "0.2.0"
- echo "0.3.0"
- ;;
- osmo-hlr)
- # Not using autotools
- echo "0.0.1"
- ;;
- osmo-mgw)
- # openbsc
- echo "1.0.1"
- ;;
- osmo-msc)
- # openbsc
- echo "1.0.1"
- ;;
- osmo-pcu)
- # Duplicates of 0.1.0, 0.2.0
- echo "0.1"
- echo "0.2"
- ;;
- osmo-sgsn)
- # openbsc
- echo "0.9.0 0.9.1 0.9.2 0.9.3 0.9.4 0.9.5 0.9.6 0.9.8 0.9.9 0.9.10 0.9.11 0.9.12 0.9.13 0.9.14"
- echo "0.9.15 0.9.16 0.10.0 0.10.1 0.11.0 0.12.0 0.13.0 0.14.0 0.15.0 1.0.1"
- ;;
- osmo-sip-connector)
- # make: *** No rule to make target 'osmoappdesc.py'
- echo "0.0.1"
- ;;
- osmo-trx)
- # cp: cannot stat './/home/user/code/osmo-dev/src/osmo-ci/_temp/repos/osmo-trx/configure'
- echo "0.2.0"
- echo "0.3.0"
- echo "1.3.0"
- ;;
- esac
-}
-
-# Clone dependency repositories.
-# $1: Osmocom repository
-prepare_depends() {
- case "$1" in
- osmo-bts)
- # Includes openbsc/gsm_data_shared.h
- prepare_repo "openbsc"
- ;;
- esac
-}
-
-# Apply workarounds for bugs that break too many releases. This function runs between ./configure and make dist-bzip2.
-# $1: Osmocom repository
-fix_repo() {
- case "$1" in
- osmo-mgw)
- # No rule to make target 'osmocom/mgcp_client/mgcp_common.h' (OS#4084)
- make -C "$TEMP/repos/$1/include/osmocom/mgcp_client" mgcp_common.h || true
- ;;
- esac
-}
-
-# Check if one specific tag should be ignored.
-# $1: Osmocom repository
-# $2: tag (e.g. "1.0.0")
-ignore_tag() {
- local repo="$1"
- local tag="$2"
- local tags="$(tags_to_ignore "$repo")"
- for tag_i in $tags; do
- if [ "$tag" = "$tag_i" ]; then
- return 0
- fi
- done
- return 1
-}
-
-# Delete existing temp dir (unless KEEP_TEMP is set). If all repos were checked out, this restores ~500 MB of space.
-remove_temp_dir() {
- if [ -n "$KEEP_TEMP" ]; then
- echo "NOTE: not removing temp dir, because KEEP_TEMP is set: $TEMP"
- elif [ -d "$TEMP" ]; then
- rm -rf "$TEMP"
- fi
-}
-
-# Clone an Osmocom repository to $TEMP/repos/$repo, clean it, checkout a tag.
-# $1: Osmocom repository
-# $2: tag (optional, default: master)
-prepare_repo() {
- local repo="$1"
- local tag="${2:-master}"
-
- if ! [ -d "$TEMP/repos/$repo" ]; then
- git -C "$TEMP/repos" clone "$OSMO_GIT_URL/$repo"
- fi
-
- cd "$TEMP/repos/$repo"
- git clean -qdxf
- git reset --hard HEAD # in case the tracked files were modified (e.g. libsmpp34 1.10)
- git checkout -q "$tag"
-}
-
-# Checkout a given tag and build a release tarball.
-# $1: Osmocom repository
-# $2: tag
-create_tarball() {
- local repo="$1"
- local tag="$2"
- local tarball="$repo-$tag.tar.bz2"
-
- # Be verbose during the tarball build and preparation. Everything else is not verbose, so we can generate an
- # easy to read overview of tarballs that are already built or are ignored.
- set -x
-
- prepare_repo "$repo" "$tag"
- prepare_depends "$repo"
-
- cd "$TEMP/repos/$repo"
- autoreconf -fi
- ./configure
- fix_repo "$repo"
- make dist-bzip2
-
- # Back to non-verbose mode
- set +x
-
- if ! [ -e "$tarball" ]; then
- echo "NOTE: tarball has a different name (wrong version in configure.ac?), renaming."
- mv -v *.tar.bz2 "$tarball"
- fi
-}
-
-# Move a generated release tarball to the output dir.
-move_tarball() {
- local repo="$1"
- local tag="$2"
- local tarball="$repo-$tag.tar.bz2"
-
- cd "$TEMP/repos/$repo"
- mkdir -p "$OUTPUT/$repo"
- mv "$tarball" "$OUTPUT/$repo/$tarball"
-}
-
-remove_temp_dir
-mkdir -p "$TEMP/repos"
-echo "Temp dir: $TEMP"
-
-for repo in $OSMO_RELEASE_REPOS; do
- echo "$repo"
- tags="$(osmo_git_last_commits_tags "$repo" "all" | cut -d / -f 3)"
-
- # Skip untagged repos
- if [ -z "$tags" ]; then
- echo " (repository has no release tags)"
- continue
- fi
-
- # Build missing tarballs for each tag
- for tag in $tags; do
- tarball="$repo-$tag.tar.bz2"
- if ignore_tag "$repo" "$tag"; then
- echo " $tarball (ignored)"
- continue
- elif [ -e "$OUTPUT/$repo/$tarball" ]; then
- echo " $tarball (exists)"
- continue
- fi
-
- echo " $tarball (creating)"
- create_tarball "$repo" "$tag"
- move_tarball "$repo" "$tag"
- done
-done
-
-remove_temp_dir
-echo "done!"
diff --git a/scripts/repo-install-test.sh b/scripts/repo-install-test.sh
index d13cfc8..7bef5bf 100755
--- a/scripts/repo-install-test.sh
+++ b/scripts/repo-install-test.sh
@@ -1,19 +1,29 @@
#!/bin/sh -ex
# Environment variables:
-# * INTERACTIVE: set to 1 to keep an interactive shell open after the script ran (for debugging)
+# * DOMAIN: default is downloads.osmocom.org, set to people.osmocom.org for testing pkgs from home:…
# * FEED: binary package feed (e.g. "latest", "nightly")
-# * PROJ: OBS project namespace (e.g. "network:osmocom:latest")
-# * PROJ_CONFLICT: Conflicting OBS project namespace (e.g. "network:osmocom:nightly")
-# * KEEP_CACHE: set to 1 to keep downloaded binary packages (for development)
+# * INTERACTIVE: set to 1 to keep an interactive shell open after the script ran (for debugging)
+# * KEEP_VM: for development: don't kill/start VM if still running
+# * PROJ: OBS project namespace (e.g. "osmocom:latest")
+# * PROJ_CONFLICT: Conflicting OBS project namespace (e.g. "osmocom:nightly")
+# * SKIP_PREPARE_VM: for development, skip the prepare_vm code
# * TESTS: which tests to run (all by default, see below for possible values)
. "$(dirname "$0")/common.sh"
+DOMAIN="${DOMAIN:-downloads.osmocom.org}"
DISTRO="$1"
DISTROS="
centos8
- debian9
debian10
+ debian11
+ debian12
"
+IMG_DIR="/opt/qemu"
+TEST_DIR="scripts/repo-install-test"
+IMG_PATH="_repo_install_test_data/temp.qcow2"
+PID_FILE="_repo_install_test_data/qemu.pid"
+PORT_FILE="_repo_install_test_data/qemu.port"
+LOG_FILE="_repo_install_test_data/qemu.log"
check_usage() {
local i
@@ -22,17 +32,204 @@ check_usage() {
return
fi
done
+ set +x
+ echo
echo "usage: repo-install-test.sh DISTRO"
echo "DISTRO: one of: $DISTROS"
exit 1
}
+get_backing_img_path() {
+ local ret=""
+
+ case "$DISTRO" in
+ centos8)
+ ret="$IMG_DIR/alma-8.5.qcow2"
+ ;;
+ debian10)
+ ret="$IMG_DIR/debian-10.qcow2"
+ ;;
+ debian11)
+ ret="$IMG_DIR/debian-11.qcow2"
+ ;;
+ debian12)
+ ret="$IMG_DIR/debian-12.qcow2"
+ ;;
+ *)
+ set +x
+ echo "ERROR: script error, missing img path for $DISTRO" >&2
+ exit 1
+ ;;
+ esac
+
+ if [ -e "$ret" ]; then
+ echo "$ret"
+ else
+ set +x
+ echo "ERROR: file not found: $ret" >&2
+ echo "ERROR: qemu images not installed via ansible?" >&2
+ exit 1
+ fi
+}
+
+find_free_ssh_port() {
+ SSH_PORT="$(echo "($PPID % 1000) + 22022" | bc)"
+ while nc -z 127.0.0.1 "$SSH_PORT"; do
+ SSH_PORT=$((SSH_PORT + 1))
+ done
+
+ echo "$SSH_PORT" > "$PORT_FILE"
+}
+
+prepare_img() {
+ mkdir -p "$(dirname "$IMG_PATH")"
+
+ qemu-img \
+ create \
+ -f qcow2 \
+ -b "$(get_backing_img_path)" \
+ -F qcow2 \
+ "$IMG_PATH"
+}
+
+qemu_start() {
+ if [ -n "$KEEP_VM" ] && [ -e "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE")"; then
+ SSH_PORT="$(cat "$PORT_FILE")"
+ return
+ fi
+
+ prepare_img
+ find_free_ssh_port
+
+ (timeout 1h qemu-system-x86_64 \
+ -cpu host \
+ -device "virtio-net-pci,netdev=net" \
+ -display none \
+ -drive "file=$IMG_PATH,format=qcow2" \
+ -enable-kvm \
+ -m 1024 \
+ -netdev "user,id=net,hostfwd=tcp:127.0.0.1:$SSH_PORT-:22" \
+ -nodefaults \
+ -pidfile "$PID_FILE" \
+ -serial stdio \
+ -smp 16 >"$LOG_FILE" 2>&1) &
+}
+
+qemu_ssh() {
+ timeout "${TIMEOUT:-1m}" \
+ sshpass -p root \
+ ssh \
+ -p "$SSH_PORT" \
+ -o StrictHostKeyChecking=no \
+ -o UserKnownHostsFile=/dev/null \
+ root@127.0.0.1 \
+ -- \
+ "$@"
+}
+
+qemu_scp() {
+ timeout "${TIMEOUT:-1m}" \
+ sshpass -p root \
+ scp \
+ -P "$SSH_PORT" \
+ -o StrictHostKeyChecking=no \
+ -o UserKnownHostsFile=/dev/null \
+ "$@"
+}
+
+qemu_run_test_script() {
+ cat <<- EOF > "$TEST_DIR/run-inside-env.sh"
+ #!/bin/sh -ex
+
+ export DISTRO="$DISTRO"
+ export DOMAIN="$DOMAIN"
+ export FEED="$FEED"
+ export PROJ="$PROJ"
+ export PROJ_CONFLICT="$PROJ_CONFLICT"
+ export SKIP_PREPARE_VM="$SKIP_PREPARE_VM"
+ export TESTS="$TESTS"
+
+ /repo-install-test/run-inside.sh
+ EOF
+
+ qemu_ssh rm -rf /repo-install-test/
+ qemu_ssh mkdir /repo-install-test
+ qemu_scp -r "$TEST_DIR"/* "root@127.0.0.1:/repo-install-test"
+
+ TIMEOUT="1h" qemu_ssh sh -ex /repo-install-test/run-inside-env.sh
+}
+
+qemu_print_log() {
+ echo
+ echo "Contents of $LOG_FILE:"
+ echo
+ cat "$LOG_FILE"
+}
+
+qemu_ssh_wait() {
+ set +x
+ echo
+ echo "Waiting for VM to boot up..."
+ echo
+ set -x
+
+ # PID file does not get created immediately
+ sleep 1
+ local pid="$(cat "$PID_FILE")"
+
+ for i in $(seq 1 6); do
+ if [ -z "$pid" ] || ! kill -0 "$pid"; then
+ set +x
+ echo "ERROR: qemu failed, pid: $pid"
+ qemu_print_log
+ exit 1
+ fi
+
+ if TIMEOUT=10s qemu_ssh true; then
+ return
+ fi
+
+ sleep 1
+ done
+
+ set +x
+ echo "ERROR: timeout, VM did not boot up. Log file contents:"
+ qemu_print_log
+ exit 1
+}
+
+clean_up() {
+ if [ -n "$KEEP_VM" ]; then
+ return
+ fi
+
+ if [ -e "$PID_FILE" ]; then
+ kill $(cat "$PID_FILE") || true
+ fi
+
+ rm -f "$IMG_PATH"
+}
+
+clean_up_trap() {
+ if [ -n "$INTERACTIVE" ]; then
+ TIMEOUT="1h" qemu_ssh bash -i
+ fi
+
+ set +x
+ echo
+ echo "### Clean up ###"
+ echo
+ set -x
+
+ trap - EXIT INT TERM 0
+
+ clean_up
+}
+
check_usage
-docker_images_require "$DISTRO-repo-install-test"
FEED="${FEED:-nightly}"
-PROJ="${PROJ:-network:osmocom:$FEED}"
-CONTAINER="$DISTRO-repo-install-test-$FEED"
+PROJ="${PROJ:-osmocom:$FEED}"
if [ -z "$TESTS" ]; then
TESTS="
@@ -46,84 +243,29 @@ fi
if [ -z "$PROJ_CONFLICT" ]; then
case "$FEED" in
latest)
- PROJ_CONFLICT="network:osmocom:nightly"
+ PROJ_CONFLICT="osmocom:nightly"
;;
nightly)
- PROJ_CONFLICT="network:osmocom:latest"
- if [ "$DISTRO" = "centos8" ]; then
- # Doesn't have packages built for "latest" yet
- PROJ_CONFLICT="network:osmocom:next"
- fi
+ PROJ_CONFLICT="osmocom:latest"
;;
next)
- PROJ_CONFLICT="network:osmocom:nightly"
+ PROJ_CONFLICT="osmocom:nightly"
;;
esac
fi
-# Try to run "systemctl status" 10 times, kill the container on failure
-check_if_systemd_is_running() {
- for i in $(seq 1 10); do
- sleep 1
- if docker exec "$CONTAINER" systemctl status; then
- return
- fi
- done
-
- echo "ERROR: systemd is not running properly."
- docker container kill "$CONTAINER"
- exit 1
-}
-
-# Kill already running container
-if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2> /dev/null)" = "true" ]; then
- docker container kill "$CONTAINER"
- sleep 1
-fi
-# Additional docker run arguments
-args=""
-if [ -n "$KEEP_CACHE" ]; then
- args="$args -e KEEP_CACHE=1"
- args="$args -v $OSMO_CI_DIR/_repo_install_test_cache/debian/apt:/var/cache/apt"
- args="$args -v $OSMO_CI_DIR/_repo_install_test_cache/centos/dnf:/var/cache/dnf"
-fi
+clean_up
+trap clean_up_trap EXIT INT TERM 0
-# Run the container
-# * This does not output anything, for debugging add -it and remove &.
-# * /run, /tmp, cgroups, SYS_ADMIN: needed for systemd
-# * SYS_NICE: needed for changing CPUScheduling{Policy,Priority} (osmo-bts systemd service files)
-docker run --rm \
- -v "$OSMO_CI_DIR/scripts/repo-install-test:/repo-install-test:ro" \
- --name "$CONTAINER" \
- -e FEED="$FEED" \
- -e PROJ="$PROJ" \
- -e PROJ_CONFLICT="$PROJ_CONFLICT" \
- -e DISTRO="$DISTRO" \
- -e TESTS="$TESTS" \
- -e container=docker \
- --tmpfs /run \
- --tmpfs /run/lock \
- --tmpfs /tmp \
- -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
- --cap-add SYS_ADMIN \
- --cap-add SYS_NICE \
- $args \
- "$USER/$DISTRO-repo-install-test" \
- /lib/systemd/systemd &
-check_if_systemd_is_running
-
-# Run the test script
-ret=0
-if ! docker exec "$CONTAINER" /repo-install-test/run-inside-docker.sh; then
- ret=1
-fi
+qemu_start
+qemu_ssh_wait
-# Interactive shell
-if [ -n "$INTERACTIVE" ]; then
- docker exec -it "$CONTAINER" bash || true
-fi
-docker container kill "$CONTAINER"
+set +x
+echo
+echo "VM is running!"
+echo
+set -x
-exit $ret
+qemu_run_test_script
diff --git a/scripts/repo-install-test/blacklist_centos8.txt b/scripts/repo-install-test/blacklist_centos8.txt
deleted file mode 100644
index 83d2cbd..0000000
--- a/scripts/repo-install-test/blacklist_centos8.txt
+++ /dev/null
@@ -1,11 +0,0 @@
-# These packages will not get explicitly installed in this test.
-# Package lines must have nothing but the package (no comment,
-# no additional space etc).
-
-# The ulfius package in network:osmocom:* is linked to the opensuse version,
-# which has dependencies that exist under another name in centos
-# (libgnutls-devel, libjansson-devel). Linking to the opensuse version has the
-# advantage that we don't need to worry about updating the package. However,
-# the dependency problem means that we can't simply install the package in
-# centos. In OBS we work around this in the project config.
-ulfius-devel
diff --git a/scripts/repo-install-test/blacklist_debian10.txt b/scripts/repo-install-test/blacklist_debian10.txt
deleted file mode 100644
index 6570de6..0000000
--- a/scripts/repo-install-test/blacklist_debian10.txt
+++ /dev/null
@@ -1,63 +0,0 @@
-# These packages will not get explicitly installed in this test.
-# Package lines must have nothing but the package (no comment,
-# no additional space etc).
-
-# OpenBSC
-# This is legacy, we aren't really interested in testing openbsc.git
-# derived packages. Packages are found in openbsc/debian/control.
-openbsc-dev
-osmo-bsc-mgcp
-osmo-bsc-mgcp-dbg
-osmocom-bs11-utils
-osmocom-bs11-utils-dbg
-osmocom-bsc-nat
-osmocom-bsc-nat-dbg
-osmocom-bsc-sccplite
-osmocom-bsc-sccplite-dbg
-osmocom-ipaccess-utils
-osmocom-ipaccess-utils-dbg
-osmocom-nitb
-osmocom-nitb-dbg
-
-# SoapySDR is not used anymore (see OS#3542)
-soapysdr-module-lms7
-soapysdr0.6-module-lms7
-soapysdr0.6-module-lms7-dbgsym
-soapysdr0.7-module-lms7
-soapysdr0.7-module-lms7-dbgsym
-
-# Depends on specific verions 0.5.4.38.0847 of rtl-sdr, which we won't install
-librtlsdr0-dbgsym
-rtl-sdr-dbgsym
-
-# Depends on mongodb, which was droppend from debian 10 onwards
-open5gs
-open5gs-amf
-open5gs-amf-dbgsym
-open5gs-ausf
-open5gs-ausf-dbgsym
-open5gs-common
-open5gs-common-dbgsym
-open5gs-dbg
-open5gs-hss
-open5gs-hss-dbgsym
-open5gs-mme
-open5gs-mme-dbgsym
-open5gs-nrf
-open5gs-nrf-dbgsym
-open5gs-pcf
-open5gs-pcf-dbgsym
-open5gs-pcrf
-open5gs-pcrf-dbgsym
-open5gs-sgwc
-open5gs-sgwc-dbgsym
-open5gs-sgwu
-open5gs-sgwu-dbgsym
-open5gs-smf
-open5gs-smf-dbgsym
-open5gs-udm
-open5gs-udm-dbgsym
-open5gs-udr
-open5gs-udr-dbgsym
-open5gs-upf
-open5gs-upf-dbgsym
diff --git a/scripts/repo-install-test/blacklist_debian9.txt b/scripts/repo-install-test/blacklist_debian9.txt
deleted file mode 100644
index cb7db6a..0000000
--- a/scripts/repo-install-test/blacklist_debian9.txt
+++ /dev/null
@@ -1,27 +0,0 @@
-# These packages will not get explicitly installed in this test.
-# Package lines must have nothing but the package (no comment,
-# no additional space etc).
-
-# OpenBSC
-# This is legacy, we aren't really interested in testing openbsc.git
-# derived packages. Packages are found in openbsc/debian/control.
-openbsc-dev
-osmo-bsc-mgcp
-osmo-bsc-mgcp-dbg
-osmocom-bs11-utils
-osmocom-bs11-utils-dbg
-osmocom-bsc-nat
-osmocom-bsc-nat-dbg
-osmocom-bsc-sccplite
-osmocom-bsc-sccplite-dbg
-osmocom-ipaccess-utils
-osmocom-ipaccess-utils-dbg
-osmocom-nitb
-osmocom-nitb-dbg
-
-# SoapySDR is not used anymore (see OS#3542)
-soapysdr-module-lms7
-soapysdr0.6-module-lms7
-soapysdr0.6-module-lms7-dbgsym
-soapysdr0.7-module-lms7
-soapysdr0.7-module-lms7-dbgsym
diff --git a/scripts/repo-install-test/run-inside-docker.sh b/scripts/repo-install-test/run-inside.sh
index 1ba94d9..425e49e 100755
--- a/scripts/repo-install-test/run-inside-docker.sh
+++ b/scripts/repo-install-test/run-inside.sh
@@ -1,37 +1,46 @@
#!/bin/sh -ex
# Environment variables:
+# * DISTRO: linux distribution name (e.g. "centos8")
# * FEED: binary package feed (e.g. "latest", "nightly")
-# * PROJ: OBS project namespace (e.g. "network:osmocom:latest")
-# * PROJ_CONFLICT: Conflicting OBS project namespace (e.g. "network:osmocom:nightly")
# * KEEP_CACHE: set to 1 to keep downloaded binary packages (for development)
-# * DISTRO: linux distribution name (e.g. "centos8")
+# * PROJ: OBS project namespace (e.g. "osmocom:latest")
+# * PROJ_CONFLICT: Conflicting OBS project namespace (e.g. "osmocom:nightly")
+# * SKIP_PREPARE_VM: for development, skip the prepare_vm code
# * TESTS: which tests to run (see repo-install-test.sh)
# Systemd services that must start up successfully after installing all packages (OS#3369)
# Disabled services:
# * osmo-ctrl2cgi (missing config: /etc/osmocom/ctrl2cgi.ini, OS#4108)
+# * osmo-remsim-client (exits immediately without USB device)
# * osmo-trap2cgi (missing config: /etc/osmocom/%N.ini, OS#4108)
-# * osmo-ggsn (no tun device in docker)
+# * osmo-trx-* (exits immediately without trx device)
+# * osmo-upf (not available for debian 10, gets added in services_check())
SERVICES="
osmo-bsc
osmo-bts-virtual
+ osmo-cbc
+ osmo-e1d
osmo-gbproxy
+ osmo-ggsn
osmo-gtphub
osmo-hlr
osmo-hnbgw
+ osmo-hnodeb
osmo-mgw
osmo-msc
osmo-pcap-client
+ osmo-pcap-server
osmo-pcu
+ osmo-remsim-bankd
+ osmo-remsim-server
osmo-sgsn
osmo-sip-connector
+ osmo-smlc
osmo-stp
"
# Services working in nightly, but not yet in latest
-# * osmo-pcap-server 0.2.0: VTY port in default config conflicts with osmo-bts (OS#5203)
SERVICES_NIGHTLY="
- osmo-pcap-server
- osmo-hnodeb
+ osmo-bsc-nat
"
distro_obsdir() {
@@ -39,12 +48,15 @@ distro_obsdir() {
centos8)
echo "CentOS_8"
;;
- debian9)
- echo "Debian_9.0"
- ;;
debian10)
echo "Debian_10"
;;
+ debian11)
+ echo "Debian_11"
+ ;;
+ debian12)
+ echo "Debian_12"
+ ;;
*)
echo "ERROR: unknown obsdir for '$DISTRO'." >&2
exit 1
@@ -54,12 +66,12 @@ distro_obsdir() {
DISTRO_OBSDIR="$(distro_obsdir)"
-# $1: OBS project (e.g. "network:osmocom:nightly" -> "network:/osmocom:/nightly")
+# $1: OBS project (e.g. "osmocom:nightly" -> "osmocom:/nightly")
proj_with_slashes() {
echo "$1" | sed "s.:.:/.g"
}
-# $1: OBS project (e.g. "network:osmocom:nightly" -> "network_osmocom_nightly")
+# $1: OBS project (e.g. "osmocom:nightly" -> "osmocom_nightly")
proj_with_underscore() {
echo "$1" | tr : _
}
@@ -96,40 +108,38 @@ check_env() {
fi
}
-# $1: OBS project (e.g. "network:osmocom:nightly")
+# $1: OBS project (e.g. "osmocom:nightly")
configure_osmocom_repo_debian() {
local proj="$1"
- local obs_repo="download.opensuse.org/repositories/$(proj_with_slashes "$proj")/$DISTRO_OBSDIR/"
- local release_key="/var/cache/apt/${proj}_Release.key"
+ local obs_repo="$DOMAIN/packages/$(proj_with_slashes "$proj")/$DISTRO_OBSDIR/"
echo "Configuring Osmocom repository"
# Add repository key
- if ! [ -e "$release_key" ]; then
- apt-get update
- apt install -y wget
- wget -O "$release_key" "https://$obs_repo/Release.key"
+ if ! [ -e /tmp/Release.key ]; then
+ wget -O /tmp/Release.key "https://obs.osmocom.org/projects/$proj/public_key"
fi
- apt-key add "$release_key"
+
+ apt-key add /tmp/Release.key
echo "deb http://$obs_repo ./" > "/etc/apt/sources.list.d/$proj.list"
apt-get update
}
-# $1: OBS project (e.g. "network:osmocom:nightly")
+# $1: OBS project (e.g. "osmocom:nightly")
configure_osmocom_repo_debian_remove() {
local proj="$1"
rm "/etc/apt/sources.list.d/$proj.list"
}
-# $1: OBS project (e.g. "network:osmocom:nightly")
+# $1: OBS project (e.g. "osmocom:nightly")
configure_osmocom_repo_centos() {
local proj="$1"
- local baseurl="https://download.opensuse.org/repositories/$(proj_with_slashes "$proj")/$DISTRO_OBSDIR"
+ local baseurl="https://$DOMAIN/packages/$(proj_with_slashes "$proj")/$DISTRO_OBSDIR"
echo "Configuring Osmocom repository"
# Generate this file, based on the feed:
- # https://download.opensuse.org/repositories/network:osmocom:latest/CentOS_8/network:osmocom:latest.repo
+ # https://downloads.osmocom.org/packages/osmocom:/latest/CentOS_8/osmocom:latest.repo
cat << EOF > "/etc/yum.repos.d/$proj.repo"
[$(proj_with_underscore "$proj")]
name=$proj
@@ -141,13 +151,13 @@ enabled=1
EOF
}
-# $1: OBS project (e.g. "network:osmocom:nightly")
+# $1: OBS project (e.g. "osmocom:nightly")
configure_osmocom_repo_centos_remove() {
local proj="$1"
rm "/etc/yum.repos.d/$proj.repo"
}
-# $1: OBS project (e.g. "network:osmocom:nightly")
+# $1: OBS project (e.g. "osmocom:nightly")
configure_osmocom_repo() {
case "$DISTRO" in
debian*)
@@ -159,32 +169,54 @@ configure_osmocom_repo() {
esac
}
-configure_keep_cache_debian() {
- rm /etc/apt/apt.conf.d/docker-clean
+prepare_vm_debian() {
+ # fmtutil fails in tex-common postinst script. This gets installed as
+ # dependency of osmo-gsm-manuals-dev, but is completely unrelated to
+ # what we want to test here so just stub it out.
+ ln -sf /bin/true /usr/bin/fmtutil
+ echo "path-exclude=/usr/bin/fmtutil" >> /etc/dpkg/dpkg.cfg.d/stub
+
+ apt-get update --allow-releaseinfo-change
+ apt-get install -y --no-install-recommends \
+ aptitude \
+ ca-certificates \
+ gnupg2 \
+ wget
- # "apt" will actually remove the cache by default, even if "apt-get" keeps it.
- # https://unix.stackexchange.com/a/447607
- echo "Binary::apt::APT::Keep-Downloaded-Packages "true";" \
- > /etc/apt/apt.conf.d/01keep-debs
+ case "$DISTRO" in
+ debian10)
+ # libgnutls30: can't access https://osmocom.org otherwise
+ # ca-certificates-java: fails if installed after java
+ apt-get install -y --no-install-recommends \
+ ca-certificates-java \
+ libgnutls30
+ ;;
+ esac
}
-configure_keep_cache_centos() {
- echo "keepcache=1" >> /etc/dnf/dnf.conf
+prepare_vm_centos() {
+ # Install dnf-utils for repoquery
+ dnf install -y dnf-utils
+
+ # Make additional development libraries available
+ yum config-manager --set-enabled powertools
}
-configure_keep_cache() {
- if [ -z "$KEEP_CACHE" ]; then
+prepare_vm() {
+ if [ -n "$SKIP_PREPARE_VM" ]; then
return
fi
case "$DISTRO" in
debian*)
- configure_keep_cache_debian
+ prepare_vm_debian
;;
centos*)
- configure_keep_cache_centos
+ prepare_vm_centos
;;
esac
+
+ configure_osmocom_repo "$PROJ"
}
# $1: file
@@ -220,9 +252,21 @@ test_conflict_debian() {
find_patterns_or_exit \
/tmp/out \
"requested an impossible situation" \
- "^The following packages have unmet dependencies:" \
- "Depends: osmocom-" \
- "but it is not going to be installed"
+ "^The following packages have unmet dependencies:"
+
+ case "$DISTRO" in
+ debian10)
+ find_patterns_or_exit \
+ /tmp/out \
+ "Depends: osmocom-" \
+ "but it is not going to be installed"
+ ;;
+ debian11|debian12)
+ find_patterns_or_exit \
+ /tmp/out \
+ "Conflicts: osmocom-"
+ ;;
+ esac
configure_osmocom_repo_debian_remove "$PROJ_CONFLICT"
configure_osmocom_repo_debian "$PROJ"
@@ -262,45 +306,77 @@ test_conflict() {
esac
}
-# Filter $PWD/osmocom_packages_all.txt through a blacklist_$DISTRO.txt and store the result in
-# $PWD/osmocom_packages.txt.
-filter_packages_txt() {
- # Copy distro specific blacklist file, remove comments and sort it
- grep -v "^#" /repo-install-test/blacklist_$DISTRO.txt | sort -u > blacklist.txt
-
- # Generate list of pkgs to be installed from available pkgs minus the ones blacklisted
- comm -23 osmocom_packages_all.txt \
- blacklist.txt > osmocom_packages.txt
+filter_packages() {
+ for i in "$@"; do
+ case "$i" in
+ # OpenBSC
+ # This is legacy, we aren't really interested in testing
+ # openbsc.git derived packages. Packages are found in
+ # openbsc/debian/control.
+ osmo-bsc-dev) ;;
+ osmo-bsc-mgcp*) ;;
+ osmocom-bs11-utils*) ;;
+ osmocom-bsc-nat*) ;;
+ osmocom-bsc-sccplite*) ;;
+ osmocom-ipaccess-utils*) ;;
+ osmocom-nitb*) ;;
+
+ # Causing conflicts, not relevant for the test
+ liblimesuite*) ;;
+ liborcania*) ;;
+ libulfius*) ;;
+ libhyder*) ;;
+ limesuite*) ;;
+ soapysdr*-module-lms7*) ;;
+
+ # Depends on specific verions 0.5.4.38.0847 of rtl-sdr, which
+ # we won't install
+ librtlsdr0-dbgsym) ;;
+ rtl-sdr-dbgsym) ;;
+
+ # Depends on mongodb, which was droppend from debian 10 onwards
+ open5gs*) ;;
+
+ # Dependencies that have a different name in centos8/almalinux8
+ # but are pulled in by linking to opensuse packages. In OBS we
+ # work around this in the project config.
+ ulfius-devel) ;;
+ nftables-devel) ;;
+ python3-nftables) ;;
+
+ # All other packages are not filtered
+ *) echo "$i" ;;
+ esac
+ done
}
install_repo_packages_debian() {
- local obs="obs://build.opensuse.org/$PROJ/$DISTRO_OBSDIR"
-
+ local packages
echo "Installing all repository packages"
# Get a list of all packages from the repository. Reference:
# https://www.debian.org/doc/manuals/aptitude/ch02s04s05.en.html
- aptitude search -F%p \
- "?origin($obs) ?architecture(native)" | sort \
- > osmocom_packages_all.txt
+ packages="$(aptitude search -F%p \
+ "?origin(.*$PROJ.*) ?architecture(native)" | sort)"
+ packages="$(filter_packages $packages)"
- filter_packages_txt
- apt install -y $(cat osmocom_packages.txt)
+ apt-get install -y --no-install-recommends -- $packages
}
install_repo_packages_centos() {
+ local packages
echo "Installing all repository packages"
# Get a list of all packages from the repository
- LANG=C.UTF-8 repoquery \
+ packages=$(LANG=C.UTF-8 repoquery \
--quiet \
--repoid="$(proj_with_underscore "$PROJ")" \
--archlist="x86_64,noarch" \
--qf="%{name}" \
- > osmocom_packages_all.txt
+ | sort)
- filter_packages_txt
- dnf install -y $(cat osmocom_packages.txt)
+ packages="$(filter_packages $packages)"
+ dnf install -y -- $packages
}
install_repo_packages() {
@@ -341,23 +417,54 @@ test_binaries() {
osmo-bsc \
osmo-bts-trx \
osmo-bts-virtual \
+ osmo-cbc \
+ osmo-e1d \
osmo-gbproxy \
- osmo-gtphub \
osmo-ggsn \
+ osmo-gtphub \
osmo-hlr \
osmo-hlr-db-tool \
osmo-hnbgw \
+ osmo-hnodeb \
osmo-mgw \
osmo-msc \
+ osmo-mslookup-client \
+ osmo-pcap-client \
+ osmo-pcap-server \
osmo-pcu \
+ osmo-remsim-bankd \
+ osmo-remsim-client-shell \
+ osmo-remsim-client-st2 \
+ osmo-remsim-server \
osmo-sgsn \
osmo-sip-connector \
+ osmo-smlc \
osmo-stp \
- osmo-trx-uhd
+ osmo-trx-ipc \
+ osmo-trx-uhd \
+ osmo-uecups-daemon
- if [ "$DISTRO" = "debian" ]; then
+ case "$DISTRO" in
+ debian*)
test_binaries_version \
osmo-trx-usrp1
+ ;;
+ esac
+
+ if [ "$DISTRO" != "debian10" ]; then
+ test_binaries_version \
+ osmo-upf
+
+ # OS#5817: not packaged for debian, needs osmo-upf release
+ if [ "$FEED" = "nightly" ] || [ "$DISTRO" = "centos8" ]; then
+ test_binaries_version \
+ osmo-pfcp-tool
+ fi
+ fi
+
+ if [ "$FEED" = "nightly" ]; then
+ test_binaries_version \
+ osmo-bsc-nat
fi
}
@@ -370,6 +477,16 @@ services_check() {
services_feed="$services_feed $SERVICES_NIGHTLY"
fi
+ # We don't build osmo-upf for debian 10
+ if [ "$DISTRO" != "debian10" ]; then
+ # osmo-upf <= 0.1.1 needs GTP kernel module
+ if [ "$FEED" = "nightly" ]; then
+ # osmo-upf nightly needs a newer kernel (OS#5905)
+ # services_feed="$services_feed osmo-upf"
+ true
+ fi
+ fi
+
systemctl start $services_feed
sleep 2
@@ -392,8 +509,9 @@ services_check() {
}
check_env
-configure_keep_cache
-configure_osmocom_repo "$PROJ"
+prepare_vm
+
+uname -a
for test in $TESTS; do
set +x
diff --git a/scripts/tarballs/install-depends.sh b/scripts/tarballs/install-depends.sh
new file mode 100755
index 0000000..0e7dc13
--- /dev/null
+++ b/scripts/tarballs/install-depends.sh
@@ -0,0 +1,31 @@
+#!/bin/sh -ex
+# For releases where no debian/control exists, install the dependencies to pass
+# the checks in configure.ac so we can run it before building a release
+# tarball.
+
+PROJECT="$1"
+TAG="$2"
+DEPENDS=""
+
+case "$PROJECT" in
+osmo-e1-recorder)
+ DEPENDS="
+ libosmo-abis-dev
+ libosmocore-dev
+ "
+ ;;
+gapk)
+ DEPENDS="
+ libasound2-dev
+ libosmocore-dev
+ "
+ ;;
+*)
+ DEPENDS=""
+ ;;
+esac
+
+if [ -n "$DEPENDS" ]; then
+ apt-get update
+ apt-get install -y --no-install-recommends $DEPENDS
+fi
diff --git a/scripts/tarballs/prefer-configure.sh b/scripts/tarballs/prefer-configure.sh
new file mode 100755
index 0000000..295b239
--- /dev/null
+++ b/scripts/tarballs/prefer-configure.sh
@@ -0,0 +1,34 @@
+#!/bin/sh -ex
+# Decide whether to build a tarball from autotools logic (exit 0) or by
+# creating a simple git archive (exit 1)
+
+PROJECT="$1"
+TAG="$2"
+
+if ! [ -e configure.ac ]; then
+ exit 1
+fi
+
+case "$PROJECT" in
+gapk)
+ case "$TAG" in
+ v0.*|v1.0)
+ # Running gapk's configure involves running libgsmhr/fetch_sources.py,
+ # which according to git log doesn't really work unless using the
+ # version from master and it looks like we don't want to distribute
+ # these sources directly... or else we should just add them to the git
+ # repository and not rely on downloading a remote archive that may just
+ # change at any time. So create a simple git archive instead.
+ exit 1
+ ;;
+ *)
+ # Fixed above v1.0
+ # https://gerrit.osmocom.org/c/gapk/+/34892/1
+ exit 0
+ ;;
+ esac
+ ;;
+*)
+ exit 0
+ ;;
+esac
diff --git a/scripts/tarballs/publish-tarballs-for-tags.sh b/scripts/tarballs/publish-tarballs-for-tags.sh
new file mode 100755
index 0000000..46a2e17
--- /dev/null
+++ b/scripts/tarballs/publish-tarballs-for-tags.sh
@@ -0,0 +1,365 @@
+#!/bin/sh -e
+. "$(dirname "$0")/../common.sh"
+OSMO_CI_DIR="$(realpath $(dirname "$0")/../..)"
+TEMP="$OSMO_CI_DIR/_temp_releases"
+WEB_PATH="/downloads/home/releases/web-files"
+SSH_COMMAND="ssh -o UserKnownHostsFile=$OSMO_CI_DIR/contrib/known_hosts -p 48"
+DOCKER_IMAGE="$USER/debian-bookworm-build"
+LOG_PREFIX="::"
+
+OSMO_RELEASE_REPOS="
+ gapk
+ libasn1c
+ libgtpnl
+ libosmo-abis
+ libosmo-netif
+ libosmo-pfcp
+ libosmo-sccp
+ libosmocore
+ libsmpp34
+ libusrp
+ osmo-bsc
+ osmo-bts
+ osmo-cbc
+ osmo-e1-recorder
+ osmo-e1d
+ osmo-fl2k
+ osmo-gbproxy
+ osmo-ggsn
+ osmo-gsm-manuals
+ osmo-hlr
+ osmo-hnbgw
+ osmo-hnodeb
+ osmo-iuh
+ osmo-mgw
+ osmo-msc
+ osmo-pcap
+ osmo-pcu
+ osmo-remsim
+ osmo-sgsn
+ osmo-sip-connector
+ osmo-smlc
+ osmo-sysmon
+ osmo-trx
+ osmo-uecups
+ osmo_dia2gsup
+ osmocom-bb
+ rtl-sdr
+ simtrace2
+"
+
+# Old release tags that are duplicates or where generating the tarball fails
+TAGS_IGNORE="
+ libosmocore:0.5.0,
+ libosmocore:0.5.1,
+
+ libsmpp34:1.12,
+
+ osmo-bsc:1.0.1,
+ osmo-bsc:1.1.0,
+ osmo-bsc:1.1.1,
+ osmo-bsc:1.1.2,
+ osmo-bsc:1.2.0,
+ osmo-bsc:1.2.1,
+ osmo-bsc:1.2.2,
+
+ osmo-bts:0.2.0,
+ osmo-bts:0.3.0,
+
+ osmo-hlr:0.0.1,
+
+ osmo-mgw:1.0.1,
+
+ osmo-msc:1.0.1,
+
+ osmo-pcap:0.0.3,
+
+ osmo-pcu:0.1,
+ osmo-pcu:0.2,
+
+ osmo-sgsn:0.9.0,
+ osmo-sgsn:0.9.1,
+ osmo-sgsn:0.9.2,
+ osmo-sgsn:0.9.3,
+ osmo-sgsn:0.9.4,
+ osmo-sgsn:0.9.5,
+ osmo-sgsn:0.9.6,
+ osmo-sgsn:0.9.8,
+ osmo-sgsn:0.9.9,
+ osmo-sgsn:0.9.10,
+ osmo-sgsn:0.9.11,
+ osmo-sgsn:0.9.12,
+ osmo-sgsn:0.9.13,
+ osmo-sgsn:0.9.14,
+ osmo-sgsn:0.9.15,
+ osmo-sgsn:0.9.16,
+ osmo-sgsn:0.10.0,
+ osmo-sgsn:0.10.1,
+ osmo-sgsn:0.11.0,
+ osmo-sgsn:0.12.0,
+ osmo-sgsn:0.13.0,
+ osmo-sgsn:0.14.0,
+ osmo-sgsn:0.15.0,
+ osmo-sgsn:1.0.1,
+
+ osmo-sip-connector:0.0.1,
+
+ osmo-trx:0.2.0,
+ osmo-trx:0.3.0,
+ osmo-trx:1.3.0,
+"
+
+mkdir -p \
+ "$TEMP" \
+ "$TEMP/src"
+
+check_ssh_auth_sock() {
+ if [ -z "$SSH_AUTH_SOCK" ]; then
+ echo "ERROR: SSH_AUTH_SOCK is not set"
+ exit 1
+ fi
+}
+
+# $1: path on server, e.g. "/releases/osmo-bsc"
+get_server_ls() {
+ local dir="$1"
+ local out="$TEMP/ls$(echo "$dir" | tr / _)"
+
+ echo "$LOG_PREFIX Listing files on server: $dir"
+
+ if [ -e "$out" ]; then
+ echo "Skipped, file exists: $out"
+ return
+ fi
+
+ dir="$(echo "$dir" | sed "s.^/releases.$WEB_PATH.")"
+ $SSH_COMMAND releases@ftp.osmocom.org "ls -1 $dir" >"$out"
+}
+
+# $1: repository
+get_git_tags() {
+ local repo="$1"
+ local out="$TEMP/git_tags_$repo"
+
+ echo "$LOG_PREFIX Getting git tags"
+
+ if [ -e "$out" ]; then
+ echo "Skipped, file exists: $out"
+ return
+ fi
+
+ osmo_git_last_tags "$repo" "all" >"$out"
+}
+
+# $1: repository
+# $2: tag
+tarball_exists() {
+ local repo="$1"
+ local tag="$2"
+
+ grep -q "^$repo-$tag\.tar\." "$TEMP"/ls_releases_"$repo"
+}
+
+# $1: repository
+# $2: tag
+is_tag_ignored() {
+ local repo="$1"
+ local tag="$2"
+
+ case "$TAGS_IGNORE" in
+ *"$repo:$tag,"*)
+ return 0
+ ;;
+ esac
+
+ return 1
+}
+
+# $1: repository
+# $2: tag
+clone_repo() {
+ local repo="$1"
+ local tag="$2"
+ local gitdir="$TEMP/src/$repo"
+
+ if ! [ -d "$gitdir" ]; then
+ local url="$(osmo_git_clone_url "$repo")"
+ echo "$LOG_PREFIX Cloning $url"
+ git -C "$TEMP/src" clone "$url" "$repo"
+ fi
+
+ echo "$LOG_PREFIX Checkout $tag"
+ cd "$gitdir"
+ git reset --hard HEAD
+ git checkout "$tag"
+ git submodule update --init
+ git clean -dxf
+
+ # Fix depends on packages that don't exist anymore
+ if [ -e debian/control ]; then
+ sed -i 's/dh-systemd \(.*\),//g' debian/control
+ sed -i 's/python-minimal,//g' debian/control
+ fi
+}
+
+# Get the desired tarball name, replace / with - in $1.
+# $1: project path (osmo-bsc, osmo-msc, simtrace/host, etc.)
+# $2: tag
+get_tarball_name() {
+ local project_path="$1"
+ local tag="$2"
+
+ echo "$(echo "$project_path" | tr / -)-$tag.tar.bz2"
+}
+
+# $1: repository
+# $2: project path (osmo-bsc, osmo-msc, simtrace/host, etc.)
+# $3: tag
+build_tarball() {
+ local repo="$1"
+ local project_path="$2"
+ local tag="$3"
+ local tarball_name="$(get_tarball_name "$project_path" "$tag")"
+ local prefix="$(echo "$tarball_name" | sed s/\.tar\.bz2//)"
+ local uid_user="$(id -u)"
+ echo "$LOG_PREFIX Building release tarball: $tarball_name"
+
+ if ! docker run \
+ --rm \
+ -e "DEBIAN_FRONTEND=noninteractive" \
+ -v "$OSMO_CI_DIR/scripts/tarballs:/tarballs" \
+ -v "$TEMP/src:/src" \
+ "$DOCKER_IMAGE" \
+ sh -ex -c "
+ cd /src/$repo
+
+ if [ -e debian/control ]; then
+ apt-get update
+ apt-get -y build-dep .
+ else
+ /tarballs/install-depends.sh \"$repo\" \"$tag\"
+ fi
+
+ cd /src/$project_path
+
+ # Erlang projects: download build depends
+ if [ -e contrib/generate_build_dep.sh ]; then
+ su build -c \"HOME=/build contrib/generate_build_dep.sh\"
+ fi
+
+ if /tarballs/prefer-configure.sh \"$repo\" \"$tag\"; then
+ su build -c \"autoreconf -fi\"
+ case \"$repo\" in
+ osmo-trx)
+ su build -c \"autoreconf -fi osmocom-bb/src/host/trxcon\"
+ su build -c \"./configure --with-mstrx\"
+ ;;
+ *)
+ su build -c \"./configure\"
+ ;;
+ esac
+ su build -c \"make dist-bzip2\"
+ else
+ su build -c \"git archive --prefix=$prefix/ -o $prefix.tar $tag\"
+
+ # Erlang projects: add build depends to release tarball
+ if [ -e build_dep.tar.gz ]; then
+ su build -c \"mkdir $prefix\"
+ su build -c \"mv build_dep.tar.gz $prefix\"
+ su build -c \"tar -rf $prefix.tar $prefix/build_dep.tar.gz\"
+ fi
+
+ su build -c \"bzip2 -9 $prefix.tar\"
+ fi
+ "; then
+ echo "$LOG_PREFIX Building tarball failed!"
+ exit 1
+ fi
+
+ cd "$TEMP/src/$project_path"
+
+ # Adjust the tarball name, e.g. for simtrace2-host-*.tar.bz2
+ if ! [ -e "$tarball_name" ]; then
+ echo
+ mv -v *.tar.bz2 "$tarball_name"
+ echo
+ fi
+}
+
+# $1: repository
+# $2: tarball path within the repository dir
+publish_tarball() {
+ local repo="$1"
+ local tarball="$2"
+ local tarball_path="$TEMP/src/$repo/$tarball"
+ local tarball_path_remote="releases@ftp.osmocom.org:$WEB_PATH/$repo/$(basename "$tarball")"
+
+ echo "$LOG_PREFIX Publishing $tarball"
+
+ if [ ! -e "$tarball_path" ]; then
+ echo "$LOG_PREFIX ERROR: tarball not found: $tarball_path"
+ exit 1
+ fi
+
+ if [ "$PUBLISH" != 1 ]; then
+ echo "$LOG_PREFIX Skipping, PUBLISH != 1"
+ return
+ fi
+
+ $SSH_COMMAND releases@ftp.osmocom.org -- mkdir -p "$WEB_PATH/$repo"
+ rsync -vz -e "$SSH_COMMAND" "$tarball_path" $tarball_path_remote
+}
+
+# $1: repository
+# $2: tag
+build_publish_tarballs() {
+ local repo="$1"
+ local tag="$2"
+ local tarballs="$repo-$tag.tar.bz2"
+
+ build_tarball "$repo" "$repo" "$tag"
+
+ case "$repo" in
+ simtrace2)
+ if [ -e "$TEMP"/src/simtrace2/host/configure.ac ]; then
+ build_tarball "simtrace2" "simtrace2/host" "$tag"
+ tarballs="$tarballs host/simtrace2-host-$tag.tar.bz2"
+ fi
+ ;;
+ esac
+
+ for tarball in $tarballs; do
+ publish_tarball "$repo" "$tarball"
+ done
+}
+
+check_ssh_auth_sock
+get_server_ls "/releases"
+
+for repo in $OSMO_RELEASE_REPOS; do
+ LOG_PREFIX=":: ($repo)"
+
+ if grep -q "^$repo$" "$TEMP/ls_releases"; then
+ get_server_ls "/releases/$repo"
+ else
+ echo "$LOG_PREFIX No release directory on server"
+ touch "$TEMP/ls_releases_$repo"
+ fi
+
+ get_git_tags "$repo"
+
+ echo "$LOG_PREFIX Building missing tarballs"
+ for tag in $(cat "$TEMP"/git_tags_"$repo"); do
+ LOG_PREFIX=":: ($repo, $tag)"
+ if tarball_exists "$repo" "$tag"; then
+ echo "$LOG_PREFIX Skipping, tarball exists"
+ continue
+ elif is_tag_ignored "$repo" "$tag"; then
+ echo "$LOG_PREFIX Skipping, tag is ignored"
+ continue
+ fi
+
+ clone_repo "$repo" "$tag"
+ build_publish_tarballs "$repo" "$tag"
+ done
+done
diff --git a/scripts/verify_endian_header.sh b/scripts/verify_endian_header.sh
new file mode 100755
index 0000000..9dcbc0e
--- /dev/null
+++ b/scripts/verify_endian_header.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+# Usage: ./verify_endian_header.sh $(find . -name "*.[hc]")
+
+HEADER="osmocom/core/endian.h"
+COUNT=0
+
+for f in $*; do
+ # Obviously, ignore the header file defining the macros
+ if [ $(basename $f) = $(basename $HEADER) ]; then
+ continue
+ fi
+ # Match files using either of OSMO_IS_{LITTLE,BIG}_ENDIAN
+ if grep -q "OSMO_IS_\(LITTLE\|BIG\)_ENDIAN" $f; then
+ # The header file must be included
+ if ! grep -q "#include <$HEADER>" $f; then
+ echo "File '$f' does not #include <$HEADER>"
+ COUNT=$((COUNT + 1))
+ fi
+ fi
+done
+
+exit $COUNT
diff --git a/scripts/verify_value_string_arrays_are_terminated.py b/scripts/verify_value_string_arrays_are_terminated.py
index f6dc545..abeff4d 100755
--- a/scripts/verify_value_string_arrays_are_terminated.py
+++ b/scripts/verify_value_string_arrays_are_terminated.py
@@ -19,7 +19,7 @@ value_string_array_re = re.compile(
re.MULTILINE | re.DOTALL)
members = r'(\.(value|str)\s*=\s*)?'
-terminator_re = re.compile('{}|{\s*' + members + '(0|NULL)\s*,'
+terminator_re = re.compile('{\s*}|{\s*0\s*}|{\s*' + members + '(0|NULL)\s*,'
'\s*' + members + '(0|NULL)\s*}')
errors_found = 0
diff --git a/scripts/xgoldmon-jenkins.sh b/scripts/xgoldmon-jenkins.sh
new file mode 100755
index 0000000..0635db4
--- /dev/null
+++ b/scripts/xgoldmon-jenkins.sh
@@ -0,0 +1,23 @@
+#!/bin/sh -ex
+# Used by jobs/master-builds.yml for xgoldmon
+TOPDIR=/build
+
+if ! [ -x "$(command -v osmo-build-dep.sh)" ]; then
+ echo "Error: missing scripts from osmo-ci.git in PATH!"
+ exit 2
+fi
+
+set -x
+
+osmo-clean-workspace.sh
+
+export deps="$TOPDIR/deps"
+export inst="$deps/install"
+export PKG_CONFIG_PATH="$inst/lib/pkgconfig:$PKG_CONFIG_PATH"
+export LD_LIBRARY_PATH="$inst/lib"
+
+mkdir -p deps
+osmo-build-dep.sh libosmocore "" '--disable-doxygen'
+
+cd "$TOPDIR"
+$MAKE