diff options
62 files changed, 1259 insertions, 463 deletions
@@ -30,7 +30,7 @@ build4-deb12build-ansible : ok=9 changed=4 unreachable=0 failed=0 s ``` ## jobs -Jenkins Job Builder YAML files defining jenkins jobs. Read `jobs/README.adoc` +Jenkins Job Builder YAML files defining jenkins jobs. Read `jobs/README.md` for more information about deployment. ## lint diff --git a/ansible/hosts b/ansible/hosts index 51e26b9..f68ea52 100644 --- a/ansible/hosts +++ b/ansible/hosts @@ -4,29 +4,25 @@ osmo-gsm-tester-prod ansible_host=10.9.25.107 [coverity_slaves] build2-deb11build-ansible ansible_host=2a01:4f8:10b:2ad9::1:13 -build3-deb11build-ansible ansible_host=2a01:4f8:212:3762::1:3 build4-deb12build-ansible ansible_host=2a01:4f8:121:200a::1:3 [jenkins_slaves] build2-deb10build-ansible ansible_host=2a01:4f8:10b:2ad9::1:10 osmocom_jenkins_slave_fstrim=True docker_max_image_space="80 GB" build2-deb11build-ansible ansible_host=2a01:4f8:10b:2ad9::1:13 osmocom_jenkins_slave_fstrim=True docker_max_image_space="80 GB" -build3-deb11build-ansible ansible_host=2a01:4f8:212:3762::1:3 osmocom_jenkins_slave_fstrim=True docker_max_image_space="80 GB" build4-deb12build-ansible ansible_host=2a01:4f8:121:200a::1:3 osmocom_jenkins_slave_fstrim=True docker_max_image_space="80 GB" host2-deb11build-ansible ansible_host=2a01:4f8:120:8470::1:4 osmocom_jenkins_slave_fstrim=True docker_max_image_space="80 GB" -rpi4-raspbian11build-ansible-1 ansible_host=10.9.25.50 docker_max_image_space="20 GB" -rpi4-raspbian11build-ansible-2 ansible_host=10.9.25.60 docker_max_image_space="20 GB" -rpi4-raspbian11build-ansible-3 ansible_host=10.9.25.70 docker_max_image_space="20 GB" -lx2-raspbian11build-ansible ansible_host=10.34.2.153 docker_max_image_space="20 GB" +rpi4-raspbian12build-ansible-1 ansible_host=10.9.25.50 docker_max_image_space="20 GB" +rpi4-raspbian12build-ansible-2 ansible_host=10.9.25.60 docker_max_image_space="20 GB" +rpi4-raspbian12build-ansible-3 ansible_host=10.9.25.70 docker_max_image_space="20 GB" +lx2-raspbian11build-ansible ansible_host=10.9.25.28 docker_max_image_space="20 GB" jenkins_tmpfs=True [simtester] simtest ansible_host=10.9.25.80 [obs_workers] -build3-obsworker1 ansible_host=2a01:4f8:212:3762::1:2 obs_worker_instances=8 obs_worker_jobs=8 build4-obsworker1 ansible_host=2a01:4f8:121:200a::1:2 obs_worker_instances=8 obs_worker_jobs=8 lx2-1-obs-aarch64-1 ansible_host=2a02:8106:13:1e34:4e9:e7ff:fe97:fe92 obs_worker_instances=1 obs_worker_jobs=4 # Need to have kvm inside lxc, see OS#5365 [qemu] -build3-deb11build-ansible ansible_host=2a01:4f8:212:3762::1:3 build4-deb12build-ansible ansible_host=2a01:4f8:121:200a::1:3 diff --git a/ansible/roles/gsm-tester-bts/files/Release.key b/ansible/roles/gsm-tester-bts/files/Release.key index ecca084..b4c019a 100644 --- a/ansible/roles/gsm-tester-bts/files/Release.key +++ b/ansible/roles/gsm-tester-bts/files/Release.key @@ -8,13 +8,13 @@ e93fLa2Cu8vhFBcVn6CuHeEhsmuMf6NLbQRNfNNCEEUYaZn7beMYtpZ7t1djsKx5 1xGm50OzI22FLu8lELQ9d7qMVGRG3WHYawX9BDteRybiyqxfwUHm1haWazRJtlGt UWyzvwAb80BK1J2Nu5fbAa3w5CoEPAbUuCyrABEBAAG0JW9zbW9jb20gT0JTIFBy b2plY3QgPG9zbW9jb21Ab3Ntb2NvbT6JAVQEEwEIAD4WIQRrKp83ktFetw1Oao+G -pzC2U3JZcwUCYrMV4wIbAwUJBB6yjwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAK -CRCGpzC2U3JZc4FRCACQQkKIrnvQ7n2u7GSmyVZa3I+oLoFXSGqaGyey5TW/nrMm -vFDKU3qliHiuNSmUY35SnAhXUsvqOYppxVRoO1MLrqUvzMOnIWqkJpf8mtjGUnsW -jyVeto7Rsjs75y2i1Hk+e7ljb/V65J3NlfrfEYWbqR9AKd53ReNXTdrQ0J05A38N -GdI4Ld/2lNISAwaBmGhqdeKsLHpQw/JERU1TApVJR1whFiIwDF1rOCg9GPnNKIk7 -yRZdK267XzztrainX/cbPILyzUZEDhYs6wQuyACyQ1YUxZIxrwVfk7PMNay8CrLH -z42B73Ne5IAj8+op/3iJafFONLm7YXiDUFN+QDYAiQEzBBMBCAAdFiEExoiYhHND +pzC2U3JZcwUCZrNL8wIbAwUJCB7onwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAK +CRCGpzC2U3JZc11RB/9oogcqTF30pPEwjjGvXF+WMiRFHl8aVvQIk9m+2KUcjlVk +0q8+/guOg7KcVejPxyn2AQbTByWWgpH1o3BoDKWnvypURAV2uGOtCo6mUQop5yK0 +d50j9hwIXSwJWoD2r5OY+Gbs4dBzMGkKbnCED1n8324b8EN6m96eoL570Uq9vy4G +gIp+QCpePu8AMdoCLR2w3J+TA4gLdDeH2cr7TgmwlDCp8e6R8FVmRt3SvunPmFz3 +FHDicv2f9ZoHXVLQ20DgWUyI1xOPlzBNJowgK9YnLPdIidE9i+EbvC1YA+6O/WMa +kSUEyJLZS5mv2+T6ptgnCIgxrIoTu63O4Rsf5zV+iQEzBBMBCAAdFiEExoiYhHND S7aVYlnqa51NyAUyjdsFAmKzE1UACgkQa51NyAUyjdvuZgf+OXmr//i7u7Gg7eWB 7e0qUsyCId9lXS8J437x3K6ciJfD7/6RSy8TFW5Nglm/uSkbyq582I8t+SoOirMD E6cg9U/5+h5s46bAf+Kd2XS/6tLGeNLM18i4el8CP06NpFzDrsKu76uYFpyRiiHD @@ -22,5 +22,5 @@ otBdtgxeLJ83LugGfZslF+/5cigJkAJMhAdVvGO8h85R6fba8ZSOKtMKkaQRfi76 nhyOrJPlLuS+DLEnHwdkOFgtKnxHdjM97K+Tx0gisb6uwaWroXfSLnhP8RTLLZZy Z+noU1Hw3c+mn4c/NYbcC/uwHYHKRzuf9gHnQ3dGgv0Z5sbeLRVo92hjGj7Ftlyd 4hmKBg== -=HxK4 +=5Cwc -----END PGP PUBLIC KEY BLOCK----- diff --git a/ansible/roles/osmocom-jenkins-slave/README.md b/ansible/roles/osmocom-jenkins-slave/README.md index 81f68b3..fc62472 100644 --- a/ansible/roles/osmocom-jenkins-slave/README.md +++ b/ansible/roles/osmocom-jenkins-slave/README.md @@ -6,3 +6,4 @@ Support the following variables: * `generic_slave`: (true) contains tasks used by the most osmocom jenkins slaves * `osmocom_jenkins_slave_fstrim`: (false) calls fstrim periodical * `ttcn3_slave`: (true) install titan ttcn3 compiler and prepares the docker-playground +* `jenkins_tmpfs`: (false) put jenkins workspaces and ccache dirs into tmpfs diff --git a/ansible/roles/osmocom-jenkins-slave/defaults/main.yml b/ansible/roles/osmocom-jenkins-slave/defaults/main.yml index a4ce8ac..fc6f9fa 100644 --- a/ansible/roles/osmocom-jenkins-slave/defaults/main.yml +++ b/ansible/roles/osmocom-jenkins-slave/defaults/main.yml @@ -15,3 +15,6 @@ ttcn3_slave: true # call fstrim peridically osmocom_jenkins_slave_fstrim: false + +# put jenkins workspaces and ccache dirs into tmpfs +jenkins_tmpfs: false diff --git a/ansible/roles/osmocom-jenkins-slave/tasks/main.yml b/ansible/roles/osmocom-jenkins-slave/tasks/main.yml index 1868d97..46da076 100644 --- a/ansible/roles/osmocom-jenkins-slave/tasks/main.yml +++ b/ansible/roles/osmocom-jenkins-slave/tasks/main.yml @@ -58,3 +58,23 @@ minute: 0 job: "fstrim / >/dev/null 2>/dev/null || true" when: osmocom_jenkins_slave_fstrim + +- name: add tmpfs for /home/osmocom-build/jenkins + when: jenkins_tmpfs + ansible.posix.mount: + boot: true + path: /home/osmocom-build/jenkins + src: tmpfs + fstype: tmpfs + state: mounted + opts: "uid=1000,gid=1000" + +- name: add tmpfs for /home/osmocom-build/ccache + when: jenkins_tmpfs + ansible.posix.mount: + boot: true + path: /home/osmocom-build/ccache + src: tmpfs + fstype: tmpfs + state: mounted + opts: "uid=1000,gid=1000" diff --git a/ansible/roles/podman/files/mknod-fuse.service b/ansible/roles/podman/files/mknod-fuse.service new file mode 100644 index 0000000..b796f4f --- /dev/null +++ b/ansible/roles/podman/files/mknod-fuse.service @@ -0,0 +1,9 @@ +[Unit] +Description=Create /dev/fuse (OS#6524) + +[Service] +ExecStart=sh -c 'test -e /dev/fuse || mknod -m 666 /dev/fuse c 10 229' +RemainAfterExit=true + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/podman/files/storage.conf b/ansible/roles/podman/files/storage.conf new file mode 100644 index 0000000..f212591 --- /dev/null +++ b/ansible/roles/podman/files/storage.conf @@ -0,0 +1,5 @@ +[storage] +driver = "overlay" + +[storage.options] +mount_program = "/usr/bin/fuse-overlayfs" diff --git a/ansible/roles/podman/tasks/main.yml b/ansible/roles/podman/tasks/main.yml index 45a712d..102b37e 100644 --- a/ansible/roles/podman/tasks/main.yml +++ b/ansible/roles/podman/tasks/main.yml @@ -5,5 +5,26 @@ pkg: - podman - buildah + - fuse-overlayfs # Has reasonable recommends needed for dns resolving etc. install_recommends: yes + +- name: install mknod-fuse.service + copy: + src: mknod-fuse.service + dest: /etc/systemd/system/ + mode: 0644 + +- name: enable mknod-fuse.service + systemd_service: + daemon_reload: true + name: mknod-fuse + enabled: true + state: started + +- name: install storage.conf + copy: + src: storage.conf + dest: "/home/{{ jenkins_user }}/.config/containers/" + mode: 0644 + owner: "{{ jenkins_user }}" diff --git a/ansible/setup-jenkins-slave.yml b/ansible/setup-jenkins-slave.yml index 03afe65..2ed3bc7 100644 --- a/ansible/setup-jenkins-slave.yml +++ b/ansible/setup-jenkins-slave.yml @@ -8,11 +8,14 @@ - name: install common utilities apt: name: - - vim - - tmux - - screen - ca-certificates + - etckeeper + - ncdu - rsync + - screen + - tmux + - tree + - vim cache_valid_time: 3600 update_cache: yes roles: @@ -23,6 +26,12 @@ tags: - docker + - name: podman + when: ansible_distribution == 'Debian' and ansible_distribution_version|int >= 11 + jenkins_user: osmocom-build + tags: + - podman + - name: install-poky-sdk when: ansible_architecture == "x86_64" jenkins_user: osmocom-build diff --git a/ansible/util/known_hosts b/ansible/util/known_hosts index 76e2a64..6067e86 100644 --- a/ansible/util/known_hosts +++ b/ansible/util/known_hosts @@ -2,9 +2,9 @@ 2a01:4f8:10b:2ad9::1:13 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEfg7NNBKjVJGoL3oUDrJCccI2D8podsSPKnRn1aTUge 2a01:4f8:212:3762::1:3 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICl7tuOgN64+iYuYCtfCKaUbN9IWzKycpKVRgzsjVCPY 2a01:4f8:120:8470::1:4 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIPKRz2Eg+h6RTREn52n/ip3pzUjRu9pw1XdjWUrvvi/ -10.9.25.50 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPowg8lZeIGJ6LCLA8U4eSHnnPX90FaVgInDuRi5U9Zq -10.9.25.60 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOAK6SQXmifMamE1VHL/1GmPkk6KhplzT20jSOwb8PyZ -10.9.25.70 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILioWTUt6CyZTceSKqy09mtlNUif+ETKTF6whnWuIROx +10.9.25.50 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKn5hZBA8i3kcCuTORaQtLXf/NqpXxhkoldj9i60GOLo +10.9.25.60 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMB+zhsSo20QYPGPNNgqknnbx3g8V+MBksHIF3GuPihu +10.9.25.70 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJ4KE0enH9e76eWTy8HLFO/2pzrZaP8G93mtWvfD8//W 2a01:4f8:121:200a::1:3 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGafw2VdeZo4YNvbu8VEBjDrsCFL+ZSbHtvhspNIQ9PR 2a01:4f8:121:200a::1:2 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFyp0daWkfsRHErCVCcP/FZmtiRvFNAsK2A5TlQ6Ja+P -10.34.2.153 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIG0rQGxSxWJHwNNT5XdeuMPSYCAXpcDnFDzvyw5+sH3l +10.9.25.28 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIG0rQGxSxWJHwNNT5XdeuMPSYCAXpcDnFDzvyw5+sH3l diff --git a/coverity/build_Osmocom.sh b/coverity/build_Osmocom.sh index 61bd2b7..22ca629 100755 --- a/coverity/build_Osmocom.sh +++ b/coverity/build_Osmocom.sh @@ -77,7 +77,7 @@ build_default libosmo-abis build_default libosmo-gprs build_default libosmo-netif build_default libosmo-pfcp -build_default libosmo-sccp +build_default libosmo-sigtran build_default libsmpp34 build_default libgtpnl build_default osmo-ggsn \ diff --git a/coverity/jenkins.sh b/coverity/jenkins.sh index b33ae86..d19b93c 100755 --- a/coverity/jenkins.sh +++ b/coverity/jenkins.sh @@ -7,7 +7,7 @@ readlink /opt/coverity/current export PATH=$PATH:/opt/coverity/current/bin -base_dir="$HOME/osmo-ci/coverity" +base_dir="/opt/osmo-ci/coverity" src_dir="$PWD/source-Osmocom" cov_dir="$src_dir/cov-int" diff --git a/coverity/prepare_source_Osmocom.sh b/coverity/prepare_source_Osmocom.sh index 2c69356..d7ac90b 100755 --- a/coverity/prepare_source_Osmocom.sh +++ b/coverity/prepare_source_Osmocom.sh @@ -18,7 +18,7 @@ PROJECTS=" libosmo-gprs libosmo-netif libosmo-pfcp - libosmo-sccp + libosmo-sigtran libsmpp34 libusrp osmo-bsc diff --git a/jenkins-jobs-osmo.sh b/jenkins-jobs-osmo.sh new file mode 100755 index 0000000..245ac90 --- /dev/null +++ b/jenkins-jobs-osmo.sh @@ -0,0 +1,32 @@ +#!/bin/sh -e +# jenkins-job-builder wrapper for deploying to the Osmocom jenkins server + +CONFIG="$HOME/.config/jenkins_jobs/jenkins_jobs_osmo-ci.ini" +GLOBAL_CONFIGS=" + $HOME/.config/jenkins_jobs/jenkins_jobs.ini + /etc/jenkins_jobs/jenkins_jobs.ini +" + +if [ "$(basename "$PWD")" != "osmo-ci" ]; then + echo "ERROR: run this script from the osmo-ci dir" + exit 1 +fi + +for i in $GLOBAL_CONFIGS; do + if [ -e "$i" ]; then + echo "ERROR: global config found: $i" + GLOBAL_CONFIG_FOUND=1 + fi +done +if [ "$GLOBAL_CONFIG_FOUND" = 1 ]; then + echo "Please rename/remove global config files to prevent deploying to the wrong jenkins server by accident." + exit 1 +fi + +if ! [ -e "$CONFIG" ]; then + echo "ERROR: config not found: $CONFIG" + echo "You need to create it first, see: $PWD/jobs/README.md" + exit 1 +fi + +jenkins-jobs --conf "$CONFIG" "$@" diff --git a/jobs/README.adoc b/jobs/README.md index eb14152..7c81af5 100644 --- a/jobs/README.adoc +++ b/jobs/README.md @@ -1,78 +1,100 @@ +# Osmocom jenkins jobs + These jenkins.osmocom.org job definitions, managed by -https://docs.openstack.org/infra/jenkins-job-builder/index.html[Jenkins Job Builder] +[Jenkins Job Builder](https://docs.openstack.org/infra/jenkins-job-builder/index.html) -*Prepare:* +## Prepare Install jenkins-job-builder: - apt-get install jenkins-job-builder - -Have a jenkins-job-builder.ini file. One of +``` +# apt-get install jenkins-job-builder +``` - ~/.config/jenkins_jobs/jenkins_jobs.ini - /etc/jenkins_jobs/jenkins_jobs.ini +Create the following file: -or place one in here and pass it to jenkins-jobs using the --conf file. +``` +~/.config/jenkins_jobs/jenkins_jobs_osmo-ci.ini +``` Make sure the file not world readable to minimally safeguard your jenkins password. Instead of using your jenkins password, use an *API Token*. To retrieve your token go to Jenkins via a Webbrowser, click on your Username in the right corner, click on configure, click on *Show API Toke...*. -jenkins_jobs.ini: +`jenkins_jobs_osmo-ci.ini`: - [jenkins] - user=my_user_name - password=my_api_token - url=https://jenkins.osmocom.org/jenkins +``` +[jenkins] +user=my_user_name +password=my_api_token +url=https://jenkins.osmocom.org/jenkins +``` and - chmod go-rwx jenkins_jobs.ini +``` +$ chmod go-rwx jenkins_jobs_osmo-ci.ini +``` -*Update a single job on jenkins.osmocom.org:* +## Update a single job on jenkins.osmocom.org - jenkins-jobs --conf jenkins_jobs.ini update gerrit-verifications.yml gerrit-osmo-msc +``` +$ cd .. +$ ./jenkins-jobs-osmo.sh update jobs/gerrit-verifications.yml gerrit-osmo-msc +``` NOTE: when you supply a name not defined in that yml file, you will not get an error message, just nothing will happen. -*Update all jobs of one file:* +## Update all jobs of one file - jenkins-jobs --conf jenkins_jobs.ini update gerrit-verifications.yml +``` +$ cd .. +$ ./jenkins-jobs-osmo.sh update jobs/gerrit-verifications.yml +``` -*Update all jobs in all files:* +## Update all jobs in all files - jenkins-jobs --conf jenkins_jobs.ini update ./ +``` +$ cd .. +$ ./jenkins-jobs-osmo.sh update jobs/ +``` -*Troubleshooting:* +## Troubleshooting -- 'jenkins.JenkinsException: create[gerrit-osmo-msc] failed' +### jenkins.JenkinsException: create[gerrit-osmo-msc] failed jenkins.osmocom.org is not reachable, or URL in the config file is erratic. Make sure it is exactly - url=https://jenkins.osmocom.org/jenkins +``` +url=https://jenkins.osmocom.org/jenkins +``` -- newlines: +### Newlines Use 'key: |' to keep new lines in multiline values, e.g.: - - shell: | - echo hello - echo world + +``` +- shell: | + echo hello + echo world +``` See also: + * https://yaml-multiline.info/ * https://stackoverflow.com/a/21699210 -- jobs named on cmdline are not updated: +### Jobs named on cmdline are not updated Make sure the job name is correct, or just issue an entire yml file without individual job names. Also be aware that jobs are only actually updated when anything changed. -*Jenkins labels* +## Jenkins labels Most jenkins jobs should run a docker container and install all required dependencies inside that, so we don't need to install them on the jenkins node. @@ -81,39 +103,47 @@ node that has docker available. So if you add a new job, you probably don't need a label at all. Existing jobs typically have a label set by the topic they belong to, e.g.: + - osmocom-master - osmocom-gerrit - ttcn3 Other labels indicate specific software/hardware works here, e.g.: + - coverity - hdlc - osmo-gsm-tester - podman -*ccache* +## ccache The jobs from master-builds and gerrit-verifications use ccache. View the statistics with SSH on the build nodes with: - $ CCACHE_DIR=~/ccache/gerrit-verifications ccache -s - $ CCACHE_DIR=~/ccache/master-builds ccache -s +``` +$ CCACHE_DIR=~/ccache/gerrit-verifications ccache -s +$ CCACHE_DIR=~/ccache/master-builds ccache -s +``` Note that running multiple jobs in parallel influence the ccache statistics, and it's impossible to tell which job caused which change in the stats (that's why they are not printed at the end of each job, it would be confusing). -*timers* +## Timers A lot of the jenkins jobs run daily with a timer: - triggers: - - timed: "H 20 * * *" +``` +triggers: + - timed: "H 20 * * *" +``` or weekly: - triggers: - - timed: "H 20 * * H" +``` +triggers: + - timed: "H 20 * * H" +``` Use H for the minute / day of week, to have it derivated as hash of the job name. Replace 20 with the hour (UTC) the job should run. @@ -121,29 +151,31 @@ name. Replace 20 with the hour (UTC) the job should run. The jobs follow this timetable, to ensure we don't attempt to use binary packages before they have been built (leading to failing jobs). - 18:00 - 21:00 OBS related - 18:XX osmocom-obs-sync (sync Debian:12 etc. with openSUSE OBS) - 19:XX osmocom-obs-check-new-distros - 19:XX osmocom-obs-wireshark - 20:XX osmocom-obs (new binary packages start building on OBS!) - - 22:00 - 03:00 Jobs that don't need binary packages - 22:XX coverity - 22:XX octsim_osmo-ccid-firmware - 22:XX osmo-gsm-tester-runner (virtual) - 23:XX master-builds-dahdi - 00:XX osmocom-api - 00:XX registry-update-base-images - 01:XX osmocom-build-tags-against-master - 01:XX osmocom-list-commits - 02:XX registry-triggers - 02:XX registry-rebuild-upload-fpga-build (weekly) - - 04:00 - 18:00 Jobs that need binary packages - ttcn3-testsuites - 04:XX osmocom-release-manuals - 05:XX osmocom-release-tarballs - 06:XX repo-install-test - 06:XX coverity-status (runs intentionally much later than the coverity job) - -master-builds: to avoid complexity, these run throughout the day (H H * * *). +``` +18:00 - 21:00 OBS related + 18:XX osmocom-obs-sync (sync Debian:12 etc. with openSUSE OBS) + 19:XX osmocom-obs-check-new-distros + 19:XX osmocom-obs-wireshark + 20:XX osmocom-obs (new binary packages start building on OBS!) + +22:00 - 03:00 Jobs that don't need binary packages + 22:XX coverity + 22:XX octsim_osmo-ccid-firmware + 22:XX osmo-gsm-tester-runner (virtual) + 23:XX master-builds-dahdi + 00:XX osmocom-api + 00:XX registry-update-base-images + 01:XX osmocom-build-tags-against-master + 01:XX osmocom-list-commits + 02:XX registry-triggers + 02:XX registry-rebuild-upload-fpga-build (weekly) + +03:00 - 18:00 Jobs that need binary packages + ttcn3-testsuites + 04:XX osmocom-release-manuals + 05:XX osmocom-release-tarballs + 06:XX repo-install-test + 06:XX coverity-status (runs intentionally much later than the coverity job) +``` + +`master-builds`: to avoid complexity, these run throughout the day (H H * * *). diff --git a/jobs/coverity.yml b/jobs/coverity.yml index 42c060a..d7492c4 100644 --- a/jobs/coverity.yml +++ b/jobs/coverity.yml @@ -17,7 +17,7 @@ -e PARALLEL_MAKE="$PARALLEL_MAKE" \ -u build \ -v "$PWD:/build" \ - -v "$HOME/osmo-ci/coverity:/home/build/osmo-ci/coverity:ro" \ + -v "$HOME/osmo-ci/coverity:/opt/osmo-ci/coverity:ro" \ -v "/opt/coverity:/opt/coverity:ro" \ -w /build/coverity \ "$USER/debian-bookworm-build" \ diff --git a/jobs/gerrit-lint.yml b/jobs/gerrit-lint.yml index 3039829..a607268 100644 --- a/jobs/gerrit-lint.yml +++ b/jobs/gerrit-lint.yml @@ -34,7 +34,6 @@ - $GERRIT_BRANCH refspec: $GERRIT_REFSPEC choosing-strategy: gerrit - wipe-workspace: false skip-tag: true submodule: recursive: false diff --git a/jobs/gerrit-verifications-dahdi.yml b/jobs/gerrit-verifications-dahdi.yml index 5a8b3a6..2cf9e95 100644 --- a/jobs/gerrit-verifications-dahdi.yml +++ b/jobs/gerrit-verifications-dahdi.yml @@ -23,7 +23,12 @@ name: 'gerrit-dahdi-linux-{type}-{linux-ver}' project-type: freestyle node: osmocom-gerrit - concurrent: true + # These jobs need an entire linux tree. We try to avoid cloning it more + # often than necessary, therefore we keep the workspace directory between + # job runs. Jenkins creates a new workspace for each concurrent job, which + # ends up consuming a lot of disk space so we disable it here. Besides disk + # space considerations, this job would run fine concurrently. + concurrent: false retry-count: 0 # scm checkout properties: - build-discarder: diff --git a/jobs/gerrit-verifications.yml b/jobs/gerrit-verifications.yml index 607e909..0e05918 100644 --- a/jobs/gerrit-verifications.yml +++ b/jobs/gerrit-verifications.yml @@ -113,7 +113,7 @@ - libosmo-gprs - libosmo-netif - libosmo-pfcp - - libosmo-sccp + - libosmo-sigtran - libosmocore: slave_axis: !!python/tuple [io_uring] # OS#6405 @@ -121,6 +121,7 @@ a1: !!python/tuple [arm-none-eabi, amd64] combination_filter: '!(arch=="arm-none-eabi" && label=="FreeBSD_amd64")' cmd: '{docker_run_ccache} {docker_img} {timeout_cmd} /build/contrib/jenkins_arch.sh "$arch"' + pipeline_binpkgs: "centos:7 debian:10 debian:12" # centos7: SYS#6760 - libsmpp34 @@ -129,21 +130,6 @@ - libusrp - - openbsc: - a1_name: SMPP - a1: !!python/tuple [--enable-smpp] - a2_name: MGCP - a2: !!python/tuple [--enable-mgcp-transcoding, --disable-mgcp-transcoding] - a3_name: IU - a3: !!python/tuple [--disable-iu] - cmd: | - {docker_run_ccache} \ - -e IU="$IU" \ - -e SMPP="$SMPP" \ - -e MGCP="$MGCP" \ - {docker_img} {timeout_cmd} /build/contrib/jenkins.sh - pipeline_binpkgs: "" - - openggsn: a1_name: GTP a1: !!python/tuple [--enable-gtp-linux,--disable-gtp-linux] @@ -154,7 +140,7 @@ - osmo-bsc-nat - osmo-bts: - slave_axis: !!python/tuple [osmocom-gerrit, rpi4-raspbian11] + slave_axis: !!python/tuple [osmocom-gerrit, raspbian] a1_name: FIRMWARE_VERSION a1: !!python/tuple [master, femtobts_v2.7, superfemto_v2.4, superfemto_v3.0.1pre, superfemto_v3.1, superfemto_v5.1, v2017.01, litecell15, oc2g, oc2g-next] a2_name: BTS_MODEL @@ -163,7 +149,7 @@ a3: !!python/tuple ["1", "0"] combination_filter: > (FIRMWARE_VERSION == "master" && WITH_MANUALS == "0" && label == "osmocom-gerrit") || - (FIRMWARE_VERSION == "master" && BTS_MODEL == "trx" && WITH_MANUALS == "0" && label == "rpi4-raspbian11") || + (FIRMWARE_VERSION == "master" && BTS_MODEL == "trx" && WITH_MANUALS == "0" && label == "raspbian") || (FIRMWARE_VERSION == "femtobts_v2.7" && BTS_MODEL == "sysmo" && WITH_MANUALS == "1" && label == "osmocom-gerrit") || (FIRMWARE_VERSION == "superfemto_v2.4" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-gerrit") || (FIRMWARE_VERSION == "superfemto_v3.0.1pre" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-gerrit") || @@ -177,12 +163,15 @@ cmd: | DOCKER_IMG="{docker_img}" case "$(arch)" in - arm*) DOCKER_IMG="$DOCKER_IMG-arm" ;; + arm*|aarch64) + DOCKER_IMG="$DOCKER_IMG-arm" + SETARCH_CMD="setarch linux32" + ;; esac {docker_run_ccache} \ -e FIRMWARE_VERSION="$FIRMWARE_VERSION" \ -e WITH_MANUALS="$WITH_MANUALS" \ - "$DOCKER_IMG" {timeout_cmd} /build/contrib/jenkins_bts_model.sh "$BTS_MODEL" + "$DOCKER_IMG" {timeout_cmd} $SETARCH_CMD /build/contrib/jenkins_bts_model.sh "$BTS_MODEL" - osmo_dia2gsup: repos_url: '{gerrit_url}/erlang/{repos}' @@ -193,8 +182,24 @@ - osmo-epdg: repos_url: '{gerrit_url}/erlang/{repos}' gerrit_project: 'erlang/{repos}' - cmd: '{docker_run} {docker_img_erlang} {timeout_cmd} /build/contrib/jenkins.sh' pipeline_binpkgs: "debian:12" + a2_name: JOB_TYPE + a2: !!python/tuple ["build", "manuals"] + cmd: | + case "$JOB_TYPE" in + build) + {docker_run} {docker_img_erlang} {timeout_cmd} /build/contrib/jenkins.sh + ;; + manuals) + {docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins_manuals.sh + ;; + esac + + - osmo-s1gw: + repos_url: '{gerrit_url}/erlang/{repos}' + gerrit_project: 'erlang/{repos}' + pipeline_binpkgs: "debian:12" + cmd: '{docker_run} {docker_img_erlang} {timeout_cmd} /build/contrib/jenkins.sh' - osmo-e1-hardware: a2_name: JOB_TYPE @@ -258,10 +263,11 @@ -e WITH_MANUALS="$WITH_MANUALS" \ {docker_img} {timeout_cmd} /build/contrib/jenkins.sh - - osmo-pcap + - osmo-pcap: + pipeline_binpkgs: "centos:7 debian:10 debian:12" # centos7: SYS#6760 - osmo-pcu: - slave_axis: !!python/tuple [osmocom-gerrit, rpi4-raspbian11] + slave_axis: !!python/tuple [osmocom-gerrit, raspbian] a1_name: FIRMWARE_VERSION a1: !!python/tuple [master, v2017.01, litecell15, oc2g] a2_name: with_vty @@ -277,24 +283,36 @@ (with_vty == "True" && with_dsp == "lc15" && FIRMWARE_VERSION=="litecell15" && WITH_MANUALS == "0" && label == "osmocom-gerrit") || (with_vty == "True" && with_dsp == "oc2g" && FIRMWARE_VERSION=="oc2g" && WITH_MANUALS == "0" && label == "osmocom-gerrit") || (with_vty == "False" && with_dsp == "none" && FIRMWARE_VERSION=="master" && WITH_MANUALS == "1" && label == "osmocom-gerrit") || - (with_vty == "False" && with_dsp == "none" && FIRMWARE_VERSION=="master" && WITH_MANUALS == "0" && label == "rpi4-raspbian11") + (with_vty == "False" && with_dsp == "none" && FIRMWARE_VERSION=="master" && WITH_MANUALS == "0" && label == "raspbian") cmd: | DOCKER_IMG="{docker_img}" case "$(arch)" in - arm*) DOCKER_IMG="$DOCKER_IMG-arm" ;; + arm*|aarch64) + DOCKER_IMG="$DOCKER_IMG-arm" + SETARCH_CMD="setarch linux32" + ;; esac {docker_run_ccache} \ -e FIRMWARE_VERSION="$FIRMWARE_VERSION" \ -e with_vty="$with_vty" \ -e with_dsp="$with_dsp" \ -e WITH_MANUALS="$WITH_MANUALS" \ - "$DOCKER_IMG" {timeout_cmd} /build/contrib/jenkins.sh + "$DOCKER_IMG" {timeout_cmd} $SETARCH_CMD /build/contrib/jenkins.sh - osmo-python-tests: repos_url: '{gerrit_url}/python/{repos}' gerrit_project: 'python/{repos}' pipeline_binpkgs: "debian:10 debian:12" + - pyosmocom: + repos_url: '{gerrit_url}/python/{repos}' + gerrit_project: 'python/{repos}' + pipeline_binpkgs: "debian:12" + a1_name: JOB_TYPE + a1: !!python/tuple ["test", "pylint", "docs"] + cmd: | + {docker_run} -e JOB_TYPE="$JOB_TYPE" {docker_img} {timeout_cmd} /build/contrib/jenkins.sh + - osmo-sgsn: a1_name: IU a1: !!python/tuple [--enable-iu, --disable-iu] @@ -318,7 +336,7 @@ - osmo-sip-connector - osmo-trx: - slave_axis: !!python/tuple [osmocom-gerrit,rpi4-raspbian11] + slave_axis: !!python/tuple [osmocom-gerrit,raspbian] a1_name: INSTR a1: !!python/tuple [--with-sse, --without-sse, --with-neon, --with-neon-vfpv4] a2_name: WITH_MANUALS @@ -326,17 +344,20 @@ combination_filter: > (INSTR == "--without-sse" && WITH_MANUALS == "1" && label == "osmocom-gerrit") || (INSTR == "--with-sse" && WITH_MANUALS == "0" && label == "osmocom-gerrit") || - (INSTR == "--with-neon" && WITH_MANUALS == "0" && label == "rpi4-raspbian11") || - (INSTR == "--with-neon-vfpv4" && WITH_MANUALS == "0" && label == "rpi4-raspbian11") + (INSTR == "--with-neon" && WITH_MANUALS == "0" && label == "raspbian") || + (INSTR == "--with-neon-vfpv4" && WITH_MANUALS == "0" && label == "raspbian") cmd: | DOCKER_IMG="{docker_img}" case "$(arch)" in - arm*) DOCKER_IMG="$DOCKER_IMG-arm" ;; + arm*|aarch64) + DOCKER_IMG="$DOCKER_IMG-arm" + SETARCH_CMD="setarch linux32" + ;; esac {docker_run_ccache} \ -e INSTR="$INSTR" \ -e WITH_MANUALS="$WITH_MANUALS" \ - "$DOCKER_IMG" {timeout_cmd} /build/contrib/jenkins.sh + "$DOCKER_IMG" {timeout_cmd} $SETARCH_CMD /build/contrib/jenkins.sh - osmocom-bb: pipeline_binpkgs: "debian:12" @@ -430,13 +451,15 @@ -v "$PWD:/build" \ "registry.osmocom.org/osmocom-build/debian-bookworm-titan" \ {timeout_cmd} \ - sh -e -x -c " - useradd --uid=1000 build; - cd /build; - su build -c 'make deps'; - su build -c 'make clean'; - su build -c 'make compile'; - " + sh -e -x -c ' + useradd --uid=1000 build + for i in /osmo-ttcn3-hacks/deps/*/; do + if ! [ -e /build/deps/"$(basename "$i")" ]; then + ln -s "$i" /build/deps/ + fi + done + su build -c "make -C /build compile" + ' pipeline_binpkgs: "" - dahdi-tools: @@ -702,6 +725,8 @@ builders: - shell: '{obj:cmd}' + # Remove workspace on success to free up space + - shell: 'rm -rf "$WORKSPACE"' publishers: - warnings: diff --git a/jobs/master-builds.yml b/jobs/master-builds.yml index a902be8..971b642 100644 --- a/jobs/master-builds.yml +++ b/jobs/master-builds.yml @@ -99,7 +99,7 @@ # Full triggers would be: # trigger: > # master-libosmo-netif, master-osmo-bsc, master-osmo-bts, master-osmo-hlr, - # master-osmo-msc, master-osmo-sgsn, master-openbsc + # master-osmo-msc, master-osmo-sgsn # Optimized: # (see comment at libosmocore) trigger: > @@ -129,13 +129,13 @@ - libosmo-netif: # Full triggers would be: # trigger: > - # master-libosmo-sccp, master-openbsc, mastere-osmo-hlr, master-osmo-iuh, + # master-libosmo-sigtran, mastere-osmo-hlr, master-osmo-iuh, # master-osmo-mgw, master-osmo-msc, master-osmo-sgsn, master-osmo-hnodeb, # master-osmo-hnbgw # Optimized: # (see comment at libosmocore) trigger: > - master-libosmo-sccp, + master-libosmo-sigtran, master-osmo-hlr, master-osmo-mgw, master-osmo-sgsn @@ -150,15 +150,14 @@ master-osmo-upf, master-osmo-hnbgw - - libosmo-sccp: + - libosmo-sigtran: # Full triggers would be: # trigger: > - # master-openbsc, master-osmo-bsc, master-osmo-iuh, master-osmo-msc, + # master-osmo-bsc, master-osmo-iuh, master-osmo-msc, # master-osmo-sgsn, master-osmo-hnodeb, master-osmo-hnbgw # Optimized: # (see comment at libosmocore) trigger: > - master-openbsc, master-osmo-bsc, master-osmo-bsc-nat, master-osmo-iuh, @@ -173,7 +172,7 @@ cmd: '{docker_run_ccache} {docker_img} {timeout_cmd} /build/contrib/jenkins_arch.sh "$arch"' # Full triggers would be: # trigger: > - # master-libosmo-abis, master-libosmo-netif, master-libosmo-sccp, master-openbsc, + # master-libosmo-abis, master-libosmo-netif, master-libosmo-sigtran, # master-osmo-bsc, master-osmo-bts, master-osmo-ggsn, master-osmo-gmr, master-osmo-hlr, # master-osmo-iuh, master-osmo-mgw, master-osmo-msc, master-osmo-pcap, master-osmo-pcu, # master-osmo-sgsn, master-osmo-sip-connector, master-osmo-tetra, master-osmo-hnodeb @@ -182,8 +181,7 @@ # libosmocore # -> master-libosmo-abis # -> master-libosmo-netif - # -> master-libosmo-sccp - # -> master-openbsc + # -> master-libosmo-sigtran # -> master-osmo-bsc # -> master-osmo-bsc-nat # -> master-osmo-iuh @@ -219,31 +217,17 @@ master-xgoldmon - libsmpp34: - trigger: master-osmo-msc, master-openbsc + trigger: master-osmo-msc - libtelnet - libusrp: trigger: master-osmo-trx - - openbsc: - a1_name: SMPP - a1: !!python/tuple [--enable-smpp] - a2_name: MGCP - a2: !!python/tuple [--enable-mgcp-transcoding, --disable-mgcp-transcoding] - a3_name: IU - a3: !!python/tuple [--disable-iu] - cmd: | - {docker_run_ccache} \ - -e IU="$IU" \ - -e SMPP="$SMPP" \ - -e MGCP="$MGCP" \ - {docker_img} {timeout_cmd} /build/contrib/jenkins.sh - - osmo-bsc - osmo-bsc-nat - osmo-bts: - slave_axis: !!python/tuple [osmocom-master, rpi4-raspbian11] + slave_axis: !!python/tuple [osmocom-master, raspbian] a1_name: FIRMWARE_VERSION a1: !!python/tuple [master, femtobts_v2.7, superfemto_v2.4, superfemto_v3.0.1pre, superfemto_v3.1, superfemto_v5.1, v2017.01, origin/nrw/litecell15, origin/nrw/oc2g] a2_name: BTS_MODEL @@ -252,7 +236,7 @@ a3: !!python/tuple ["1", "0"] combination_filter: > (FIRMWARE_VERSION == "master" && WITH_MANUALS == "1" && label == "osmocom-master") || - (FIRMWARE_VERSION == "master" && BTS_MODEL == "trx" && WITH_MANUALS == "0" && label == "rpi4-raspbian11") || + (FIRMWARE_VERSION == "master" && BTS_MODEL == "trx" && WITH_MANUALS == "0" && label == "raspbian") || (FIRMWARE_VERSION == "femtobts_v2.7" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-master") || (FIRMWARE_VERSION == "superfemto_v2.4" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-master") || (FIRMWARE_VERSION == "superfemto_v3.0.1pre" && BTS_MODEL == "sysmo" && WITH_MANUALS == "0" && label == "osmocom-master") || @@ -265,12 +249,15 @@ cmd: | DOCKER_IMG="{docker_img}" case "$(arch)" in - arm*) DOCKER_IMG="$DOCKER_IMG-arm" ;; + arm*|aarch64) + DOCKER_IMG="$DOCKER_IMG-arm" + SETARCH_CMD="setarch linux32" + ;; esac {docker_run_ccache} \ -e FIRMWARE_VERSION="$FIRMWARE_VERSION" \ -e WITH_MANUALS="$WITH_MANUALS" \ - "$DOCKER_IMG" {timeout_cmd} /build/contrib/jenkins_bts_model.sh "$BTS_MODEL" + "$DOCKER_IMG" {timeout_cmd} $SETARCH_CMD /build/contrib/jenkins_bts_model.sh "$BTS_MODEL" - osmo_dia2gsup: repos_url: '{gerrit_url}/erlang/{repos}' @@ -278,6 +265,20 @@ - osmo-epdg: repos_url: '{gerrit_url}/erlang/{repos}' + a2_name: JOB_TYPE + a2: !!python/tuple ["build", "manuals"] + cmd: | + case "$JOB_TYPE" in + build) + {docker_run} {docker_img_erlang} {timeout_cmd} /build/contrib/jenkins.sh + ;; + manuals) + {docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins_manuals.sh + ;; + esac + + - osmo-s1gw: + repos_url: '{gerrit_url}/erlang/{repos}' cmd: '{docker_run} {docker_img_erlang} {timeout_cmd} /build/contrib/jenkins.sh' - osmo-e1-hardware: @@ -384,7 +385,7 @@ - osmo-pcap - osmo-pcu: - slave_axis: !!python/tuple [osmocom-master, rpi4-raspbian11] + slave_axis: !!python/tuple [osmocom-master, raspbian] a1_name: FIRMWARE_VERSION a1: !!python/tuple [master, v2017.01, origin/nrw/litecell15] a2_name: with_vty @@ -399,22 +400,32 @@ (with_vty == "True" && with_dsp == "lc15" && FIRMWARE_VERSION=="v2017.01" && WITH_MANUALS == "0" && label == "osmocom-master") || (with_vty == "True" && with_dsp == "lc15" && FIRMWARE_VERSION=="origin/nrw/litecell15" && WITH_MANUALS == "0" && label == "osmocom-master") || (with_vty == "False" && with_dsp == "none" && FIRMWARE_VERSION=="master" && WITH_MANUALS == "1" && label == "osmocom-master") || - (with_vty == "False" && with_dsp == "none" && FIRMWARE_VERSION=="master" && WITH_MANUALS == "0" && label == "rpi4-raspbian11") + (with_vty == "False" && with_dsp == "none" && FIRMWARE_VERSION=="master" && WITH_MANUALS == "0" && label == "raspbian") cmd: | DOCKER_IMG="{docker_img}" case "$(arch)" in - arm*) DOCKER_IMG="$DOCKER_IMG-arm" ;; + arm*|aarch64) + DOCKER_IMG="$DOCKER_IMG-arm" + SETARCH_CMD="setarch linux32" + ;; esac {docker_run_ccache} \ -e FIRMWARE_VERSION="$FIRMWARE_VERSION" \ -e with_vty="$with_vty" \ -e with_dsp="$with_dsp" \ -e WITH_MANUALS="$WITH_MANUALS" \ - "$DOCKER_IMG" {timeout_cmd} /build/contrib/jenkins.sh + "$DOCKER_IMG" {timeout_cmd} $SETARCH_CMD /build/contrib/jenkins.sh - osmo-python-tests: repos_url: '{gerrit_url}/python/{repos}' + - pyosmocom: + repos_url: '{gerrit_url}/python/{repos}' + a1_name: JOB_TYPE + a1: !!python/tuple ["test", "pylint", "docs"] + cmd: | + {docker_run} -e JOB_TYPE="$JOB_TYPE" {docker_img} {timeout_cmd} /build/contrib/jenkins.sh + - osmo-sgsn: a1_name: IU a1: !!python/tuple [--enable-iu, --disable-iu] @@ -436,7 +447,7 @@ email: jenkins-notifications@lists.osmocom.org laforge@gnumonks.org 246tnt@gmail.com - osmo-trx: - slave_axis: !!python/tuple [osmocom-master,rpi4-raspbian11] + slave_axis: !!python/tuple [osmocom-master,raspbian] a1_name: INSTR a1: !!python/tuple [--with-sse, --without-sse, --with-neon, --with-neon-vfpv4] a2_name: WITH_MANUALS @@ -444,17 +455,20 @@ combination_filter: > (INSTR == "--without-sse" && WITH_MANUALS == "1" && label == "osmocom-master") || (INSTR == "--with-sse" && WITH_MANUALS == "0" && label == "osmocom-master") || - (INSTR == "--with-neon" && WITH_MANUALS == "0" && label == "rpi4-raspbian11") || - (INSTR == "--with-neon-vfpv4" && WITH_MANUALS == "0" && label == "rpi4-raspbian11") + (INSTR == "--with-neon" && WITH_MANUALS == "0" && label == "raspbian") || + (INSTR == "--with-neon-vfpv4" && WITH_MANUALS == "0" && label == "raspbian") cmd: | DOCKER_IMG="{docker_img}" case "$(arch)" in - arm*) DOCKER_IMG="$DOCKER_IMG-arm" ;; + arm*|aarch64) + DOCKER_IMG="$DOCKER_IMG-arm" + SETARCH_CMD="setarch linux32" + ;; esac {docker_run_ccache} \ -e INSTR="$INSTR" \ -e WITH_MANUALS="$WITH_MANUALS" \ - "$DOCKER_IMG" {timeout_cmd} /build/contrib/jenkins.sh + "$DOCKER_IMG" {timeout_cmd} $SETARCH_CMD /build/contrib/jenkins.sh - osmocom-bb - osmo-sysmon @@ -620,6 +634,11 @@ builders: - shell: '{obj:cmd}' + - shell: | + # Remove workspace on success to free up space. Create an empty dir + # so the job does not abort when attempting to collect artifacts. + rm -rf "$WORKSPACE" + mkdir -p "$WORKSPACE" publishers: - archive: diff --git a/jobs/osmo-gsm-tester-builder.yml b/jobs/osmo-gsm-tester-builder.yml index cd50cb8..ec9c29f 100644 --- a/jobs/osmo-gsm-tester-builder.yml +++ b/jobs/osmo-gsm-tester-builder.yml @@ -17,10 +17,8 @@ - osmo-sgsn - osmo-trx - osmocom-bb - - osmo-nitb: - triggered-by: master-openbsc - osmo-stp: - triggered-by: master-libosmo-sccp + triggered-by: master-libosmo-sigtran - osmo-bts-sysmo: triggered-by: master-osmo-bts - osmo-pcu-sysmo: @@ -108,17 +106,10 @@ - add_param_build_branch: name: OSMO_GSM_TESTER_BUILD_srslte scm: - - osmo-gsm-tester-repo - -# The repo must be used seperate to workaround the bug "Can not expand OSMO_GSM_TESTER_BRANCH". -# The safe-guard check to not use un-defined variables seems to be broken. -- scm: - name: osmo-gsm-tester-repo - scm: - git: url: https://gerrit.osmocom.org/osmo-gsm-tester branches: - - ${OSMO_GSM_TESTER_BRANCH} + - '$OSMO_GSM_TESTER_BRANCH' wipe-workspace: false skip-tag: true basedir: osmo-gsm-tester diff --git a/jobs/osmo-gsm-tester-runner.yml b/jobs/osmo-gsm-tester-runner.yml index 4909fe7..5bc7140 100644 --- a/jobs/osmo-gsm-tester-runner.yml +++ b/jobs/osmo-gsm-tester-runner.yml @@ -34,7 +34,7 @@ - git: url: https://gerrit.osmocom.org/osmo-gsm-tester branches: - - ${OSMO_GSM_TESTER_BRANCH} + - '$OSMO_GSM_TESTER_BRANCH' wipe-workspace: false skip-tag: true basedir: osmo-gsm-tester @@ -137,7 +137,7 @@ - osmo-gsm-tester_build-srslte builders: - copy_artifact_all - - shell: !include-raw: osmo-gsm-tester_run-{stage}.sh + - shell: !include-raw-expand: osmo-gsm-tester_run-{stage}.sh publishers: - archive: artifacts: '*-run.tgz, *-bin.tgz' @@ -154,6 +154,7 @@ # gerrit job - job: name: 'osmo-gsm-tester_gerrit' + disabled: true # osmo-gsm-tester nodes are currently offline defaults: runner scm: - 'osmo-gsm-tester-gerrit' @@ -175,7 +176,7 @@ # make sure no test results/bin artifacts from a previous run remain rm -rf trial-* *.tgz *.md5 - copy_artifact_all - - shell: !include-raw: osmo-gsm-tester_run-gerrit.sh + - shell: !include-raw-expand: osmo-gsm-tester_run-gerrit.sh publishers: - archive: artifacts: '*-run.tgz, *-bin.tgz' @@ -200,7 +201,7 @@ - copy_artifact_all - copy_artifact: repo: osmo-gsm-tester_build-osmocom-bb - - shell: !include-raw: osmo-gsm-tester_ttcn3.sh + - shell: !include-raw-expand: osmo-gsm-tester_ttcn3.sh publishers: - archive: artifacts: '*-run.tgz, *-bin.tgz' @@ -227,14 +228,14 @@ repo: osmo-gsm-tester_build-osmocom-bb - shell: | # Set a trap to fix workspace permissions / kill the docker container on exit - clean_up() { + clean_up() {{ docker kill "osmo-gsm-tester-virtual" || true docker run --rm \ -v "$WORKSPACE":/workspace \ debian:bullseye \ chmod -R a+rwX /workspace/ - } - clean_up_trap() { + }} + clean_up_trap() {{ set +x echo echo "### Clean up ###" @@ -243,7 +244,7 @@ trap - EXIT INT TERM 0 clean_up - } + }} trap clean_up_trap EXIT INT TERM 0 # Make sure no test results from a previous run remain @@ -281,7 +282,7 @@ --name=osmo-gsm-tester-virtual \ --cap-add=sys_nice \ $USER/osmo-gsm-tester \ - /bin/bash -c 'LANG="en_US.utf8" LC_ALL="en_US.UTF-8" LC_LANG="en_US.UTF-8" PATH="$PWD/osmo-gsm-tester/src:${PATH}" ./osmo-gsm-tester/contrib/jenkins-run.sh' + /bin/bash -c 'LANG="en_US.utf8" LC_ALL="en_US.UTF-8" LC_LANG="en_US.UTF-8" PATH="$PWD/osmo-gsm-tester/src:${{PATH}}" ./osmo-gsm-tester/contrib/jenkins-run.sh' publishers: - archive: artifacts: '*-run.tgz, *-bin.tgz' diff --git a/jobs/osmocom-build-tags-against-master.yml b/jobs/osmocom-build-tags-against-master.yml index e33c9a3..b242103 100644 --- a/jobs/osmocom-build-tags-against-master.yml +++ b/jobs/osmocom-build-tags-against-master.yml @@ -32,6 +32,11 @@ -w /build \ "$USER/debian-bookworm-build" \ scripts/osmocom-build-old-tags-against-master.sh + + # Remove git repos and build results on success to free up space + # Logs are kept + rm -rf "$WORKSPACE"/_deps + rm -rf "$WORKSPACE"/_temp/_inst* scm: - git: branches: diff --git a/jobs/osmocom-depcheck.yml b/jobs/osmocom-depcheck.yml index 5e20f95..5e9d247 100644 --- a/jobs/osmocom-depcheck.yml +++ b/jobs/osmocom-depcheck.yml @@ -25,7 +25,7 @@ empty to default to all projects (!), default revision is "master". Examples: "osmo-hlr", "osmo-hlr:0.2.1 osmo-bts:0.8.1" - default: 'osmo-hlr:0.2.1' + default: 'osmo-hlr:master' - string: name: GIT_URL_PREFIX description: | @@ -73,9 +73,4 @@ git-config-name: 'Jenkins Builder' git-config-email: 'jenkins@osmocom.org' - publishers: - - email: - notify-every-unstable-build: true - recipients: 'jenkins-notifications@lists.osmocom.org' - # vim: expandtab tabstop=2 shiftwidth=2 diff --git a/jobs/osmocom-obs-nightly-asan.yml b/jobs/osmocom-obs-nightly-asan.yml new file mode 100644 index 0000000..e3bf098 --- /dev/null +++ b/jobs/osmocom-obs-nightly-asan.yml @@ -0,0 +1,65 @@ +- job: + name: 'Osmocom_OBS_nightly_asan' + project-type: freestyle + concurrent: false + defaults: global + description: | + Binary repository with address sanitizer enabled + (<a href="https://osmocom.org/issues/5301">OS#5301</a>) + properties: + - build-discarder: + days-to-keep: 30 + num-to-keep: 30 + node: obs + parameters: + - string: + name: BRANCH + description: osmo-ci.git branch + default: 'refs/remotes/origin/master' + builders: + - shell: | + export PYTHONUNBUFFERED=1 + + # We only care about Osmocom packages that run in the TTCN-3 testsuites + # for the asan repository. + EXCLUDE_PACKAGES=" + erlang/osmo_dia2gsup + erlang/osmo-epdg + erlang/osmo-s1gw + osmo-bsc-nat + osmo-fl2k + osmo-gsm-manuals + python/osmo-python-tests + rtl-sdr + " + + ./scripts/obs/update_obs_project.py \ + --apiurl https://obs.osmocom.org \ + --configure-append " --enable-sanitize" \ + --conflict-pkgname "osmocom-nightly-asan" \ + --conflict-version "$(date +%Y%m%d%H%M)" \ + --disable-manuals \ + --docker \ + --feed "nightly" \ + --meta \ + --version-append "~asan" \ + osmocom:nightly:asan \ + ALL_OSMOCOM_PACKAGES \ + --exclude-package $EXCLUDE_PACKAGES + + # Remove workspace on success to free up space + rm -rf "$WORKSPACE" + scm: + - git: + branches: + - '$BRANCH' + url: https://gerrit.osmocom.org/osmo-ci + git-config-name: 'Jenkins Builder' + git-config-email: 'jenkins@osmocom.org' + triggers: + - timed: "H 20 * * *" + + publishers: + - email: + notify-every-unstable-build: true + recipients: 'jenkins-notifications@lists.osmocom.org' diff --git a/jobs/osmocom-obs-sync.yml b/jobs/osmocom-obs-sync.yml index 3776688..7f13e02 100644 --- a/jobs/osmocom-obs-sync.yml +++ b/jobs/osmocom-obs-sync.yml @@ -36,7 +36,6 @@ Raspbian:11 Raspbian:12 Ubuntu:22.04 - Ubuntu:23.10 Ubuntu:24.04 " ./scripts/obs/sync_obs_projects.py \ diff --git a/jobs/osmocom-obs.yml b/jobs/osmocom-obs.yml index eb77f4d..e61b16b 100644 --- a/jobs/osmocom-obs.yml +++ b/jobs/osmocom-obs.yml @@ -42,6 +42,10 @@ --feed {type} \ --meta \ osmocom:{type} + + # Remove workspace on success to free up space + rm -rf "$WORKSPACE" + scm: - git: branches: diff --git a/jobs/registry-rebuild-upload-titan.yml b/jobs/registry-rebuild-upload-titan.yml index a535920..8fed502 100644 --- a/jobs/registry-rebuild-upload-titan.yml +++ b/jobs/registry-rebuild-upload-titan.yml @@ -3,6 +3,7 @@ jobs: - registry-rebuild-upload-titan - registry-rebuild-upload-ttcn3-bts-tests + - registry-rebuild-upload-testenv - scm: name: osmocom-docker-playground @@ -69,4 +70,35 @@ notify-every-unstable-build: true recipients: 'jenkins-notifications@lists.osmocom.org' +- job: + name: registry-rebuild-upload-testenv + description: Rebuild + publish podman images for testenv (OS#6494) + parameters: + - string: + name: BRANCH + description: osmo-ttcn3-hacks.git branch + default: 'master' + scm: + - git: + url: https://gerrit.osmocom.org/osmo-ttcn3-hacks + branches: + - '$BRANCH' + builders: + - shell: | + REGISTRY="registry.osmocom.org" + DISTRO="debian:bookworm" + IMAGE="registry.osmocom.org/osmocom-build/"$(echo "$DISTRO" | tr : -)"-osmo-ttcn3-testenv" + + podman build \ + --build-arg DISTRO="$DISTRO" \ + --build-arg REGISTRY="$REGISTRY" \ + -t "$IMAGE" \ + _testenv/data/podman + + podman push "$IMAGE" + publishers: + - email: + notify-every-unstable-build: true + recipients: 'jenkins-notifications@lists.osmocom.org' + # vim: expandtab tabstop=2 shiftwidth=2 diff --git a/jobs/ttcn3-testsuites-testenv.yml b/jobs/ttcn3-testsuites-testenv.yml new file mode 100644 index 0000000..b5180f1 --- /dev/null +++ b/jobs/ttcn3-testsuites-testenv.yml @@ -0,0 +1,278 @@ +--- +- project: + name: 'ttcn3-testsuites' + concurrent: false + disabled: false + description: | + Run the <code>{testsuite}</code> testsuite from + <a href="https://gitea.osmocom.org/ttcn3/osmo-ttcn3-hacks"> + osmo-ttcn3-hacks.git</a> with testenv.py.<br> + See <a href="https://osmocom.org/projects/cellular-infrastructure/wiki/Titan_TTCN3_Testsuites"> + TTCN3 Testsuite</a> for more information.<br> + <br> + Arguments: <code>{args}</code> + timer: "@midnight" + node: ttcn3 + jobs: + - "{job-name}" + # timers: see jobs/README.adoc + job-name: + # --------------------------------------------- + # Debian nightly + # --------------------------------------------- + - ttcn3-hlr-test: + testsuite: hlr + args: -b osmocom:nightly + timer: H 05 * * * + + - ttcn3-mgw-test: + testsuite: mgw + args: -b osmocom:nightly + timer: H 05 * * * + + - ttcn3-sip-test: + testsuite: sip + args: -b osmocom:nightly + timer: H 05 * * * + + - ttcn3-sgsn-test: + testsuite: sgsn + args: -b osmocom:nightly -c generic + timer: H 05 * * * + + - ttcn3-sgsn-test-sns: + testsuite: sgsn + args: -b osmocom:nightly -c sns + timer: H 05 * * * + + - ttcn3-smlc-test: + testsuite: smlc + args: -b osmocom:nightly + timer: H 06 * * * + + - ttcn3-cbc-test: + testsuite: cbc + args: -b osmocom:nightly + timer: H 06 * * * + + - ttcn3-hnodeb-test: + testsuite: hnodeb + args: -b osmocom:nightly + timer: H 07 * * * + + - ttcn3-dia2gsup-test: + testsuite: dia2gsup + args: -b osmocom:nightly + timer: H 07 * * * + + - ttcn3-upf-test: + testsuite: upf + args: -b osmocom:nightly + timer: H 07 * * * + + - ttcn3-hss-test-ogs: + testsuite: hss + args: -b osmocom:nightly + timer: H 07 * * * + email: jenkins-notifications@lists.osmocom.org acetcom@gmail.com + + - ttcn3-mme-test-ogs: + testsuite: mme + args: -b osmocom:nightly + timer: H 07 * * * + email: jenkins-notifications@lists.osmocom.org acetcom@gmail.com + + - ttcn3-pcrf-test-ogs: + testsuite: pcrf + args: -b osmocom:nightly + timer: H 07 * * * + email: jenkins-notifications@lists.osmocom.org acetcom@gmail.com + + - ttcn3-s1gw-test: + testsuite: s1gw + args: -b osmocom:nightly + timer: H 07 * * * + + # --------------------------------------------- + # Debian latest + # --------------------------------------------- + - ttcn3-hlr-test-latest: + testsuite: hlr + args: -b osmocom:latest + timer: H 08 * * * + + - ttcn3-mgw-test-latest: + testsuite: mgw + args: -b osmocom:latest + timer: H 08 * * * + + - ttcn3-sip-test-latest: + testsuite: sip + args: -b osmocom:latest + timer: H 08 * * * + + - ttcn3-sgsn-test-latest: + testsuite: sgsn + args: -b osmocom:latest -c generic + timer: H 08 * * * + + - ttcn3-sgsn-test-sns-latest: + testsuite: sgsn + args: -b osmocom:latest -c sns + timer: H 08 * * * + + - ttcn3-smlc-test-latest: + testsuite: smlc + args: -b osmocom:latest + timer: H 09 * * * + + - ttcn3-cbc-test-latest: + testsuite: cbc + args: -b osmocom:latest + timer: H 09 * * * + + - ttcn3-hnodeb-test-latest: + testsuite: hnodeb + args: -b osmocom:latest + timer: H 10 * * * + + - ttcn3-upf-test-latest: + testsuite: upf + args: -b osmocom:latest + timer: H 10 * * * + disabled: true + + - ttcn3-s1gw-test-latest: + testsuite: s1gw + args: -b osmocom:latest + timer: H 10 * * * + disabled: true # disable until > 0.1.x release + + # --------------------------------------------- + # Debian nightly + io_uring (OS#6357) + # --------------------------------------------- + - ttcn3-mgw-test-io_uring: + testsuite: mgw + args: -b osmocom:nightly --io-uring + timer: H 11 * * * + + - ttcn3-sgsn-test-io_uring: + testsuite: sgsn + args: -b osmocom:nightly --io-uring -c generic + timer: H 11 * * * + + # --------------------------------------------- + # Debian nightly + asan (OS#5301) + # --------------------------------------------- + - ttcn3-mgw-test-asan: + testsuite: mgw + args: -b osmocom:nightly:asan + timer: H 12 * * * + + - ttcn3-hlr-test-asan: + testsuite: hlr + args: -b osmocom:nightly:asan + timer: H 12 * * * + + - ttcn3-sip-test-asan: + testsuite: sip + args: -b osmocom:nightly:asan + timer: H 12 * * * + + - ttcn3-hnodeb-test-asan: + testsuite: hnodeb + args: -b osmocom:nightly:asan + timer: H 12 * * * + + - ttcn3-smlc-test-asan: + testsuite: smlc + args: -b osmocom:nightly:asan + timer: H 12 * * * + + - ttcn3-cbc-test-asan: + testsuite: cbc + args: -b osmocom:nightly:asan + timer: H 12 * * * + + - ttcn3-sgsn-test-asan: + testsuite: sgsn + args: -b osmocom:nightly:asan -c generic + timer: H 12 * * * + +- job-template: + name: '{job-name}' + project-type: freestyle + disabled: '{obj:disabled}' + defaults: global + description: '{obj:description}' + node: '{obj:node}' + parameters: + - string: + name: BRANCH + description: | + Branch of <code>osmo-ttcn3-hacks.git</code> + default: 'master' + builders: + - shell: |- + DISTRO="debian:bookworm" + IMAGE="registry.osmocom.org/osmocom-build/"$(echo "$DISTRO" | tr : -)"-osmo-ttcn3-testenv" + + podman pull "$IMAGE" + + rm -rf logs _cache + + set +x # Don't output the color codes we set here + export TESTENV_COLOR_DEBUG="$(printf '\e[0;94m')" # light blue + export TESTENV_COLOR_INFO="$(printf '\e[1;34m')" # bold, blue + export TESTENV_COLOR_WARNING="$(printf '\e[1;35m')" # bold, purple + export TESTENV_COLOR_ERROR="$(printf '\e[1;91m')" # bold, red + export TESTENV_COLOR_CRITICAL="$(printf '\e[1;91m')" # bold, red + export TESTENV_SOURCE_HIGHLIGHT_COLORS="esc" + export TESTENV_NO_IMAGE_UP_TO_DATE_CHECK=1 + set -x + + ./testenv.py run \ + {testsuite} \ + --podman \ + --cache "$PWD"/_cache \ + --ccache ~/ccache/testenv \ + --log-dir "$PWD"/logs \ + {args} + + # Free up space from downloaded apt packages, build artifacts etc. + # The osmo-ttcn3-hacks deps dir and ccache are persistent. + rm -rf _cache + + scm: + - git: + branches: + - '$BRANCH' + url: https://gerrit.osmocom.org/osmo-ttcn3-hacks + git-config-name: 'Jenkins Builder' + git-config-email: 'jenkins@osmocom.org' + wipe-workspace: false + triggers: + - timed: "{obj:timer}" + publishers: + - junit: + results: '**/junit-xml-*.log' + allow-empty-results: false + - email: + notify-every-unstable-build: false + recipients: '{obj:email}' + send-to-individuals: false + - archive: + allow-empty: false + artifacts: logs/**/* + case-sensitive: true + default-excludes: true + fingerprint: false + only-if-success: false + properties: + - build-discarder: + days-to-keep: 30 + num-to-keep: 120 + wrappers: + - ansicolor: + colormap: xterm + email: jenkins-notifications@lists.osmocom.org diff --git a/jobs/ttcn3-testsuites.yml b/jobs/ttcn3-testsuites.yml index ac79f4d..98becb5 100644 --- a/jobs/ttcn3-testsuites.yml +++ b/jobs/ttcn3-testsuites.yml @@ -20,6 +20,8 @@ # timer: 10 per hour from 04:00 - 18:00, see jobs/README.adoc job-name: # debian nightly - order by time + - ttcn3-msc-test: + timer: H 03 * * * - nplab-m3ua-test: timer: H 04 * * * - nplab-sua-test: @@ -30,16 +32,24 @@ timer: H 04 * * * description: | Test osmo-ggsn master with the Debian kernel. - - ttcn3-ggsn-test-kernel-pablo-gtp: + - ttcn3-ggsn-test-kernel-net-next: + timer: H 04 * * * + # Cache the linux source tree in the workspace of one node + wipe_workspace: false + node: ttcn3-ggsn-test-kernel-git + description: | + Test osmo-ggsn master with the Linux kernel built from master of + <a href="https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git/"> + net-next.git</a>. + - ttcn3-ggsn-test-kernel-torvalds: timer: H 04 * * * # Cache the linux source tree in the workspace of one node wipe_workspace: false + node: ttcn3-ggsn-test-kernel-git description: | - Test osmo-ggsn master with the Linux kernel built from main of - <a href="https://git.kernel.org/pub/scm/linux/kernel/git/pablo/gtp.git/">pablo/gtp.git</a>. - <br> - This is a temporary job, to be removed once the IPv6 related GTP - patches are upstreamed (OS#1952). + Test osmo-ggsn master with the Linux kernel built from master of + <a href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/"> + torvalds/linux.git</a>. - ttcn3-ggsn-test-kernel-latest: timer: H 04 * * * description: | @@ -67,26 +77,14 @@ description: | Test open5gs-smfd + open5gs-upfd main. email: jenkins-notifications@lists.osmocom.org acetcom@gmail.com - - ttcn3-msc-test: - timer: H 04 * * * - ttcn3-pcu-test: timer: H 04 * * * - ttcn3-pcu-test-sns: timer: H 05 * * * - - ttcn3-sgsn-test: - timer: H 05 * * * - - ttcn3-sgsn-test-sns: - timer: H 05 * * * - ttcn3-bsc-test: timer: H 05 * * * - - ttcn3-mgw-test: - timer: H 05 * * * - - ttcn3-hlr-test: - timer: H 05 * * * - ttcn3-bts-test: timer: H 05 * * * - - ttcn3-sip-test: - timer: H 05 * * * - ttcn3-bscnat-test: timer: H 05 * * * disabled: true @@ -98,15 +96,11 @@ timer: H 06 * * * - ttcn3-sccp-test: timer: H 06 * * * - - ttcn3-smlc-test: - timer: H 06 * * * - ttcn3-gbproxy-test: timer: H 06 * * * - ttcn3-gbproxy-test-fr: timer: H 06 * * * node: hdlc - - ttcn3-cbc-test: - timer: H 06 * * * - ttcn3-ns-test: timer: H 06 * * * - ttcn3-ns-test-sns: @@ -120,10 +114,6 @@ timer: H 07 * * * - ttcn3-bsc-test-vamos: timer: H 07 * * * - - ttcn3-hnodeb-test: - timer: H 07 * * * - - ttcn3-upf-test: - timer: H 07 * * * - ttcn3-hnbgw-test: timer: H 07 * * * - ttcn3-fr-test: @@ -134,18 +124,6 @@ description: | Test open5gs-smfd + open5gs-upfd PGW main. email: jenkins-notifications@lists.osmocom.org acetcom@gmail.com - - ttcn3-mme-test-ogs: - timer: H 07 * * * - description: | - Test open5gs-mmed - email: jenkins-notifications@lists.osmocom.org acetcom@gmail.com - - ttcn3-hss-test-ogs: - timer: H 07 * * * - description: | - Test open5gs-hssd - email: jenkins-notifications@lists.osmocom.org acetcom@gmail.com - - ttcn3-dia2gsup-test: - timer: H 07 * * * - ttcn3-epdg-test: timer: H 07 * * * # osmo-epdg requires OTP and kernel >= debian12: @@ -166,20 +144,10 @@ timer: H 08 * * * - ttcn3-pcu-test-sns-latest: timer: H 08 * * * - - ttcn3-sgsn-test-latest: - timer: H 08 * * * - - ttcn3-sgsn-test-sns-latest: - timer: H 08 * * * - ttcn3-bsc-test-latest: timer: H 08 * * * - - ttcn3-mgw-test-latest: - timer: H 08 * * * - - ttcn3-hlr-test-latest: - timer: H 08 * * * - ttcn3-bts-test-latest: timer: H 08 * * * - - ttcn3-sip-test-latest: - timer: H 09 * * * - ttcn3-bscnat-test-latest: timer: H 09 * * * disabled: true @@ -191,24 +159,19 @@ timer: H 09 * * * - ttcn3-sccp-test-latest: timer: H 09 * * * - - ttcn3-smlc-test-latest: - timer: H 09 * * * - ttcn3-gbproxy-test-latest: timer: H 09 * * * - ttcn3-gbproxy-test-fr-latest: timer: H 09 * * * node: hdlc - - ttcn3-cbc-test-latest: - timer: H 09 * * * - ttcn3-pcap-client-test-latest: timer: H 10 * * * - - ttcn3-hnodeb-test-latest: - timer: H 10 * * * - - ttcn3-upf-test-latest: - timer: H 10 * * * - disabled: true - ttcn3-hnbgw-test-latest: timer: H 10 * * * + - ttcn3-epdg-test-latest: + timer: H 07 * * * + # osmo-epdg requires OTP and kernel >= debian12: + node: build4-deb12build-ansible # Jobs below use uppercase TTCN3-$DISTRO, so they get listed separately from the ttcn3-* (debian) jobs @@ -230,13 +193,36 @@ timer: H 11 * * * - ttcn3-gbproxy-test-io_uring: timer: H 11 * * * - - ttcn3-sgsn-test-io_uring: - timer: H 11 * * * - ttcn3-stp-test-io_uring: timer: H 11 * * * - - ttcn3-mgw-test-io_uring: + - ttcn3-bts-test-io_uring: timer: H 11 * * * + # With sanitizer (OS#5301) + - ttcn3-bsc-test-asan: + timer: H 12 * * * + - ttcn3-bts-test-asan: + timer: H 12 * * * + - ttcn3-gbproxy-test-asan: + timer: H 12 * * * + - ttcn3-ggsn-test-asan: + timer: H 12 * * * + - ttcn3-hnbgw-test-asan: + timer: H 12 * * * + - ttcn3-msc-test-asan: + timer: H 13 * * * + - ttcn3-pcap-client-test-asan: + timer: H 13 * * * + - ttcn3-pcu-test-asan: + timer: H 13 * * * + - ttcn3-remsim-test-asan: + timer: H 13 * * * + - ttcn3-sccp-test-asan: + timer: H 13 * * * + - ttcn3-stp-test-asan: + timer: H 13 * * * + + - job-template: name: '{job-name}' project-type: freestyle @@ -264,15 +250,24 @@ # Run against debian's kernel export KERNEL_TEST=1 ;; - ttcn3-ggsn-test-kernel-pablo-gtp) + ttcn3-ggsn-test-kernel-net-next) DIR="ttcn3-ggsn-test" export IMAGE_SUFFIX="master" export KERNEL_TEST=1 export KERNEL_BUILD=1 - export KERNEL_URL="https://git.kernel.org/pub/scm/linux/kernel/git/pablo/gtp.git" - export KERNEL_REMOTE_NAME="pablo-gtp" + export KERNEL_URL="https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git" + export KERNEL_REMOTE_NAME="net-next" export KERNEL_BRANCH="main" ;; + ttcn3-ggsn-test-kernel-torvalds) + DIR="ttcn3-ggsn-test" + export IMAGE_SUFFIX="master" + export KERNEL_TEST=1 + export KERNEL_BUILD=1 + export KERNEL_URL="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" + export KERNEL_REMOTE_NAME="torvalds" + export KERNEL_BRANCH="master" + ;; ttcn3-ggsn-test-kernel-latest) DIR="ttcn3-ggsn-test" export IMAGE_SUFFIX="latest" @@ -304,10 +299,20 @@ DIR="$(echo "{job-name}" | sed s/^TTCN3-centos7\-/ttcn3-/)" export IMAGE_SUFFIX="master-centos7" ;; + *-asan) + DIR="$(echo "{job-name}" | sed s/\-asan$//)" + export IMAGE_SUFFIX="asan" + export OSMOCOM_REPO_VERSION="asan" + ;; *-io_uring) DIR="$(echo "{job-name}" | sed s/\-io_uring$//)" - # OS#6405: set memlock to unlimited for older kernels - export DOCKER_ARGS="-e LIBOSMO_IO_BACKEND=IO_URING --ulimit memlock=-1" + # OS#6405: set memlock to unlimited for older kernels and + # enable io_uring syscalls in the seccomp profile + export DOCKER_ARGS=" + -e LIBOSMO_IO_BACKEND=IO_URING + --ulimit memlock=-1 + --security-opt seccomp=../seccomp_profile.json + " ;; *-latest) DIR="$(echo "{job-name}" | sed s/\-latest$//)" diff --git a/jobs/update-containers-osmo-python-tests.yml b/jobs/update-containers-osmo-python-tests.yml new file mode 100644 index 0000000..63b2929 --- /dev/null +++ b/jobs/update-containers-osmo-python-tests.yml @@ -0,0 +1,25 @@ +- job: + name: update-containers-osmo-python-tests + description: | + Trigger rebuild of containers after osmo-python-tests.git has changed. + properties: + - build-discarder: + days-to-keep: 30 + num-to-keep: 120 + scm: + - git: + url: https://gerrit.osmocom.org/python/osmo-python-tests + branches: + - 'origin/master' + skip-tag: true + wipe-workspace: true + triggers: + - pollscm: + cron: "H/5 * * * *" + ignore-post-commit-hooks: false + publishers: + - trigger: + project: update-osmo-ci-on-slaves + threshold: FAILURE + - email: + recipients: jenkins-notifications@lists.osmocom.org diff --git a/jobs/update-osmo-ci-on-slaves.yml b/jobs/update-osmo-ci-on-slaves.yml index c826175..b4994f7 100644 --- a/jobs/update-osmo-ci-on-slaves.yml +++ b/jobs/update-osmo-ci-on-slaves.yml @@ -11,9 +11,10 @@ - build4-deb12build-ansible - gtp0-deb10fr - host2-deb11build-ansible - - rpi4-raspbian11build-ansible-1 - - rpi4-raspbian11build-ansible-2 - - rpi4-raspbian11build-ansible-3 + - lx2-raspbian11build-ansible + - rpi4-raspbian12build-ansible-1 + - rpi4-raspbian12build-ansible-2 + - rpi4-raspbian12build-ansible-3 - simtester properties: - build-discarder: diff --git a/jobs/update-osmo-python-on-slaves.yml b/jobs/update-osmo-python-on-slaves.yml deleted file mode 100644 index fad1cf6..0000000 --- a/jobs/update-osmo-python-on-slaves.yml +++ /dev/null @@ -1,46 +0,0 @@ -- job: - name: update-osmo-python-on-slaves - project-type: matrix - axes: - - axis: - type: slave - name: label - values: - - build2-deb10build-ansible - - build2-deb11build-ansible - - build4-deb12build-ansible - - host2-deb11build-ansible - - rpi4-raspbian11build-ansible-1 - - rpi4-raspbian11build-ansible-2 - - rpi4-raspbian11build-ansible-3 - properties: - - build-discarder: - days-to-keep: 30 - num-to-keep: 120 - scm: - - git: - url: https://gerrit.osmocom.org/python/osmo-python-tests - branches: - - 'origin/master' - skip-tag: true - wipe-workspace: true - - triggers: - - pollscm: - cron: "H/5 * * * *" - ignore-post-commit-hooks: false - - builders: - - shell: './contrib/jenkins.sh' - description: | - <b>Auto-generated using Jenkins Job Builder. DO NOT EDIT MANUALLY!</b> - - # make sure to update the osmo-python in docker images, rebuilt from osmo-ci/contrib/jenkins.sh - publishers: - - trigger: - project: update-osmo-ci-on-slaves - threshold: FAILURE - - email: - recipients: jenkins-notifications@lists.osmocom.org - -# vim: expandtab tabstop=2 shiftwidth=2 diff --git a/lint/checkpatch/checkpatch.pl b/lint/checkpatch/checkpatch.pl index ff6eea1..eada2a9 100755 --- a/lint/checkpatch/checkpatch.pl +++ b/lint/checkpatch/checkpatch.pl @@ -4888,7 +4888,6 @@ sub process { # Ignore those directives where spaces _are_ permitted. if ($name =~ /^(?: if|for|while|switch|return|case| - [a-z_]*for[_]?each[a-z_]*| volatile|__volatile__| __attribute__|format|__extension__| asm|__asm__)$/x) @@ -5514,7 +5513,7 @@ sub process { } # Need a space before open parenthesis after if, while etc - if ($line =~ /\b(if|while|for|switch|[a-z_]*for[_]?each[a-z_]*)\(/) { + if ($line =~ /\b(if|while|for|switch)\(/) { if (ERROR("SPACING", "space required before the open parenthesis '('\n" . $herecurr) && $fix) { diff --git a/lint/checkpatch/spelling.txt b/lint/checkpatch/spelling.txt index f60a02f..6cfb584 100644 --- a/lint/checkpatch/spelling.txt +++ b/lint/checkpatch/spelling.txt @@ -1439,7 +1439,6 @@ tansmit||transmit targetted||targeted targetting||targeting taskelt||tasklet -teh||the temorary||temporary temproarily||temporarily temperture||temperature diff --git a/qemu-kvm/ggsn-kernel-gtp/postinst/Release.key b/qemu-kvm/ggsn-kernel-gtp/postinst/Release.key index ecca084..b4c019a 100644 --- a/qemu-kvm/ggsn-kernel-gtp/postinst/Release.key +++ b/qemu-kvm/ggsn-kernel-gtp/postinst/Release.key @@ -8,13 +8,13 @@ e93fLa2Cu8vhFBcVn6CuHeEhsmuMf6NLbQRNfNNCEEUYaZn7beMYtpZ7t1djsKx5 1xGm50OzI22FLu8lELQ9d7qMVGRG3WHYawX9BDteRybiyqxfwUHm1haWazRJtlGt UWyzvwAb80BK1J2Nu5fbAa3w5CoEPAbUuCyrABEBAAG0JW9zbW9jb20gT0JTIFBy b2plY3QgPG9zbW9jb21Ab3Ntb2NvbT6JAVQEEwEIAD4WIQRrKp83ktFetw1Oao+G -pzC2U3JZcwUCYrMV4wIbAwUJBB6yjwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAK -CRCGpzC2U3JZc4FRCACQQkKIrnvQ7n2u7GSmyVZa3I+oLoFXSGqaGyey5TW/nrMm -vFDKU3qliHiuNSmUY35SnAhXUsvqOYppxVRoO1MLrqUvzMOnIWqkJpf8mtjGUnsW -jyVeto7Rsjs75y2i1Hk+e7ljb/V65J3NlfrfEYWbqR9AKd53ReNXTdrQ0J05A38N -GdI4Ld/2lNISAwaBmGhqdeKsLHpQw/JERU1TApVJR1whFiIwDF1rOCg9GPnNKIk7 -yRZdK267XzztrainX/cbPILyzUZEDhYs6wQuyACyQ1YUxZIxrwVfk7PMNay8CrLH -z42B73Ne5IAj8+op/3iJafFONLm7YXiDUFN+QDYAiQEzBBMBCAAdFiEExoiYhHND +pzC2U3JZcwUCZrNL8wIbAwUJCB7onwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAK +CRCGpzC2U3JZc11RB/9oogcqTF30pPEwjjGvXF+WMiRFHl8aVvQIk9m+2KUcjlVk +0q8+/guOg7KcVejPxyn2AQbTByWWgpH1o3BoDKWnvypURAV2uGOtCo6mUQop5yK0 +d50j9hwIXSwJWoD2r5OY+Gbs4dBzMGkKbnCED1n8324b8EN6m96eoL570Uq9vy4G +gIp+QCpePu8AMdoCLR2w3J+TA4gLdDeH2cr7TgmwlDCp8e6R8FVmRt3SvunPmFz3 +FHDicv2f9ZoHXVLQ20DgWUyI1xOPlzBNJowgK9YnLPdIidE9i+EbvC1YA+6O/WMa +kSUEyJLZS5mv2+T6ptgnCIgxrIoTu63O4Rsf5zV+iQEzBBMBCAAdFiEExoiYhHND S7aVYlnqa51NyAUyjdsFAmKzE1UACgkQa51NyAUyjdvuZgf+OXmr//i7u7Gg7eWB 7e0qUsyCId9lXS8J437x3K6ciJfD7/6RSy8TFW5Nglm/uSkbyq582I8t+SoOirMD E6cg9U/5+h5s46bAf+Kd2XS/6tLGeNLM18i4el8CP06NpFzDrsKu76uYFpyRiiHD @@ -22,5 +22,5 @@ otBdtgxeLJ83LugGfZslF+/5cigJkAJMhAdVvGO8h85R6fba8ZSOKtMKkaQRfi76 nhyOrJPlLuS+DLEnHwdkOFgtKnxHdjM97K+Tx0gisb6uwaWroXfSLnhP8RTLLZZy Z+noU1Hw3c+mn4c/NYbcC/uwHYHKRzuf9gHnQ3dGgv0Z5sbeLRVo92hjGj7Ftlyd 4hmKBg== -=HxK4 +=5Cwc -----END PGP PUBLIC KEY BLOCK----- diff --git a/scripts/common.sh b/scripts/common.sh index 1081859..46bf9c3 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -13,7 +13,8 @@ OSMO_RELEASE_REPOS=" libosmo-gprs libosmo-netif libosmo-pfcp - libosmo-sccp + libosmo-sccp-legacy + libosmo-sigtran libosmocore libsmpp34 libusrp @@ -80,6 +81,9 @@ osmo_git_last_commits_tags() { local url ret pattern case "$project" in + strongswan-epdg) + pattern='refs/tags/osmo-epdg-[0-9.]*$' + ;; gapk|osmo-fl2k|rtl-sdr) pattern='refs/tags/v[0-9.]*$' ;; @@ -130,10 +134,7 @@ osmo_git_last_tags() { # $1: Osmocom project (e.g. "osmo-hlr") osmo_git_clone_url() { case "$1" in - libgtpnl|libasn1c|libsmpp34) - echo "$OSMO_GIT_URL_GITEA"/cellular-infrastructure/"$1" - ;; - rtl-sdr|osmo-fl2k|libosmo-dsp|libusrp) + rtl-sdr|osmo-fl2k|libosmo-dsp) echo "$OSMO_GIT_URL_GITEA"/sdr/"$1" ;; osmo-gmr) @@ -142,9 +143,18 @@ osmo_git_clone_url() { osmo-isdntap) echo "$OSMO_GIT_URL_GITEA"/retronetworking/"$1" ;; - osmo_dia2gsup) + strongswan-epdg) + echo "$OSMO_GIT_URL_GITEA"/ims-volte-vowifi/strongswan + ;; + libosmo-sccp-legacy) + echo "$OSMO_GIT_URL_GITEA"/osmocom/"$1" + ;; + osmo_dia2gsup|osmo-epdg|osmo-s1gw) echo "$OSMO_GIT_URL_GERRIT"/erlang/"$1" ;; + pyosmocom) + echo "$OSMO_GIT_URL_GERRIT"/python/"$1" + ;; *) echo "$OSMO_GIT_URL_GERRIT"/"$1" ;; diff --git a/scripts/jenkins-gerrit/comment_generate.py b/scripts/jenkins-gerrit/comment_generate.py index e127c4c..a2aec0e 100755 --- a/scripts/jenkins-gerrit/comment_generate.py +++ b/scripts/jenkins-gerrit/comment_generate.py @@ -6,6 +6,7 @@ import io import json import re import urllib.request +import sys jenkins_url = "https://jenkins.osmocom.org" re_start_build = re.compile("Starting building: gerrit-[a-zA-Z-_0-9]* #[0-9]*") @@ -94,11 +95,17 @@ def parse_pipeline(build_url): if stage.startswith("comment_"): # Jobs that run this script, not relevant for summary continue - assert stage in ret, f"found result for stage {stage}, but" \ - " didn't find where it was started. The" \ - " re_start_build regex probably needs to be adjusted" \ - " to match the related gerrit-*-build job.\n\n" \ - f"ret: {ret}" + if stage not in ret: + print(f"URL: {url}") + print() + print(f"ERROR: found result for stage {stage}, but didn't" + " find where it was started. Possible reasons:") + print("* The re_stat_build regex needs to be adjusted" + " to match the related gerrit-*-build job") + print("* The gerrit-*-build job has not been deployed," + " and therefore could not be started by the" + " gerrit-* job.") + sys.exit(1) ret[stage]["passed"] = (match.group(2) == "SUCCESS") return ret diff --git a/scripts/manuals/publish-manuals-for-tags.sh b/scripts/manuals/publish-manuals-for-tags.sh index 3f95ce7..50b7a3b 100755 --- a/scripts/manuals/publish-manuals-for-tags.sh +++ b/scripts/manuals/publish-manuals-for-tags.sh @@ -200,6 +200,9 @@ TAGS_IGNORE=" osmo-trx:0.4.0, pysim:1.0, + + pyosmocom:0.0.1, + pyosmocom:0.0.2, " mkdir -p \ @@ -230,7 +233,7 @@ get_configure_opts_from_repo_name() { get_repo_name_from_docs_dir() { case "$1" in osmo-stp) - echo "libosmo-sccp" + echo "libosmo-sigtran" ;; *) echo "$1" @@ -241,7 +244,7 @@ get_repo_name_from_docs_dir() { # $1: repo name get_docs_dir_from_repo_name() { case "$1" in - libosmo-sccp) + libosmo-sigtran) echo "osmo-stp" ;; *) @@ -327,8 +330,10 @@ clone_repo() { git clean -dxf # Fix depends on packages that don't exist anymore - sed -i 's/dh-systemd \(.*\),//g' debian/control - sed -i 's/python-minimal,//g' debian/control + if [ -e debian/control ]; then + sed -i 's/dh-systemd \(.*\),//g' debian/control + sed -i 's/python-minimal,//g' debian/control + fi } # $1: repository @@ -363,7 +368,9 @@ build_publish_manuals() { # Install dependencies case $repo in *) - apt-get -y build-dep /build + if [ -e debian/control ]; then + apt-get -y build-dep /build + fi ;; esac @@ -379,6 +386,12 @@ build_publish_manuals() { su build -c \"make -C \$dir\" done ;; + osmo-epdg) + su build -c \"make -C docs/manuals\" + ;; + pyosmocom) + su build -c \"make -C docs html latexpdf\" + ;; *) su build -c \"autoreconf -fi\" su build -c \"./configure $configure_opts\" @@ -394,6 +407,12 @@ build_publish_manuals() { su build -c \"make -C \$dir publish\" done ;; + osmo-epdg) + su build -c \"make -C docs/manuals publish\" + ;; + pyosmocom) + su build -c \"make -C docs publish publish-html\" + ;; *) su build -c \"make -C doc/manuals publish\" ;; diff --git a/scripts/obs/build_binpkg.py b/scripts/obs/build_binpkg.py index ae2fe53..5c8cd01 100755 --- a/scripts/obs/build_binpkg.py +++ b/scripts/obs/build_binpkg.py @@ -69,6 +69,7 @@ def main(): env = {"JOBS": str(args.jobs), "PACKAGE": args.package, "BUILDUSER": os.environ["USER"], + "DISTRO": args.docker, "PACKAGEFORMAT": "deb"} docker_args = [] diff --git a/scripts/obs/build_srcpkg.py b/scripts/obs/build_srcpkg.py index a5feeea..86655f5 100755 --- a/scripts/obs/build_srcpkg.py +++ b/scripts/obs/build_srcpkg.py @@ -18,9 +18,9 @@ def main(): " code that runs to generate source packages which we" " upload to https://obs.osmocom.org." f" Output dir: {lib.config.path_temp}/srcpkgs") - lib.add_shared_arguments(parser) - parser.add_argument("-g", "--gerrit-id", type=int, default=0, - help="clone particular revision from gerrit using given ID") + groups = lib.add_shared_arguments(parser) + groups["git"].add_argument("-g", "--gerrit-id", type=int, default=0, + help="clone particular revision from gerrit using given ID") parser.add_argument("package", nargs="?", help="package name, e.g. libosmocore or open5gs") args = parser.parse_args() diff --git a/scripts/obs/data/Release.key b/scripts/obs/data/Release.key index ecca084..b4c019a 100644 --- a/scripts/obs/data/Release.key +++ b/scripts/obs/data/Release.key @@ -8,13 +8,13 @@ e93fLa2Cu8vhFBcVn6CuHeEhsmuMf6NLbQRNfNNCEEUYaZn7beMYtpZ7t1djsKx5 1xGm50OzI22FLu8lELQ9d7qMVGRG3WHYawX9BDteRybiyqxfwUHm1haWazRJtlGt UWyzvwAb80BK1J2Nu5fbAa3w5CoEPAbUuCyrABEBAAG0JW9zbW9jb20gT0JTIFBy b2plY3QgPG9zbW9jb21Ab3Ntb2NvbT6JAVQEEwEIAD4WIQRrKp83ktFetw1Oao+G -pzC2U3JZcwUCYrMV4wIbAwUJBB6yjwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAK -CRCGpzC2U3JZc4FRCACQQkKIrnvQ7n2u7GSmyVZa3I+oLoFXSGqaGyey5TW/nrMm -vFDKU3qliHiuNSmUY35SnAhXUsvqOYppxVRoO1MLrqUvzMOnIWqkJpf8mtjGUnsW -jyVeto7Rsjs75y2i1Hk+e7ljb/V65J3NlfrfEYWbqR9AKd53ReNXTdrQ0J05A38N -GdI4Ld/2lNISAwaBmGhqdeKsLHpQw/JERU1TApVJR1whFiIwDF1rOCg9GPnNKIk7 -yRZdK267XzztrainX/cbPILyzUZEDhYs6wQuyACyQ1YUxZIxrwVfk7PMNay8CrLH -z42B73Ne5IAj8+op/3iJafFONLm7YXiDUFN+QDYAiQEzBBMBCAAdFiEExoiYhHND +pzC2U3JZcwUCZrNL8wIbAwUJCB7onwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAK +CRCGpzC2U3JZc11RB/9oogcqTF30pPEwjjGvXF+WMiRFHl8aVvQIk9m+2KUcjlVk +0q8+/guOg7KcVejPxyn2AQbTByWWgpH1o3BoDKWnvypURAV2uGOtCo6mUQop5yK0 +d50j9hwIXSwJWoD2r5OY+Gbs4dBzMGkKbnCED1n8324b8EN6m96eoL570Uq9vy4G +gIp+QCpePu8AMdoCLR2w3J+TA4gLdDeH2cr7TgmwlDCp8e6R8FVmRt3SvunPmFz3 +FHDicv2f9ZoHXVLQ20DgWUyI1xOPlzBNJowgK9YnLPdIidE9i+EbvC1YA+6O/WMa +kSUEyJLZS5mv2+T6ptgnCIgxrIoTu63O4Rsf5zV+iQEzBBMBCAAdFiEExoiYhHND S7aVYlnqa51NyAUyjdsFAmKzE1UACgkQa51NyAUyjdvuZgf+OXmr//i7u7Gg7eWB 7e0qUsyCId9lXS8J437x3K6ciJfD7/6RSy8TFW5Nglm/uSkbyq582I8t+SoOirMD E6cg9U/5+h5s46bAf+Kd2XS/6tLGeNLM18i4el8CP06NpFzDrsKu76uYFpyRiiHD @@ -22,5 +22,5 @@ otBdtgxeLJ83LugGfZslF+/5cigJkAJMhAdVvGO8h85R6fba8ZSOKtMKkaQRfi76 nhyOrJPlLuS+DLEnHwdkOFgtKnxHdjM97K+Tx0gisb6uwaWroXfSLnhP8RTLLZZy Z+noU1Hw3c+mn4c/NYbcC/uwHYHKRzuf9gHnQ3dGgv0Z5sbeLRVo92hjGj7Ftlyd 4hmKBg== -=HxK4 +=5Cwc -----END PGP PUBLIC KEY BLOCK----- diff --git a/scripts/obs/data/build_binpkg.Dockerfile b/scripts/obs/data/build_binpkg.Dockerfile index 55bfe0c..66f1feb 100644 --- a/scripts/obs/data/build_binpkg.Dockerfile +++ b/scripts/obs/data/build_binpkg.Dockerfile @@ -51,6 +51,29 @@ RUN case "$DISTRO" in \ yum config-manager --set-enabled powertools && \ su user -c rpmdev-setuptree \ ;; \ + centos:7) \ + set -x && \ + sed -i 's/mirror.centos.org/vault.centos.org/g' /etc/yum.repos.d/*.repo && \ + sed -i 's/^#.*baseurl=http/baseurl=http/g' /etc/yum.repos.d/*.repo && \ + sed -i 's/^mirrorlist=http/#mirrorlist=http/g' /etc/yum.repos.d/*.repo && \ + \ + yum -y install \ + autoconf \ + automake \ + binutils \ + dnf-utils \ + gcc \ + gcc-c++ \ + glibc-devel \ + iproute \ + libtool \ + make \ + redhat-rpm-config \ + rpm-build \ + rpmdevtools \ + wget && \ + su user -c rpmdev-setuptree \ + ;; \ esac # Add master repository, where packages immediately get updated after merging @@ -71,7 +94,7 @@ RUN set -x; \ echo "deb https://downloads.osmocom.org/packages/osmocom:/$FEED/xUbuntu_$VERSION/ ./" \ > /etc/apt/sources.list.d/osmocom-$FEED.list \ ;; \ - almalinux:*) \ + almalinux:*|centos:*) \ { echo "[network_osmocom_$FEED]"; \ echo "name=osmocom:$FEED"; \ echo "type=rpm-md"; \ diff --git a/scripts/obs/data/build_rpm.sh b/scripts/obs/data/build_rpm.sh index a73d164..7247999 100755 --- a/scripts/obs/data/build_rpm.sh +++ b/scripts/obs/data/build_rpm.sh @@ -22,7 +22,23 @@ su "$BUILDUSER" -c "cp _temp/srcpkgs/$PACKAGE/rpmlintrc ~/rpmbuild/SOURCES" su "$BUILDUSER" -c "cp /obs/data/rpmmacros ~/.rpmmacros" # Force refresh of package index data (OS#6038) -dnf makecache --refresh +if command -v dnf; then + dnf makecache --refresh +else + yum clean expire-cache +fi + +case "$DISTRO" in + centos:7) + # HACK: remove pkg-config as centos7 has pkgconfig instead + # (which will get pulled in automatically). This could be + # solved more elegantly, but we only build very few centos7 + # packages, so let's not spend too much time on this. In OBS + # this is handled in the centos7 prjconf: + # https://build.opensuse.org/projects/CentOS:CentOS-7/prjconf + sed -i '/^BuildRequires:.*pkg-config/d' "/home/$BUILDUSER/rpmbuild/SPECS/$spec" + ;; +esac $yum_builddep "/home/$BUILDUSER/rpmbuild/SPECS/$spec" diff --git a/scripts/obs/data/build_srcpkg.Dockerfile b/scripts/obs/data/build_srcpkg.Dockerfile index 80488ea..4822450 100644 --- a/scripts/obs/data/build_srcpkg.Dockerfile +++ b/scripts/obs/data/build_srcpkg.Dockerfile @@ -24,6 +24,7 @@ RUN apt-get update && \ quilt \ rebar3 \ sed \ + sphinx-common \ && \ apt-get clean diff --git a/scripts/obs/gerrit_binpkgs.sh b/scripts/obs/gerrit_binpkgs.sh index 1e4ae26..4e3b0f3 100755 --- a/scripts/obs/gerrit_binpkgs.sh +++ b/scripts/obs/gerrit_binpkgs.sh @@ -32,9 +32,12 @@ echo ":: Building the source package" --feed master \ --git-skip-fetch \ --git-skip-checkout \ + --no-meta \ "$PROJECT_NAME" echo ":: Building the binary packages" "$SCRIPTS_OBS_DIR"/build_binpkg.py \ --docker "$DISTRO" \ "$PROJECT_NAME" + +echo ":: Find binary packages in: $SCRIPTS_OBS_DIR/_temp/binpkgs" diff --git a/scripts/obs/lib/__init__.py b/scripts/obs/lib/__init__.py index 5292dc5..66090f2 100644 --- a/scripts/obs/lib/__init__.py +++ b/scripts/obs/lib/__init__.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-or-later # Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de> -import importlib +import importlib.util import os import shutil import subprocess @@ -19,60 +19,81 @@ cmds_verbose = False def add_shared_arguments(parser): """ Arguments shared between build_srcpkg.py and update_obs_project.py. """ - parser.add_argument("-f", "--feed", - help="package feed (default: nightly). The feed" - " determines the git revision to be built:" - " 'nightly' and 'master' build 'origin/master'," - " 'latest' builds the last signed tag," - " other feeds build their respective branch.", - metavar="FEED", default="nightly", - choices=lib.config.feeds) - parser.add_argument("-a", "--allow-unknown-package", action="store_true", - help="don't complain if the name of the package is not" - " stored in lib/config.py") - parser.add_argument("-b", "--git-branch", help="instead of using a branch" - " based on the feed, checkout this git branch", - metavar="BRANCH", default=None) - parser.add_argument("-d", "--docker", - help="run in docker to avoid installing required pkgs", - action="store_true") - parser.add_argument("-s", "--git-skip-fetch", - help="do not fetch already cloned git repositories", - action="store_false", dest="git_fetch") - parser.add_argument("-S", "--git-skip-checkout", - help="do not checkout and reset to a branch/tag", - action="store_false", dest="git_checkout") - parser.add_argument("-m", "--meta", action="store_true", - help="build a meta package (e.g. osmocom-nightly)") - parser.add_argument("-i", "--ignore-req", action="store_true", - help="skip required programs check") - parser.add_argument("-c", "--conflict-version", nargs="?", - help="Of the generated source packages, all Osmocom" - " packages (not e.g. open5gs, see lib/config.py" - " for full list) depend on a meta-package such as" - " osmocom-nightly, osmocom-latest, osmocom-2021q1" - " etc. These meta-packages conflict with each" - " other to ensure that one does not mix e.g." - " latest and nightly packages by accident." - " With this -c argument, it is possible to let" - " these packages depend on a meta-package of a" - " specific version. This is used for nightly and" - " 20YYqX packages to ensure they are not mixed" - " from different build dates (ABI compatibility" - " is only on latest).") - parser.add_argument("-p", "--conflict-pkgname", nargs="?", - help="name of the meta-package to depend on (default:" - " osmocom-$feed)") - parser.add_argument("-M", "--no-meta", action="store_true", - help="Don't depend on the meta package (helpful when" - " building one-off packages for development)") - parser.add_argument("-v", "--verbose", action="store_true", + + group_reqprog = parser.add_argument_group("required program options") + group_reqprog.add_argument("-d", "--docker", help="run in docker to avoid" + " installing required pkgs", + action="store_true") + group_reqprog.add_argument("-i", "--ignore-req", action="store_true", + help="skip required programs check") + + group_feed = parser.add_argument_group("feed options", + "The feed option implies the source revision (nightly/master: build" + " the master branch; latest: build the last tag) and when a package" + " is considered outdated (latest/master: when the commit changes on" + " the latest tag or on master; nightly: always).") + group_feed.add_argument("-f", "--feed", help="package feed (default:" + " nightly, can also be master or latest)", + metavar="FEED", default="nightly", + choices=lib.config.feeds) + + group_pkg = parser.add_argument_group("package options") + group_pkg.add_argument("-a", "--allow-unknown-package", + action="store_true", + help="don't complain if the name of the package is" + " not stored in lib/config.py") + group_pkg.add_argument("-e", "--version-append", + help="add a string at the end of the version, e.g." + " '~osmocom' for the wireshark package") + group_pkg.add_argument("-C", "--configure-append", + help="add these parameters to the ./configure line" + " (e.g. ' --enable-sanitize')") + group_pkg.add_argument("-D", "--disable-manuals", action="store_true", + help="speed up build by removing the" + " osmo-gsm-manuals dependency and not building" + " asciidoc and doxygen manuals") + + group_git = parser.add_argument_group("git options") + group_git.add_argument("-b", "--git-branch", help="instead of using a" + " branch based on the feed, checkout this git" + " branch", metavar="BRANCH", default=None) + group_git.add_argument("-s", "--git-skip-fetch", + help="do not fetch already cloned git repositories", + action="store_false", dest="git_fetch") + group_git.add_argument("-S", "--git-skip-checkout", + help="do not checkout and reset to a branch/tag", + action="store_false", dest="git_checkout") + + group_meta = parser.add_argument_group("meta package options", + "Osmocom packages depend on a meta-package such as osmocom-nightly." + " These meta-packages conflict with each other to ensure e.g." + " incompatible nightly and latest packages are not mixed by accident." + " For nightly we don't have API compatibility and so we let these" + " depend on a meta package of a specific build date (-c).") + group_meta.add_argument("-m", "--meta", action="store_true", + help="build a meta package (e.g. osmocom-nightly)") + group_meta.add_argument("-c", "--conflict-version", nargs="?", + help="depend on meta-package of specific version") + group_meta.add_argument("-p", "--conflict-pkgname", nargs="?", + help="name of the meta-package to depend on" + " (default: osmocom-$feed)") + group_meta.add_argument("-M", "--no-meta", action="store_true", + help="Don't depend on the meta package (use when" + " building one-off packages for development)") + + group_devel = parser.add_argument_group("development options") + group_devel.add_argument("-v", "--verbose", action="store_true", help="always print shell commands and their output," " instead of only printing them on error") - parser.add_argument("-e", "--version-append", - help="add a string at the end of the version, e.g." - " '~osmocom' for the wireshark package to" - " indicate that it is the version from our repo") + + return { + "devel": group_devel, + "feed": group_feed, + "git": group_git, + "meta": group_meta, + "pkg": group_pkg, + "reqprog": group_reqprog, + } def set_cmds_verbose(new_val): @@ -95,7 +116,7 @@ def check_required_programs(): ok = False for module in lib.config.required_python_modules: - if not importlib.find_loader(module): + if not importlib.util.find_spec(module): print(f"ERROR: missing python3 module: {module}") ok = False diff --git a/scripts/obs/lib/config.py b/scripts/obs/lib/config.py index b4e23a0..c6f373e 100644 --- a/scripts/obs/lib/config.py +++ b/scripts/obs/lib/config.py @@ -44,9 +44,12 @@ feeds = [ # prevents that packages from different feeds are mixed by accident. # NOTE: Before adding new projects, add them to jobs/gerrit-verifications.yml # and ensure the rpm and deb packages build successfully in jenkins. +# NOTE: Consider whether new packages should be added to EXCLUDE_PACKAGES in +# osmocom-obs-nightly-asan.yml. projects_osmocom = [ "erlang/osmo_dia2gsup", "erlang/osmo-epdg", + "erlang/osmo-s1gw", "gapk", "libasn1c", "libgtpnl", @@ -55,7 +58,8 @@ projects_osmocom = [ "libosmo-gprs", "libosmo-netif", "libosmo-pfcp", - "libosmo-sccp", + "libosmo-sccp-legacy", # can be dropped after osmo-msc >= 1.12.x release + "libosmo-sigtran", "libosmocore", "libsmpp34", "libusrp", @@ -86,8 +90,10 @@ projects_osmocom = [ "osmo-upf", "osmocom-bb", "python/osmo-python-tests", + "python/pyosmocom", "rtl-sdr", "simtrace2", + "strongswan-epdg", ] projects_other = [ "limesuite", @@ -103,6 +109,8 @@ git_url_other = { "open5gs": "https://github.com/open5gs/open5gs", "osmo-fl2k": "https://gitea.osmocom.org/sdr/osmo-fl2k", "rtl-sdr": "https://gitea.osmocom.org/sdr/rtl-sdr", + "strongswan-epdg": "https://gitea.osmocom.org/ims-volte-vowifi/strongswan-epdg", + "libosmo-sccp-legacy": "https://gitea.osmocom.org/osmocom/libosmo-sccp-legacy", } git_branch_default = "master" @@ -110,18 +118,27 @@ git_branch_other = { "open5gs": "main", } -git_latest_tag_pattern_default = "^[0-9]*\\.[0-9]*\\.[0-9]*$" +def tag_pattern(prefix: str = '', + a: str = r'\d+', + b: str = r'\.\d+', + c: str = r'\.\d+') -> str: + return rf'^{prefix}{a}{b}{c}$' + +git_latest_tag_pattern_default = tag_pattern() git_latest_tag_pattern_other = { - "limesuite": "^v[0-9]*\\.[0-9]*\\.[0-9]*$", - "open5gs": "^v[0-9]*\\.[0-9]*\\.[0-9]*$", - "osmo-fl2k": "^v[0-9]*\\.[0-9]*\\.[0-9]*$", - "rtl-sdr": "^v[0-9]*\\.[0-9]*\\.[0-9]*$", - "wireshark": "^v[0-9]*\\.[0-9]*\\.[0-9a-z]*$", + "gapk": tag_pattern('v', c=r'(\.\d+)?'), + "limesuite": tag_pattern('v'), + "open5gs": tag_pattern('v'), + "osmo-fl2k": tag_pattern('v'), + "rtl-sdr": tag_pattern('v'), + "strongswan-epdg": tag_pattern('osmo-epdg-', c=r'\.[0-9a-z]+'), + "wireshark": tag_pattern('v', c=r'\.[0-9a-z]+'), } docker_distro_default = "debian:12" docker_distro_other = [ "almalinux:*", # instead of centos (SYS#5818) + "centos:7", # SYS#6760 "debian:*", "ubuntu:*", ] diff --git a/scripts/obs/lib/debian.py b/scripts/obs/lib/debian.py index 6703f8b..85fcc5f 100644 --- a/scripts/obs/lib/debian.py +++ b/scripts/obs/lib/debian.py @@ -92,6 +92,17 @@ def get_last_version_from_changelog(project): return ret +def transform_version(version): + if version is None: + return None + + if "osmo-epdg-" in version: + version = f"{version.replace('osmo-epdg-', '', 1)}-osmo-epdg" + + version = version.replace("-", ".") + return version + + def changelog_add_entry_if_needed(project, version): """ Adjust the changelog if the version in the changelog is different from the given version. """ @@ -112,6 +123,9 @@ def changelog_add_entry_if_needed(project, version): # from wireshark. Don't abort here if that is the case. pass + # Debian versions must start with a digit + version = transform_version(version) + if version_changelog == version: return @@ -120,6 +134,54 @@ def changelog_add_entry_if_needed(project, version): changelog_add_entry(project, version) +def configure_append(project, parameters): + """ Add one or more configure parameters like --enable-sanitize to the + dh_auto_configure line, also add the override_dh_auto_configure block + if missing. """ + print(f"{project}: adding configure parameters: {parameters}") + rules = f"{lib.git.get_repo_path(project)}/debian/rules" + override_found = False + replaced = False + with open(rules, "r") as f: + lines = f.readlines() + for i in range(len(lines)): + line = lines[i] + if line.startswith("override_dh_auto_configure:"): + override_found = True + continue + if "dh_auto_configure" not in line: + continue + assert override_found + if " -- " in line.replace("\t", " "): + lines[i] = line.replace(" --", f" -- {parameters}", 1) + else: + lines[i] = line.replace("dh_auto_configure", + f"dh_auto_configure -- {parameters}", 1) + replaced = True + break + if not override_found: + lines += ["\n", + "override_dh_auto_configure:\n", + f"\tdh_auto_configure -- {parameters}\n"] + with open(rules, "w") as f: + f.writelines(lines) + + +def disable_manuals(project): + """ Disabling manuals speeds up the build time significantly, we don't + need them for e.g. the asan repository. """ + print(f"{project}: disabling manuals") + debian = f"{lib.git.get_repo_path(project)}/debian" + # Remove dependencies + lib.run_cmd(["sed", "-i", "/osmo-gsm-manuals-dev/d", f"{debian}/control"]) + lib.run_cmd(["sed", "-i", "/doxygen/d", f"{debian}/control"]) + # Remove debian/*-doc.install + lib.run_cmd(f"rm -rf {shlex.quote(debian)}/*-doc.install", shell=True) + # debian/rules: remove --enable-manuals/doxygen, add --disable-doxygen + lib.run_cmd(["sed", "-i", "s/--enable-manuals//g", f"{debian}/rules"]) + lib.run_cmd(["sed", "-i", "s/--enable-doxygen//g", f"{debian}/rules"]) + + def build_source_package(project): fix_source_format(project) print(f"{project}: building debian source package") diff --git a/scripts/obs/lib/metapkg.py b/scripts/obs/lib/metapkg.py index 38ed930..8a9922d 100644 --- a/scripts/obs/lib/metapkg.py +++ b/scripts/obs/lib/metapkg.py @@ -7,17 +7,17 @@ import lib.debian import lib.rpm_spec -def get_conflicts(): +def get_conflicts(pkgname): ret = [] for f in lib.config.feeds: - if f == lib.args.feed: + if "osmocom-{f}" == pkgname: continue ret += [f"osmocom-{f}"] return ret -def prepare_source_dir(): - path = f"{lib.config.path_cache}/osmocom-{lib.args.feed}" +def prepare_source_dir(pkgname): + path = f"{lib.config.path_cache}/{pkgname}" if os.path.exists(path): lib.run_cmd(["rm", "-rf", path]) @@ -26,27 +26,27 @@ def prepare_source_dir(): os.makedirs(f"{path}/contrib") -def generate_debian_pkg(version): +def generate_debian_pkg(pkgname, version): feed = lib.args.feed - path = f"{lib.config.path_cache}/osmocom-{feed}" - conflicts = get_conflicts() + path = f"{lib.config.path_cache}/{pkgname}" + conflicts = get_conflicts(pkgname) with open(f"{path}/debian/control", "w") as f: - f.write(f"Source: osmocom-{feed}\n") + f.write(f"Source: {pkgname}\n") f.write("Section: unknown\n") f.write("Priority: optional\n") f.write("Maintainer: Osmocom OBS scripts <info@osmocom.org>\n") f.write("Build-Depends: debhelper (>= 10)\n") f.write("Standards-Version: 3.9.8\n") f.write("\n") - f.write(f"Package: osmocom-{feed}\n") + f.write(f"Package: {pkgname}\n") f.write("Depends: ${misc:Depends}\n") f.write("Architecture: any\n") f.write(f"Conflicts: {', '.join(conflicts)}\n") f.write(f"Description: Dummy package, conflicts with {conflicts}\n") with open(f"{path}/debian/changelog", "w") as f: - f.write(f"osmocom-{feed} ({version}) unstable; urgency=medium\n") + f.write(f"{pkgname} ({version}) unstable; urgency=medium\n") f.write("\n") f.write(f" * Dummy package, which conflicts with: {conflicts}\n") f.write("\n") @@ -64,15 +64,14 @@ def generate_debian_pkg(version): f.write("10\n") -def generate_rpm_spec(version): +def generate_rpm_spec(pkgname, version): feed = lib.args.feed - print(f"osmocom-{feed}: generating rpm spec file") - path = (f"{lib.config.path_cache}/osmocom-{feed}/contrib/osmocom-{feed}" - ".spec.in") - conflicts = get_conflicts() + print(f"{pkgname}: generating rpm spec file") + path = f"{lib.config.path_cache}/{pkgname}/contrib/{pkgname}.spec.in" + conflicts = get_conflicts(pkgname) with open(path, "w") as f: - f.write(f"Name: osmocom-{feed}\n") + f.write(f"Name: {pkgname}\n") f.write(f"Version: {version}\n") f.write(f"Summary: Dummy package, conflicts with: {conflicts}\n") f.write("Release: 0\n") @@ -92,8 +91,8 @@ def build(): version = conflict_version if conflict_version else "1.0.0" print(f"{pkgname}: generating meta package {version}") - prepare_source_dir() - generate_debian_pkg(version) + prepare_source_dir(pkgname) + generate_debian_pkg(pkgname, version) os.makedirs(lib.get_output_path(pkgname)) lib.remove_cache_extra_files() @@ -101,7 +100,7 @@ def build(): lib.debian.build_source_package(pkgname) lib.debian.move_files_to_output(pkgname) - generate_rpm_spec(version) + generate_rpm_spec(pkgname, version) lib.rpm_spec.copy_to_output(pkgname) lib.remove_cache_extra_files() diff --git a/scripts/obs/lib/osc.py b/scripts/obs/lib/osc.py index ef2fc4b..57be783 100644 --- a/scripts/obs/lib/osc.py +++ b/scripts/obs/lib/osc.py @@ -86,8 +86,9 @@ def get_package_version(package): if line.endswith(".dsc"): return line.split("_")[-1][:-4] - lib.exit_error_cmd(ret, "failed to find package version on OBS by" - " extracting the version from the file list") + print(f"{package}: WARNING: failed to find package version on OBS!") + print(f"{package}: assuming the package is outdated") + return "0" def create_package(package): diff --git a/scripts/obs/lib/srcpkg.py b/scripts/obs/lib/srcpkg.py index aef7732..9dcb33f 100644 --- a/scripts/obs/lib/srcpkg.py +++ b/scripts/obs/lib/srcpkg.py @@ -31,7 +31,7 @@ def get_git_version(project): # Run git-version-gen if it is in the repository script_path = f"{repo_path}/git-version-gen" if os.path.exists(script_path): - ret = lib.run_cmd([script_path, "."], cwd=repo_path).output + ret = lib.run_cmd([script_path, "."], cwd=repo_path).output.rstrip() if not ret: lib.exit_error_cmd(ret, "empty output from git-version-gen") return ret @@ -49,12 +49,18 @@ def get_git_version(project): "HEAD"], cwd=repo_path, check=False) if result.returncode == 128: - print(f"{project}: has no git tags, using 0.0.0 as version") + print(f"{project}: has no git tags") commit = lib.run_cmd(["git", "rev-parse", "HEAD"], cwd=repo_path).output[0:4] count = lib.run_cmd(["git", "rev-list", "--count", "HEAD"], cwd=repo_path).output.rstrip() - return f"0.0.0.{count}-{commit}" + try: + print(f"{project}: getting version from debian/changelog") + version = lib.debian.get_last_version_from_changelog(project) + return f"{version}.{count}-{commit}" + except: + print(f"{project}: using 0.0.0 as version") + return f"0.0.0.{count}-{commit}" if result.returncode != 0: lib.exit_error_cmd(result, "command failed unexpectedly") @@ -204,6 +210,12 @@ def build(project, gerrit_id=0): if project in lib.config.projects_osmocom: run_generate_build_dep(project) + if lib.args.configure_append: + lib.debian.configure_append(project, lib.args.configure_append) + + if lib.args.disable_manuals: + lib.debian.disable_manuals(project) + lib.debian.build_source_package(project) lib.debian.move_files_to_output(project) diff --git a/scripts/obs/sync_obs_projects.py b/scripts/obs/sync_obs_projects.py index cc4635a..beb3e09 100755 --- a/scripts/obs/sync_obs_projects.py +++ b/scripts/obs/sync_obs_projects.py @@ -182,15 +182,16 @@ def rewrite_meta(project): # (e.g. <pubkey>debian-archive-12</pubkey>): # https://github.com/openSUSE/open-build-service/pull/14528 # Also we don't have such a pubkeydir set up on our OBS server. Assume - # ftp.de.debian.org is a trusted mirror, switch to HTTPS and skip the - # PGP verification by removing the pubkey blocks. + # https://debian.inf.tu-dresden.de/ is a trusted mirror, switch to + # HTTPS and skip the PGP verification by removing the pubkey blocks. if project.startswith("Debian:"): for download in repository.findall(".download"): url = download.get("url") print(f" changing url to https: {url}") assert url.startswith("http://ftp.de.debian.org/debian"), \ f"unexpected mirror URL" - download.set("url", url.replace("http://", "https://")) + download.set("url", url.replace("http://ftp.de.debian.org/debian", + "https://debian.inf.tu-dresden.de/debian")) for pubkey in download.findall("pubkey"): download.remove(pubkey) diff --git a/scripts/obs/update_obs_eclipse_titan.sh b/scripts/obs/update_obs_eclipse_titan.sh new file mode 100755 index 0000000..6efb7c6 --- /dev/null +++ b/scripts/obs/update_obs_eclipse_titan.sh @@ -0,0 +1,40 @@ +#!/bin/sh -e +# https://osmocom.org/projects/cellular-infrastructure/wiki/Upgrading_eclipse-titan_in_the_Osmocom_OBS +DIR="$(realpath "$(dirname "$0")")" +PROJ="$1" +GIT_URL="https://gitea.osmocom.org/osmith/titan.core" +CHECKOUT="osmocom/9.0.0" + +prepare_git_repo() { + cd "$DIR" + if ! [ -d _cache/eclipse-titan ]; then + mkdir -p _cache + git -C _cache clone "$GIT_URL" eclipse-titan + fi + + cd _cache/eclipse-titan + git fetch + git clean -fdx + git checkout -f -B "$CHECKOUT" + git reset --hard origin/"$CHECKOUT" +} + +update_obs_project() { + cd "$DIR" + ./update_obs_project.py \ + --apiurl https://obs.osmocom.org \ + --docker \ + --allow-unknown-package \ + --git-skip-checkout \ + --git-skip-fetch \ + --version-append "~osmocom" \ + "$PROJ" \ + eclipse-titan +} + +set -x +prepare_git_repo + +if [ -n "$PROJ" ]; then + update_obs_project +fi diff --git a/scripts/obs/update_obs_project.py b/scripts/obs/update_obs_project.py index a456218..0852693 100755 --- a/scripts/obs/update_obs_project.py +++ b/scripts/obs/update_obs_project.py @@ -7,6 +7,7 @@ import sys import traceback import lib import lib.config +import lib.debian import lib.docker import lib.git import lib.metapkg @@ -21,7 +22,7 @@ srcpkgs_failed_upload = [] # list of pkgnames srcpkgs_updated = [] # list of pkgnames -def parse_packages(packages_arg): +def parse_packages(packages_arg, excludes_arg): ret = [] if packages_arg: for package in packages_arg: @@ -29,11 +30,15 @@ def parse_packages(packages_arg): ret += lib.config.projects_osmocom else: ret += [lib.set_proper_package_name(package)] - return ret + else: # Default to all + ret += lib.config.projects_osmocom + ret += lib.config.projects_other + + # Apply --exclude-package + for package in excludes_arg: + if package in ret: + ret.remove(package) - # Default to all - ret += lib.config.projects_osmocom - ret += lib.config.projects_other return ret @@ -93,6 +98,7 @@ def build_srcpkg_if_needed(pkgs_remote, package, is_meta_pkg): branch_missing_ok=delete) else: latest_version = lib.git.get_latest_tag_remote(package) + latest_version = lib.debian.transform_version(latest_version) if latest_version is None: if delete and os.path.basename(package) in pkgs_remote: @@ -138,7 +144,8 @@ def build_srcpkgs(pkgs_remote, packages): if lib.args.meta: feed = lib.args.feed - build_srcpkg_if_needed(pkgs_remote, f"osmocom-{feed}", True) + metapkg = lib.args.conflict_pkgname or f"osmocom-{feed}" + build_srcpkg_if_needed(pkgs_remote, metapkg, True) for package in packages: build_srcpkg_if_needed(pkgs_remote, package, False) @@ -209,16 +216,23 @@ def validate_args(args): def main(): parser = argparse.ArgumentParser( description="Generate source packages and upload them to OBS.") - lib.add_shared_arguments(parser) - parser.add_argument("-A", "--apiurl", help="OBS API URL or .oscrc alias" - " (e.g. https://obs.osmocom.org)") - parser.add_argument("-n", "--no-skip-up-to-date", - dest="skip_up_to_date", action="store_false", - help="for latest feed, build and upload packages even" - " if the version did not change") - parser.add_argument("--delete", action="store_true", - help="remove packages from OBS if the git branch (-b)" - " does not exist anymore") + groups = lib.add_shared_arguments(parser) + + groups["pkg"].add_argument("-E", "--exclude-package", nargs="*", + default=[], + help="do not build and update this package") + + group_obs = parser.add_argument_group("OBS options") + group_obs.add_argument("-A", "--apiurl", help="OBS API URL or .oscrc alias" + " (e.g. https://obs.osmocom.org)") + group_obs.add_argument("-n", "--no-skip-up-to-date", + dest="skip_up_to_date", action="store_false", + help="for latest feed, build and upload packages" + " even if the version did not change") + group_obs.add_argument("--delete", action="store_true", + help="remove packages from OBS if the git branch" + " (-b) does not exist anymore") + parser.add_argument("obs_project", help="OBS project, e.g. home:osmith:nightly") parser.add_argument("package", nargs="*", @@ -227,7 +241,7 @@ def main(): args = parser.parse_args() validate_args(args) lib.set_args(args) - packages = parse_packages(args.package) + packages = parse_packages(args.package, args.exclude_package) if args.docker: lib.docker.run_in_docker_and_exit("update_obs_project.py", True) diff --git a/scripts/osmo-ci-docker-rebuild.sh b/scripts/osmo-ci-docker-rebuild.sh index c9e2b63..78bf21c 100755 --- a/scripts/osmo-ci-docker-rebuild.sh +++ b/scripts/osmo-ci-docker-rebuild.sh @@ -9,7 +9,7 @@ x86_64) "debian-bookworm-build" \ "debian-bookworm-erlang" ;; -arm*) +arm*|aarch64) docker_images_require \ "debian-bookworm-build-arm" ;; diff --git a/scripts/osmo-depcheck/config.py b/scripts/osmo-depcheck/config.py index 94a1cb5..39c79b4 100644 --- a/scripts/osmo-depcheck/config.py +++ b/scripts/osmo-depcheck/config.py @@ -30,15 +30,20 @@ repos = {"libosmocore": ("libosmocodec", "libosmoctrl", "libosmogb", "libosmogsm", + "libosmoisdn", "libosmosim", + "libosmousb", "libosmovty"), "libosmo-abis": ("libosmoabis", "libosmotrau"), - "libosmo-sccp": ("libosmo-mtp", - "libosmo-sigtran", - "libosmo-xua"), + "libosmo-sigtran": ("libosmo-sigtran"), + "libosmo-sccp-legacy": ("libosmo-mtp", + "libosmo-sccp", + "libosmo-xua"), "osmo-ggsn": ("libgtp"), "osmo-hlr": ("libosmo-gsup-client"), "osmo-iuh": ("libosmo-ranap"), "osmo-mgw": ("libosmo-mgcp-client", - "libosmo-legacy-mgcp")} + "libosmo-legacy-mgcp"), + "osmo-e1d": ("libosmo-e1d", + "libosmo-octoi")} diff --git a/scripts/osmocom-api-doxygen.sh b/scripts/osmocom-api-doxygen.sh index cfa5e61..0b9a965 100755 --- a/scripts/osmocom-api-doxygen.sh +++ b/scripts/osmocom-api-doxygen.sh @@ -8,7 +8,7 @@ repos_api=" libosmo-abis libosmo-dsp libosmo-netif - libosmo-sccp + libosmo-sigtran osmo-gmr " diff --git a/scripts/osmocom-build-old-tags-against-master.sh b/scripts/osmocom-build-old-tags-against-master.sh index 5f0d04d..eaf7308 100755 --- a/scripts/osmocom-build-old-tags-against-master.sh +++ b/scripts/osmocom-build-old-tags-against-master.sh @@ -16,7 +16,8 @@ REPOS_MASTER=" libosmocore libosmo-abis libosmo-netif - libosmo-sccp + libosmo-sigtran + libosmo-sccp-legacy libsmpp34 libasn1c osmo-ggsn @@ -30,7 +31,7 @@ REPOS_TAGS=" libosmocore libosmo-netif libosmo-sccp - openbsc + libosmo-sigtran osmo-bsc osmo-bts osmo-ggsn diff --git a/scripts/tarballs/prefer-configure.sh b/scripts/tarballs/prefer-configure.sh index 295b239..c344e48 100755 --- a/scripts/tarballs/prefer-configure.sh +++ b/scripts/tarballs/prefer-configure.sh @@ -28,6 +28,12 @@ gapk) ;; esac ;; +strongswan-epdg) + # Use a simple git archive instead of running configure and "make dist-bzip2" + # The configure script requires tags to be in a specific format and clashs + # with our custom tags. + exit 1 + ;; *) exit 0 ;; diff --git a/scripts/tarballs/publish-tarballs-for-tags.sh b/scripts/tarballs/publish-tarballs-for-tags.sh index 46a2e17..0335bd3 100755 --- a/scripts/tarballs/publish-tarballs-for-tags.sh +++ b/scripts/tarballs/publish-tarballs-for-tags.sh @@ -14,7 +14,8 @@ OSMO_RELEASE_REPOS=" libosmo-abis libosmo-netif libosmo-pfcp - libosmo-sccp + libosmo-sccp-legacy + libosmo-sigtran libosmocore libsmpp34 libusrp @@ -23,6 +24,7 @@ OSMO_RELEASE_REPOS=" osmo-cbc osmo-e1-recorder osmo-e1d + osmo-epdg osmo-fl2k osmo-gbproxy osmo-ggsn @@ -36,6 +38,7 @@ OSMO_RELEASE_REPOS=" osmo-pcap osmo-pcu osmo-remsim + osmo-s1gw osmo-sgsn osmo-sip-connector osmo-smlc @@ -46,6 +49,7 @@ OSMO_RELEASE_REPOS=" osmocom-bb rtl-sdr simtrace2 + strongswan-epdg " # Old release tags that are duplicates or where generating the tarball fails @@ -107,6 +111,10 @@ TAGS_IGNORE=" osmo-trx:0.2.0, osmo-trx:0.3.0, osmo-trx:1.3.0, + + libosmo-sccp-legacy:2.0.0, + libosmo-sccp-legacy:2.0.1, + libosmo-sccp-legacy:2.0.2, " mkdir -p \ |