summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--ansible/hosts1
-rw-r--r--ansible/roles/osmocom-jenkins-slave/tasks/osmo-ci.yml4
-rw-r--r--ansible/roles/osmocom-jenkins-slave/tasks/ttcn3-slave.yml11
-rw-r--r--ansible/setup-gsm-tester.yml2
-rw-r--r--ansible/setup-simtest.yml20
-rwxr-xr-xcontrib/jenkins.sh17
-rwxr-xr-xcontrib/obs-mirror/rm-old-nightly-archives.sh7
l---------coverity/common.sh1
-rwxr-xr-xcoverity/local_test.sh1
-rwxr-xr-xcoverity/prepare_source_Osmocom.sh9
-rw-r--r--jobs/coverity.yml1
-rw-r--r--jobs/gerrit-binpkgs.yml120
-rw-r--r--jobs/gerrit-lint.yml169
-rw-r--r--jobs/gerrit-pipeline-result.yml80
-rw-r--r--jobs/gerrit-verifications-dahdi.yml33
-rw-r--r--jobs/gerrit-verifications.yml355
-rw-r--r--jobs/master-builds-dahdi.yml24
-rw-r--r--jobs/master-builds.yml77
-rw-r--r--jobs/osmocom-api.yml89
-rw-r--r--jobs/osmocom-build-tags-against-master.yml2
-rw-r--r--jobs/osmocom-depcheck.yml4
-rw-r--r--jobs/osmocom-list-commits.yml2
-rw-r--r--jobs/osmocom-obs-check-builders.yml28
-rw-r--r--jobs/osmocom-obs.yml7
-rw-r--r--jobs/ttcn3-testsuites-kernel-git.yml2
-rw-r--r--jobs/ttcn3-testsuites.yml6
-rw-r--r--jobs/update-osmo-ci-on-slaves.yml1
-rwxr-xr-xlint/checkpatch/checkpatch_osmo.sh4
-rw-r--r--lint/checkpatch/spelling.txt1
-rwxr-xr-xlint/lint_diff.sh3
-rwxr-xr-xqemu-kvm/ggsn-kernel-gtp/postinst.sh2
-rw-r--r--qemu-kvm/ggsn-kernel-gtp/postinst/Release.key38
-rw-r--r--qemu-kvm/ggsn-kernel-gtp/postinst/authorized_keys3
-rw-r--r--scripts/common.sh30
-rwxr-xr-xscripts/jenkins-gerrit/pipeline_summary.py227
-rwxr-xr-xscripts/jenkins-gerrit/pipeline_summary_send.sh32
-rw-r--r--scripts/obs/README21
-rwxr-xr-xscripts/obs/build_binpkg.py90
-rwxr-xr-xscripts/obs/build_srcpkg.py12
-rwxr-xr-xscripts/obs/check_builders.sh19
-rwxr-xr-xscripts/obs/data/build.sh15
-rw-r--r--scripts/obs/data/build_binpkg.Dockerfile78
-rw-r--r--scripts/obs/data/build_binpkg_manuals.Dockerfile20
-rwxr-xr-xscripts/obs/data/build_deb.sh14
-rwxr-xr-xscripts/obs/data/build_rpm.sh31
-rw-r--r--scripts/obs/data/build_srcpkg.Dockerfile (renamed from scripts/obs/data/Dockerfile)1
-rw-r--r--scripts/obs/data/rpmmacros5
-rw-r--r--scripts/obs/lib/__init__.py33
-rw-r--r--scripts/obs/lib/binpkg_deb.py29
-rw-r--r--scripts/obs/lib/binpkg_rpm.py29
-rw-r--r--scripts/obs/lib/config.py12
-rw-r--r--scripts/obs/lib/docker.py72
-rw-r--r--scripts/obs/lib/git.py38
-rw-r--r--scripts/obs/lib/osc.py28
-rw-r--r--scripts/obs/lib/srcpkg.py41
-rwxr-xr-xscripts/obs/update_obs_project.py29
-rw-r--r--scripts/osmo-build.sh2
-rw-r--r--scripts/osmo-depcheck/config.py2
-rw-r--r--scripts/osmo-depcheck/dependencies.py4
-rwxr-xr-xscripts/osmo-layer1-headers.sh2
60 files changed, 1699 insertions, 341 deletions
diff --git a/ansible/hosts b/ansible/hosts
index c6164a3..1c1ed4d 100644
--- a/ansible/hosts
+++ b/ansible/hosts
@@ -11,6 +11,7 @@ host2-deb9build-ansible ansible_host=2a01:4f8:120:8470::1:3
admin2-deb9build ansible_host=2a01:4f8:13b:828::1:300 osmocom_jenkins_slave_fstrim=True
build2-deb9build-ansible ansible_host=2a01:4f8:10b:2ad9::1:6 osmocom_jenkins_slave_fstrim=True
build2-deb10build-ansible ansible_host=2a01:4f8:10b:2ad9::1:10 osmocom_jenkins_slave_fstrim=True
+build2-deb11build-ansible ansible_host=2a01:4f8:10b:2ad9::1:13 osmocom_jenkins_slave_fstrim=True
host2-deb9build-ansible ansible_host=2a01:4f8:120:8470::1:3 osmocom_jenkins_slave_fstrim=True
gtp0-deb9build ansible_host=10.34.2.102 osmocom_jenkins_slave_fstrim=True
gtp0-deb10build ansible_host=10.34.2.101 osmocom_jenkins_slave_fstrim=True
diff --git a/ansible/roles/osmocom-jenkins-slave/tasks/osmo-ci.yml b/ansible/roles/osmocom-jenkins-slave/tasks/osmo-ci.yml
index b26d6ab..a19dcaf 100644
--- a/ansible/roles/osmocom-jenkins-slave/tasks/osmo-ci.yml
+++ b/ansible/roles/osmocom-jenkins-slave/tasks/osmo-ci.yml
@@ -8,14 +8,14 @@
become: yes
become_user: "{{ jenkins_user }}"
git:
- repo: "https://git.osmocom.org/osmo-ci"
+ repo: "https://gerrit.osmocom.org/osmo-ci"
dest: "/home/{{ jenkins_user }}/osmo-ci"
- name: deploy osmo-gsm-manuals to home directory
become: yes
become_user: "{{ jenkins_user }}"
git:
- repo: "https://git.osmocom.org/osmo-gsm-manuals"
+ repo: "https://gerrit.osmocom.org/osmo-gsm-manuals"
dest: "/home/{{ jenkins_user }}/osmo-gsm-manuals"
- name: rebuild osmocom docker images for jenkins build tests
diff --git a/ansible/roles/osmocom-jenkins-slave/tasks/ttcn3-slave.yml b/ansible/roles/osmocom-jenkins-slave/tasks/ttcn3-slave.yml
index 0910102..67717fc 100644
--- a/ansible/roles/osmocom-jenkins-slave/tasks/ttcn3-slave.yml
+++ b/ansible/roles/osmocom-jenkins-slave/tasks/ttcn3-slave.yml
@@ -11,16 +11,21 @@
become: yes
become_user: "{{ jenkins_user }}"
git:
- repo: "https://git.osmocom.org/docker-playground"
+ repo: "https://gerrit.osmocom.org/docker-playground"
dest: "/home/{{ jenkins_user }}/jenkins/docker-playground"
- name: add apt.key
- apt_key:
+ get_url:
url: https://obs.osmocom.org/projects/osmocom/public_key
+ dest: /etc/apt/trusted.gpg.d/obs.osmocom.org.asc
+ # OBS webserver needs this header, returns 404 otherwise
+ # https://github.com/ansible/ansible/issues/50436#issuecomment-451205256
+ headers:
+ Accept: '*/*'
- name: add apt repository
apt_repository:
- repo: "deb https://downloads.osmocom.org/packages/osmocom:/latest/{{ obs_distro }}/ ./"
+ repo: "deb [signed-by=/etc/apt/trusted.gpg.d/obs.osmocom.org.asc] https://downloads.osmocom.org/packages/osmocom:/latest/{{ obs_distro }}/ ./"
filename: obs_osmocom
update_cache: yes
diff --git a/ansible/setup-gsm-tester.yml b/ansible/setup-gsm-tester.yml
index 34976d7..9d27256 100644
--- a/ansible/setup-gsm-tester.yml
+++ b/ansible/setup-gsm-tester.yml
@@ -18,7 +18,7 @@
become: yes
become_user: "jenkins"
git:
- repo: "https://git.osmocom.org/osmo-ci"
+ repo: "https://gerrit.osmocom.org/osmo-ci"
dest: "/home/jenkins/osmo-ci"
roles:
- name: gsm-tester
diff --git a/ansible/setup-simtest.yml b/ansible/setup-simtest.yml
index 6eb67fc..edc250d 100644
--- a/ansible/setup-simtest.yml
+++ b/ansible/setup-simtest.yml
@@ -21,6 +21,26 @@
- virtualenv
install_recommends: no
+ - name: "pysim: get requirements.txt"
+ get_url:
+ url: https://gitea.osmocom.org/sim-card/pysim/raw/branch/master/requirements.txt
+ dest: /tmp/pysim_requirements.txt
+
+ - name: "pysim: install deps from requirements.txt"
+ pip:
+ executable: pip3
+ requirements: /tmp/pysim_requirements.txt
+
+ # https://gitea.osmocom.org/sim-card/pysim/src/branch/master/contrib/jenkins.sh
+ - name: "pysim: install other deps needed for contrib/jenkins.sh"
+ pip:
+ executable: pip3
+ name:
+ - "git+https://github.com/osmocom/sphinx-argparse@master#egg=sphinx-argparse"
+ - pylint
+ - sphinx
+ - sphinxcontrib-napoleon
+
roles:
- name: osmocom-jenkins-slave
generic_slave: false
diff --git a/contrib/jenkins.sh b/contrib/jenkins.sh
index a15ea05..96becc7 100755
--- a/contrib/jenkins.sh
+++ b/contrib/jenkins.sh
@@ -1,10 +1,22 @@
#!/bin/sh
set -e -x
+WORKSPACE_DIR="$(realpath "$(dirname "$0")/..")"
# Clone repository to ~/, or update existing
# $1: name of osmocom project
clone_repo() {
- cd ~/"$1" || (cd ~/ && git clone https://git.osmocom.org/"$1" && cd ~/"$1")
+ local project="$1"
+ local url="https://gerrit.osmocom.org/$project"
+
+ if [ -d ~/"$project" ]; then
+ cd ~/"$project"
+ git remote set-url origin "$url"
+ else
+ cd ~
+ git clone "$url"
+ cd "$project"
+ fi
+
git rev-parse HEAD
git status
@@ -12,12 +24,13 @@ clone_repo() {
git rev-parse HEAD
git status
+
+ cd "$WORKSPACE_DIR"
}
clone_repo osmo-ci
clone_repo osmo-gsm-manuals
if [ `uname` = "Linux" ] && [ "x${OSMO_CI_NO_DOCKER}" != "x1" ]; then
- cd ~/osmo-ci
scripts/osmo-ci-docker-rebuild.sh
fi
diff --git a/contrib/obs-mirror/rm-old-nightly-archives.sh b/contrib/obs-mirror/rm-old-nightly-archives.sh
index 485bf01..98c81ee 100755
--- a/contrib/obs-mirror/rm-old-nightly-archives.sh
+++ b/contrib/obs-mirror/rm-old-nightly-archives.sh
@@ -1,9 +1,6 @@
-#!/bin/sh -e
+#!/bin/sh -ex
# Remove nightly archives older than one month (OS#4862)
-echo "Redirecting all output to: /home/pkgmirror/rm-old-nightly-archives.log"
-exec >"/home/pkgmirror/rm-old-nightly-archives.log" 2>&1
-
-DRY=0
+DRY=1
# Get removal date in seconds since epoch and display it
DATE_RM_SEC=$(expr $(date +%s) - 3600 \* 24 \* 32)
diff --git a/coverity/common.sh b/coverity/common.sh
new file mode 120000
index 0000000..8f435c6
--- /dev/null
+++ b/coverity/common.sh
@@ -0,0 +1 @@
+../scripts/common.sh \ No newline at end of file
diff --git a/coverity/local_test.sh b/coverity/local_test.sh
index 4ac1307..dffb101 100755
--- a/coverity/local_test.sh
+++ b/coverity/local_test.sh
@@ -7,6 +7,7 @@ export SRC_CLEAN=1
mkdir -p /tmp/coverity
cp "$SCRIPT_DIR"/* /tmp/coverity
+ln -sf "$SCRIPT_DIR/../scripts/common.sh" /tmp/coverity/common.sh
cd /tmp/coverity
diff --git a/coverity/prepare_source_Osmocom.sh b/coverity/prepare_source_Osmocom.sh
index 122fc1c..526e9d3 100755
--- a/coverity/prepare_source_Osmocom.sh
+++ b/coverity/prepare_source_Osmocom.sh
@@ -1,4 +1,5 @@
-#!/bin/sh
+#!/bin/sh -xe
+. $(realpath common.sh)
BASEDIR=source-Osmocom
# How to add a new project:
@@ -45,7 +46,7 @@ PROJECTS_DONT_BUILD_TEST="
asn1c
"
-[ -d $BASEDIR ] || mkdir -p $BASEDIR
+mkdir -p $BASEDIR
cd $BASEDIR
for proj in $PROJECTS $PROJECTS_DONT_BUILD_TEST; do
@@ -57,14 +58,14 @@ for proj in $PROJECTS $PROJECTS_DONT_BUILD_TEST; do
git -C "$proj" clean -ffxd
fi
else
- git clone https://git.osmocom.org/$proj
+ git clone "$(osmo_git_clone_url "$proj")"
fi
done
# We want to compile tests, but not execute them. Using 'noinst_PROGRAMS'
# instead of 'check_PROGRAMS' allows building test binaries during 'make all'.
for proj in $PROJECTS; do
- files="$(git -C $proj grep -l check_PROGRAMS)"
+ files="$(git -C $proj grep -l check_PROGRAMS)" || true
if [ -n "$files" ]; then
(cd $proj && sed -i "s/check_PROGRAMS/noinst_PROGRAMS/" $files)
fi
diff --git a/jobs/coverity.yml b/jobs/coverity.yml
index 74803c7..242d1aa 100644
--- a/jobs/coverity.yml
+++ b/jobs/coverity.yml
@@ -9,7 +9,6 @@
project-type: freestyle
defaults: global
description: 'Generated by job-builder'
- display-name: 'Coverity-Upload'
node: coverity
builders:
- shell: |
diff --git a/jobs/gerrit-binpkgs.yml b/jobs/gerrit-binpkgs.yml
new file mode 100644
index 0000000..8408950
--- /dev/null
+++ b/jobs/gerrit-binpkgs.yml
@@ -0,0 +1,120 @@
+# This file holds all gerrit verifications for building deb and rpm packages.
+# https://jenkins.osmocom.org/jenkins/view/Jenkins-Gerrit/.
+# One can simply add a gerrit job by adding project's repository to repos list.
+
+- project:
+ name: gerrit-binpkgs
+ jobs:
+ - 'gerrit-binpkgs-{type}'
+ type:
+ - 'deb'
+ - 'rpm'
+
+- job-template:
+ name: 'gerrit-binpkgs-{type}'
+ project-type: freestyle
+ node: osmocom-gerrit-debian10 || osmocom-gerrit-debian11
+ concurrent: true
+ retry-count: 3 # scm checkout
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 120
+ artifact-days-to-keep: -1
+ artifact-num-to-keep: -1
+ description: |
+ {type} package job of CI for patches sent to
+ <a href="https://gerrit.osmocom.org">gerrit</a>
+ </br></br>
+ Related issue: <a href="https://osmocom.org/issues/2385">OS#2385</a>
+
+ parameters:
+ - string:
+ name: BRANCH_CI
+ description: |
+ osmo-ci.git branch
+ default: 'master'
+ - string:
+ name: GERRIT_REPO_URL
+ description: set by gerrit verification pipeline job
+ - string:
+ name: GERRIT_BRANCH
+ description: set by gerrit verification pipeline job
+ - string:
+ name: GERRIT_REFSPEC
+ description: set by gerrit verification pipeline job
+ - string:
+ name: PROJECT_NAME
+ description: project to test, e.g. 'libosmocore' (set by pipeline)
+
+ scm:
+ - git:
+ basedir: 'code-from-gerrit'
+ url: '$GERRIT_REPO_URL'
+ credentials-id: d5eda5e9-b59d-44ba-88d2-43473cb6e42d
+ branches:
+ - $GERRIT_BRANCH
+ refspec: $GERRIT_REFSPEC
+ name:
+ choosing-strategy: gerrit
+ wipe-workspace: true
+ skip-tag: true
+ submodule:
+ recursive: false
+
+ builders:
+ - shell: |-
+ # Skip rpm build if .spec.in file is missing
+ if [ "{type}" = "rpm" ] && [ -z "$(find code-from-gerrit -name '*.spec.in')" ]; then
+ set +x
+ echo
+ echo "No *.spec.in file found, skipping rpm build."
+ echo
+ exit 0
+ fi
+
+ # Clone osmo-ci.git
+ rm -rf osmo-ci
+ git clone \
+ --depth=1 \
+ --branch="$BRANCH_CI" \
+ https://gerrit.osmocom.org/osmo-ci \
+ osmo-ci
+ git -C osmo-ci log --oneline
+
+ # Get distro from type
+ case "{type}" in
+ deb) distro="debian:11" ;;
+ rpm) distro="almalinux:8" ;;
+ *) echo "unknown package format: '{type}'"; exit 1 ;;
+ esac
+
+ # Move code from gerrit to build_srcpkg.py's git cache
+ cache_dir=osmo-ci/scripts/obs/_cache
+ mkdir -p $cache_dir
+ mv code-from-gerrit "$cache_dir/$PROJECT_NAME"
+
+ # Set a known branch name
+ git -C "$cache_dir/$PROJECT_NAME" checkout -B "origin/gerrit"
+
+ # Build source package
+ cd osmo-ci/scripts/obs/
+ ./build_srcpkg.py \
+ --docker \
+ --feed master \
+ --git-branch gerrit \
+ --git-skip-fetch \
+ "$PROJECT_NAME"
+
+ # Build binary package
+ ./build_binpkg.py \
+ --docker "$distro" \
+ "$PROJECT_NAME"
+ wrappers:
+ - ansicolor:
+ colormap: xterm
+ - ssh-agent-credentials:
+ users:
+ - d5eda5e9-b59d-44ba-88d2-43473cb6e42d
+
+# vim: expandtab tabstop=2 shiftwidth=2
diff --git a/jobs/gerrit-lint.yml b/jobs/gerrit-lint.yml
index 602ce68..dfdab5b 100644
--- a/jobs/gerrit-lint.yml
+++ b/jobs/gerrit-lint.yml
@@ -1,99 +1,17 @@
# This file holds all gerrit lint verifications https://jenkins.osmocom.org/jenkins/view/Jenkins-Gerrit/.
-# One can simply add a gerrit job by adding project's repository to repos list.
+# Add project's repositories to jobs/gerrit-verifications.yml. When
+# pipeline_lint is set for that project, the pipeline will call the gerrit-lint
+# job for it.
- project:
name: gerrit-lint
- # following default values can be overridden by each repo
- disabled: false
- gerrit_url: 'ssh://jenkins@gerrit.osmocom.org:29418'
- repos_url: '{gerrit_url}/{repos}'
- gerrit_project: '{repos}'
-
- # in alphabetical order
- repos:
- - asn1c
- - cellmgr-ng
- - docker-playground
- - libasn1c
- - libgtpnl
- - libosmo-abis
- - libosmo-gprs
- - libosmo-netif
- - libosmo-pfcp
- - libosmo-sccp
- - libosmocore
- - libsmpp34
- - libtelnet
- - libusrp
- - openbsc
- - openggsn
- - osmo-bsc
- - osmo-bsc-nat
- - osmo-bts
- - osmo-hnodeb
- - osmo-hnbgw
- - osmo-upf
- - osmo-ci
-
- - osmo_dia2gsup:
- repos_url: '{gerrit_url}/erlang/{repos}'
- gerrit_project: 'erlang/{repos}'
-
- - osmo-e1-hardware
- - osmo-ggsn
- - osmo-gsm-manuals
-
- - osmo_gsup:
- repos_url: '{gerrit_url}/erlang/{repos}'
- gerrit_project: 'erlang/{repos}'
-
- - osmo-hlr
- - osmo-iuh
- - osmo-mgw
- - osmo-msc
- - osmo-pcap
- - osmo-pcu
-
- - osmo-python-tests:
- repos_url: '{gerrit_url}/python/{repos}'
- gerrit_project: 'python/{repos}'
-
- - osmo-sgsn
-
- - osmo_ss7:
- repos_url: '{gerrit_url}/erlang/{repos}'
- gerrit_project: 'erlang/{repos}'
-
- - osmo-sip-connector
- - osmo-trx
- - osmocom-bb
- - osmo-tetra
- - osmo-sysmon
- - osmo-remsim
- - simtrace2
- - osmo-asf4-dfu
- - osmo-ccid-firmware
- - osmo-e1d
- - osmo-cbc
- - osmo-e1-recorder
- - gapk
- - osmo-uecups
- - osmo-el2tpd
- - osmo-smlc
- - osmo-gbproxy
- - pysim
-
- - osmo-ttcn3-hacks:
- repos_url: 'https://gerrit.osmocom.org/{repos}'
-
jobs:
- - 'gerrit-{repos}-lint'
+ - 'gerrit-lint'
-- job-template:
- name: 'gerrit-{repos}-lint'
+- job:
+ name: 'gerrit-lint'
project-type: freestyle
node: osmocom-gerrit-debian9
- disabled: '{obj:disabled}'
retry-count: 3 # scm checkout
properties:
- build-discarder:
@@ -102,8 +20,7 @@
artifact-days-to-keep: -1
artifact-num-to-keep: -1
description: |
- Run linting on patches submitted to gerrit for
- <a href="https://gerrit.osmocom.org/#/q/status:open+project:{repos}">{repos}</a>
+ Lint job of CI for patches sent to <a href="https://gerrit.osmocom.org">gerrit</a>.
</br></br>
Related issue: <a href="https://osmocom.org/issues/5087">OS#5087</a>
@@ -113,54 +30,60 @@
description: |
osmo-ci.git branch
default: 'master'
+ - string:
+ name: GERRIT_PROJECT
+ description: set by gerrit verification pipeline job
+ - string:
+ name: GERRIT_CHANGE_NUMBER
+ description: set by gerrit verification pipeline job
+ - string:
+ name: GERRIT_PATCHSET_NUMBER
+ description: set by gerrit verification pipeline job
+ - string:
+ name: GERRIT_BRANCH
+ description: set by gerrit verification pipeline job
+ - string:
+ name: GERRIT_HOST
+ description: set by gerrit verification pipeline job
+ - string:
+ name: GERRIT_PATCHSET_REVISION
+ description: set by gerrit verification pipeline job
+ - string:
+ name: GERRIT_PORT
+ description: set by gerrit verification pipeline job
+ - string:
+ name: GERRIT_REFSPEC
+ description: set by gerrit verification pipeline job
+ - string:
+ name: GERRIT_REPO_URL
+ description: set by gerrit verification pipeline job
scm:
- git:
basedir: 'code-from-gerrit'
- url: '{obj:repos_url}'
+ url: '$GERRIT_REPO_URL'
credentials-id: d5eda5e9-b59d-44ba-88d2-43473cb6e42d
branches:
- $GERRIT_BRANCH
refspec: $GERRIT_REFSPEC
- name:
choosing-strategy: gerrit
wipe-workspace: false
skip-tag: true
submodule:
recursive: false
- - git:
- basedir: 'osmo-ci'
- url: '{gerrit_url}/osmo-ci'
- credentials-id: d5eda5e9-b59d-44ba-88d2-43473cb6e42d
- branches:
- - '$BRANCH_CI'
- wipe-workspace: true
-
- triggers:
- - gerrit:
- trigger-on:
- - patchset-created-event:
- exclude-drafts: true
- exclude-no-code-change: true
- projects:
- - project-compare-type: 'PLAIN'
- project-pattern: '{obj:gerrit_project}'
- branches:
- - branch-compare-type: 'ANT'
- branch-pattern: '**'
- skip-vote:
- successful: false
- failed: false
- unstable: false
- notbuilt: false
- silent: false
- escape-quotes: false
- no-name-and-email: false
- trigger-for-unreviewed-patches: true
- server-name: gerrit.osmocom.org
builders:
- - shell: 'cd code-from-gerrit && ../osmo-ci/lint/lint_diff.sh HEAD~1'
+ - shell: |
+ rm -rf osmo-ci
+ git clone \
+ --depth=1 \
+ --branch="$BRANCH_CI" \
+ https://gerrit.osmocom.org/osmo-ci \
+ osmo-ci
+ git -C osmo-ci log --oneline
+
+ cd code-from-gerrit
+ ../osmo-ci/lint/lint_diff.sh HEAD~1
wrappers:
- ansicolor:
diff --git a/jobs/gerrit-pipeline-result.yml b/jobs/gerrit-pipeline-result.yml
new file mode 100644
index 0000000..94fc88c
--- /dev/null
+++ b/jobs/gerrit-pipeline-result.yml
@@ -0,0 +1,80 @@
+# This job runs at the end of the pipeline in gerrit-verififactions.yml, to
+# post a list of failed/successful job links to gerrit and to vote +V/-V.
+
+- project:
+ name: gerrit-pipeline-result
+ jobs:
+ - 'gerrit-pipeline-result'
+
+- job:
+ name: 'gerrit-pipeline-result'
+ project-type: freestyle
+ node: osmocom-gerrit-debian10 || osmocom-gerrit-debian11
+ retry-count: 3 # scm checkout
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 120
+ artifact-days-to-keep: -1
+ artifact-num-to-keep: -1
+ description: |
+ Result job of CI for patches sent to <a href="https://gerrit.osmocom.org">gerrit</a>.
+ </br></br>
+ Related issue: <a href="https://osmocom.org/issues/2385">OS#2385</a>
+
+ parameters:
+ - string:
+ name: BRANCH_CI
+ description: |
+ osmo-ci.git branch
+ default: 'master'
+ - string:
+ name: GERRIT_PROJECT
+ description: set by gerrit verification pipeline job
+ - string:
+ name: GERRIT_CHANGE_NUMBER
+ description: set by gerrit verification pipeline job
+ - string:
+ name: GERRIT_PATCHSET_NUMBER
+ description: set by gerrit verification pipeline job
+ - string:
+ name: GERRIT_BRANCH
+ description: set by gerrit verification pipeline job
+ - string:
+ name: GERRIT_HOST
+ description: set by gerrit verification pipeline job
+ - string:
+ name: GERRIT_PATCHSET_REVISION
+ description: set by gerrit verification pipeline job
+ - string:
+ name: GERRIT_PATCHSET_UPLOADER_NAME
+ description: set by gerrit verification pipeline job
+ - string:
+ name: GERRIT_PORT
+ description: set by gerrit verification pipeline job
+ - string:
+ name: GERRIT_REFSPEC
+ description: set by gerrit verification pipeline job
+ - string:
+ name: PIPELINE_BUILD_URL
+ description: set by gerrit verification pipeline job
+
+ scm:
+ - git:
+ url: 'https://gerrit.osmocom.org/osmo-ci'
+ credentials-id: d5eda5e9-b59d-44ba-88d2-43473cb6e42d
+ branches:
+ - '$BRANCH_CI'
+ wipe-workspace: true
+
+ builders:
+ - shell: 'cd scripts/jenkins-gerrit && ./pipeline_summary_send.sh'
+
+ wrappers:
+ - ansicolor:
+ colormap: xterm
+ - ssh-agent-credentials:
+ users:
+ - d5eda5e9-b59d-44ba-88d2-43473cb6e42d
+
+# vim: expandtab tabstop=2 shiftwidth=2
diff --git a/jobs/gerrit-verifications-dahdi.yml b/jobs/gerrit-verifications-dahdi.yml
index 51bfb04..5343096 100644
--- a/jobs/gerrit-verifications-dahdi.yml
+++ b/jobs/gerrit-verifications-dahdi.yml
@@ -29,23 +29,9 @@
<b>Auto-generated using Jenkins Job Builder. DO NOT EDIT MANUALLY!</b>
scm:
- - git:
- basedir: 'dahdi-linux'
- url: 'https://gerrit.osmocom.org/dahdi-linux'
- credentials-id: d5eda5e9-b59d-44ba-88d2-43473cb6e42d
- branches:
- - $GERRIT_BRANCH
- refspec: $GERRIT_REFSPEC
- choosing-strategy: gerrit
- # When the gerrit git repository is configured to wipe-workspace, the
- # linux.git clone gets removed as well. Cloning the linux repo takes a
- # long time, so only clean both repos, don't remove them.
- wipe-workspace: false
- clean:
- before: true
- skip-tag: true
- submodule:
- recursive: true
+ # Let jenkins clone the big linux repository and keep it between builds
+ # (wipe-workspace is false). Clone dahdi-linux with code from gerrit in
+ # the shell part below.
- git:
basedir: 'linux'
url: 'https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git'
@@ -76,12 +62,21 @@
notbuilt: false
silent: false
escape-quotes: false
- no-name-and-email: false
- trigger-for-unreviewed-patches: true
server-name: gerrit.osmocom.org
builders:
- shell: |
+ rm -rf dahdi-linux
+ git init dahdi-linux
+ cd dahdi-linux
+ git config advice.detachedHead false
+ git fetch \
+ --depth=1 \
+ https://gerrit.osmocom.org/dahdi-linux \
+ "$GERRIT_REFSPEC"
+ git checkout FETCH_HEAD
+ cd ..
+
docker run \
--rm=true \
-e "KSRC=/linux" \
diff --git a/jobs/gerrit-verifications.yml b/jobs/gerrit-verifications.yml
index 85c78b4..2bbfe4e 100644
--- a/jobs/gerrit-verifications.yml
+++ b/jobs/gerrit-verifications.yml
@@ -1,5 +1,30 @@
# This file holds all gerrit build verifications https://jenkins.osmocom.org/jenkins/view/Jenkins-Gerrit/.
-# One cane simply add a gerrit job by adding project's repository to repos list.
+# One can simply add a gerrit job by adding project's repository to repos list.
+#
+# Overview of jobs involved in the pipeline:
+# https://osmocom.org/projects/osmocom-servers/wiki/Jenkins_gerrit-verifications_Pipeline
+#
+# How it works:
+# * Two jobs get added for each repository:
+# gerrit-{repos}
+# gerrit-{repos}-build
+# * The gerrit-{repos} job is a pipeline that runs the gerrit-{repos}-build job
+# and other jobs (linting, building debian packages, ...). The purpose of
+# this job is to reduce the amount of comments (and resulting mails) in
+# gerrit coming from the jenkins build verification. If each job ran
+# separately without this pipeline job, they would each generate mails.
+# The gerrit-{repos} job gets triggered from gerrit whenever a new patch to
+# test is available. It passes the GERRIT_BRANCH and GERRIT_REFSPEC variables
+# from gerrit to the jobs called in the pipeline.
+# * The gerrit-{repos}-build job runs contrib/jenkins.sh in docker in a matrix.
+# By default the matrix results in only one job, however for some projects we
+# use the matrix to build for multiple CPU architectures (x86_64, arm) or use
+# it to build multiple configurations of the same project (see osmo-bts).
+#
+# NOTE: after updating the job with Jenkins Job Builder as usual, check if a
+# new pipeline script was generated and approve it here:
+# https://jenkins.osmocom.org/jenkins/scriptApproval/
+# This happens when changing the pipeline script, when adding new projects etc.
- project:
name: gerrit
@@ -47,11 +72,22 @@
gerrit_url: 'ssh://jenkins@gerrit.osmocom.org:29418'
repos_url: '{gerrit_url}/{repos}'
gerrit_project: '{repos}'
+ # Which jobs to run in the pipeline
+ pipeline_build: true
+ pipeline_lint: true
+ pipeline_binpkgs: true
# in alphabetical order
repos:
- - asn1c
+ - asn1c:
+ pipeline_binpkgs: false
+
- cellmgr-ng
+
+ - docker-playground:
+ pipeline_build: false
+ pipeline_binpkgs: false
+
- libasn1c
- libgtpnl
- libosmo-abis
@@ -67,7 +103,10 @@
cmd: '{docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins_arch.sh "$arch"'
- libsmpp34
- - libtelnet
+
+ - libtelnet:
+ pipeline_binpkgs: false
+
- libusrp
- openbsc:
@@ -126,8 +165,25 @@
cmd: '{docker_run} {docker_img_erlang} {timeout_cmd} /build/contrib/jenkins.sh'
- osmo-e1-hardware:
- docker_img: 'registry.osmocom.org/$USER/fpga-build'
- cmd: 'docker pull {docker_img} && {docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins.sh'
+ a2_name: JOB_TYPE
+ a2: !!python/tuple ["firmware", "gateware", "manuals", "software"]
+ cmd: |
+ DOCKER_IMG="{docker_img}"
+ case "$JOB_TYPE" in
+ firmware|gateware|manuals)
+ DOCKER_IMG="registry.osmocom.org/$USER/fpga-build"
+ docker pull "$DOCKER_IMG"
+ ;;
+ esac
+ {docker_run} \
+ -e JOB_TYPE="$JOB_TYPE" \
+ "$DOCKER_IMG" \
+ {timeout_cmd} /build/contrib/jenkins.sh
+ pipeline_binpkgs: false
+
+ - osmo-ci:
+ pipeline_build: false
+ pipeline_binpkgs: false
- osmo-ggsn:
a1_name: GTP
@@ -149,6 +205,7 @@
repos_url: '{gerrit_url}/erlang/{repos}'
gerrit_project: 'erlang/{repos}'
cmd: '{docker_run} {docker_img_erlang} {timeout_cmd} /build/contrib/jenkins.sh'
+ pipeline_binpkgs: false
- osmo-hlr
- osmo-iuh
@@ -218,6 +275,7 @@
repos_url: '{gerrit_url}/erlang/{repos}'
gerrit_project: 'erlang/{repos}'
cmd: '{docker_run} {docker_img_erlang} {timeout_cmd} /build/contrib/jenkins.sh'
+ pipeline_binpkgs: false
- osmo-sip-connector
@@ -238,22 +296,41 @@
-e WITH_MANUALS="$WITH_MANUALS" \
{docker_img} {timeout_cmd} /build/contrib/jenkins.sh
- - osmocom-bb
- - osmo-tetra
+ - osmocom-bb:
+ pipeline_binpkgs: false
+
+ - osmo-tetra:
+ pipeline_binpkgs: false
+
- osmo-sysmon
- osmo-remsim
- simtrace2
+
- osmo-opencm3-projects:
docker_img: '$USER/debian-bullseye-jenkins'
cmd: '{docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins.sh'
- - osmo-asf4-dfu
- - osmo-ccid-firmware
+ pipeline_binpkgs: false
+
+ - osmo-asf4-dfu:
+ pipeline_binpkgs: false
+
+ - osmo-ccid-firmware:
+ pipeline_binpkgs: false
+
- osmo-e1d
- osmo-cbc
- - osmo-e1-recorder
- - gapk
+
+ - osmo-e1-recorder:
+ pipeline_binpkgs: false
+
+ - gapk:
+ pipeline_binpkgs: false
+
- osmo-uecups
- - osmo-el2tpd
+
+ - osmo-el2tpd:
+ pipeline_binpkgs: false
+
- osmo-smlc
- osmo-gbproxy
- osmo-hnodeb
@@ -261,9 +338,27 @@
- osmo-upf
- pysim:
- slave_axis: !!python/tuple [simtester]
concurrent: false
- cmd: '{timeout_cmd} ./contrib/jenkins.sh'
+ slave_axis: !!python/tuple [simtester,osmocom-master]
+ a2_name: JOB_TYPE
+ a2: !!python/tuple ["test", "pylint", "docs"]
+ combination_filter: >
+ (JOB_TYPE == "test" && label == "simtester") ||
+ (JOB_TYPE == "pylint" && label == "osmocom-master") ||
+ (JOB_TYPE == "docs" && label == "osmocom-master")
+ cmd: |
+ case "$JOB_TYPE" in
+ "test")
+ {timeout_cmd} ./contrib/jenkins.sh
+ ;;
+ *)
+ {docker_run} \
+ -e JOB_TYPE="$JOB_TYPE" \
+ {docker_img} \
+ {timeout_cmd} /build/contrib/jenkins.sh
+ ;;
+ esac
+ pipeline_binpkgs: false
- osmo-ttcn3-hacks:
slave_axis: !!python/tuple [ttcn3]
@@ -280,14 +375,213 @@
su build -c 'make clean';
su build -c 'make compile';
"
+ pipeline_binpkgs: false
- - dahdi-tools
+ - dahdi-tools:
+ pipeline_lint: false
+ pipeline_binpkgs: false
jobs:
- 'gerrit-{repos}'
+ - 'gerrit-{repos}-build'
- job-template:
name: 'gerrit-{repos}'
+ project-type: pipeline
+ concurrent: true
+ parameters:
+ - bool:
+ name: PIPELINE_BUILD
+ description: Enable the build job (runs contrib/jenkins.sh)
+ default: '{obj:pipeline_build}'
+ - bool:
+ name: PIPELINE_LINT
+ description: Enable the lint job
+ default: '{obj:pipeline_lint}'
+ - string:
+ name: REPO_URL
+ description: Clone URL, to be passed to jobs started in the pipeline
+ default: '{obj:repos_url}'
+ - bool:
+ name: PIPELINE_BINPKGS
+ description: Enable the binpkgs job (build deb/rpm packages)
+ default: '{obj:pipeline_binpkgs}'
+ # NOTE: jenkins pipelines don't run unless the dsl-script was approved.
+ # Sadly this doesn't happen automatically when updating the job with
+ # Jenkins Job Builder. So if you change the pipeline script, update the job
+ # as usually with JJB and then approve the scripts here:
+ # https://jenkins.osmocom.org/jenkins/scriptApproval/
+ dsl: |
+ pipeline {{
+ agent none
+ stages {{
+ stage("Verification") {{
+ parallel {{
+ stage("Build") {{
+ when {{
+ expression {{ params.PIPELINE_BUILD }}
+ }}
+ steps {{
+ script {{
+ // Run the build job for this repository and keep going on failure
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {{
+ env.PIPELINE_BUILD_PASSED = "0"
+ build job: 'gerrit-{repos}-build', parameters: [
+ string(name: "GERRIT_BRANCH", value: "${{env.GERRIT_BRANCH}}"),
+ string(name: "GERRIT_REFSPEC", value: "${{env.GERRIT_REFSPEC}}"),
+ ]
+ env.PIPELINE_BUILD_PASSED = "1"
+ }}
+ }}
+ }}
+ }}
+ stage("Lint") {{
+ when {{
+ expression {{ params.PIPELINE_LINT }}
+ }}
+ steps {{
+ script {{
+ // Run the lint job for this repository and keep going on failure
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {{
+ env.PIPELINE_LINT_PASSED = "0"
+ build job: 'gerrit-lint', parameters: [
+ string(name: "GERRIT_PROJECT", value: "${{env.GERRIT_PROJECT}}"),
+ string(name: "GERRIT_CHANGE_NUMBER", value: "${{env.GERRIT_CHANGE_NUMBER}}"),
+ string(name: "GERRIT_PATCHSET_NUMBER", value: "${{env.GERRIT_PATCHSET_NUMBER}}"),
+ string(name: "GERRIT_BRANCH", value: "${{env.GERRIT_BRANCH}}"),
+ string(name: "GERRIT_HOST", value: "${{env.GERRIT_HOST}}"),
+ string(name: "GERRIT_PATCHSET_REVISION", value: "${{env.GERRIT_PATCHSET_REVISION}}"),
+ string(name: "GERRIT_PORT", value: "${{env.GERRIT_PORT}}"),
+ string(name: "GERRIT_REFSPEC", value: "${{env.GERRIT_REFSPEC}}"),
+ string(name: "GERRIT_REPO_URL", value: "${{env.REPO_URL}}"),
+ ]
+ env.PIPELINE_LINT_PASSED = "1"
+ }}
+ }}
+ }}
+ }}
+ stage("deb") {{
+ when {{
+ expression {{ params.PIPELINE_BINPKGS }}
+ }}
+ steps {{
+ script {{
+ // Run the binpkgs job for this repository and keep going on failure
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {{
+ env.PIPELINE_DEB_PASSED = "0"
+ build job: 'gerrit-binpkgs-deb', parameters: [
+ string(name: "GERRIT_BRANCH", value: "${{env.GERRIT_BRANCH}}"),
+ string(name: "GERRIT_REFSPEC", value: "${{env.GERRIT_REFSPEC}}"),
+ string(name: "GERRIT_REPO_URL", value: "${{env.REPO_URL}}"),
+ string(name: "PROJECT_NAME", value: "{repos}"),
+ ]
+ env.PIPELINE_DEB_PASSED = "1"
+ }}
+ }}
+ }}
+ }}
+ stage("rpm") {{
+ when {{
+ expression {{ params.PIPELINE_BINPKGS }}
+ }}
+ steps {{
+ script {{
+ // Run the binpkgs job for this repository and keep going on failure
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {{
+ env.PIPELINE_RPM_PASSED = "0"
+ build job: 'gerrit-binpkgs-rpm', parameters: [
+ string(name: "GERRIT_BRANCH", value: "${{env.GERRIT_BRANCH}}"),
+ string(name: "GERRIT_REFSPEC", value: "${{env.GERRIT_REFSPEC}}"),
+ string(name: "GERRIT_REPO_URL", value: "${{env.REPO_URL}}"),
+ string(name: "PROJECT_NAME", value: "{repos}"),
+ ]
+ env.PIPELINE_RPM_PASSED = "1"
+ }}
+ }}
+ }}
+ }}
+ }}
+ }}
+ stage("Result") {{
+ steps {{
+ echo "PIPELINE_BUILD_PASSED=${{env.PIPELINE_BUILD_PASSED}}"
+ echo "PIPELINE_LINT_PASSED=${{env.PIPELINE_LINT_PASSED}}"
+ echo "PIPELINE_DEB_PASSED=${{env.PIPELINE_DEB_PASSED}}"
+ echo "PIPELINE_RPM_PASSED=${{env.PIPELINE_RPM_PASSED}}"
+
+ // Run the result job to get successful/failed links and add a
+ // comment + vote to gerrit
+ script {{
+ build job: 'gerrit-pipeline-result', parameters: [
+ string(name: "GERRIT_PROJECT", value: "${{env.GERRIT_PROJECT}}"),
+ string(name: "GERRIT_CHANGE_NUMBER", value: "${{env.GERRIT_CHANGE_NUMBER}}"),
+ string(name: "GERRIT_PATCHSET_NUMBER", value: "${{env.GERRIT_PATCHSET_NUMBER}}"),
+ string(name: "GERRIT_BRANCH", value: "${{env.GERRIT_BRANCH}}"),
+ string(name: "GERRIT_HOST", value: "${{env.GERRIT_HOST}}"),
+ string(name: "GERRIT_PATCHSET_REVISION", value: "${{env.GERRIT_PATCHSET_REVISION}}"),
+ string(name: "GERRIT_PATCHSET_UPLOADER_NAME", value: "${{env.GERRIT_PATCHSET_UPLOADER_NAME}}"),
+ string(name: "GERRIT_PORT", value: "${{env.GERRIT_PORT}}"),
+ string(name: "GERRIT_REFSPEC", value: "${{env.GERRIT_REFSPEC}}"),
+ string(name: "PIPELINE_BUILD_URL", value: "${{env.BUILD_URL}}"),
+ ]
+ }}
+ }}
+ }}
+ // The end result is success if all started jobs were successful,
+ // and failed otherwise.
+ }}
+ }}
+ disabled: '{obj:disabled}'
+ retry-count: 3 # scm checkout
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 120
+ artifact-days-to-keep: -1
+ artifact-num-to-keep: -1
+ description: |
+ Pipeline of CI for patches sent to
+ <a href="https://gerrit.osmocom.org/#/q/status:open+project:{repos}">{repos}</a>
+ <br/><br/>
+ <b>Auto-generated using Jenkins Job Builder. DO NOT EDIT MANUALLY!</b>
+
+ scm:
+ - git:
+ url: '{obj:repos_url}'
+ credentials-id: d5eda5e9-b59d-44ba-88d2-43473cb6e42d
+ branches:
+ - $GERRIT_BRANCH
+ refspec: $GERRIT_REFSPEC
+ name:
+ choosing-strategy: gerrit
+ wipe-workspace: false
+ skip-tag: true
+ submodule:
+ recursive: true
+
+ triggers:
+ - gerrit:
+ trigger-on:
+ - patchset-created-event:
+ exclude-drafts: true
+ exclude-no-code-change: true
+ projects:
+ - project-compare-type: 'PLAIN'
+ project-pattern: '{obj:gerrit_project}'
+ branches:
+ - branch-compare-type: 'ANT'
+ branch-pattern: '**'
+ skip-vote:
+ successful: false
+ failed: false
+ unstable: false
+ notbuilt: false
+ silent: true # comment + vote is done in gerrit-pipeline-result.yml
+ escape-quotes: false
+ server-name: gerrit.osmocom.org
+
+- job-template:
+ name: 'gerrit-{repos}-build'
project-type: matrix
node: '{node}'
concurrent: '{obj:concurrent}'
@@ -300,7 +594,7 @@
artifact-days-to-keep: -1
artifact-num-to-keep: -1
description: |
- Pending gerrit code reviews of
+ Build job of CI for patches sent to
<a href="https://gerrit.osmocom.org/#/q/status:open+project:{repos}">{repos}</a>
<br/><br/>
<b>Auto-generated using Jenkins Job Builder. DO NOT EDIT MANUALLY!</b>
@@ -345,28 +639,13 @@
submodule:
recursive: true
- triggers:
- - gerrit:
- trigger-on:
- - patchset-created-event:
- exclude-drafts: true
- exclude-no-code-change: true
- projects:
- - project-compare-type: 'PLAIN'
- project-pattern: '{obj:gerrit_project}'
- branches:
- - branch-compare-type: 'ANT'
- branch-pattern: '**'
- skip-vote:
- successful: false
- failed: false
- unstable: false
- notbuilt: false
- silent: false
- escape-quotes: false
- no-name-and-email: false
- trigger-for-unreviewed-patches: true
- server-name: gerrit.osmocom.org
+ parameters:
+ - string:
+ name: GERRIT_BRANCH
+ description: set by gerrit-{repos} job
+ - string:
+ name: GERRIT_REFSPEC
+ description: set by gerrit-{repos} job
builders:
- shell: '{obj:cmd}'
diff --git a/jobs/master-builds-dahdi.yml b/jobs/master-builds-dahdi.yml
index a404487..e7c85b7 100644
--- a/jobs/master-builds-dahdi.yml
+++ b/jobs/master-builds-dahdi.yml
@@ -29,21 +29,9 @@
<b>Auto-generated using Jenkins Job Builder. DO NOT EDIT MANUALLY!</b>
scm:
- - git:
- basedir: 'dahdi-linux'
- url: 'https://gerrit.osmocom.org/dahdi-linux'
- credentials-id: d5eda5e9-b59d-44ba-88d2-43473cb6e42d
- branches:
- - 'origin/master'
- # When the gerrit git repository is configured to wipe-workspace, the
- # linux.git clone gets removed as well. Cloning the linux repo takes a
- # long time, so only clean both repos, don't remove them.
- wipe-workspace: false
- clean:
- before: true
- skip-tag: true
- submodule:
- recursive: true
+ # Let jenkins clone the big linux repository and keep it between builds
+ # (wipe-workspace is false). Clone dahdi-linux with code from gerrit in
+ # the shell part below.
- git:
basedir: 'linux'
url: 'https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git'
@@ -60,6 +48,12 @@
builders:
- shell: |
+ rm -rf dahdi-linux
+ git clone \
+ --depth=1 \
+ https://gerrit.osmocom.org/dahdi-linux
+ git -C dahdi-linux log --oneline
+
docker run \
--rm=true \
-e "KSRC=/linux" \
diff --git a/jobs/master-builds.yml b/jobs/master-builds.yml
index 7fc306f..3ef3f03 100644
--- a/jobs/master-builds.yml
+++ b/jobs/master-builds.yml
@@ -42,7 +42,8 @@
-v "$ARTIFACT_STORE:/artifact_store" \
-v "$HOME/.ssh:/home/build/.ssh:ro" \
docker_img: '$USER/debian-buster-jenkins'
- cmd: '{docker_run} {docker_img} /build/contrib/jenkins.sh'
+ timeout_cmd: '/usr/bin/timeout 30m'
+ cmd: '{docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins.sh'
wrappers: []
trigger:
email: jenkins-notifications@lists.osmocom.org laforge@gnumonks.org
@@ -76,7 +77,7 @@
git_base_url: https://gitea.osmocom.org/sdr
cmd: |
{docker_run} \
- {docker_img} sh -ex -c \
+ {docker_img} {timeout_cmd} sh -ex -c \
'autoreconf -fi; \
./configure; \
$MAKE $PARALLEL_MAKE; \
@@ -136,7 +137,7 @@
a1_name: arch
a1: !!python/tuple [arm-none-eabi, amd64]
combination_filter: '!(arch=="arm-none-eabi" && label=="FreeBSD_amd64")'
- cmd: '{docker_run} {docker_img} /build/contrib/jenkins_arch.sh "$arch"'
+ cmd: '{docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins_arch.sh "$arch"'
# Full triggers would be:
# trigger: >
# master-libosmo-abis, master-libosmo-netif, master-libosmo-sccp, master-openbsc,
@@ -203,7 +204,7 @@
-e IU="$IU" \
-e SMPP="$SMPP" \
-e MGCP="$MGCP" \
- {docker_img} /build/contrib/jenkins.sh
+ {docker_img} {timeout_cmd} /build/contrib/jenkins.sh
- osmo-bsc
- osmo-bsc-nat
@@ -232,15 +233,31 @@
{docker_run} \
-e FIRMWARE_VERSION="$FIRMWARE_VERSION" \
-e WITH_MANUALS="$WITH_MANUALS" \
- {docker_img} /build/contrib/jenkins_bts_model.sh "$BTS_MODEL"
+ {docker_img} {timeout_cmd} /build/contrib/jenkins_bts_model.sh "$BTS_MODEL"
- osmo-e1-hardware:
+ a2_name: JOB_TYPE
+ a2: !!python/tuple ["firmware", "gateware", "manuals", "software"]
+ cmd: |
+ DOCKER_IMG="{docker_img}"
+ case "$JOB_TYPE" in
+ firmware|gateware|manuals)
+ DOCKER_IMG="registry.osmocom.org/$USER/fpga-build"
+ docker pull "$DOCKER_IMG"
+ ;;
+ esac
+ {docker_run} \
+ -e JOB_TYPE="$JOB_TYPE" \
+ "$DOCKER_IMG" \
+ {timeout_cmd} /build/contrib/jenkins.sh --publish
+ - ice40-usbtrace:
+ git_base_url: https://gitea.osmocom.org/electronics
docker_img: 'registry.osmocom.org/$USER/fpga-build'
- cmd: 'docker pull {docker_img} && {docker_run} {docker_img} /build/contrib/jenkins.sh --publish'
+ cmd: 'docker pull {docker_img} && {docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins.sh --publish'
- no2bootloader:
git_base_url: https://github.com/no2fpga
docker_img: 'registry.osmocom.org/$USER/fpga-build'
- cmd: 'docker pull {docker_img} && {docker_run} {docker_img} /build/contrib/jenkins.sh --publish'
+ cmd: 'docker pull {docker_img} && {docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins.sh --publish'
email: jenkins-notifications@lists.osmocom.org laforge@gnumonks.org kredon@sysmocom.de 246tnt@gmail.com
- osmo-fl2k:
@@ -257,7 +274,7 @@
{docker_run} \
-e GTP="$GTP" \
-e WITH_MANUALS="$WITH_MANUALS" \
- {docker_img} /build/contrib/jenkins.sh
+ {docker_img} {timeout_cmd} /build/contrib/jenkins.sh
trigger: master-osmo-sgsn
- osmo-upf
@@ -270,7 +287,7 @@
node: 'osmocom-master-debian9'
- osmo-gsm-tester:
- cmd: '{docker_run} {docker_img} /build/contrib/jenkins-build-manuals.sh'
+ cmd: '{docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins-build-manuals.sh'
- osmo-hlr
@@ -300,7 +317,7 @@
{docker_run} \
-e IU="$IU" \
-e WITH_MANUALS="$WITH_MANUALS" \
- {docker_img} /build/contrib/jenkins.sh
+ {docker_img} {timeout_cmd} /build/contrib/jenkins.sh
- osmo-pcap
@@ -327,7 +344,7 @@
-e with_vty="$with_vty" \
-e with_dsp="$with_dsp" \
-e WITH_MANUALS="$WITH_MANUALS" \
- {docker_img} /build/contrib/jenkins.sh
+ {docker_img} {timeout_cmd} /build/contrib/jenkins.sh
- osmo-sgsn:
a1_name: IU
@@ -341,7 +358,7 @@
{docker_run} \
-e IU="$IU" \
-e WITH_MANUALS="$WITH_MANUALS" \
- {docker_img} /build/contrib/jenkins.sh
+ {docker_img} {timeout_cmd} /build/contrib/jenkins.sh
- osmo-sip-connector:
email: jenkins-notifications@lists.osmocom.org laforge@gnumonks.org holger@freyther.de
@@ -364,24 +381,24 @@
{docker_run} \
-e INSTR="$INSTR" \
-e WITH_MANUALS="$WITH_MANUALS" \
- {docker_img} /build/contrib/jenkins.sh
+ {docker_img} {timeout_cmd} /build/contrib/jenkins.sh
- osmocom-bb
- osmo-sysmon
- osmo-remsim
- osmo-asf4-dfu:
- cmd: '{docker_run} {docker_img} /build/contrib/jenkins.sh --publish'
+ cmd: '{docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins.sh --publish'
email: jenkins-notifications@lists.osmocom.org laforge@gnumonks.org ewild@sysmocom.de
- osmo-ccid-firmware:
- cmd: '{docker_run} {docker_img} /build/contrib/jenkins.sh --publish'
+ cmd: '{docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins.sh --publish'
email: jenkins-notifications@lists.osmocom.org laforge@gnumonks.org ewild@sysmocom.de
- simtrace2:
- cmd: '{docker_run} {docker_img} /build/contrib/jenkins.sh --publish'
+ cmd: '{docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins.sh --publish'
email: jenkins-notifications@lists.osmocom.org laforge@gnumonks.org kredon@sysmocom.de
- osmo-opencm3-projects:
git_base_url: https://gitea.osmocom.org/electronics
docker_img: '$USER/debian-bullseye-jenkins'
- cmd: '{docker_run} {docker_img} /build/contrib/jenkins.sh --publish'
+ cmd: '{docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins.sh --publish'
email: jenkins-notifications@lists.osmocom.org laforge@gnumonks.org kredon@sysmocom.de
- osmo-e1d
@@ -401,12 +418,30 @@
- pysim:
concurrent: false
- slave_axis: !!python/tuple [simtester]
- cmd: 'PUBLISH=1 WITH_MANUALS=1 /usr/bin/timeout 30m ./contrib/jenkins.sh'
+ # OS#5767: only build2-deb9build-ansible is able to upload docs
+ slave_axis: !!python/tuple [simtester,osmocom-master,build2-deb9build-ansible]
+ a2_name: JOB_TYPE
+ a2: !!python/tuple ["test", "pylint", "docs"]
+ combination_filter: >
+ (JOB_TYPE == "test" && label == "simtester") ||
+ (JOB_TYPE == "pylint" && label == "osmocom-master") ||
+ (JOB_TYPE == "docs" && label == "build2-deb9build-ansible")
+ cmd: |
+ case "$JOB_TYPE" in
+ "test")
+ {timeout_cmd} ./contrib/jenkins.sh
+ ;;
+ *)
+ {docker_run} \
+ -e JOB_TYPE="$JOB_TYPE" \
+ {docker_img} \
+ {timeout_cmd} /build/contrib/jenkins.sh
+ ;;
+ esac
- sysmo-usim-tool:
concurrent: false
- cmd: cd ./tests; ./run-tests
+ cmd: cd ./tests; {timeout_cmd} ./run-tests
slave_axis: !!python/tuple [simtester]
git_base_url: https://gitea.sysmocom.de/sysmocom
@@ -491,6 +526,8 @@
resolve-relative-paths: true
- trigger:
project: '{obj:trigger}'
+ - trigger:
+ project: 'Osmocom_OBS_master_obs.osmocom.org'
- email:
recipients: '{obj:email}'
send-to-individuals: true
diff --git a/jobs/osmocom-api.yml b/jobs/osmocom-api.yml
new file mode 100644
index 0000000..f91779d
--- /dev/null
+++ b/jobs/osmocom-api.yml
@@ -0,0 +1,89 @@
+- job:
+ concurrent: false
+ description: This job builds the doxygen API documentation for latest libosmo*
+ and uploads it to <a href="https://downloads.osmocom.org/api/latest/"
+ >https://downloads.osmocom.org/api/latest/</a>
+ name: Osmocom_API
+ node: osmocom-master-debian9
+ project-type: freestyle
+ publishers:
+ - email:
+ notify-every-unstable-build: true
+ recipients: jenkins-notifications@lists.osmocom.org
+ triggers:
+ - timed: "H H * * *"
+ wrappers:
+ - ssh-agent-credentials:
+ users:
+ - api@osmocom
+ parameters:
+ - string:
+ name: BRANCH
+ description: osmo-ci.git branch
+ default: 'master'
+ scm:
+ - git:
+ branches:
+ - '$BRANCH'
+ url: https://gerrit.osmocom.org/osmo-ci
+ git-config-name: 'Jenkins Builder'
+ git-config-email: 'jenkins@osmocom.org'
+ wipe-workspace: true
+ builders:
+ - shell: |
+ # Repositories for which doxygen documentation will be generated and
+ # uploaded, also dependencies which need to be built
+ repos_api="
+ libosmocore
+ libosmo-abis
+ libosmo-dsp
+ libosmo-netif
+ libosmo-sccp
+ osmo-gmr"
+
+ # Source common.sh from osmo-ci.git for osmo_git_clone_url()
+ . scripts/common.sh
+
+ # Put git repos and install data in a subdir, so it isn't in the root
+ # of the cloned osmo-ci.git repository
+ mkdir _osmocom_api
+ cd _osmocom_api
+
+ # Prepare pkgconfig path
+ export PKG_CONFIG_PATH=$PWD/install/lib/pkgconfig
+ mkdir -p "$PKG_CONFIG_PATH"
+
+ # Prepare known_hosts
+ cat > known_hosts <<EOF
+ [ftp.osmocom.org]:48 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDDgQ9HntlpWNmh953a2Gc8NysKE4orOatVT1wQkyzhARnfYUerRuwyNr1GqMyBKdSI9amYVBXJIOUFcpV81niA7zQRUs66bpIMkE9/rHxBd81SkorEPOIS84W4vm3SZtuNqa+fADcqe88Hcb0ZdTzjKILuwi19gzrQyME2knHY71EOETe9Yow5RD2hTIpB5ecNxI0LUKDq+Ii8HfBvndPBIr0BWYDugckQ3Bocf+yn/tn2/GZieFEyFpBGF/MnLbAAfUKIdeyFRX7ufaiWWz5yKAfEhtziqdAGZaXNaLG6gkpy3EixOAy6ZXuTAk3b3Y0FUmDjhOHllbPmTOcKMry9
+ [ftp.osmocom.org]:48 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPdWn1kEousXuKsZ+qJEZTt/NSeASxCrUfNDW3LWtH+d8Ust7ZuKp/vuyG+5pe5pwpPOgFu7TjN+0lVjYJVXH54=
+ [ftp.osmocom.org]:48 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK8iivY70EiR5NiGChV39gRLjNpC8lvu1ZdHtdMw2zuX
+ EOF
+
+ # Clone and build the repositories
+ for i in $repos_api; do
+ git clone "$(osmo_git_clone_url "$i")"
+ cd "$i"
+ autoreconf -fi
+ ./configure \
+ --prefix=$PWD/../install \
+ --with-systemdsystemunitdir=no
+ make $PARALLEL_MAKE install
+ cd ..
+ done
+
+ # Upload all docs
+ for i in $repos_api; do
+ if ! [ -d "$i"/doc ]; then
+ # e.g. libosmo-abis is built as dependency for others but doesn't
+ # have its own doxygen documentation as of writing
+ continue
+ fi
+
+ rsync \
+ -avz \
+ --delete \
+ -e "ssh -o 'UserKnownHostsFile=known_hosts' -p 48" \
+ ./"$i"/doc/ \
+ api@ftp.osmocom.org:web-files/latest/"$i"/
+ done
diff --git a/jobs/osmocom-build-tags-against-master.yml b/jobs/osmocom-build-tags-against-master.yml
index 22ac9ae..351aff2 100644
--- a/jobs/osmocom-build-tags-against-master.yml
+++ b/jobs/osmocom-build-tags-against-master.yml
@@ -25,7 +25,7 @@
- git:
branches:
- '$BRANCH'
- url: https://git.osmocom.org/osmo-ci
+ url: https://gerrit.osmocom.org/osmo-ci
git-config-name: 'Jenkins Builder'
git-config-email: 'jenkins@osmocom.org'
triggers:
diff --git a/jobs/osmocom-depcheck.yml b/jobs/osmocom-depcheck.yml
index 1eadd0e..4ca04e2 100644
--- a/jobs/osmocom-depcheck.yml
+++ b/jobs/osmocom-depcheck.yml
@@ -26,7 +26,7 @@
name: GIT_URL_PREFIX
description: |
Where to clone the sources from
- default: 'https://git.osmocom.org/'
+ default: 'https://gerrit.osmocom.org/'
- bool:
name: BUILD
description: |
@@ -65,7 +65,7 @@
- git:
branches:
- '$BRANCH'
- url: https://git.osmocom.org/osmo-ci
+ url: https://gerrit.osmocom.org/osmo-ci
git-config-name: 'Jenkins Builder'
git-config-email: 'jenkins@osmocom.org'
diff --git a/jobs/osmocom-list-commits.yml b/jobs/osmocom-list-commits.yml
index c9995c6..65bec81 100644
--- a/jobs/osmocom-list-commits.yml
+++ b/jobs/osmocom-list-commits.yml
@@ -26,7 +26,7 @@
- git:
branches:
- '$BRANCH'
- url: https://git.osmocom.org/osmo-ci
+ url: https://gerrit.osmocom.org/osmo-ci
git-config-name: 'Jenkins Builder'
git-config-email: 'jenkins@osmocom.org'
triggers:
diff --git a/jobs/osmocom-obs-check-builders.yml b/jobs/osmocom-obs-check-builders.yml
new file mode 100644
index 0000000..f378ec1
--- /dev/null
+++ b/jobs/osmocom-obs-check-builders.yml
@@ -0,0 +1,28 @@
+---
+- project:
+ name: Osmocom_OBS_check_builders
+ jobs:
+ - Osmocom_OBS_check_builders
+
+- job-template:
+ name: 'Osmocom_OBS_check_builders'
+ project-type: freestyle
+ defaults: global
+ description: |
+ Verify that the expected amount of builders are connected to
+ obs.osmocom.org.
+ builders:
+ - shell: |
+ scripts/obs/check_builders.sh
+ scm:
+ - git:
+ branches:
+ - 'origin/master'
+ url: https://gerrit.osmocom.org/osmo-ci
+ triggers:
+ - timed: "@hourly"
+ node: obs
+ publishers:
+ - email:
+ notify-every-unstable-build: true
+ recipients: 'jenkins-notifications@lists.osmocom.org'
diff --git a/jobs/osmocom-obs.yml b/jobs/osmocom-obs.yml
index 06b4ecb..25e48c3 100644
--- a/jobs/osmocom-obs.yml
+++ b/jobs/osmocom-obs.yml
@@ -4,6 +4,8 @@
jobs:
- Osmocom_OBS_{type}_{server}
type:
+ - master:
+ conflict_version: ""
- nightly:
# For nightly we don't provide ABI compatibility, make sure packages
# from different build dates are not mixed by accident
@@ -11,14 +13,13 @@
- latest:
conflict_version: ""
server:
- - build.opensuse.org:
- proj: "network:osmocom"
- obs.osmocom.org:
proj: "osmocom"
- job-template:
name: 'Osmocom_OBS_{type}_{server}'
project-type: freestyle
+ concurrent: false
defaults: global
description: |
See <a href="https://osmocom.org/projects/cellular-infrastructure/wiki/Binary_Packages">Wiki: binary packages</a>
@@ -44,7 +45,7 @@
- git:
branches:
- '$BRANCH'
- url: https://git.osmocom.org/osmo-ci
+ url: https://gerrit.osmocom.org/osmo-ci
git-config-name: 'Jenkins Builder'
git-config-email: 'jenkins@osmocom.org'
triggers:
diff --git a/jobs/ttcn3-testsuites-kernel-git.yml b/jobs/ttcn3-testsuites-kernel-git.yml
index 9d644ec..e378122 100644
--- a/jobs/ttcn3-testsuites-kernel-git.yml
+++ b/jobs/ttcn3-testsuites-kernel-git.yml
@@ -61,7 +61,7 @@
description: |
Version of the Osmocom stack to test against the kernel. Either
<code>latest</code> for the latest release version, or
- <code>master</code> for HEAD of git.osmocom.org.
+ <code>master</code> for HEAD of Osmocom git repositories.
default: 'latest'
- string:
name: BRANCH
diff --git a/jobs/ttcn3-testsuites.yml b/jobs/ttcn3-testsuites.yml
index 9eef9f8..17a0942 100644
--- a/jobs/ttcn3-testsuites.yml
+++ b/jobs/ttcn3-testsuites.yml
@@ -5,7 +5,7 @@
disabled: false
description: |
Run the a testsuite from
- <a href="https://git.osmocom.org/docker-playground/">
+ <a href="https://gitea.osmocom.org/osmocom/docker-playground">
docker-playground.git</a>.<br>
If this job ends in <code>-latest</code>, the job runs the latest
stable versions of the Osmocom stack. Otherwise the nightly builds.<br>
@@ -146,13 +146,9 @@
- ttcn3-upf-test:
blocking: "^(ttcn3|TTCN3-.*)-upf-test.*"
timer: 00 13 * * *
- disabled: true
- ttcn3-hnbgw-test:
blocking: "^(ttcn3|TTCN3-.*)-hnbgw-test.*"
timer: 40 13 * * *
- - ttcn3-hnbgw-test-pfcp:
- blocking: "^(ttcn3|TTCN3-.*)-hnbgw-test.*"
- timer: 40 15 * * *
- ttcn3-fr-test:
blocking: "^(ttcn3|TTCN3-.*)-fr-test.*"
timer: 40 10 * * *
diff --git a/jobs/update-osmo-ci-on-slaves.yml b/jobs/update-osmo-ci-on-slaves.yml
index 41181cd..39afb46 100644
--- a/jobs/update-osmo-ci-on-slaves.yml
+++ b/jobs/update-osmo-ci-on-slaves.yml
@@ -30,6 +30,7 @@
skip-tag: true
branches:
- 'origin/master'
+ wipe-workspace: true
triggers:
- pollscm:
diff --git a/lint/checkpatch/checkpatch_osmo.sh b/lint/checkpatch/checkpatch_osmo.sh
index b0c3985..bc45f85 100755
--- a/lint/checkpatch/checkpatch_osmo.sh
+++ b/lint/checkpatch/checkpatch_osmo.sh
@@ -61,13 +61,13 @@ exclude_paths_project() {
# * EMBEDDED_FUNCTION_NAME: often __func__ isn't used, arguably not much benefit in changing this when touching code
# * EXECUTE_PERMISSIONS: not followed, files need to be executable: git-version-gen, some in debian/
# * FILE_PATH_CHANGES: we don't use a MAINTAINERS file
-# * FUNCTION_WITHOUT_ARGS: not followed: warns about func() instead of func(void)
# * GLOBAL_INITIALISERS: we initialise globals to NULL for talloc ctx (e.g. *tall_lapd_ctx = NULL)
# * IF_0: used intentionally
# * INITIALISED_STATIC: we use this, see also http://lkml.iu.edu/hypermail/linux/kernel/0808.1/2235.html
# * LINE_CONTINUATIONS: false positives
# * LINE_SPACING: we don't always put a blank line after declarations
# * LONG_LINE*: should be 120 chars, but exceptions are done often so don't fail here
+# * MACRO_WITH_FLOW_CONTROL: not followed
# * MISSING_SPACE: warns about breaking strings at space characters, not useful for long strings of hex chars
# * PREFER_DEFINED_ATTRIBUTE_MACRO: macros like __packed not defined in libosmocore
# * PREFER_FALLTHROUGH: pseudo keyword macro "fallthrough" is not defined in libosmocore
@@ -99,7 +99,6 @@ $SCRIPT_DIR/checkpatch.pl \
--ignore EMBEDDED_FUNCTION_NAME \
--ignore EXECUTE_PERMISSIONS \
--ignore FILE_PATH_CHANGES \
- --ignore FUNCTION_WITHOUT_ARGS \
--ignore GLOBAL_INITIALISERS \
--ignore IF_0 \
--ignore INITIALISED_STATIC \
@@ -108,6 +107,7 @@ $SCRIPT_DIR/checkpatch.pl \
--ignore LONG_LINE \
--ignore LONG_LINE_COMMENT \
--ignore LONG_LINE_STRING \
+ --ignore MACRO_WITH_FLOW_CONTROL \
--ignore MISSING_SPACE \
--ignore PREFER_DEFINED_ATTRIBUTE_MACRO \
--ignore PREFER_FALLTHROUGH \
diff --git a/lint/checkpatch/spelling.txt b/lint/checkpatch/spelling.txt
index 7b6a012..7f0745c 100644
--- a/lint/checkpatch/spelling.txt
+++ b/lint/checkpatch/spelling.txt
@@ -50,7 +50,6 @@ acient||ancient
acitions||actions
acitve||active
acknowldegement||acknowledgment
-acknowledgement||acknowledgment
ackowledge||acknowledge
ackowledged||acknowledged
acording||according
diff --git a/lint/lint_diff.sh b/lint/lint_diff.sh
index 7ae186f..662bdd3 100755
--- a/lint/lint_diff.sh
+++ b/lint/lint_diff.sh
@@ -68,7 +68,8 @@ if [ "$ERROR" = 1 ]; then
"$GERRIT_HOST" \
gerrit \
review \
- "$GERRIT_PATCHSET_REVISION" \
+ --project "$GERRIT_PROJECT" \
+ "$GERRIT_CHANGE_NUMBER,$GERRIT_PATCHSET_NUMBER" \
--json \
< gerrit_report.json
fi
diff --git a/qemu-kvm/ggsn-kernel-gtp/postinst.sh b/qemu-kvm/ggsn-kernel-gtp/postinst.sh
index cbee562..27df200 100755
--- a/qemu-kvm/ggsn-kernel-gtp/postinst.sh
+++ b/qemu-kvm/ggsn-kernel-gtp/postinst.sh
@@ -48,7 +48,7 @@ apt-get clean
# add osmocom:nightly feed + install libosmocore-dev
apt-key add /tmp/postinst/Release.key
-echo "deb http://download.opensuse.org/repositories/network:/osmocom:/nightly/Debian_9.0/ ./" > /etc/apt/sources.list.d/osmocom-nightly.list
+echo "deb http://downloads.osmocom.org/packages/osmocom:/nightly/Debian_9.0/ ./" > /etc/apt/sources.list.d/osmocom-nightly.list
apt-get update
apt-get install -y --no-install-recommends \
libosmocore-dev
diff --git a/qemu-kvm/ggsn-kernel-gtp/postinst/Release.key b/qemu-kvm/ggsn-kernel-gtp/postinst/Release.key
index a737316..ecca084 100644
--- a/qemu-kvm/ggsn-kernel-gtp/postinst/Release.key
+++ b/qemu-kvm/ggsn-kernel-gtp/postinst/Release.key
@@ -1,20 +1,26 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: GnuPG v1.4.5 (GNU/Linux)
-mQENBFJBt/wBCADAht3d/ilNuyzaXYw/QwTRvmjyoDvfXw+H/3Fvk1zlDZoiKPPc
-a1wCVBINUZl7vYM2OXqbJwYa++JP2Q48xKSvC6thbRc/YLievkbcvTemf7IaREfl
-CTjoYpoqXHa9kHMw1aALDm8CNU88jZmnV7v9L6hKkbYDxie+jpoj7D6B9JlxgNJ4
-5dQyRNsFGVcIl4Vplt1HyGc5Q5nQI/VgS2rlF/IOXmhRQBc4LEDdU8R2IKnkU4ee
-S7TWanAigGAQhxGuCkS39/CWzc1DhLhjlNhBl/+RTPejkqJtAy00ZLps3+RqUN1Y
-CU/Fsr7aRlYVGqQ/BlptwV0XQ2VVYJX2oEBBABEBAAG0MG5ldHdvcmsgT0JTIFBy
-b2plY3QgPG5ldHdvcmtAYnVpbGQub3BlbnN1c2Uub3JnPokBPAQTAQIAJgUCXm/4
-pgIbAwUJEEzwqgYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEGLrGgkXKA3f/1AH
-/A7WVSpfM4wV/DMqZPTsSjChB4JyDotxpV7qHZzBC5aaP2dINZyi9PayIwZWbvCY
-VKvt+Fw8oCGC9F9mdh10Xe+ElHeVNSihzABPuu1RkRkb1nvkymScy0yxydodYOBi
-K4WQ+BhpijXWmYvOekIwbS5Hi9BHpfgK4TinK0xsvh1bVLeQJ8YjrnNFIAR2CnBa
-X7Y72Up/kKL08DdQzuS+mKrJtAQlGMtIsukWC2ajYQMkNwm8Gvhpn8za113dCkBW
-XAFnlQqQobKwC7b19QgEtJI/YpGSrRc6WaZxPyAjscbWQlFEAB900sVj4BWT55ig
-7O2uSdsCVhTuU7T0ztwsgvmIRgQTEQIABgUCUkG3/AAKCRA7MBG3a51lIzhdAJ9v
-d6XPffMZRcCGgDEY5OaTn/MsCQCgrXbeZpFJgnirSrc8rRonvzYFiF4=
-=/Tek
+mQENBGKzE1QBCADFcM3ZzggvgxNRNNqDGWf5xIDAiK5qzFLdGes7L6F9VCHdaPy0
+RAOB5bFb/Q1tSDFNEBLtaauXKz+4iGL6qMVjZcyjzpB5w4jKN+kkrFRhjDNUv/SH
+BX6d+P7v5WBGSNArNgA8D1BGzckp5a99EZ0okMJFEqIcN40PD6OGugpq5XnVV5Nk
+e93fLa2Cu8vhFBcVn6CuHeEhsmuMf6NLbQRNfNNCEEUYaZn7beMYtpZ7t1djsKx5
+1xGm50OzI22FLu8lELQ9d7qMVGRG3WHYawX9BDteRybiyqxfwUHm1haWazRJtlGt
+UWyzvwAb80BK1J2Nu5fbAa3w5CoEPAbUuCyrABEBAAG0JW9zbW9jb20gT0JTIFBy
+b2plY3QgPG9zbW9jb21Ab3Ntb2NvbT6JAVQEEwEIAD4WIQRrKp83ktFetw1Oao+G
+pzC2U3JZcwUCYrMV4wIbAwUJBB6yjwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAK
+CRCGpzC2U3JZc4FRCACQQkKIrnvQ7n2u7GSmyVZa3I+oLoFXSGqaGyey5TW/nrMm
+vFDKU3qliHiuNSmUY35SnAhXUsvqOYppxVRoO1MLrqUvzMOnIWqkJpf8mtjGUnsW
+jyVeto7Rsjs75y2i1Hk+e7ljb/V65J3NlfrfEYWbqR9AKd53ReNXTdrQ0J05A38N
+GdI4Ld/2lNISAwaBmGhqdeKsLHpQw/JERU1TApVJR1whFiIwDF1rOCg9GPnNKIk7
+yRZdK267XzztrainX/cbPILyzUZEDhYs6wQuyACyQ1YUxZIxrwVfk7PMNay8CrLH
+z42B73Ne5IAj8+op/3iJafFONLm7YXiDUFN+QDYAiQEzBBMBCAAdFiEExoiYhHND
+S7aVYlnqa51NyAUyjdsFAmKzE1UACgkQa51NyAUyjdvuZgf+OXmr//i7u7Gg7eWB
+7e0qUsyCId9lXS8J437x3K6ciJfD7/6RSy8TFW5Nglm/uSkbyq582I8t+SoOirMD
+E6cg9U/5+h5s46bAf+Kd2XS/6tLGeNLM18i4el8CP06NpFzDrsKu76uYFpyRiiHD
+otBdtgxeLJ83LugGfZslF+/5cigJkAJMhAdVvGO8h85R6fba8ZSOKtMKkaQRfi76
+nhyOrJPlLuS+DLEnHwdkOFgtKnxHdjM97K+Tx0gisb6uwaWroXfSLnhP8RTLLZZy
+Z+noU1Hw3c+mn4c/NYbcC/uwHYHKRzuf9gHnQ3dGgv0Z5sbeLRVo92hjGj7Ftlyd
+4hmKBg==
+=HxK4
-----END PGP PUBLIC KEY BLOCK-----
diff --git a/qemu-kvm/ggsn-kernel-gtp/postinst/authorized_keys b/qemu-kvm/ggsn-kernel-gtp/postinst/authorized_keys
index 817a537..6a7cc8a 100644
--- a/qemu-kvm/ggsn-kernel-gtp/postinst/authorized_keys
+++ b/qemu-kvm/ggsn-kernel-gtp/postinst/authorized_keys
@@ -1,2 +1 @@
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCmaAZWIdZZjFZHq7Xm5OCUz/Cm46haad+pm2iZ2uSQCIXubjN8HGz1XujBMH1DHhSDK+L0JOBEtxHaY4JR7F177V5R8o50VNwmtTrTaNBPS8WPyVBDuHjjDVSZA+d6QT3Y+3X/FhfoFKmGc2a8MZRmE8yDGW484y7MTZ4XBHqy5vWnnywxWCZm7kVqNccM4oQ4tPWlMTxk6+tgWC4wvk3axsUVUm0ak/WKuZLoX+jBWlsWXR4uwaOEqblpNMv98CxjLgpjEBTvZxgrgY35eAm11S9lmvxFSjLIZA1V/2huAA/9MSNvI4dX4ZQPHfLCsMqFjhkIY0hvFK2uX8hlxHhOR6GkOq3n3MdcKCMI+JAISCP6g++DdQO5wzfOLeSdpTETNpIxeFkjQLS+HnuAcgjRK6O7czI+xYfCiBcaoslsTIYi33Pg7gPuDjwAyVdKqQPVdYPluR8W5+FT1lFtf1jwIB3jgt+v9q+r6d4LfkOCAU17yxG6b40P+FqK9O7X+iUkva/lbghdAw7DimczDIuEMpVVVL3/WhmCIMrfJPG/RtMdoczcAw4q4FxBretZR1Ke4AomUvUklFwfMtLqijxBJN2Owpy7jtqggs+v6cUX54fKh7/SeVrgyIgeUtgJp6lqaxlMao8SoR59YoqLqZueYOp9QxYujA70AGoMMwxsEw== laforge@nataraja
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDAX61B6rs76JfKmtqqFoR6o9KAP1/WVz9/COtjNrBDc6IfqDy46ZTlKwud4Nm3O3QF5cbdxx/hnOvep4SsLTArYuD8EcBtMoKhh8+c6Y30wR9l3kUWen4JEUJjFOCX/jr2SG4tikF/0S2zUJmILyFMw53pc1KSOmEewiWI8Z6U15ntLWEljWg+BWPs25kgQ5qeA3TO9ygNPFUrEQDGJlzLVubENwZOweTMll89bFaD+6jbjBG4ViPsSlzSvan9CThrTCrVCSmTObfOz3cEci0Wvgh1mBuOyfo5GyXQ7V8tHTF9AU5lGLybeyiXOki3f+bf9da8rnHuswR6ap8VsXrra2QFr40ro7ACCc7TC5Sa62c2Y8pC2Buuv3WcZ1x5PG7gRbU3iZqR0r7HeQGP/lVD0Z9uSV+9dSvg4uhsSS59XzRoI4gElHBiSGdxm0EKB+WI/0y+G9z10qyZggDPHqD67BY1Qpi5uQxQrFiOc5eadC0ucjjA2o4KqEid2cHI4qu8YLBns5X+GE1BB9o/ix5eSYFTBju0FlgzLaYEmo6zIURpKKvKfPEgpU/ZYWRX1SaUlohXaJxB60k87eyHNDCFwyIFDklZ8LB0WBvTJGqJaDmRcR9Idz4dP8UWCHD0hiShSLKW5sOFzkJflFRmaTWmBV8W/mwc9lVJrYuaSd7S4w== laforge@sunbeam.rsa
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF+89UJw1295JWouFrLnIVbxsX0i5X46J16olfb5gqyL laforge@nataraja
diff --git a/scripts/common.sh b/scripts/common.sh
index 23a3380..23d58e7 100644
--- a/scripts/common.sh
+++ b/scripts/common.sh
@@ -1,7 +1,7 @@
#!/bin/sh
# Various functions and variables used in multiple osmo-ci shell scripts
OSMO_CI_DIR="$(realpath "$(dirname "$0")/..")"
-OSMO_GIT_URL="https://git.osmocom.org"
+OSMO_GIT_URL_GITEA="https://gitea.osmocom.org"
OSMO_GIT_URL_GERRIT="https://gerrit.osmocom.org"
# Osmocom repositories of which we want to build release tarballs automatically, and list the current versions at
@@ -110,23 +110,21 @@ osmo_git_last_tags() {
echo "$ret" | cut -d/ -f 3
}
-# Pass all arguments to "git clone", but write the current date and time before the clone and on failure.
-# This helps analyzing errors with git.osmocom.org (OS#4083).
-osmo_git_clone_date() {
- date "+%Y-%m-%d %H:%M:%S"
- if ! git clone "$@"; then
- date "+%Y-%m-%d %H:%M:%S"
- exit 1
- fi
-}
-
-# Print git clone URL for an Osmocom git repository. Prefer the gerrit clone URL, because cloning from the regular URL
-# sometimes results in the "garbage at end of loose object" error (OS#4083).
+# Echo git clone URL for an Osmocom git repository. For projects developed on
+# gerrit, use the gerrit URL to avoid the mirror sync delay, for other
+# repositories use the gitea URL.
+# https://osmocom.org/projects/cellular-infrastructure/wiki/Git_infrastructure
# $1: Osmocom project (e.g. "osmo-hlr")
osmo_git_clone_url() {
case "$1" in
- rtl-sdr|osmo-fl2k|libosmo-dsp|libgtpnl|libasn1c|libusrp|libsmpp34)
- echo "$OSMO_GIT_URL"/"$1"
+ libgtpnl|libasn1c|libsmpp34)
+ echo "$OSMO_GIT_URL_GITEA"/cellular-infrastructure/"$1"
+ ;;
+ rtl-sdr|osmo-fl2k|libosmo-dsp|libusrp)
+ echo "$OSMO_GIT_URL_GITEA"/sdr/"$1"
+ ;;
+ osmo-gmr)
+ echo "$OSMO_GIT_URL_GITEA"/satellite/"$1"
;;
*)
echo "$OSMO_GIT_URL_GERRIT"/"$1"
@@ -160,7 +158,7 @@ docker_images_require() {
if [ -d "_docker_playground" ]; then
git -C _docker_playground fetch
else
- git clone https://git.osmocom.org/docker-playground/ _docker_playground
+ git clone https://gerrit.osmocom.org/docker-playground/ _docker_playground
fi
cd _docker_playground
diff --git a/scripts/jenkins-gerrit/pipeline_summary.py b/scripts/jenkins-gerrit/pipeline_summary.py
new file mode 100755
index 0000000..53c0f4a
--- /dev/null
+++ b/scripts/jenkins-gerrit/pipeline_summary.py
@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import argparse
+import io
+import json
+import re
+import urllib.request
+
+jenkins_url = "https://jenkins.osmocom.org"
+re_start_build = re.compile("Starting building: gerrit-[a-zA-Z-_0-9]* #[0-9]*")
+re_result = re.compile("^PIPELINE_[A-Z]*_PASSED=[01]$")
+re_job_type = re.compile("JOB_TYPE=([a-zA-Z-_0-9]*),")
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(
+ description="Get a summary of failed / successful builds from the CI"
+ " pipeline we run for patches submitted to gerrit.")
+ parser.add_argument("build_url",
+ help="$BUILD_URL of the pipeline job, e.g."
+ " https://jenkins.osmocom.org/jenkins/job/gerrit-osmo-bsc-nat/17/")
+ parser.add_argument("-o", "--output", help="output json file")
+ parser.add_argument("-n", "--notify-on-success", action="store_true",
+ help="always indicate in json that the owner should be"
+ " notified via mail, not only on failure")
+ return parser.parse_args()
+
+
+def stage_from_job_name(job_name):
+ if job_name == "gerrit-pipeline-result":
+ # The job that runs this script. Don't include it in the summary.
+ return None
+ if job_name == "gerrit-lint":
+ return "lint"
+ if job_name == "gerrit-binpkgs-deb":
+ return "deb"
+ if job_name == "gerrit-binpkgs-rpm":
+ return "rpm"
+ if job_name.endswith("-build"):
+ return "build"
+ assert False, f"couldn't figure out stage from job_name: {job_name}"
+
+
+def parse_pipeline(build_url):
+ """ Parse started jobs and result from the pipeline log.
+ :returns: a dict that looks like:
+ {"build": {"name": "gerrit-osmo-bsc-nat-build", id=7,
+ "passed": True, "url": "https://..."},
+ "lint": {...},
+ "deb": {...},
+ "rpm: {...}} """
+ global re_start_build
+ global re_result
+ global jenkins_url
+ ret = {}
+
+ url = f"{build_url}/consoleText"
+ with urllib.request.urlopen(url) as response:
+ for line in io.TextIOWrapper(response, encoding='utf-8'):
+ # Parse start build lines
+ for match in re_start_build.findall(line):
+ job_name = match.split(" ")[2]
+ job_id = int(match.split(" ")[3].replace("#", ""))
+ job_url = f"{jenkins_url}/jenkins/job/{job_name}/{job_id}"
+ stage = stage_from_job_name(job_name)
+ if stage:
+ ret[stage] = {"url": job_url, "name": job_name, "id": job_id}
+
+ # Parse result lines
+ if re_result.match(line):
+ stage = line.split("_")[1].lower()
+ assert stage in ret, f"found result for stage {stage}, but" \
+ " didn't find where it was started. The" \
+ " re_start_build regex probably needs to be adjusted" \
+ " to match the related gerrit-*-build job."
+ passed = line.split("=")[1].rstrip() == "1"
+ ret[stage]["passed"] = passed
+
+ return ret
+
+
+def parse_build_matrix(job):
+ """ Parse started jobs and result from the matrix of the build job. Usually
+ it is only one job, but for some projects we build for multiple arches
+ (x86_64, arm) or build multiple times with different configure flags.
+ :param job: "build" dict from parse_pipeline()
+ :returns: a list of jobs in the matrix, looks like:
+ [{"passed": True, "url": "https://..."}, ...]
+ """
+ global jenkins_url
+
+ ret = []
+ url = f"{job['url']}/consoleFull"
+ with urllib.request.urlopen(url) as response:
+ for line in io.TextIOWrapper(response, encoding='utf-8'):
+ if " completed with result " in line:
+ url = line.split("<a href='", 1)[1].split("'", 1)[0]
+ url = f"{jenkins_url}{url}{job['id']}"
+ result = line.split(" completed with result ")[1].rstrip()
+ passed = result == "SUCCESS"
+ ret += [{"passed": passed, "url": url}]
+ return ret
+
+
+def jobs_for_summary(pipeline, build_matrix):
+ """ Sort the jobs from pipeline and build matrix into passed/failed lists.
+ :returns: a dict that looks like:
+ {"passed": [{"stage": "build", "url": "https://..."}, ...],
+ "failed": [...]} """
+ ret = {"passed": [], "failed": []}
+
+ # Build errors are most interesting, display them first
+ for job in build_matrix:
+ category = "passed" if job["passed"] else "failed"
+ ret[category] += [{"stage": "build", "url": job["url"]}]
+
+ # Hide the build matrix job (we show the jobs started by it instead), as
+ # long as there is at least one failed started job when the matrix failed
+ matrix_failed = "build" in pipeline and not pipeline["build"]["passed"]
+ show_build_matrix_job = matrix_failed and not ret["failed"]
+
+ # Add jobs from the pipeline
+ for stage, job in pipeline.items():
+ if stage == "build" and not show_build_matrix_job:
+ continue
+ category = "passed" if job["passed"] else "failed"
+ ret[category] += [{"stage": stage, "url": job["url"]}]
+
+ return ret
+
+
+def get_job_short_name(job):
+ """ :returns: a short job name, usually the stage (lint, deb, rpm, build).
+ Or in case of build a more useful name like the JOB_TYPE part
+ of the URL if it is found. For osmo-e1-hardware it could be
+ one of: manuals, gateware, firmware, software """
+ global re_job_type
+ stage = job["stage"]
+
+ if stage == "build":
+ match = re_job_type.search(job["url"])
+ if match:
+ return match.group(1)
+
+ return stage
+
+
+def get_jobs_list_str(jobs):
+ ret = ""
+ for job in jobs:
+ ret += f" [{get_job_short_name(job)}] {job['url']}/consoleFull\n"
+ return ret
+
+
+def get_pipeline_summary(build_url, notify_on_success):
+ """ Generate a summary of failed and successful builds for gerrit.
+ :returns: a dict that is expected by gerrit's set-review api, e.g.
+ {"tag": "jenkins",
+ "message": "...",
+ "labels": {"Code-Review": -1},
+ "notify": "OWNER"} """
+ summary = ""
+ pipeline = parse_pipeline(build_url)
+
+ build_matrix = []
+ if "build" in pipeline:
+ build_matrix = parse_build_matrix(pipeline["build"])
+
+ jobs = jobs_for_summary(pipeline, build_matrix)
+
+ if jobs["failed"]:
+ summary += f"{len(jobs['failed'])} failed:\n"
+ summary += get_jobs_list_str(jobs["failed"])
+ summary += "\n"
+
+ summary += f"{len(jobs['passed'])} passed:\n"
+ summary += get_jobs_list_str(jobs["passed"])
+
+ if "build" in pipeline and "deb" in pipeline and "rpm" in pipeline and \
+ not pipeline["build"]["passed"] and pipeline["deb"]["passed"] \
+ and pipeline["rpm"]["passed"]:
+ summary += "\n"
+ summary += "The build job(s) failed, but deb/rpm jobs passed.\n"
+ summary += "We don't enable external/vty tests when building\n"
+ summary += "packages, so maybe those failed. Check the logs.\n"
+
+ if "lint" in pipeline and not pipeline["lint"]["passed"]:
+ summary += "\n"
+ summary += "Please fix the linting errors. More information:\n"
+ summary += "https://osmocom.org/projects/cellular-infrastructure/wiki/Linting\n"
+
+ summary += "\n"
+ if jobs["failed"]:
+ summary += "Build Failed\n"
+ summary += "\n"
+ summary += f"Find the Retrigger button here:\n{build_url}\n"
+ vote = -1
+ notify = "OWNER"
+ else:
+ summary += "Build Successful\n"
+ vote = 1
+ notify = "OWNER" if notify_on_success else "NONE"
+
+ # Reference:
+ # https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#set-review
+ # https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#review-input
+ return {"tag": "jenkins",
+ "message": summary,
+ "labels": {"Verified": vote},
+ "notify": notify}
+
+
+def main():
+ args = parse_args()
+ summary = get_pipeline_summary(args.build_url, args.notify_on_success)
+
+ print()
+ print(summary["message"])
+ print(f"notify: {summary['notify']}")
+
+ if args.output:
+ with open(args.output, "w") as handle:
+ json.dump(summary, handle, indent=4)
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/jenkins-gerrit/pipeline_summary_send.sh b/scripts/jenkins-gerrit/pipeline_summary_send.sh
new file mode 100755
index 0000000..29ea860
--- /dev/null
+++ b/scripts/jenkins-gerrit/pipeline_summary_send.sh
@@ -0,0 +1,32 @@
+#!/bin/sh -e
+
+# By default, a mail notification will only be sent if the gerrit verification
+# failed. Add yourself here to also receive notifications on successs.
+notify_on_success_users="
+ pespin
+"
+
+arg_notify=""
+for i in $notify_on_success_users; do
+ if [ "$GERRIT_PATCHSET_UPLOADER_NAME" = "$i" ]; then
+ arg_notify="--notify-on-success"
+ break
+ fi
+done
+
+set -x
+
+./pipeline_summary.py "$PIPELINE_BUILD_URL" \
+ -o gerrit_report.json \
+ $arg_notify
+
+ssh \
+ -p "$GERRIT_PORT" \
+ -l jenkins \
+ "$GERRIT_HOST" \
+ gerrit \
+ review \
+ --project "$GERRIT_PROJECT" \
+ "$GERRIT_CHANGE_NUMBER,$GERRIT_PATCHSET_NUMBER" \
+ --json \
+ < gerrit_report.json
diff --git a/scripts/obs/README b/scripts/obs/README
index 67d2a2b..f761f1e 100644
--- a/scripts/obs/README
+++ b/scripts/obs/README
@@ -1,3 +1,22 @@
+Overview
+========
+This directory contains the following scripts, related to building Osmocom
+projects in OBS (Open Build Service) and building binary packages. Here is an
+overview of the scripts, run them with -h to get a more detailed description.
+
+* build_srcpkg.py: build one source package for an Osmocom project
+* update_obs_project.py: generate source packages and upload them to OBS
+* build_binpkg.py: build rpm/deb packages for one Osmocom project
+
+Docker
+------
+The above scripts have a -d parameter, that allows running them inside docker.
+This is how we run it in jenkins to avoid installing dependencies on the host.
+
+Note that when building binary packages, libosmocore socket tests and possibly
+others will fail unless docker has ipv6 configured:
+https://osmocom.org/issues/4700#note-3
+
Usage Example: Submitting source packages to Osmocom's OBS build server
=======================================================================
@@ -29,7 +48,7 @@ Verify that it worked:
libosmo-pfcp
osmo-hnbgw
-FYI, the config file will look like this:
+FYI, the config file (oscrc) will look like this:
[general]
apiurl = https://obs.osmocom.org
diff --git a/scripts/obs/build_binpkg.py b/scripts/obs/build_binpkg.py
new file mode 100755
index 0000000..a34a699
--- /dev/null
+++ b/scripts/obs/build_binpkg.py
@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import argparse
+import multiprocessing
+import os
+import lib
+import lib.binpkg_deb
+import lib.config
+import lib.docker
+import lib.git
+import lib.metapkg
+import lib.srcpkg
+
+
+def main():
+ distro_default = lib.config.docker_distro_default
+ distro_choices = [distro_default] + lib.config.docker_distro_other
+ jobs_default = multiprocessing.cpu_count() + 1
+
+ parser = argparse.ArgumentParser(
+ description="Build a deb or rpm package as it would be done on"
+ " obs.osmocom.org. Use after building a source package"
+ " with build_srcpkg.py."
+ f" Output dir: {lib.config.path_temp}/binpkgs")
+ parser.add_argument("-d", "--docker", choices=distro_choices,
+ const=distro_default, nargs="?",
+ help="build the package in docker for a specific"
+ f" distro (default: {distro_default})")
+ parser.add_argument("-j", "--jobs", type=int, default=jobs_default,
+ help=f"parallel running jobs (default: {jobs_default})")
+ parser.add_argument("-r", "--run-shell-on-error", action="store_true",
+ help="run an interactive shell if the build fails")
+ parser.add_argument("-v", "--verbose", action="store_true",
+ help="always print shell commands and their output,"
+ " instead of only printing them on error")
+ parser.add_argument("package",
+ help="package name, e.g. libosmocore")
+ args = parser.parse_args()
+
+ lib.set_cmds_verbose(args.verbose)
+
+ srcdir = f"{lib.config.path_temp}/srcpkgs/{args.package}"
+ if not os.path.exists(srcdir):
+ print(f"ERROR: {args.package}: no srcpkg found, run build_srcpkg.py"
+ " first!")
+ exit(1)
+
+ bindir = f"{lib.config.path_temp}/binpkgs"
+ lib.run_cmd(["rm", "-rf", bindir])
+ os.makedirs(bindir)
+
+ distro = args.docker if args.docker else distro_default
+
+ env = {"JOBS": str(args.jobs),
+ "PACKAGE": args.package,
+ "BUILDUSER": os.environ["USER"],
+ "PACKAGEFORMAT": "deb"}
+
+ docker_args = []
+ if args.run_shell_on_error:
+ env["RUN_SHELL_ON_ERROR"] = "1"
+ docker_args += ["-i", "-t"]
+
+ script_path = "data/build.sh"
+
+ if not distro.startswith("debian:"):
+ env["PACKAGEFORMAT"] = "rpm"
+
+ if args.docker:
+ image_type = "build_binpkg"
+
+ # Optimization: use docker container with osmo-gsm-manuals-dev already
+ # installed if it is in build depends
+ if distro.startswith("debian:") \
+ and lib.srcpkg.requires_osmo_gsm_manuals_dev(args.package):
+ image_type += "_manuals"
+
+ env["BUILDUSER"] = "user"
+ lib.docker.run_in_docker_and_exit(script_path,
+ image_type=image_type,
+ distro=distro,
+ pass_argv=False, env=env,
+ docker_args=docker_args)
+ else:
+ lib.run_cmd(["sudo", "-E", script_path], env=env,
+ cwd=lib.config.path_top)
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/obs/build_srcpkg.py b/scripts/obs/build_srcpkg.py
index ea7ab34..ad21927 100755
--- a/scripts/obs/build_srcpkg.py
+++ b/scripts/obs/build_srcpkg.py
@@ -18,6 +18,8 @@ def main():
" upload to https://obs.osmocom.org."
f" Output dir: {lib.config.path_temp}/srcpkgs")
lib.add_shared_arguments(parser)
+ parser.add_argument("-g", "--gerrit-id", type=int, default=0,
+ help="clone particular revision from gerrit using given ID")
parser.add_argument("package", nargs="?",
help="package name, e.g. libosmocore or open5gs")
args = parser.parse_args()
@@ -29,11 +31,13 @@ def main():
lib.set_cmds_verbose(args.verbose)
if args.docker:
- lib.docker.run_in_docker_and_exit(__file__, args)
+ lib.docker.run_in_docker_and_exit("build_srcpkg.py")
+
+ if not args.ignore_req:
+ lib.check_required_programs()
- lib.check_required_programs()
if args.package:
- lib.check_package(args.package)
+ args.package = lib.set_proper_package_name(args.package)
lib.remove_temp()
if args.meta:
@@ -41,7 +45,7 @@ def main():
if args.package:
lib.srcpkg.build(args.package, args.feed, args.git_branch, args.conflict_version,
- args.git_fetch)
+ args.git_fetch, args.gerrit_id)
if __name__ == "__main__":
diff --git a/scripts/obs/check_builders.sh b/scripts/obs/check_builders.sh
new file mode 100755
index 0000000..65a0d76
--- /dev/null
+++ b/scripts/obs/check_builders.sh
@@ -0,0 +1,19 @@
+#!/bin/sh -ex
+min=10
+max=500
+wget -q https://obs.osmocom.org -O index.html
+
+set +x
+for i in $(seq $min $max); do
+ if grep -q " of $i build hosts" index.html; then
+ echo
+ echo "Check successful, $i builders are connected to OBS"
+ echo
+ exit 0
+ fi
+done
+
+echo
+echo "ERROR: expected at least $min builders to be connected to OBS!"
+echo
+exit 1
diff --git a/scripts/obs/data/build.sh b/scripts/obs/data/build.sh
new file mode 100755
index 0000000..7dfbda7
--- /dev/null
+++ b/scripts/obs/data/build.sh
@@ -0,0 +1,15 @@
+#!/bin/sh -e
+
+if ! data/build_"$PACKAGEFORMAT".sh; then
+ echo
+ echo "ERROR: build failed!"
+ echo
+ if [ -n "$RUN_SHELL_ON_ERROR" ]; then
+ bash
+ fi
+ exit 1
+fi
+
+echo
+echo "Build successful!"
+echo
diff --git a/scripts/obs/data/build_binpkg.Dockerfile b/scripts/obs/data/build_binpkg.Dockerfile
new file mode 100644
index 0000000..925680d
--- /dev/null
+++ b/scripts/obs/data/build_binpkg.Dockerfile
@@ -0,0 +1,78 @@
+ARG DISTRO_FROM
+FROM ${DISTRO_FROM}
+ARG DISTRO
+ARG UID
+
+COPY Release.key /tmp/Release.key
+
+RUN useradd --uid=${UID} -m user
+
+# Only install build-essential here, and what's needed to add the Osmocom
+# repository. Everything else must be defined as dependency in the package
+# build recipe. For rpm-based distributions, there is no build-essential or
+# similar package. Instead add relevant packages from prjconf, e.g.:
+# https://build.opensuse.org/projects/CentOS:CentOS-8/prjconf
+# For debian, make sure we don't have man pages as otherwise it takes some time
+# to regenerate the manuals database when installing build dependencies.
+# SYS#5818: using almalinux:8 instead of centos:8
+RUN case "$DISTRO" in \
+ debian*) \
+ echo "path-exclude=/usr/share/man/*" \
+ > /etc/dpkg/dpkg.cfg.d/exclude-man-pages && \
+ rm -rf /usr/share/man/ && \
+ apt-get update && \
+ apt-get install -y --no-install-recommends \
+ build-essential \
+ ca-certificates \
+ fakeroot \
+ git \
+ gnupg2 \
+ && \
+ apt-get clean \
+ ;; \
+ almalinux*) \
+ dnf -y install \
+ autoconf \
+ automake \
+ binutils \
+ dnf-utils \
+ gcc \
+ gcc-c++ \
+ glibc-devel \
+ libtool \
+ make \
+ redhat-rpm-config \
+ rpm-build \
+ rpmdevtools \
+ wget && \
+ yum config-manager --set-enabled powertools && \
+ su user -c rpmdev-setuptree \
+ ;; \
+ esac
+
+# Add master repository, where packages immediately get updated after merging
+# patches to master.
+RUN case "$DISTRO" in \
+ debian:11) \
+ apt-key add /tmp/Release.key && \
+ rm /tmp/Release.key && \
+ echo "deb https://downloads.osmocom.org/packages/osmocom:/master/Debian_11/ ./" \
+ > /etc/apt/sources.list.d/osmocom-master.list \
+ ;; \
+ almalinux:8) \
+ { echo "[network_osmocom_master]"; \
+ echo "name=osmocom:master"; \
+ echo "type=rpm-md"; \
+ echo "baseurl=https://downloads.osmocom.org/packages/osmocom:/master/CentOS_8/"; \
+ echo "gpgcheck=1"; \
+ echo "gpgkey=https://downloads.osmocom.org/packages/osmocom:/master/CentOS_8/repodata/repomd.xml.key"; \
+ echo "enabled=1"; \
+ } > /etc/yum.repos.d/network:osmocom:master.repo \
+ ;; \
+ *) \
+ echo "can't install repo for $DISTRO" && \
+ exit 1 \
+ ;; \
+ esac
+
+WORKDIR /obs/
diff --git a/scripts/obs/data/build_binpkg_manuals.Dockerfile b/scripts/obs/data/build_binpkg_manuals.Dockerfile
new file mode 100644
index 0000000..fd4c709
--- /dev/null
+++ b/scripts/obs/data/build_binpkg_manuals.Dockerfile
@@ -0,0 +1,20 @@
+# Optimization: installing osmo-gsm-manuals-dev and its many, many dependencies
+# takes quite a long time - sometimes longer than building the package itself
+# (related: OS#4132). Instead of doing this every time before starting a build,
+# here is a second docker container that already has it installed. This gets
+# used by build_binpkg.py in case the package to build depends on
+# osmo-gsm-manuals-dev and the build is done for Debian. Note that right now we
+# don't build the manuals for rpm-based distributions.
+ARG DISTRO_FROM
+FROM ${DISTRO_FROM}
+ARG DISTRO
+
+RUN case "$DISTRO" in \
+ debian*) \
+ apt-get update && \
+ apt-get install -y --no-install-recommends \
+ osmo-gsm-manuals-dev \
+ && \
+ apt-get clean \
+ ;; \
+ esac
diff --git a/scripts/obs/data/build_deb.sh b/scripts/obs/data/build_deb.sh
new file mode 100755
index 0000000..0030278
--- /dev/null
+++ b/scripts/obs/data/build_deb.sh
@@ -0,0 +1,14 @@
+#!/bin/sh -ex
+
+apt_get="apt-get"
+if [ -n "$INSIDE_DOCKER" ]; then
+ export DEBIAN_FRONTEND=noninteractive
+ apt_get="apt-get -y"
+fi
+
+su "$BUILDUSER" -c "tar -C _temp/binpkgs -xvf _temp/srcpkgs/$PACKAGE/*.tar.*"
+cd _temp/binpkgs/*
+
+$apt_get update
+$apt_get build-dep .
+su "$BUILDUSER" -c "dpkg-buildpackage -us -uc -j$JOBS"
diff --git a/scripts/obs/data/build_rpm.sh b/scripts/obs/data/build_rpm.sh
new file mode 100755
index 0000000..3ade17e
--- /dev/null
+++ b/scripts/obs/data/build_rpm.sh
@@ -0,0 +1,31 @@
+#!/bin/sh -ex
+
+if ! [ -d /home/$BUILDUSER/rpmbuild/SOURCES ]; then
+ set +x
+ echo "ERROR: rpmdev-setuptree did not run"
+ echo "If this is an rpm based system and you want to build the package"
+ echo "here, run rpmdev-setuptree. Otherwise consider building the"
+ echo "package in docker (-d)."
+ exit 1
+fi
+
+yum_builddep="yum-builddep"
+if [ -n "$INSIDE_DOCKER" ]; then
+ yum_builddep="yum-builddep -y"
+fi
+
+spec="$(basename "$(find _temp/srcpkgs/"$PACKAGE" -name '*.spec')")"
+
+su "$BUILDUSER" -c "cp _temp/srcpkgs/$PACKAGE/$spec ~/rpmbuild/SPECS"
+su "$BUILDUSER" -c "cp _temp/srcpkgs/$PACKAGE/*.tar.* ~/rpmbuild/SOURCES"
+su "$BUILDUSER" -c "cp _temp/srcpkgs/$PACKAGE/rpmlintrc ~/rpmbuild/SOURCES"
+su "$BUILDUSER" -c "cp /obs/data/rpmmacros ~/.rpmmacros"
+
+$yum_builddep "/home/$BUILDUSER/rpmbuild/SPECS/$spec"
+
+su "$BUILDUSER" -c "rpmbuild -bb ~/rpmbuild/SPECS/$spec"
+
+# Make built rpms available outside of docker
+if [ -n "$INSIDE_DOCKER" ]; then
+ su "$BUILDUSER" -c "mv ~/rpmbuild/RPMS/*/*.rpm _temp/binpkgs/"
+fi
diff --git a/scripts/obs/data/Dockerfile b/scripts/obs/data/build_srcpkg.Dockerfile
index d8bd9c5..f5d72cd 100644
--- a/scripts/obs/data/Dockerfile
+++ b/scripts/obs/data/build_srcpkg.Dockerfile
@@ -23,6 +23,7 @@ RUN apt-get update && \
dpkg-dev \
fakeroot \
git \
+ git-review \
meson \
osc \
python3-setuptools \
diff --git a/scripts/obs/data/rpmmacros b/scripts/obs/data/rpmmacros
new file mode 100644
index 0000000..5ac26c7
--- /dev/null
+++ b/scripts/obs/data/rpmmacros
@@ -0,0 +1,5 @@
+# Default values for macros from OBS project config:
+# https://build.opensuse.org/projects/CentOS:CentOS-8/prjconf
+
+%ext_info .gz
+%ext_man .gz
diff --git a/scripts/obs/lib/__init__.py b/scripts/obs/lib/__init__.py
index a07a072..479c1c1 100644
--- a/scripts/obs/lib/__init__.py
+++ b/scripts/obs/lib/__init__.py
@@ -7,16 +7,18 @@ import shutil
import subprocess
import sys
import tempfile
+import inspect
import lib.config
cmds_verbose = False
def add_shared_arguments(parser):
+ """ Arguments shared between build_srcpkg.py and update_obs_project.py. """
parser.add_argument("-f", "--feed",
help="package feed (default: nightly). The feed"
" determines the git revision to be built:"
- " 'nightly' builds 'origin/master',"
+ " 'nightly' and 'master' build 'origin/master',"
" 'latest' builds the last signed tag,"
" other feeds build their respective branch.",
metavar="FEED", default="nightly",
@@ -32,6 +34,8 @@ def add_shared_arguments(parser):
action="store_false", dest="git_fetch")
parser.add_argument("-m", "--meta", action="store_true",
help="build a meta package (e.g. osmocom-nightly)")
+ parser.add_argument("-i", "--ignore-req", action="store_true",
+ help="skip required programs check")
parser.add_argument("-c", "--conflict-version", nargs="?",
help="Of the generated source packages, all Osmocom"
" packages (not e.g. open5gs, see lib/config.py"
@@ -74,11 +78,16 @@ def check_required_programs():
exit(1)
-def check_package(package):
+def set_proper_package_name(package):
if package in lib.config.projects_osmocom:
- return
+ return package
if package in lib.config.projects_other:
- return
+ return package
+
+ # Add prefix to Osmocom package if missing
+ for package_cfg in lib.config.projects_osmocom:
+ if os.path.basename(package_cfg) == package:
+ return package_cfg
print(f"ERROR: unknown package: {package}")
print("See packages_osmocom and packages_other in obs/lib/config.py")
@@ -87,12 +96,17 @@ def check_package(package):
def exit_error_cmd(completed, error_msg):
""" :param completed: return from run_cmd() below """
+ global cmds_verbose
+
print()
print(f"ERROR: {error_msg}")
print()
print(f"*** command ***\n{completed.args}\n")
print(f"*** returncode ***\n{completed.returncode}\n")
- print(f"*** output ***\n{completed.output}")
+
+ if not cmds_verbose:
+ print(f"*** output ***\n{completed.output}")
+
print("*** python trace ***")
raise RuntimeError("shell command related error, find details right above"
" this python trace")
@@ -108,13 +122,14 @@ def run_cmd(cmd, check=True, *args, **kwargs):
:param check: stop with error if exit code is not 0 """
global cmds_verbose
+ caller = inspect.stack()[2][3]
if cmds_verbose:
- print(f"+ {cmd}")
+ print(f"+ {caller}(): {cmd}")
with tempfile.TemporaryFile(encoding="utf8", mode="w+") as output_buf:
- p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT, text=True, bufsize=1,
- *args, **kwargs)
+ p = subprocess.Popen(cmd, stdin=subprocess.DEVNULL,
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+ text=True, bufsize=1, *args, **kwargs)
while True:
out = p.stdout.read(1)
diff --git a/scripts/obs/lib/binpkg_deb.py b/scripts/obs/lib/binpkg_deb.py
new file mode 100644
index 0000000..b26623c
--- /dev/null
+++ b/scripts/obs/lib/binpkg_deb.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import os
+import glob
+import lib.config
+
+
+def extract_source(srcdir, bindir):
+ tarball = glob.glob(f"{srcdir}/*.tar.*")[0]
+
+ print(f"extracting {os.path.basename(tarball)}")
+ lib.run_cmd(["tar", "-xf", tarball], cwd=bindir)
+
+ return glob.glob(f"{bindir}/*/")[0]
+
+
+def build(srcdir, jobs):
+ bindir = f"{lib.config.path_temp}/binpkg"
+ extractdir = extract_source(srcdir, bindir)
+
+ lib.set_cmds_verbose(True)
+
+ # install deps
+ lib.run_cmd(["apt-get", "-y", "build-dep", "."], cwd=extractdir)
+
+ print("running dpkg-buildpackage")
+ lib.run_cmd(["dpkg-buildpackage", "-us", "-uc", f"-j{jobs}"],
+ cwd=extractdir)
diff --git a/scripts/obs/lib/binpkg_rpm.py b/scripts/obs/lib/binpkg_rpm.py
new file mode 100644
index 0000000..b26623c
--- /dev/null
+++ b/scripts/obs/lib/binpkg_rpm.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import os
+import glob
+import lib.config
+
+
+def extract_source(srcdir, bindir):
+ tarball = glob.glob(f"{srcdir}/*.tar.*")[0]
+
+ print(f"extracting {os.path.basename(tarball)}")
+ lib.run_cmd(["tar", "-xf", tarball], cwd=bindir)
+
+ return glob.glob(f"{bindir}/*/")[0]
+
+
+def build(srcdir, jobs):
+ bindir = f"{lib.config.path_temp}/binpkg"
+ extractdir = extract_source(srcdir, bindir)
+
+ lib.set_cmds_verbose(True)
+
+ # install deps
+ lib.run_cmd(["apt-get", "-y", "build-dep", "."], cwd=extractdir)
+
+ print("running dpkg-buildpackage")
+ lib.run_cmd(["dpkg-buildpackage", "-us", "-uc", f"-j{jobs}"],
+ cwd=extractdir)
diff --git a/scripts/obs/lib/config.py b/scripts/obs/lib/config.py
index 1e569ef..4a4bf3c 100644
--- a/scripts/obs/lib/config.py
+++ b/scripts/obs/lib/config.py
@@ -17,6 +17,7 @@ required_programs = [
"fakeroot",
"find",
"git",
+ "git-review",
"meson",
"osc",
"rebar3",
@@ -31,15 +32,15 @@ feeds = [
"2022q1",
"2022q2",
"latest",
+ "master",
"nightly",
]
# Osmocom projects: generated source packages will depend on a meta package,
# such as osmocom-nightly, osmocom-latest or osmocom-2022q1. This meta package
# prevents that packages from different feeds are mixed by accident.
-# NOTE: Before adding new projects, make sure the rpm and deb build in OBS!
-# Test it in your own namespace (home:youruser), see README for
-# instructions and/or ask osmith for help.
+# NOTE: Before adding new projects, add them to jobs/gerrit-verifications.yml
+# and ensure the rpm and deb packages build successfully in jenkins.
projects_osmocom = [
"erlang/osmo_dia2gsup",
"libasn1c",
@@ -109,4 +110,7 @@ git_latest_tag_pattern_other = {
"open5gs": "^v[0-9]*\\.[0-9]*\\.[0-9]*$",
}
-docker_image_name = "debian-bullseye-osmocom-obs"
+docker_distro_default = "debian:11"
+docker_distro_other = [
+ "almalinux:8", # instead of centos:8 (SYS#5818)
+]
diff --git a/scripts/obs/lib/docker.py b/scripts/obs/lib/docker.py
index a5dbcd9..6bcf024 100644
--- a/scripts/obs/lib/docker.py
+++ b/scripts/obs/lib/docker.py
@@ -8,11 +8,32 @@ import lib
import lib.config
-def build_image():
- print(f"docker: building image {lib.config.docker_image_name}")
+def get_image_name(distro, image_type):
+ ret = f"{distro}-osmocom-obs-{image_type}"
+ ret = ret.replace(":","-").replace("_","-")
+ return ret
+
+
+def get_distro_from(distro, image_type):
+ # Manuals: depend on regular image (data/build_binpkg_manuals.Dockerfile)
+ if image_type.endswith("_manuals"):
+ return get_image_name(distro, image_type.replace("_manuals", ""))
+
+ return distro
+
+
+def build_image(distro, image_type):
+ image_name = get_image_name(distro, image_type)
+ distro_from = get_distro_from(distro, image_type)
+
+ print(f"docker: building image {image_name}")
+
lib.run_cmd(["docker", "build",
+ "--build-arg", f"DISTRO={distro}",
+ "--build-arg", f"DISTRO_FROM={distro_from}",
"--build-arg", f"UID={os.getuid()}",
- "-t", lib.config.docker_image_name,
+ "-t", image_name,
+ "-f", f"{lib.config.path_top}/data/{image_type}.Dockerfile",
f"{lib.config.path_top}/data"])
@@ -28,7 +49,18 @@ def get_oscrc():
exit(1)
-def run_in_docker_and_exit(script_path, args, add_oscrc=False):
+def run_in_docker_and_exit(script_path, add_oscrc=False,
+ image_type="build_srcpkg", distro=None,
+ pass_argv=True, env={}, docker_args=[]):
+ """
+ :param script_path: what to run inside docker, relative to scripts/obs/
+ :param add_oscrc: put user's oscrc in docker (contains obs credentials!)
+ :param image_type: which Dockerfile to use (data/{image_type}.Dockerfile)
+ :param distro: which Linux distribution to use, e.g. "debian:11"
+ :param pass_argv: pass arguments from sys.argv to the script
+ :param env: dict of environment variables
+ :param docker_args: extra arguments to pass to docker
+ """
if "INSIDE_DOCKER" in os.environ:
return
@@ -36,27 +68,43 @@ def run_in_docker_and_exit(script_path, args, add_oscrc=False):
print("ERROR: docker is not installed")
exit(1)
+ if not distro:
+ distro = lib.config.docker_distro_default
+ image_name = get_image_name(distro, image_type)
+
oscrc = None
if add_oscrc:
oscrc = get_oscrc()
- # Build the docker image. Unless it is up-to-date, this will take a few
- # minutes or so, therefore print the output.
+ # Unless the docker image is up-to-date, building will take a few
+ # minutes or so, therefore print the output. No need to restore
+ # set_cmds_verbose, as we use subprocess.run() below and exit afterwards.
lib.set_cmds_verbose(True)
- build_image()
- lib.set_cmds_verbose(args.verbose)
+
+ # Manuals: build regular image first (data/build_binpkg_manuals.Dockerfile)
+ if image_type.endswith("_manuals"):
+ build_image(distro, image_type.replace("_manuals",""))
+
+ build_image(distro, image_type)
cmd = ["docker", "run",
+ "--rm",
"-e", "INSIDE_DOCKER=1",
"-e", "PYTHONUNBUFFERED=1",
- "--rm", "-v", f"{lib.config.path_top}:/obs"]
+ "-v", f"{lib.config.path_top}:/obs"]
+
+ for env_key, env_val in env.items():
+ cmd += ["-e", f"{env_key}={env_val}"]
if oscrc:
cmd += ["-v", f"{oscrc}:/home/user/.oscrc"]
- script_path = f"/obs/{os.path.basename(script_path)}"
- cmd += [lib.config.docker_image_name, script_path] + sys.argv[1:]
+ cmd += docker_args
+ cmd += [image_name, f"/obs/{script_path}"]
+
+ if pass_argv:
+ cmd += sys.argv[1:]
- print(f"docker: running: {os.path.basename(script_path)} inside docker")
+ print(f"docker: running: {script_path} inside docker")
ret = subprocess.run(cmd)
exit(ret.returncode)
diff --git a/scripts/obs/lib/git.py b/scripts/obs/lib/git.py
index 29c183b..8081a54 100644
--- a/scripts/obs/lib/git.py
+++ b/scripts/obs/lib/git.py
@@ -58,14 +58,46 @@ def checkout(project, branch):
lib.run_cmd(["git", "reset", "--hard", branch], cwd=repo_path)
-def checkout_default_branch(project):
- branch = lib.config.git_branch_default
+def checkout_from_review(project, gerrit_id):
+ """ checkout a given gerrit ID """
+ repo_path = get_repo_path(project)
+ lib.run_cmd(["git", "review", "-s"], cwd=repo_path)
+ lib.run_cmd(["git", "review", "-d", str(gerrit_id)], cwd=repo_path)
+
+
+def get_default_branch(project):
if project in lib.config.git_branch_other:
- branch = lib.config.git_branch_other[project]
+ return lib.config.git_branch_other[project]
+ return lib.config.git_branch_default
+
+def checkout_default_branch(project):
+ branch = get_default_branch(project)
checkout(project, f"origin/{branch}")
+def get_head(project):
+ repo_path = get_repo_path(project)
+ ret = lib.run_cmd(["git", "rev-parse", "HEAD"], cwd=repo_path)
+ return ret.output.rstrip()
+
+
+def get_head_remote(project, branch):
+ if not branch:
+ branch = get_default_branch(project)
+ repo_url = get_repo_url(project)
+
+ print(f"{project}: getting head from git remote for {branch}")
+ ls_remote = lib.run_cmd(["git", "ls-remote", repo_url, f"heads/{branch}"])
+
+ ret = ls_remote.output.split("\t")[0]
+ if not ret:
+ lib.exit_error_cmd(ls_remote, f"failed to find head commit for"
+ "{project} in output")
+
+ return ret
+
+
def get_latest_tag(project):
pattern_str = get_latest_tag_pattern(project)
pattern = re.compile(pattern_str)
diff --git a/scripts/obs/lib/osc.py b/scripts/obs/lib/osc.py
index 2fcdbc6..df5c34c 100644
--- a/scripts/obs/lib/osc.py
+++ b/scripts/obs/lib/osc.py
@@ -20,6 +20,18 @@ def check_proj(obs_project):
exit(1)
+def check_oscrc():
+ configdir = os.environ.get("XDG_CONFIG_HOME", "~/.config")
+ paths = ["~/.oscrc", f"{configdir}/osc/oscrc"]
+ for path in paths:
+ if os.path.exists(os.path.expanduser(path)):
+ return
+
+ print("ERROR: oscrc does not exist, please create one as explained in the"
+ " README.")
+ exit(1)
+
+
def set_apiurl(url):
global apiurl
apiurl = url
@@ -50,7 +62,7 @@ def get_remote_pkgs(proj):
return ret.output.rstrip().split("\n")
-def get_package_version(proj, package):
+def get_package_version(proj, package, feed):
print(f"{package}: getting OBS version")
ret = run_osc(["list", proj, os.path.basename(package)])
@@ -58,15 +70,21 @@ def get_package_version(proj, package):
if ret.output == '\n':
return "0"
- # Extract the version from the dsc filename
+ # Extract the version from the file list
for line in ret.output.split('\n'):
line = line.rstrip()
- if line.endswith(".dsc"):
- return line.split("_")[-1][:-4]
+ if feed == "master" and package != "osmocom-master":
+ # Use commit_*.txt
+ if line.startswith("commit_") and line.endswith(".txt"):
+ return line.split("_")[1].split(".")[0]
+ else:
+ # Use *.dsc
+ if line.endswith(".dsc"):
+ return line.split("_")[-1][:-4]
lib.exit_error_cmd(ret, "failed to find package version on OBS by"
- " extracting the version from the .dsc filename")
+ " extracting the version from the file list")
def create_package(proj, package):
diff --git a/scripts/obs/lib/srcpkg.py b/scripts/obs/lib/srcpkg.py
index 035cf81..b37b8b3 100644
--- a/scripts/obs/lib/srcpkg.py
+++ b/scripts/obs/lib/srcpkg.py
@@ -1,7 +1,9 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0-or-later
# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import glob
import os
+import pathlib
import lib.config
import lib.debian
import lib.rpm_spec
@@ -13,7 +15,7 @@ def checkout_for_feed(project, feed, branch=None):
lib.git.checkout(project, f"origin/{branch}")
elif feed == "latest":
lib.git.checkout_latest_tag(project)
- elif feed == "nightly":
+ elif feed in ["master", "nightly"]:
lib.git.checkout_default_branch(project)
else: # 2022q1 etc
lib.git.checkout(project, f"origin/{feed}")
@@ -117,10 +119,25 @@ def write_tarball_version(project, version):
f.write(f"{version}\n")
-def build(project, feed, branch, conflict_version, fetch):
+def write_commit_txt(project):
+ """ Write the current git commit to commit_$commit.txt file, so it gets
+ uploaded to OBS along with the rest of the source package. This allows
+ figuring out if the source package is still up-to-date or not for the
+ master feed. """
+ output_path = lib.get_output_path(project)
+ commit = lib.git.get_head(project)
+
+ print(f"{project}: adding commit_{commit}.txt")
+ pathlib.Path(f"{output_path}/commit_{commit}.txt").touch()
+
+
+def build(project, feed, branch, conflict_version, fetch, gerrit_id=0):
lib.git.clone(project, fetch)
lib.git.clean(project)
- checkout_for_feed(project, feed, branch)
+ if gerrit_id > 0:
+ lib.git.checkout_from_review(project, gerrit_id)
+ else:
+ checkout_for_feed(project, feed, branch)
version = get_version_for_feed(project, feed, conflict_version)
epoch = get_epoch(project)
version_epoch = f"{epoch}:{version}" if epoch else version
@@ -152,5 +169,23 @@ def build(project, feed, branch, conflict_version, fetch):
lib.rpm_spec.generate(project, version, epoch)
lib.rpm_spec.copy_to_output(project)
+ if feed == "master":
+ write_commit_txt(project)
+
lib.remove_cache_extra_files()
return version_epoch
+
+
+def requires_osmo_gsm_manuals_dev(project):
+ """ Check if an already built source package has osmo-gsm-manuals-dev in
+ Build-Depends of the .dsc file """
+ path_dsc = glob.glob(f"{lib.get_output_path(project)}/*.dsc")
+ assert len(path_dsc) == 1, f"failed to get dsc path for {project}"
+
+ with open(path_dsc[0], "r") as handle:
+ for line in handle.readlines():
+ if line.startswith("Build-Depends:") \
+ and "osmo-gsm-manuals-dev" in line:
+ return True
+
+ return False
diff --git a/scripts/obs/update_obs_project.py b/scripts/obs/update_obs_project.py
index 02db7ce..86cf40a 100755
--- a/scripts/obs/update_obs_project.py
+++ b/scripts/obs/update_obs_project.py
@@ -20,13 +20,13 @@ srcpkgs_updated = [] # list of pkgnames
def parse_packages(packages_arg):
+ ret = []
if packages_arg:
for package in packages_arg:
- lib.check_package(package)
- return packages_arg
+ ret += [lib.set_proper_package_name(package)]
+ return ret
# Default to all
- ret = []
ret += lib.config.projects_osmocom
ret += lib.config.projects_other
return ret
@@ -66,13 +66,16 @@ def build_srcpkg_if_needed(proj, feed, branch, pkgs_remote, package, conflict_ve
fetch, is_meta_pkg, skip_up_to_date):
global srcpkgs_skipped
- if feed != "latest":
- print(f"{package}: building source package (feed is {feed})")
- else:
+ if feed in ["master", "latest"]:
+ """ Check if we can skip this package by comparing the OBS version with
+ the git remote. """
if is_meta_pkg:
latest_version = conflict_version if conflict_version else "1.0.0"
else:
- latest_version = lib.git.get_latest_tag_remote(package)
+ if feed == "master":
+ latest_version = lib.git.get_head_remote(package, branch)
+ else:
+ latest_version = lib.git.get_latest_tag_remote(package)
if latest_version is None:
print(f"{package}: skipping (no git tag found)")
@@ -82,7 +85,7 @@ def build_srcpkg_if_needed(proj, feed, branch, pkgs_remote, package, conflict_ve
if os.path.basename(package) not in pkgs_remote:
print(f"{package}: building source package (not in OBS)")
else:
- obs_version = lib.osc.get_package_version(proj, package)
+ obs_version = lib.osc.get_package_version(proj, package, feed)
if is_up_to_date(obs_version, latest_version):
if skip_up_to_date:
print(f"{package}: skipping ({obs_version} is up-to-date)")
@@ -95,6 +98,8 @@ def build_srcpkg_if_needed(proj, feed, branch, pkgs_remote, package, conflict_ve
else:
print(f"{package}: building source package (outdated:"
f" {latest_version} <=> {obs_version} in OBS)")
+ else:
+ print(f"{package}: building source package (feed is {feed})")
build_srcpkg(feed, branch, package, conflict_version, fetch, is_meta_pkg)
@@ -197,11 +202,15 @@ def main():
lib.set_cmds_verbose(args.verbose)
if args.docker:
- lib.docker.run_in_docker_and_exit(__file__, args, True)
+ lib.docker.run_in_docker_and_exit("update_obs_project.py", True)
lib.osc.check_proj(proj)
+ lib.osc.check_oscrc()
lib.osc.set_apiurl(args.apiurl)
- lib.check_required_programs()
+
+ if not args.ignore_req:
+ lib.check_required_programs()
+
lib.remove_temp()
pkgs_remote = lib.osc.get_remote_pkgs(proj)
diff --git a/scripts/osmo-build.sh b/scripts/osmo-build.sh
index 8ca8e47..43413cf 100644
--- a/scripts/osmo-build.sh
+++ b/scripts/osmo-build.sh
@@ -172,7 +172,7 @@ branch_and_rev_of_local_repo() {
branch_and_rev_of_remote_repo() {
if [ -z "${2+x}" ]; then branch="master"; else branch="$2"; fi
branch="$( echo "$branch" | tr '/' '_')"
- rev="$(git ls-remote "https://git.osmocom.org/$1" "refs/heads/$branch")"
+ rev="$(git ls-remote "https://gerrit.osmocom.org/$1" "refs/heads/$branch")"
# check whether branch is a tag
if [ "$rev" = "" ]; then
diff --git a/scripts/osmo-depcheck/config.py b/scripts/osmo-depcheck/config.py
index 28f0699..94a1cb5 100644
--- a/scripts/osmo-depcheck/config.py
+++ b/scripts/osmo-depcheck/config.py
@@ -2,7 +2,7 @@
# Copyright 2018 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
# Where to clone sources from (with trailing slash)
-git_url_prefix = "https://git.osmocom.org/"
+git_url_prefix = "https://gerrit.osmocom.org/"
# Default projects to build when none are specified on the command line
projects = ("osmo-bts",
diff --git a/scripts/osmo-depcheck/dependencies.py b/scripts/osmo-depcheck/dependencies.py
index 4e611ad..d2b9927 100644
--- a/scripts/osmo-depcheck/dependencies.py
+++ b/scripts/osmo-depcheck/dependencies.py
@@ -14,7 +14,7 @@ def git_clone(workdir, prefix, cache_git_fetch, repository, version):
""" Clone a missing git repository and checkout a specific version tag.
:param workdir: path to where all data (git, build, install) is stored
- :param prefix: git url prefix (e.g. "https://git.osmocom.org/")
+ :param prefix: git url prefix (e.g. "https://gerrit.osmocom.org/")
:param cache_git_fetch: list of repositories that have already been
fetched in this run of osmo-depcheck
:param repository: Osmocom git repository name (e.g. "libosmo-abis")
@@ -56,7 +56,7 @@ def generate(workdir, prefix, cache_git_fetch, initial, rev):
repository, parsing the "configure.ac" file, and recursing.
:param workdir: path to where all data (git, build, install) is stored
- :param prefix: git url prefix (e.g. "https://git.osmocom.org/")
+ :param prefix: git url prefix (e.g. "https://gerrit.osmocom.org/")
:param cache_git_fetch: list of repositories that have already been
fetched in this run of osmo-depcheck
:param initial: the first program to look at (e.g. "osmo-bts")
diff --git a/scripts/osmo-layer1-headers.sh b/scripts/osmo-layer1-headers.sh
index 19db1d7..b9c43c5 100755
--- a/scripts/osmo-layer1-headers.sh
+++ b/scripts/osmo-layer1-headers.sh
@@ -16,7 +16,7 @@ case "$1" in
version="origin/master"
;;
oct)
- uri="https://git.osmocom.org/octphy-2g-headers"
+ uri="https://gitea.osmocom.org/cellular-infrastructure/octphy-2g-headers"
version_prefix=""
version="origin/master"
;;