diff options
Diffstat (limited to 'ansible')
46 files changed, 1096 insertions, 377 deletions
diff --git a/ansible/.gitignore b/ansible/.gitignore index 2fa603c..b23a1b1 100644 --- a/ansible/.gitignore +++ b/ansible/.gitignore @@ -1,4 +1,4 @@ -poky-* +poky-*toolchain* cov-analysis-*.tar.gz *.retry *.swo diff --git a/ansible/README.md b/ansible/README.md index e5d146c..9085c74 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -33,4 +33,4 @@ User root # how to make slaves log-in to registry.osmocom.org: -`ansible jenkins-slaves -u root -a "su - osmocom-build -c 'docker login -u jenkins-slave -p PASSWD https://registry.osmocom.org/'"` +`ansible jenkins_slaves -u root -a "su - osmocom-build -c 'docker login -u jenkins_slave -p PASSWD https://registry.osmocom.org/'"` diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg new file mode 100644 index 0000000..6c361a3 --- /dev/null +++ b/ansible/ansible.cfg @@ -0,0 +1,8 @@ +[defaults] +stdout_callback=debug + +remote_user = root + +[ssh_connection] +ssh_args = -F util/ssh_config -o ControlMaster=auto -o ControlPersist=60s -o StrictHostKeyChecking=accept-new + diff --git a/ansible/hosts b/ansible/hosts index 892428b..51e26b9 100644 --- a/ansible/hosts +++ b/ansible/hosts @@ -1,21 +1,32 @@ -[gsm-tester] +[gsm_tester] osmo-gsm-tester-rnd ansible_host=10.9.25.101 osmo-gsm-tester-prod ansible_host=10.9.25.107 -[coverity-slaves] -admin2-deb9build ansible_host=2a01:4f8:13b:828::1:300 -build2-deb9build-ansible ansible_host=2a01:4f8:10b:2ad9::1:6 -host2-deb9build-ansible ansible_host=2a01:4f8:120:8470::1:3 +[coverity_slaves] +build2-deb11build-ansible ansible_host=2a01:4f8:10b:2ad9::1:13 +build3-deb11build-ansible ansible_host=2a01:4f8:212:3762::1:3 +build4-deb12build-ansible ansible_host=2a01:4f8:121:200a::1:3 -[jenkins-slaves] -admin2-deb9build ansible_host=2a01:4f8:13b:828::1:300 osmocom_jenkins_slave_fstrim=True -build2-deb9build-ansible ansible_host=2a01:4f8:10b:2ad9::1:6 osmocom_jenkins_slave_fstrim=True -build2-deb10build-ansible ansible_host=2a01:4f8:10b:2ad9::1:10 osmocom_jenkins_slave_fstrim=True -host2-deb9build-ansible ansible_host=2a01:4f8:120:8470::1:3 osmocom_jenkins_slave_fstrim=True -gtp0-deb9build ansible_host=10.34.2.147 osmocom_jenkins_slave_fstrim=True -rpi4-raspbian10build-ansible-1 ansible_host=10.9.25.50 -rpi4-raspbian10build-ansible-2 ansible_host=10.9.25.60 -rpi4-raspbian10build-ansible-3 ansible_host=10.9.25.70 +[jenkins_slaves] +build2-deb10build-ansible ansible_host=2a01:4f8:10b:2ad9::1:10 osmocom_jenkins_slave_fstrim=True docker_max_image_space="80 GB" +build2-deb11build-ansible ansible_host=2a01:4f8:10b:2ad9::1:13 osmocom_jenkins_slave_fstrim=True docker_max_image_space="80 GB" +build3-deb11build-ansible ansible_host=2a01:4f8:212:3762::1:3 osmocom_jenkins_slave_fstrim=True docker_max_image_space="80 GB" +build4-deb12build-ansible ansible_host=2a01:4f8:121:200a::1:3 osmocom_jenkins_slave_fstrim=True docker_max_image_space="80 GB" +host2-deb11build-ansible ansible_host=2a01:4f8:120:8470::1:4 osmocom_jenkins_slave_fstrim=True docker_max_image_space="80 GB" +rpi4-raspbian11build-ansible-1 ansible_host=10.9.25.50 docker_max_image_space="20 GB" +rpi4-raspbian11build-ansible-2 ansible_host=10.9.25.60 docker_max_image_space="20 GB" +rpi4-raspbian11build-ansible-3 ansible_host=10.9.25.70 docker_max_image_space="20 GB" +lx2-raspbian11build-ansible ansible_host=10.34.2.153 docker_max_image_space="20 GB" [simtester] simtest ansible_host=10.9.25.80 + +[obs_workers] +build3-obsworker1 ansible_host=2a01:4f8:212:3762::1:2 obs_worker_instances=8 obs_worker_jobs=8 +build4-obsworker1 ansible_host=2a01:4f8:121:200a::1:2 obs_worker_instances=8 obs_worker_jobs=8 +lx2-1-obs-aarch64-1 ansible_host=2a02:8106:13:1e34:4e9:e7ff:fe97:fe92 obs_worker_instances=1 obs_worker_jobs=4 + +# Need to have kvm inside lxc, see OS#5365 +[qemu] +build3-deb11build-ansible ansible_host=2a01:4f8:212:3762::1:3 +build4-deb12build-ansible ansible_host=2a01:4f8:121:200a::1:3 diff --git a/ansible/roles/apt-allow-relinfo-change/README.md b/ansible/roles/apt-allow-relinfo-change/README.md new file mode 100644 index 0000000..cd00a46 --- /dev/null +++ b/ansible/roles/apt-allow-relinfo-change/README.md @@ -0,0 +1,12 @@ +--- +When the suite of one debian release changes from stable to oldstable, apt +stops working with the following error: + +``` +W:This must be accepted explicitly before updates for this repository can be applied. See apt-secure(8) manpage for details. +E:Repository 'http://raspbian.raspberrypi.org/raspbian buster InRelease' changed its 'Suite' value from 'stable' to 'oldstable' +``` + +This role configures apt to allow the release info change. + +Related: https://github.com/ansible/ansible/issues/48352 diff --git a/ansible/roles/apt-allow-relinfo-change/tasks/main.yml b/ansible/roles/apt-allow-relinfo-change/tasks/main.yml new file mode 100644 index 0000000..098f5ee --- /dev/null +++ b/ansible/roles/apt-allow-relinfo-change/tasks/main.yml @@ -0,0 +1,7 @@ +--- +- name: "configure to allow release info change" + lineinfile: + path: /etc/apt/apt.conf.d/99relinfochange + state: present + create: yes + line: Acquire::AllowReleaseInfoChange::Suite "true"; diff --git a/ansible/roles/docker/defaults/main.yml b/ansible/roles/docker/defaults/main.yml index 2c03d90..9faf286 100644 --- a/ansible/roles/docker/defaults/main.yml +++ b/ansible/roles/docker/defaults/main.yml @@ -2,3 +2,7 @@ # Adds this user to the group docker which is allowed to access docker jenkins_user: jenkins + +# Keep amount of stored docker images below this size +# https://osmocom.org/projects/osmocom-servers/wiki/Docker_cache_clean_up +docker_max_image_space: "40 GB" diff --git a/ansible/roles/docker/files/Dockerfile b/ansible/roles/docker/files/Dockerfile new file mode 100644 index 0000000..d0ddf9c --- /dev/null +++ b/ansible/roles/docker/files/Dockerfile @@ -0,0 +1,33 @@ +ARG REGISTRY=docker.io +FROM ${REGISTRY}/alpine:3.15 +ARG DOCKER_GID + +RUN apk add \ + cargo \ + docker-cli + +# Create user and docker group with same group-id as on host system, create +# /opt/docuum dir owned by user +RUN set -x && \ + if getent group "${DOCKER_GID}"; then \ + delgroup $(getent group "${DOCKER_GID}" | cut -d: -f1); \ + fi && \ + addgroup -g "${DOCKER_GID}" docker && \ + adduser -D -u 1000 -G docker user && \ + mkdir /opt/docuum && \ + chown user /opt/docuum + +USER user + +ARG DOCUUM_VER=0.20.4 + +RUN set -x && \ + cd /opt/docuum && \ + wget https://github.com/stepchowfun/docuum/archive/refs/tags/v${DOCUUM_VER}.tar.gz \ + -O docuum.tar.gz && \ + tar -xf docuum.tar.gz && \ + cd docuum-${DOCUUM_VER} && \ + cargo build --release && \ + cd .. && \ + mv docuum-${DOCUUM_VER}/target/release/docuum . && \ + rm -rf ~/.cargo docuum-${DOCUUM_VER} docuum.tar.gz diff --git a/ansible/roles/docker/files/docuum.service b/ansible/roles/docker/files/docuum.service new file mode 100644 index 0000000..8c62973 --- /dev/null +++ b/ansible/roles/docker/files/docuum.service @@ -0,0 +1,11 @@ +[Unit] +Description=Docuum +After=docker.service +Wants=docker.service + +[Service] +ExecStart=/opt/docuum/docuum.sh +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/docker/files/docuum.sh b/ansible/roles/docker/files/docuum.sh new file mode 100755 index 0000000..e7d5e28 --- /dev/null +++ b/ansible/roles/docker/files/docuum.sh @@ -0,0 +1,43 @@ +#!/bin/sh -ex + +# Maximum amount of storage that docker images may consume +THRESHOLD="$(cat /opt/docuum/docker_max_image_space)" + +DIR="$(dirname "$(realpath "$0")")" +IMG="osmo-ci-docuum" +DOCUUM_UID="1000" +DOCKER_GID="$(getent group docker | cut -d : -f 3)" +PULL_ARG="" + +if [ -z "$THRESHOLD" ]; then + set +x + echo "ERROR: failed to read threshold from /opt/docuum/docker_max_image_space" + exit 1 +fi + +if [ "$INITIAL_BUILD" = 1 ]; then + PULL_ARG="--pull" +fi + +mkdir -p /var/cache/docuum +chown "$DOCUUM_UID" /var/cache/docuum + +cd "$DIR" +docker build \ + --build-arg DOCKER_GID="$DOCKER_GID" \ + $PULL_ARG \ + -t "$IMG" \ + . + +if [ "$INITIAL_BUILD" = 1 ]; then + exit 0 +fi + +docker run \ + --rm \ + --init \ + --name docuum \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /var/cache/docuum:/home/user \ + "$IMG" \ + sh -c "exec /opt/docuum/docuum --threshold '$THRESHOLD'" diff --git a/ansible/roles/docker/tasks/docuum.yml b/ansible/roles/docker/tasks/docuum.yml new file mode 100644 index 0000000..594c010 --- /dev/null +++ b/ansible/roles/docker/tasks/docuum.yml @@ -0,0 +1,40 @@ +--- +- name: "docuum : set docker_max_image_space to {{ docker_max_image_space }}" + lineinfile: + path: /opt/docuum/docker_max_image_space + state: present + create: yes + line: "{{ docker_max_image_space }}" + regexp: ".*" # replace previous content if changed + +- name: "docuum : copy Dockerfile" + copy: + src: Dockerfile + dest: /opt/docuum/ + mode: 0644 + +- name: "docuum : copy docuum.sh" + copy: + src: docuum.sh + dest: /opt/docuum/ + mode: 0755 + +- name: "docuum : build container" + shell: INITIAL_BUILD=1 /opt/docuum/docuum.sh + +- name: "docuum : copy docuum.service" + copy: + src: docuum.service + dest: /lib/systemd/system/docuum.service + register: docuumservice + +- name: "docuum : systemctl daemon-reload" + systemd: + daemon_reload: yes + when: docuumservice is changed + +- name: "docuum : ensure the systemd service is installed" + systemd: + name: docuum.service + state: started + enabled: yes diff --git a/ansible/roles/docker/tasks/main.yml b/ansible/roles/docker/tasks/main.yml index a268d2b..f64ae96 100644 --- a/ansible/roles/docker/tasks/main.yml +++ b/ansible/roles/docker/tasks/main.yml @@ -40,6 +40,7 @@ - name: cleanup old docker images cron: name: cleanup-docker-images + disabled: false minute: 0 hour: '*/3' user: "{{ jenkins_user }}" @@ -50,3 +51,10 @@ src: daemon.json dest: /etc/docker/daemon.json notify: restart docker + +# daemon.json configures overlay2 storage driver, clean up vfs leftovers +- name: cleanup vfs storage dir + shell: "docker info | grep -q 'Storage Driver: overlay2' || rm -rf /var/lib/docker/vfs" + +# After docker is set up, add docuum to clean old docker images +- import_tasks: docuum.yml diff --git a/ansible/roles/gsm-tester-bts/files/Release.key b/ansible/roles/gsm-tester-bts/files/Release.key index 4257562..ecca084 100644 --- a/ansible/roles/gsm-tester-bts/files/Release.key +++ b/ansible/roles/gsm-tester-bts/files/Release.key @@ -1,20 +1,26 @@ -----BEGIN PGP PUBLIC KEY BLOCK----- Version: GnuPG v1.4.5 (GNU/Linux) -mQENBFJBt/wBCADAht3d/ilNuyzaXYw/QwTRvmjyoDvfXw+H/3Fvk1zlDZoiKPPc -a1wCVBINUZl7vYM2OXqbJwYa++JP2Q48xKSvC6thbRc/YLievkbcvTemf7IaREfl -CTjoYpoqXHa9kHMw1aALDm8CNU88jZmnV7v9L6hKkbYDxie+jpoj7D6B9JlxgNJ4 -5dQyRNsFGVcIl4Vplt1HyGc5Q5nQI/VgS2rlF/IOXmhRQBc4LEDdU8R2IKnkU4ee -S7TWanAigGAQhxGuCkS39/CWzc1DhLhjlNhBl/+RTPejkqJtAy00ZLps3+RqUN1Y -CU/Fsr7aRlYVGqQ/BlptwV0XQ2VVYJX2oEBBABEBAAG0MG5ldHdvcmsgT0JTIFBy -b2plY3QgPG5ldHdvcmtAYnVpbGQub3BlbnN1c2Uub3JnPokBPAQTAQIAJgUCXoL5 -awIbAwUJEF/xbwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEGLrGgkXKA3fBFkH -/Ry9IT3WkpyrKLi6wlTKBZTDGz58xDEd6i4fKr1sepgfexzmu/t1125A4Jyuy5Qp -wolWdVIxmo8Mo9/u16ssGV/Iiy4y+3v+yfOExUeaM5XS4eW5cWcySBi+y3PGL17o -Qf/iZDnzJtx16kMgD+lybTFSRDhRg4BV1egMMtOV/XIhC7mp0Gt6oOU72fGuWFKs -3prY+XmL20x8+FfI35sXRAQu2L7OBDnUJoHHuHIT1s5no5JSYubctY/KNT/9YGme -DSspyOxXJTo6YgOJStyMQZ3DaE4XxSnGpzPV27OnfMWjRf+KLvgFjx9n/zPIzBTh -Q9gL7GfQUOkJ5L0GrOvVxBiIRgQTEQIABgUCUkG3/AAKCRA7MBG3a51lIzhdAJ9v -d6XPffMZRcCGgDEY5OaTn/MsCQCgrXbeZpFJgnirSrc8rRonvzYFiF4= -=zwAO +mQENBGKzE1QBCADFcM3ZzggvgxNRNNqDGWf5xIDAiK5qzFLdGes7L6F9VCHdaPy0 +RAOB5bFb/Q1tSDFNEBLtaauXKz+4iGL6qMVjZcyjzpB5w4jKN+kkrFRhjDNUv/SH +BX6d+P7v5WBGSNArNgA8D1BGzckp5a99EZ0okMJFEqIcN40PD6OGugpq5XnVV5Nk +e93fLa2Cu8vhFBcVn6CuHeEhsmuMf6NLbQRNfNNCEEUYaZn7beMYtpZ7t1djsKx5 +1xGm50OzI22FLu8lELQ9d7qMVGRG3WHYawX9BDteRybiyqxfwUHm1haWazRJtlGt +UWyzvwAb80BK1J2Nu5fbAa3w5CoEPAbUuCyrABEBAAG0JW9zbW9jb20gT0JTIFBy +b2plY3QgPG9zbW9jb21Ab3Ntb2NvbT6JAVQEEwEIAD4WIQRrKp83ktFetw1Oao+G +pzC2U3JZcwUCYrMV4wIbAwUJBB6yjwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAK +CRCGpzC2U3JZc4FRCACQQkKIrnvQ7n2u7GSmyVZa3I+oLoFXSGqaGyey5TW/nrMm +vFDKU3qliHiuNSmUY35SnAhXUsvqOYppxVRoO1MLrqUvzMOnIWqkJpf8mtjGUnsW +jyVeto7Rsjs75y2i1Hk+e7ljb/V65J3NlfrfEYWbqR9AKd53ReNXTdrQ0J05A38N +GdI4Ld/2lNISAwaBmGhqdeKsLHpQw/JERU1TApVJR1whFiIwDF1rOCg9GPnNKIk7 +yRZdK267XzztrainX/cbPILyzUZEDhYs6wQuyACyQ1YUxZIxrwVfk7PMNay8CrLH +z42B73Ne5IAj8+op/3iJafFONLm7YXiDUFN+QDYAiQEzBBMBCAAdFiEExoiYhHND +S7aVYlnqa51NyAUyjdsFAmKzE1UACgkQa51NyAUyjdvuZgf+OXmr//i7u7Gg7eWB +7e0qUsyCId9lXS8J437x3K6ciJfD7/6RSy8TFW5Nglm/uSkbyq582I8t+SoOirMD +E6cg9U/5+h5s46bAf+Kd2XS/6tLGeNLM18i4el8CP06NpFzDrsKu76uYFpyRiiHD +otBdtgxeLJ83LugGfZslF+/5cigJkAJMhAdVvGO8h85R6fba8ZSOKtMKkaQRfi76 +nhyOrJPlLuS+DLEnHwdkOFgtKnxHdjM97K+Tx0gisb6uwaWroXfSLnhP8RTLLZZy +Z+noU1Hw3c+mn4c/NYbcC/uwHYHKRzuf9gHnQ3dGgv0Z5sbeLRVo92hjGj7Ftlyd +4hmKBg== +=HxK4 -----END PGP PUBLIC KEY BLOCK----- diff --git a/ansible/roles/gsm-tester-bts/tasks/main.yml b/ansible/roles/gsm-tester-bts/tasks/main.yml index f1377f0..7ada1d8 100644 --- a/ansible/roles/gsm-tester-bts/tasks/main.yml +++ b/ansible/roles/gsm-tester-bts/tasks/main.yml @@ -42,7 +42,7 @@ - name: add apt-repo of OBS network:osmocom:nightly apt_repository: filename: obs_network_osmocom_latest - repo: "deb http://download.opensuse.org/repositories/network:/osmocom:/nightly/Debian_{{ ansible_distribution_major_version }}.0/ ./" + repo: "deb https://downloads.osmocom.org/packages/osmocom:/nightly/Debian_{{ ansible_distribution_major_version }}.0/ ./" update_cache: yes - name: install osmocom utilities diff --git a/ansible/roles/gsm-tester/tasks/main.yml b/ansible/roles/gsm-tester/tasks/main.yml index 418509e..a1fb6cd 100644 --- a/ansible/roles/gsm-tester/tasks/main.yml +++ b/ansible/roles/gsm-tester/tasks/main.yml @@ -66,7 +66,7 @@ - name: install srsLTE runtime extra dependencies (Debian) apt: name: - - soapysdr0.7-module-lms7 + - soapysdr-module-lms7 when: ansible_distribution == 'Debian' tags: - srs @@ -132,7 +132,7 @@ - name: setup ofono repository git: - repo: 'git://git.sysmocom.de/ofono' + repo: 'https://gitea.sysmocom.de/sysmocom/ofono' dest: /root/ofono version: osmo-gsm-tester clone: yes @@ -354,7 +354,7 @@ - name: setup osmo-gsm-tester repository git: - repo: 'git://git.osmocom.org/osmo-gsm-tester' + repo: 'https://gitea.osmocom.org/cellular-infrastructure/osmo-gsm-tester' dest: /root/osmo-gsm-tester version: master clone: yes diff --git a/ansible/roles/install-coverity/defaults/main.yml b/ansible/roles/install-coverity/defaults/main.yml index 6b97781..7be5be6 100644 --- a/ansible/roles/install-coverity/defaults/main.yml +++ b/ansible/roles/install-coverity/defaults/main.yml @@ -1,4 +1,4 @@ --- -coverity_version: 2019.03 +coverity_version: 2023.6.2 coverity_installer_file: "cov-analysis-linux64-{{ coverity_version }}.tar.gz" diff --git a/ansible/roles/install-coverity/tasks/main.yml b/ansible/roles/install-coverity/tasks/main.yml index 5baf6f3..ca4a24c 100644 --- a/ansible/roles/install-coverity/tasks/main.yml +++ b/ansible/roles/install-coverity/tasks/main.yml @@ -3,41 +3,46 @@ apt: name: - curl - -- name: copy coverity installer - copy: - src: "{{ coverity_installer_file }}" - dest: "/tmp/{{ coverity_installer_file }}" - mode: 0750 - register: coverity_copy - ignore_errors: yes tags: [coverity] -- name: create /opt/coverity - file: - path: /opt/coverity/ - state: directory - when: coverity_copy.failed == False +- name: check if coverity needs to be installed + stat: + path: "/opt/coverity/{{ coverity_version }}/" + register: coverity_stat tags: [coverity] -- name: unpack coverity - unarchive: - src: "/tmp/{{ coverity_installer_file }}" - dest: /opt/coverity/ - remote_src: yes - when: coverity_copy.failed == False +- name: install coverity if required + when: coverity_stat.stat.exists == False tags: [coverity] + block: + - name: copy coverity installer + copy: + src: "{{ coverity_installer_file }}" + dest: "/tmp/{{ coverity_installer_file }}" + mode: 0750 + register: coverity_copy + ignore_errors: yes -- name: create link /opt/coverity/last - file: - src: /opt/coverity/cov-analysis-linux64-{{ coverity_version }} - dest: /opt/coverity/current - state: link - when: coverity_copy.failed == False - tags: [coverity] + - name: create /opt/coverity/{{ coverity_version }}/ + file: + path: /opt/coverity/{{ coverity_version }}/ + state: directory + when: coverity_copy.failed == False -- name: "Please download {{ coverity_installer_file }} to your ansible directory to allow ansible to install coverity" - debug: - msg: "Ansible can not find {{ coverity_installer_file }}" - when: coverity_copy.failed - tags: [coverity] + - name: unpack coverity + unarchive: + src: "/tmp/{{ coverity_installer_file }}" + dest: /opt/coverity/{{ coverity_version }} + remote_src: yes + when: coverity_copy.failed == False + + - name: create link /opt/coverity/current + shell: rm -vf /opt/coverity/current && ln -svf /opt/coverity/{{ coverity_version }}/* /opt/coverity/current + args: + warn: false + when: coverity_copy.failed == False + + - name: "Please download {{ coverity_installer_file }} to your ansible directory to allow ansible to install coverity" + debug: + msg: "Ansible can not find {{ coverity_installer_file }}" + when: coverity_copy.failed diff --git a/ansible/roles/install-poky-sdk/tasks/main.yml b/ansible/roles/install-poky-sdk/tasks/main.yml index 85b3c94..57d0137 100644 --- a/ansible/roles/install-poky-sdk/tasks/main.yml +++ b/ansible/roles/install-poky-sdk/tasks/main.yml @@ -1,46 +1,54 @@ --- -- name: install bzip2 and tar +- name: install dependencies for poky installer apt: name: - - tar + - build-essential - bzip2 + - file + - tar cache_valid_time: 3600 update_cache: yes - -- name: copy poky installer - copy: - src: "{{ poky_installer_file }}" - dest: "/tmp/{{ poky_installer_file }}" - mode: 0750 - register: poky_copy - ignore_errors: yes - tags: [poky] - -- name: execute poky installer - command: "/tmp/{{ poky_installer_file }} -y -d '{{ poky_dest }}'" - args: - creates: "{{ poky_dest }}" - when: poky_copy.failed == False tags: [poky] -- name: change owner/group to jenkins user - file: +- name: check if poky needs to be installed + stat: path: "{{ poky_dest }}" - owner: "{{ jenkins_user }}" - group: "{{ jenkins_user }}" - recurse: yes - when: poky_copy.failed == False + register: poky_stat tags: [poky] -- name: remove poky installer - file: - path: "/tmp/{{ poky_installer_file }}" - state: absent - when: poky_copy.failed == False +- name: install poky if required tags: [poky] + when: poky_stat.stat.exists == False + block: + - name: copy poky installer + copy: + src: "{{ poky_installer_file }}" + dest: "/tmp/{{ poky_installer_file }}" + mode: 0750 + register: poky_copy + ignore_errors: yes -- name: "Please download {{ poky_installer_file }} to your ansible directory to allow ansible to install poky" - debug: - msg: "Ansible can not find or copy {{ poky_installer_file }}" - when: poky_copy.failed == True - tags: [poky] + - name: execute poky installer + command: "/tmp/{{ poky_installer_file }} -y -d '{{ poky_dest }}'" + args: + creates: "{{ poky_dest }}" + when: poky_copy.failed == False + + - name: change owner/group to jenkins user + file: + path: "{{ poky_dest }}" + owner: "{{ jenkins_user }}" + group: "{{ jenkins_user }}" + recurse: yes + when: poky_copy.failed == False + + - name: remove poky installer + file: + path: "/tmp/{{ poky_installer_file }}" + state: absent + when: poky_copy.failed == False + + - name: "Please download {{ poky_installer_file }} to your ansible directory to allow ansible to install poky" + debug: + msg: "Ansible can not find or copy {{ poky_installer_file }}" + when: poky_copy.failed == True diff --git a/ansible/roles/osmocom-jenkins-slave/README.md b/ansible/roles/osmocom-jenkins-slave/README.md index 50aed16..81f68b3 100644 --- a/ansible/roles/osmocom-jenkins-slave/README.md +++ b/ansible/roles/osmocom-jenkins-slave/README.md @@ -3,7 +3,6 @@ Support the following variables: * `install_jenkins_utilities`: (true) install usefull utilities including qemu/debootstrap/fakeroot -* `install_osmocom_build_deps`: (true) install all osmocom runtime and build time dependencies * `generic_slave`: (true) contains tasks used by the most osmocom jenkins slaves * `osmocom_jenkins_slave_fstrim`: (false) calls fstrim periodical * `ttcn3_slave`: (true) install titan ttcn3 compiler and prepares the docker-playground diff --git a/ansible/roles/osmocom-jenkins-slave/defaults/main.yml b/ansible/roles/osmocom-jenkins-slave/defaults/main.yml index 4e1f179..a4ce8ac 100644 --- a/ansible/roles/osmocom-jenkins-slave/defaults/main.yml +++ b/ansible/roles/osmocom-jenkins-slave/defaults/main.yml @@ -6,9 +6,6 @@ jenkins_user: jenkins # install usefull utilities including qemu/debootstrap/fakeroot install_jenkins_utilities: true -# install all osmocom runtime and build time dependencies -install_osmocom_build_deps: true - # E.g. a generic_slave requires different tasks as a # special slave including the gsm-tester generic_slave: true diff --git a/ansible/roles/osmocom-jenkins-slave/tasks/debian.yml b/ansible/roles/osmocom-jenkins-slave/tasks/debian.yml index 0c762e9..62beb8e 100644 --- a/ansible/roles/osmocom-jenkins-slave/tasks/debian.yml +++ b/ansible/roles/osmocom-jenkins-slave/tasks/debian.yml @@ -3,14 +3,15 @@ - name: install jenkins utilities apt: name: - - htop - - virt-manager - - qemu - - qemu-user-static - - qemu-system-arm - - proot + - bc - debootstrap - fakeroot + - htop + - proot + - python3-setuptools + - qemu-system-arm + - qemu-user-static + - virt-manager cache_valid_time: 3600 update_cache: yes install_recommends: no @@ -50,7 +51,7 @@ - name: install java for buster and later apt: name: - - openjdk-11-jre-headless + - default-jre-headless cache_valid_time: 3600 update_cache: yes install_recommends: no diff --git a/ansible/roles/osmocom-jenkins-slave/tasks/main.yml b/ansible/roles/osmocom-jenkins-slave/tasks/main.yml index 2623201..1868d97 100644 --- a/ansible/roles/osmocom-jenkins-slave/tasks/main.yml +++ b/ansible/roles/osmocom-jenkins-slave/tasks/main.yml @@ -22,19 +22,22 @@ shell: /bin/bash uid: 1000 -- name: setup jenkins ssh key +- name: setup jenkins ssh key (authorized_keys) authorized_key: user: "{{ jenkins_user }}" key: "{{ lookup('file', 'jenkins.osmocom.org.pub') }}" +- name: setup gerrit ssh key (known_hosts) + known_hosts: + name: '[gerrit.osmocom.org]:29418' + key: '[gerrit.osmocom.org]:29418 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQDI4wZwtcGz2V98iZWWEtsA162PolLtgQTBjnC14ONzZGQkTh9bTPsoSxnM5TKOm4F2bzq7gb+Qrsj4ZECD19qztmD37kMP9jn7/2i1V8oLAy9ojyklXAdBVTliQteVI7ieDOyKCnGbszvFWXY2isoO1k7yJLv0QyL7F5AuZlxeHw==' + path: /etc/ssh/ssh_known_hosts + state: present + - name: install ttcn3 dependencies include_tasks: ttcn3-slave.yml when: ttcn3_slave and ansible_distribution == 'Debian' and ansible_distribution_version|int >= 9 and not (ansible_architecture == "armv7l" or ansible_architecture == "arm64") -- name: include osmocom_build_deps.yml - include_tasks: osmocom_build_deps.yml - when: install_osmocom_build_deps - - name: include generic slave include_tasks: generic-slave.yml when: generic_slave diff --git a/ansible/roles/osmocom-jenkins-slave/tasks/osmo-ci.yml b/ansible/roles/osmocom-jenkins-slave/tasks/osmo-ci.yml index b26d6ab..a2f2797 100644 --- a/ansible/roles/osmocom-jenkins-slave/tasks/osmo-ci.yml +++ b/ansible/roles/osmocom-jenkins-slave/tasks/osmo-ci.yml @@ -4,18 +4,26 @@ # we need to initially install osmo-ci and the docker images *before* # that jenkins job can execute subsequent updates +- name: install osmo-ci build dependencies + apt: + name: + - make + cache_valid_time: 3600 + update_cache: yes + install_recommends: no + - name: deploy osmo-ci to home directory become: yes become_user: "{{ jenkins_user }}" git: - repo: "https://git.osmocom.org/osmo-ci" + repo: "https://gerrit.osmocom.org/osmo-ci" dest: "/home/{{ jenkins_user }}/osmo-ci" - name: deploy osmo-gsm-manuals to home directory become: yes become_user: "{{ jenkins_user }}" git: - repo: "https://git.osmocom.org/osmo-gsm-manuals" + repo: "https://gerrit.osmocom.org/osmo-gsm-manuals" dest: "/home/{{ jenkins_user }}/osmo-gsm-manuals" - name: rebuild osmocom docker images for jenkins build tests diff --git a/ansible/roles/osmocom-jenkins-slave/tasks/osmocom_build_deps.yml b/ansible/roles/osmocom-jenkins-slave/tasks/osmocom_build_deps.yml deleted file mode 100644 index e2b5a98..0000000 --- a/ansible/roles/osmocom-jenkins-slave/tasks/osmocom_build_deps.yml +++ /dev/null @@ -1,190 +0,0 @@ ---- -- name: install build utilities - apt: - name: - - asciidoc - - autoconf - - autoconf-archive - - automake - - bc - - bison - - coccinelle - - cppcheck - - debhelper - - devscripts - - dh-autoreconf - - dh-systemd - - doxygen - - flex - - g++ - - gawk - - gcc - - gcc-arm-none-eabi - - git - - git-buildpackage - - graphviz - - libcsv-dev - - libtool - - libboost-all-dev - - make - - mscgen - - osc - - pkgconf - - python-minimal - - python-setuptools - - python3 - - python3-setuptools - - stow - - texinfo - # by gsm-manuals - - python3-nwdiag - - python-pychart - - inkscape - # by gapk - - libasound-dev - - libopencore-amrnb-dev - # by LimeSuite - - cmake - - patchelf - # by libusrp - - sdcc - # by mingw upload job - - rsync - cache_valid_time: 3600 - update_cache: yes - -- name: install build dependencies and libraries - apt: - name: - - dahdi-source - - docbook5-xml - - libboost-dev - - libc-ares-dev - - libdbd-sqlite3 - - libdbi-dev - - libffi-dev - - libffi-dev - - libfftw3-dev - - libgmp-dev - - libgnutls28-dev - - libgps-dev - - libgsm1-dev - - libnl-3-dev - - libnl-route-3-dev - - libmnl-dev - # by osmo-sysmon - - liboping-dev - - libncurses5-dev - - libortp-dev - - libpcap-dev - - libpcsclite-dev - - libreadline-dev - - libsctp-dev - - libsigsegv-dev - - libsnmp-dev - - libsofia-sip-ua-glib-dev - - libsqlite3-dev - - libssl-dev - - libsystemd-dev - - libtalloc-dev - - libuhd-dev - - libusb-1.0-0-dev - - libusb-dev - - libzmq3-dev - - sqlite3 - - libaio-dev - cache_valid_time: 3600 - update_cache: yes - install_recommends: no - -- name: install liblua for jessie - apt: - name: liblua5.3-dev - cache_valid_time: 3600 - update_cache: yes - install_recommends: no - default_release: jessie-backports - when: ansible_distribution == 'Debian' and ansible_distribution_release == 'jessie' - -- name: install liblua for stretch or later - apt: - name: - - liblua5.3-dev - cache_valid_time: 3600 - update_cache: yes - install_recommends: no - when: ansible_distribution == 'Debian' and ansible_distribution_version|int >= 9 - -- name: install liblimesuite-dev for stretch or later - apt: - name: - - liblimesuite-dev - cache_valid_time: 3600 - update_cache: yes - install_recommends: no - when: ansible_distribution == 'Debian' and ansible_distribution_version|int >= 9 - -- name: install osmo-remsim dependencies - apt: - name: - - libjansson-dev - - libulfius-dev - cache_valid_time: 3600 - update_cache: yes - when: ansible_distribution == 'Debian' and ansible_distribution_version|int >= 9 - -- name: install srsLTE build dependencies - apt: - name: - - cmake - - libfftw3-dev - - libmbedtls-dev - - libboost-program-options-dev - - libconfig++-dev - - libsctp-dev - - libpcsclite-dev - - libuhd-dev - - libczmq-dev - - libsoapysdr-dev - cache_valid_time: 3600 - update_cache: yes - when: ansible_distribution == 'Debian' and ansible_distribution_version|int >= 9 - -- name: install open5gs build dependencies - apt: - name: - - meson - - ninja-build - - libgcrypt-dev - - libidn11-dev - - libmongoc-dev - - libbson-dev - - libyaml-dev - - libnghttp2-dev - cache_valid_time: 3600 - update_cache: yes - when: ansible_distribution == 'Debian' and ansible_distribution_version|int >= 10 - -- name: download osmo-python-tests for py2 - git: - repo: "https://gerrit.osmocom.org/python/osmo-python-tests" - dest: "/home/{{ jenkins_user }}/osmo-python-tests2" - version: "fb4b813d4df62b7b2445bdced961eb1847267eed" - -- name: install osmo-python-tests for py2 - shell: - cmd: "python2 setup.py clean build install && date > installed" - chdir: "/home/{{ jenkins_user }}/osmo-python-tests2" - creates: "/home/{{ jenkins_user }}/osmo-python-tests2/installed" - -- name: download osmo-python-tests for py3 - git: - repo: "https://gerrit.osmocom.org/python/osmo-python-tests" - dest: "/home/{{ jenkins_user }}/osmo-python-tests3" - version: "be7fcf5f28a5d0f9ad35510f09b5979268d76598" - -- name: install osmo-python-tests for py3 - shell: - cmd: "python3 setup.py clean build install && date > installed" - chdir: "/home/{{ jenkins_user }}/osmo-python-tests3" - creates: "/home/{{ jenkins_user }}/osmo-python-tests3/installed" diff --git a/ansible/roles/osmocom-jenkins-slave/tasks/ttcn3-slave.yml b/ansible/roles/osmocom-jenkins-slave/tasks/ttcn3-slave.yml index fbfc1ec..4237073 100644 --- a/ansible/roles/osmocom-jenkins-slave/tasks/ttcn3-slave.yml +++ b/ansible/roles/osmocom-jenkins-slave/tasks/ttcn3-slave.yml @@ -1,4 +1,12 @@ --- +- name: install programs used in jenkins.sh + apt: + name: + - netcat-openbsd + - sudo + cache_valid_time: 3600 + update_cache: yes + install_recommends: no - name: ensure jenkins folder exists become: yes @@ -11,33 +19,27 @@ become: yes become_user: "{{ jenkins_user }}" git: - repo: "https://git.osmocom.org/docker-playground" + repo: "https://gerrit.osmocom.org/docker-playground" dest: "/home/{{ jenkins_user }}/jenkins/docker-playground" - name: add apt.key - apt_key: - url: https://download.opensuse.org/repositories/network:/osmocom:/latest/{{ obs_distro }}/Release.key + get_url: + url: https://obs.osmocom.org/projects/osmocom/public_key + dest: /etc/apt/trusted.gpg.d/obs.osmocom.org.asc + # OBS webserver needs this header, returns 404 otherwise + # https://github.com/ansible/ansible/issues/50436#issuecomment-451205256 + headers: + Accept: '*/*' - name: add apt repository apt_repository: - repo: "deb http://download.opensuse.org/repositories/network:/osmocom:/latest/{{ obs_distro }}/ ./" + repo: "deb [signed-by=/etc/apt/trusted.gpg.d/obs.osmocom.org.asc] https://downloads.osmocom.org/packages/osmocom:/latest/{{ obs_distro }}/ ./" filename: obs_osmocom update_cache: yes -- name: install titan +- name: install libosmocore-utils apt: - name: eclipse-titan + name: + - libosmocore-utils update_cache: yes cache_valid_time: 3600 - -- name: Get DEB architecture - shell: dpkg --print-architecture - register: deb_architecture - -- name: Install libfftranscode0 - apt: - deb: https://ftp.osmocom.org/binaries/libfftranscode/libfftranscode0_0.3_{{deb_architecture.stdout}}.deb - -- name: Install libfftranscode-dev - apt: - deb: https://ftp.osmocom.org/binaries/libfftranscode/libfftranscode-dev_0.3_{{deb_architecture.stdout}}.deb diff --git a/ansible/roles/osmocom-obs-worker/files/obs-ca.crt b/ansible/roles/osmocom-obs-worker/files/obs-ca.crt new file mode 100644 index 0000000..271d168 --- /dev/null +++ b/ansible/roles/osmocom-obs-worker/files/obs-ca.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSzCCAjOgAwIBAgIUcLDeyxhcMpcccZ1i7AIKqvwTJ2gwDQYJKoZIhvcNAQEL +BQAwFjEUMBIGA1UEAwwLRWFzeS1SU0EgQ0EwHhcNMjIwNTEwMDkzMjQwWhcNNDIw +NTA1MDkzMjQwWjAWMRQwEgYDVQQDDAtFYXN5LVJTQSBDQTCCASIwDQYJKoZIhvcN +AQEBBQADggEPADCCAQoCggEBALUsvHoy4oqpOHuzyoIG3qyoTGQW8Zpse7RaYAWJ +CLgcAGelXWrOhPy40xAWHpLzAnAIxYR4ZSlD2Ebaj08esz4VQs2CKQ0mJEVZA7oq +84YO3WK02HzoYFLee4ySvI967FCn6BwVZQVdz/A8wOIlAbe7Hmucx6dNLTLTR+Pw +YwHMSdnj+OlC2oX2UO+ZIR7jFbSYp7jGF9V8zk74NoAFAuv5rs1UeP01ZF9xgXll +KHUajaUKMKs+6KTo4d2l9ha45LHcoAuDUURzf4GMvO7zVVGw4mFwy/YsPoLV5+vX +YT4GqkBgk9iZXirgTk6xCkiP0NUw5UNhtarUDKvR/U8wLw8CAwEAAaOBkDCBjTAd +BgNVHQ4EFgQUHdDZVpUSwtRrSKp+PuMRK12TlQYwUQYDVR0jBEowSIAUHdDZVpUS +wtRrSKp+PuMRK12TlQahGqQYMBYxFDASBgNVBAMMC0Vhc3ktUlNBIENBghRwsN7L +GFwylxxxnWLsAgqq/BMnaDAMBgNVHRMEBTADAQH/MAsGA1UdDwQEAwIBBjANBgkq +hkiG9w0BAQsFAAOCAQEAPGqIfE172P1NaQGRIHdsRJRot9NnJzZih3q9m+T/YIE8 +8T3uoxhEsMPLzP19w8wAbRy+CjpLwTuYO44kOIVLYhFdx3BTVRhw+YU2RH4PSttE +JDVyYTFPGzjxwprpfui1aGom8u0oPL5MeutMWrPH6qVlIv7JGR3EbDG0iYwE5MZJ +6kW8VW6o5fL02uT+75vNkHTipLUO6S4l6h9NrrgR0ijZ5dhyKC0FZnLnowtMCXXl +2rfHSnGKjMGcq8Jlrwo6eMVPbZM6rrU/tvrjJkqESZY0jMktd0fV903XJ6+X4U9U +stu/HUtKUQC8d3wua1PuI1uPvpr8AQDcf5LuA/STHg== +-----END CERTIFICATE----- diff --git a/ansible/roles/osmocom-obs-worker/files/obs.conf b/ansible/roles/osmocom-obs-worker/files/obs.conf new file mode 100644 index 0000000..7dc2a84 --- /dev/null +++ b/ansible/roles/osmocom-obs-worker/files/obs.conf @@ -0,0 +1,15 @@ +client +remote 2a01:4f8:201:344a::1:4 1194 +ca "/etc/openvpn/obs-ca.crt" +cert "/etc/openvpn/obs.crt" +key "/etc/openvpn/obs.key" +comp-lzo yes +cipher AES-256-CBC +dev tun +proto udp6 +nobind +auth-nocache +script-security 2 +persist-key +persist-tun +remote-cert-tls server diff --git a/ansible/roles/osmocom-obs-worker/tasks/main.yml b/ansible/roles/osmocom-obs-worker/tasks/main.yml new file mode 100644 index 0000000..b350163 --- /dev/null +++ b/ansible/roles/osmocom-obs-worker/tasks/main.yml @@ -0,0 +1,8 @@ +--- + +- name: install OBS + include_tasks: obs.yml + +- name: install openvpn + include_tasks: openvpn.yml + diff --git a/ansible/roles/osmocom-obs-worker/tasks/obs.yml b/ansible/roles/osmocom-obs-worker/tasks/obs.yml new file mode 100644 index 0000000..a0a6227 --- /dev/null +++ b/ansible/roles/osmocom-obs-worker/tasks/obs.yml @@ -0,0 +1,52 @@ +- name: add OBS rpm repository + zypper_repository: + repo: https://download.opensuse.org/repositories/OBS:/Server:/2.10/{{ ansible_distribution_version }}/OBS:Server:2.10.repo + autorefresh: yes + auto_import_keys: yes + +- name: install obs-worker + zypper: + name: + - obs-worker + +- name: Make sure an entry in /etc/hosts exists + lineinfile: + path: /etc/hosts + regexp: "^172.16.31.1" + line: "172.16.31.1 obs-backend.osmocom.org obs obs.osmocom.org" + state: present + +- name: Create /srv/obs/cache + file: + path: /srv/obs/cache + owner: obsrun + group: obsrun + recurse: yes + +- name: generate obs-server config file + template: + src: obs-server + dest: /etc/sysconfig/obs-server + +- name: install lxc + when: obs_vm_type == "lxc" + zypper: + name: + - lxc + +- name: install qemu + when: obs_vm_type != "lxc" + zypper: + name: + - qemu-arm + +- name: install qemu-arm + when: obs_vm_type == "emulator:armv7l" or obs_vm_type == "emulator:aarch64" + zypper: + name: + - qemu-arm + +- name: ensure the systemd service is installed + systemd: + name: obsworker.service + enabled: yes diff --git a/ansible/roles/osmocom-obs-worker/tasks/openvpn.yml b/ansible/roles/osmocom-obs-worker/tasks/openvpn.yml new file mode 100644 index 0000000..da53fa5 --- /dev/null +++ b/ansible/roles/osmocom-obs-worker/tasks/openvpn.yml @@ -0,0 +1,21 @@ +- name: Install openvpn + zypper: + name: + - openvpn + +- name: copy openvpn config + copy: + src: obs.conf + dest: /etc/openvpn/ + mode: 0600 + +- name: copy openvpn CA certificate + copy: + src: obs-ca.crt + dest: /etc/openvpn/ + mode: 0600 + +- name: ensure the systemd service is installed + systemd: + name: openvpn@obs.service + enabled: yes diff --git a/ansible/roles/osmocom-obs-worker/templates/obs-server b/ansible/roles/osmocom-obs-worker/templates/obs-server new file mode 100644 index 0000000..f042cd1 --- /dev/null +++ b/ansible/roles/osmocom-obs-worker/templates/obs-server @@ -0,0 +1,442 @@ +# +# NOTE: all these options can be also declared in /etc/buildhost.config on each worker differently. +# + +## Path: Applications/OBS +## Description: The OBS backend code directory +## Type: string +## Default: "" +## Config: OBS +# +# An empty dir will lead to the fall back directory, typically /usr/lib/obs/server/ +# +OBS_BACKENDCODE_DIR="" + +## Path: Applications/OBS +## Description: The base for OBS communication directory +## Type: string +## Default: "" +## Config: OBS +# +# An empty dir will lead to the fall back directory, typically /srv/obs/run +# +OBS_RUN_DIR="" + +## Path: Applications/OBS +## Description: The base for OBS logging directory +## Type: string +## Default: "" +## Config: OBS +# +# An empty dir will lead to the fall back directory, typically /srv/obs/log +# +OBS_LOG_DIR="" + +## Path: Applications/OBS +## Description: The base directory for OBS +## Type: string +## Default: "" +## Config: OBS +# +# An empty dir will lead to the fall back directory, typically /srv/obs +# +OBS_BASE_DIR="" + +## Path: Applications/OBS +## Description: Automatically setup api and webui for OBS server, be warned, this will replace config files ! +## Type: ("yes" | "no") +## Default: "no" +## Config: OBS +# +# This is usally only enabled on the OBS Appliance +# +OBS_API_AUTOSETUP="no" +# +# NOTE: all these options can be also declared in /etc/buildhost.config on each worker differently. +# + +## Path: Applications/OBS +## Description: define source server host to be used +## Type: string +## Default: "" +## Config: OBS +# +# An empty setting will point to localhost:5352 by default +# +OBS_SRC_SERVER="obs-backend.osmocom.org:5352" + +## Path: Applications/OBS +## Description: define repository server host to be used +## Type: string +## Default: "" +## Config: OBS +# +# An empty setting will point to localhost:5252 by default +# +OBS_REPO_SERVERS="obs-backend.osmocom.org:5252" + +## Path: Applications/OBS +## Description: define number of build instances +## Type: integer +## Default: 0 +## Config: OBS +# +# 0 instances will automatically use the number of CPU's +# +OBS_WORKER_INSTANCES="{{obs_worker_instances}}" + +## Path: Applications/OBS +## Description: define names of build instances for z/VM +## Type: string +## Default: "" +## Config: OBS +# +# The names of the workers as defined in z/VM. These must have two minidisks +# assigned, and have a secondary console configured to the local machine: +# 0150 is the root device +# 0250 is the swap device +# +#OBS_WORKER_INSTANCE_NAMES="LINUX075 LINUX076 LINUX077" +OBS_WORKER_INSTANCE_NAMES="" + +## Path: Applications/OBS +## Description: The base directory, where sub directories for each worker will get created +## Type: string +## Default: "" +## Config: OBS +# +# +OBS_WORKER_DIRECTORY="" + +## Path: Applications/OBS +## Description: The base for port numbers used by worker instances +## Type: integer +## Default: "0" +## Config: OBS +# +# 0 means let the operating system assign a port number +# +OBS_WORKER_PORTBASE="0" + +## Path: Applications/OBS +## Description: Number of parallel compile jobs per worker +## Type: integer +## Default: "1" +## Config: OBS +# +# this maps usually to "make -j1" during build +# +OBS_WORKER_JOBS="{{obs_worker_jobs}}" + +## Path: Applications/OBS +## Description: Run in test mode (build results will be ignore, no job blocking) +## Type: ("yes" | "") +## Default: "" +## Config: OBS +# +OBS_WORKER_TEST_MODE="" + +## Path: Applications/OBS +## Description: define one or more labels for the build host. +## Type: string +## Default: "" +## Config: OBS +# +# A label can be used to build specific packages only on dedicated hosts. +# For example for benchmarking. +# +OBS_WORKER_HOSTLABELS="" + +## Path: Applications/OBS +## Description: can be used to define a security level of the worker +## Type: string +## Default: "" +## Config: OBS +# +# This will extend the hostlabels and can be used to limit the workers +# to the hosts which have all security fixes deployed. +# +OBS_WORKER_SECURITY_LEVEL="" + +## Path: Applications/OBS +## Description: Register in SLP server +## Type: ("yes" | "no") +## Default: "yes" +## Config: OBS +# +# +OBS_USE_SLP="yes" + +## Path: Applications/OBS +## Description: Use a common cache directory for downloaded packages +## Type: string +## Default: "" +## Config: OBS +# +# Enable caching requires a given directory here. Be warned, content will be +# removed there ! +# +OBS_CACHE_DIR="/srv/obs/cache" + +## Path: Applications/OBS +## Description: Defines the package cache size +## Type: size in MB +## Default: "" +## Config: OBS +# +# Set the size to 50% of the maximum usable size of this partition +# +OBS_CACHE_SIZE="100000" + +## Path: Applications/OBS +## Description: Defines the nice level of running workers +## Type: integer +## Default: 18 +## Config: OBS +# +# Nicenesses range from -20 (most favorable scheduling) to 19 (least +# favorable). +# Default to 18 as some testsuites depend on being able to switch to +# one priority below (19) _and_ having changed the numeric level +# (so going from 19->19 makes them fail). +# +OBS_WORKER_NICE_LEVEL=18 + +## Path: Applications/OBS +## Description: Set used VM type by worker +## Type: ("auto" | "xen" | "kvm" | "lxc" | "zvm" | "emulator:$arch" | "emulator:$arch:$script" | "none" | "openstack") +## Default: "auto" +## Config: OBS +# +# +OBS_VM_TYPE="{{obs_vm_type}}" + +## Path: Applications/OBS +## Description: Set kernel used by worker (kvm) +## Type: ("none" | "/boot/vmlinuz" | "/foo/bar/vmlinuz) +## Default: "none" +## Config: OBS +# +# For z/VM this is normally /boot/image +# +OBS_VM_KERNEL="none" + +## Path: Applications/OBS +## Description: Set initrd used by worker (kvm) +## Type: ("none" | "/boot/initrd" | "/foo/bar/initrd-foo) +## Default: "none" +## Config: OBS +# +# for KVM, you have to create with (example for openSUSE 11.2): +# +# export rootfstype="ext4" +# mkinitrd -d /dev/null -m "ext4 binfmt_misc virtio_pci virtio_blk" -k vmlinuz-2.6.31.12-0.2-default -i initrd-2.6.31.12-0.2-default-obs_worker +# +# a working initrd file which includes virtio and binfmt_misc for OBS in order to work fine +# +# for z/VM, the build script will create a initrd at the given location if +# it does not yet exist. +# +OBS_VM_INITRD="none" + +## Path: Applications/OBS +## Description: Autosetup for XEN/KVM/TMPFS disk (root) - Filesize in MB +## Type: integer +## Default: "4096" +## Config: OBS +# +# +OBS_VM_DISK_AUTOSETUP_ROOT_FILESIZE="4096" + +## Path: Applications/OBS +## Description: Autosetup for XEN/KVM disk (swap) - Filesize in MB +## Type: integer +## Default: "1024" +## Config: OBS +# +# +OBS_VM_DISK_AUTOSETUP_SWAP_FILESIZE="1024" + +## Path: Applications/OBS +## Description: Default filesystem to use for autosetup. +## Type: ("ext2" | "ext3" | "ext4" | "reiserfs" | "btrfs" | "xfs") +## Default: "ext3" +## Config: OBS +# +# +# Buildflag vmfstype may overwrite this for a specific job. +OBS_VM_DISK_AUTOSETUP_FILESYSTEM="ext3" + +## Path: Applications/OBS +## Description: Filesystem mount options to use for autosetup +## Type: string +## Default: "" +## Config: OBS +# +# +OBS_VM_DISK_AUTOSETUP_MOUNT_OPTIONS="" + +## Path: Applications/OBS +## Description: Enable build in memory +## Type: ("yes" | "") +## Default: "" +## Config: OBS +# +# WARNING: this requires much memory! +# +OBS_VM_USE_TMPFS="yes" + +## Path: Applications/OBS +## Description: Specify custom options for VM handler +## Type: string +## Default: "" +## Config: OBS +# +# Can be used to workaround problems with VM handler and should not be needed usually +# +OBS_VM_CUSTOM_OPTION="" + +## Path: Applications/OBS +## Description: Memory allocated for each VM (512) if not set +## Type: integer +## Default: "" +## Config: OBS +# +# +OBS_INSTANCE_MEMORY="" + +## Path: Applications/OBS +## Description: Enable storage auto configuration +## Type: ("yes" | "") +## Default: "" +## Config: OBS +# +# WARNING: this may destroy data on your hard disk ! +# This is usually only used on mass deployed worker instances +# +OBS_STORAGE_AUTOSETUP="" + +## Path: Applications/OBS +## Description: Setup LVM via obsstoragesetup +## Type: ("take_all" | "use_obs_vg" | "none") +## Default: "use_obs_vg" +## Config: OBS +# +# take_all: WARNING: all LVM partitions will be used and all data erased ! +# use_obs_vg: A lvm volume group named "OBS" will be re-setup for the workers. +# +OBS_SETUP_WORKER_PARTITIONS="use_obs_vg" + +## Path: Applications/OBS +## Description: Size in MB when creating LVM partition for cache partition +## Type: integer +## Default: "" +## Config: OBS +# +# +OBS_WORKER_CACHE_SIZE="" + +## Path: Applications/OBS +## Description: Size in MB when creating LVM partition for each worker root partition +## Type: integer +## Default: "" +## Config: OBS +# +# +OBS_WORKER_ROOT_SIZE="" + +## Path: Applications/OBS +## Description: Size in MB when creating LVM partition for each worker swap partition +## Type: integer +## Default: "" +## Config: OBS +# +# +OBS_WORKER_SWAP_SIZE="" + +## Path: Applications/OBS +## Description: URL to a proxy service for caching binaries used by worker +## Type: string +## Default: "" +## Config: OBS +# +# +OBS_WORKER_BINARIES_PROXY="" + +## Path: Applications/OBS +## Description: URL to a ssh pub key to allow root user login +## Type: string +## Default: "" +## Config: OBS +# +# This is usually used on mass (PXE) deployed workers) +# +OBS_ROOT_SSHD_KEY_URL="" + +## Path: Applications/OBS +## Description: URL to a script to be downloaded and executed +## Type: string +## Default: "" +## Config: OBS +# +# This is a hook for doing special things in your setup at boot time +# +OBS_WORKER_SCRIPT_URL="" + +## Path: Applications/OBS +## Description: If chroot/lxc is used for build, empty it after build is finished +## Type: ("yes" | "") +## Default: "" +## Config: OBS +# +# +OBS_WORKER_CLEANUP_CHROOT="" + +##Path: Application/OBS +## Description: wipes the build environment of the worker after the build +## Type: ("yes" | "") +## Default: "" +## Config: OBS +# +# +OBS_WORKER_WIPE_AFTER_BUILD="" + +##Path: Application/OBS +## Description: name or id of openstack instance that controls the worker (building) instances +## Type: ("yes" | "") +## Default: "" +## Config: OBS +# +# +OBS_WORKER_CONTROL_INSTANCE="" + +##Path: Application/OBS +## Description: name or id flavor to create openstack worker (building) instance +## Type: ("yes" | "") +## Default: "" +## Config: OBS +# +# +OBS_WORKER_OS_FLAVOR="" + +##Path: Application/OBS +## Description: openstack environment variables. Only used when OBS_VM_TYPE=openstack +## Type: ("yes" | "") +## Default: "" +## Config: OBS +# +# +OS_AUTH_URL="" + +OS_PROJECT_ID="" +OS_PROJECT_NAME="" +OS_USER_DOMAIN_NAME="" +OS_USERNAME="" +OS_PASSWORD="" +OS_REGION_NAME="" + +OBS_WORKER_PREFIX="" + +OBS_OPENSTACK_DISK_SIZE="" +OBS_OPENSTACK_SWAP_SIZE="" +OBS_OPENSTACK_MEMORY_SIZE="" diff --git a/ansible/roles/osmocom-workstation/tasks/git-repos.yml b/ansible/roles/osmocom-workstation/tasks/git-repos.yml index 79f7f31..719b6eb 100644 --- a/ansible/roles/osmocom-workstation/tasks/git-repos.yml +++ b/ansible/roles/osmocom-workstation/tasks/git-repos.yml @@ -4,7 +4,7 @@ become: yes become_user: "{{ osmocom_user }}" git: - repo: git://git.osmocom.org/{{ item }}.git + repo: https://git.osmocom.org/{{ item }} dest: "{{ git_destination }}/{{ item }}" accept_hostkey: yes force: no @@ -36,6 +36,8 @@ - osmo-gsm-manuals - osmo-gtp-kernel - osmo-hlr + - osmo-hnodeb + - osmo-hnbgw - osmo-iuh - osmo-mgw - osmo-msc @@ -104,20 +106,20 @@ chdir: "{{ git_destination }}/osmo-ttcn3-hacks" - name: install sysmocom git repos - git: repo=git://git.sysmocom.de/{{ item }}.git + git: repo=https://gitea.sysmocom.de/{{ item }}.git dest={{ git_destination }}/{{ item }} accept_hostkey=yes force=yes recursive=no with_items: - sysmo-bts/layer1-api - - poky/meta-sysmocom-bsp - - rs-backup-suite - - sysmo-usim-tool + - sysmo-bts/meta-sysmocom-bsp + - hwelte/rs-backup-suite + - sysmocom/sysmo-usim-tool - name: install github git repos - git: repo=git://github.com/{{ item }}.git + git: repo=https://github.com/{{ item }}.git dest={{ git_destination }}/{{ item }} accept_hostkey=yes force=yes diff --git a/ansible/roles/osmocom-workstation/tasks/main.yml b/ansible/roles/osmocom-workstation/tasks/main.yml index ae29111..0331207 100644 --- a/ansible/roles/osmocom-workstation/tasks/main.yml +++ b/ansible/roles/osmocom-workstation/tasks/main.yml @@ -9,8 +9,8 @@ shell: /usr/bin/zsh groups: docker -- name: include git repos - include: git-repos.yml +- name: import git repos + import_tasks: git-repos.yml - name: install distribution packages apt: diff --git a/ansible/roles/podman/tasks/debian.yml b/ansible/roles/podman/tasks/debian.yml deleted file mode 100644 index e261a3f..0000000 --- a/ansible/roles/podman/tasks/debian.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- - -- name: add apt https support - apt: - cache_valid_time: 3600 - update_cache: yes - pkg: - - apt-transport-https - - gnupg - - cron - - ca-certificates - when: ansible_distribution_major_version < 11 - -- name: add docker gpg key to apt keyring - apt_key: - id: 2472D6D0D2F66AF87ABA8DA34D64390375060AA4 - url: https://ftp.uni-stuttgart.de/opensuse/repositories/devel:/kubic:/libcontainers:/stable/{{ obs_distro }}/Release.key - when: ansible_distribution_major_version < 11 - -- apt_repository: - repo: "deb https://ftp.uni-stuttgart.de/opensuse/repositories/devel:/kubic:/libcontainers:/stable/{{ obs_distro }}/ /" - state: present - update_cache: yes - when: ansible_distribution_major_version < 11 - -- name: install podman - apt: - pkg: - - podman - - runc - - buildah - - skopeo - - dnsmasq diff --git a/ansible/roles/podman/tasks/main.yml b/ansible/roles/podman/tasks/main.yml index 049f5c2..45a712d 100644 --- a/ansible/roles/podman/tasks/main.yml +++ b/ansible/roles/podman/tasks/main.yml @@ -1,7 +1,9 @@ --- -- name: Include tasks for Debian - include_tasks: debian.yml - when: (ansible_distribution in ['Debian', 'Raspbian']) - -# TODO: add other distributions below +- name: install podman + apt: + pkg: + - podman + - buildah + # Has reasonable recommends needed for dns resolving etc. + install_recommends: yes diff --git a/ansible/roles/qemu/files/qemu-create-vms.sh b/ansible/roles/qemu/files/qemu-create-vms.sh new file mode 100755 index 0000000..a6ee8e7 --- /dev/null +++ b/ansible/roles/qemu/files/qemu-create-vms.sh @@ -0,0 +1,85 @@ +#!/bin/sh -e +# Create qcow2 images with ssh root login enabled for repo-install-test and +# store them in /opt/qemu. +# Set KEEP_CACHE=1 during development, so virt-builder only needs to download +# the image once. + +# Distribution names, as in the base images from here: +# https://builder.libguestfs.org/ +DISTROS=" + alma-8.5 + debian-10 + debian-11 + debian-12 +" +TEMP_SCRIPT="$(mktemp)" + +if [ "$(id -u)" != 0 ]; then + echo "ERROR: run this as root" + exit 1 +fi + +mkdir -p /opt/qemu + +for distro in $DISTROS; do + img="/opt/qemu/$distro.qcow2" + + echo + echo "# $distro" + echo + + if [ -e "$img" ]; then + echo "=> File exists, skipping." + continue + fi + + case "$distro" in + alma-*) + # Install SCTP kernel module + # https://forums.centos.org/viewtopic.php?t=71818 + cat <<- EOF > "$TEMP_SCRIPT" + #!/bin/sh -ex + dnf upgrade -y kernel + dnf install -y kernel-modules-extra + rm -f /etc/modprobe.d/sctp-blacklist.conf + EOF + ;; + debian-*) + # Generate SSH server keys and allow login as root + cat <<- EOF > "$TEMP_SCRIPT" + #!/bin/sh -ex + ssh-keygen -A + echo "PermitRootLogin yes" >> /etc/ssh/sshd_config + EOF + ;; + esac + + EXTRA_ARGS="" + case "$distro" in + debian-12) + # repo-install-test runs out of space with the default size + EXTRA_ARGS="--size 8G" + ;; + esac + + virt-builder \ + "$distro" \ + -o "/opt/qemu/$distro.qcow2" \ + --format qcow2 \ + --root-password password:root \ + --run "$TEMP_SCRIPT" \ + --verbose \ + $EXTRA_ARGS + + if [ -z "$KEEP_CACHE" ]; then + virt-builder --delete-cache + fi +done + +rm "$TEMP_SCRIPT" + +# Marker for ansible main.yml to skip the script +touch /opt/qemu/.qemu-create-vms-done-v2 + +echo +echo "Done!" diff --git a/ansible/roles/qemu/tasks/main.yml b/ansible/roles/qemu/tasks/main.yml new file mode 100644 index 0000000..1678217 --- /dev/null +++ b/ansible/roles/qemu/tasks/main.yml @@ -0,0 +1,22 @@ +--- + +- name: install packages + apt: + name: + - libguestfs-tools + - netcat-openbsd + - qemu-system-x86 + - sshpass + cache_valid_time: 3600 + update_cache: yes + +- name: run qemu-create-vms.sh + ansible.builtin.script: qemu-create-vms.sh + args: + creates: /opt/qemu/.qemu-create-vms-done-v2 + +- name: add jenkins to the kvm group + user: + name: "{{ jenkins_user }}" + groups: kvm + append: yes diff --git a/ansible/setup-gsm-tester.yml b/ansible/setup-gsm-tester.yml index 26285df..526b621 100644 --- a/ansible/setup-gsm-tester.yml +++ b/ansible/setup-gsm-tester.yml @@ -1,6 +1,6 @@ --- - name: setup osmo-gsm-tester apu - hosts: gsm-tester + hosts: gsm_tester user: root tasks: - name: install common utilities @@ -14,6 +14,12 @@ - curl cache_valid_time: 3600 update_cache: yes + - name: deploy osmo-ci to home directory + become: yes + become_user: "jenkins" + git: + repo: "https://gerrit.osmocom.org/osmo-ci" + dest: "/home/jenkins/osmo-ci" roles: - name: gsm-tester tags: diff --git a/ansible/setup-jenkins-slave.yml b/ansible/setup-jenkins-slave.yml index 07414aa..5813b40 100644 --- a/ansible/setup-jenkins-slave.yml +++ b/ansible/setup-jenkins-slave.yml @@ -1,6 +1,6 @@ --- - name: setup jenkins slaves - hosts: jenkins-slaves + hosts: jenkins_slaves user: root vars_files: - "vars/{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version']}}.yml" @@ -15,6 +15,8 @@ cache_valid_time: 3600 update_cache: yes roles: + - name: apt-allow-relinfo-change + - name: docker jenkins_user: osmocom-build tags: @@ -45,9 +47,18 @@ - name: ntpd - name: setup coverity slaves - hosts: coverity-slaves + hosts: coverity_slaves user: root roles: - name: install-coverity tags: - coverity + +- name: setup qemu images + hosts: qemu + user: root + roles: + - name: qemu + jenkins_user: osmocom-build + tags: + - qemu diff --git a/ansible/setup-obs-worker.yml b/ansible/setup-obs-worker.yml new file mode 100644 index 0000000..b79e1a5 --- /dev/null +++ b/ansible/setup-obs-worker.yml @@ -0,0 +1,21 @@ +--- +- name: setup OBS worker + hosts: obs_workers + user: root + vars_files: + - "vars/{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version']}}.yml" + tasks: + - name: install common utilities + zypper: + name: + - vim + - tmux + - screen + - tcpdump + - strace + - ltrace + + roles: + - name: osmocom-obs-worker + tags: + - obs-worker diff --git a/ansible/setup-simtest.yml b/ansible/setup-simtest.yml index 6eb67fc..eb37186 100644 --- a/ansible/setup-simtest.yml +++ b/ansible/setup-simtest.yml @@ -18,14 +18,24 @@ - python3-pip - python3-serial - python3-pyscard + - tshark - virtualenv install_recommends: no + - name: "pysim: get requirements.txt" + get_url: + url: https://gitea.osmocom.org/sim-card/pysim/raw/branch/master/requirements.txt + dest: /tmp/pysim_requirements.txt + + - name: "pysim: install deps from requirements.txt" + pip: + executable: pip3 + requirements: /tmp/pysim_requirements.txt + roles: - name: osmocom-jenkins-slave generic_slave: false install_jenkins_utilities: false - install_osmocom_build_deps: false ttcn3_slave: false tags: - jenkins-slave diff --git a/ansible/util/known_hosts b/ansible/util/known_hosts new file mode 100644 index 0000000..76e2a64 --- /dev/null +++ b/ansible/util/known_hosts @@ -0,0 +1,10 @@ +2a01:4f8:10b:2ad9::1:10 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHtUqYQ2qE2w2et8frRXfauWiocse2sdqRLo1gksFaSS +2a01:4f8:10b:2ad9::1:13 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEfg7NNBKjVJGoL3oUDrJCccI2D8podsSPKnRn1aTUge +2a01:4f8:212:3762::1:3 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICl7tuOgN64+iYuYCtfCKaUbN9IWzKycpKVRgzsjVCPY +2a01:4f8:120:8470::1:4 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIPKRz2Eg+h6RTREn52n/ip3pzUjRu9pw1XdjWUrvvi/ +10.9.25.50 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPowg8lZeIGJ6LCLA8U4eSHnnPX90FaVgInDuRi5U9Zq +10.9.25.60 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOAK6SQXmifMamE1VHL/1GmPkk6KhplzT20jSOwb8PyZ +10.9.25.70 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILioWTUt6CyZTceSKqy09mtlNUif+ETKTF6whnWuIROx +2a01:4f8:121:200a::1:3 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGafw2VdeZo4YNvbu8VEBjDrsCFL+ZSbHtvhspNIQ9PR +2a01:4f8:121:200a::1:2 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFyp0daWkfsRHErCVCcP/FZmtiRvFNAsK2A5TlQ6Ja+P +10.34.2.153 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIG0rQGxSxWJHwNNT5XdeuMPSYCAXpcDnFDzvyw5+sH3l diff --git a/ansible/util/ssh_config b/ansible/util/ssh_config new file mode 100644 index 0000000..9738e83 --- /dev/null +++ b/ansible/util/ssh_config @@ -0,0 +1,3 @@ + UserKnownHostsFile=util/known_hosts + HashKnownHosts=no + diff --git a/ansible/vars/Debian-11.yml b/ansible/vars/Debian-11.yml new file mode 100644 index 0000000..8361e15 --- /dev/null +++ b/ansible/vars/Debian-11.yml @@ -0,0 +1,2 @@ +--- +obs_distro: Debian_11 diff --git a/ansible/vars/Debian-12.yml b/ansible/vars/Debian-12.yml new file mode 100644 index 0000000..4db6ac1 --- /dev/null +++ b/ansible/vars/Debian-12.yml @@ -0,0 +1,2 @@ +--- +obs_distro: Debian_12 diff --git a/ansible/vars/Suse-15.yml b/ansible/vars/Suse-15.yml new file mode 100644 index 0000000..0cf00c2 --- /dev/null +++ b/ansible/vars/Suse-15.yml @@ -0,0 +1,2 @@ +--- +obs_distro: fixme |