From 3a6250a9fbb2bf6c723eb331d8a4bc240426b24d Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Fri, 23 Sep 2022 12:29:26 +0200 Subject: obs: update_obs_project: add master feed Add a new master feed, where packages are updated as soon as patches get merged to master. Upload a commit_$COMMIT.txt file for each package in this feed and use it to determine if the package needs to be updated or not. In most packages the commit is already part of the version in the dsc file, e.g. "libosmocore_1.7.0.38.c3b90.dsc", but that's harder to parse and is more likely to have a hash collision (just 5 characters). Related: OS#2385 Change-Id: I3b0b4f4876b8c1eeb61f20d903a6f2cac6e99638 --- scripts/obs/lib/__init__.py | 2 +- scripts/obs/lib/config.py | 1 + scripts/obs/lib/git.py | 22 ++++++++++++++++++++++ scripts/obs/lib/osc.py | 16 +++++++++++----- scripts/obs/lib/srcpkg.py | 18 +++++++++++++++++- scripts/obs/update_obs_project.py | 9 ++++++--- 6 files changed, 58 insertions(+), 10 deletions(-) diff --git a/scripts/obs/lib/__init__.py b/scripts/obs/lib/__init__.py index f94c0a8..8f52dba 100644 --- a/scripts/obs/lib/__init__.py +++ b/scripts/obs/lib/__init__.py @@ -17,7 +17,7 @@ def add_shared_arguments(parser): parser.add_argument("-f", "--feed", help="package feed (default: nightly). The feed" " determines the git revision to be built:" - " 'nightly' builds 'origin/master'," + " 'nightly' and 'master' build 'origin/master'," " 'latest' builds the last signed tag," " other feeds build their respective branch.", metavar="FEED", default="nightly", diff --git a/scripts/obs/lib/config.py b/scripts/obs/lib/config.py index c9df74a..79a7849 100644 --- a/scripts/obs/lib/config.py +++ b/scripts/obs/lib/config.py @@ -32,6 +32,7 @@ feeds = [ "2022q1", "2022q2", "latest", + "master", "nightly", ] diff --git a/scripts/obs/lib/git.py b/scripts/obs/lib/git.py index aab0533..8faf5b2 100644 --- a/scripts/obs/lib/git.py +++ b/scripts/obs/lib/git.py @@ -76,6 +76,28 @@ def checkout_default_branch(project): checkout(project, f"origin/{branch}") +def get_head(project): + repo_path = get_repo_path(project) + ret = lib.run_cmd(["git", "rev-parse", "HEAD"], cwd=repo_path) + return ret.output.rstrip() + + +def get_head_remote(project, branch): + if not branch: + branch = get_default_branch(project) + repo_url = get_repo_url(project) + + print(f"{project}: getting head from git remote for {branch}") + ls_remote = lib.run_cmd(["git", "ls-remote", repo_url, branch]) + + ret = ls_remote.output.split("\t")[0] + if not ret: + lib.exit_error_cmd(ls_remote, f"failed to find head commit for" + "{project} in output") + + return ret + + def get_latest_tag(project): pattern_str = get_latest_tag_pattern(project) pattern = re.compile(pattern_str) diff --git a/scripts/obs/lib/osc.py b/scripts/obs/lib/osc.py index 51e2f5f..df5c34c 100644 --- a/scripts/obs/lib/osc.py +++ b/scripts/obs/lib/osc.py @@ -62,7 +62,7 @@ def get_remote_pkgs(proj): return ret.output.rstrip().split("\n") -def get_package_version(proj, package): +def get_package_version(proj, package, feed): print(f"{package}: getting OBS version") ret = run_osc(["list", proj, os.path.basename(package)]) @@ -70,15 +70,21 @@ def get_package_version(proj, package): if ret.output == '\n': return "0" - # Extract the version from the dsc filename + # Extract the version from the file list for line in ret.output.split('\n'): line = line.rstrip() - if line.endswith(".dsc"): - return line.split("_")[-1][:-4] + if feed == "master" and package != "osmocom-master": + # Use commit_*.txt + if line.startswith("commit_") and line.endswith(".txt"): + return line.split("_")[1].split(".")[0] + else: + # Use *.dsc + if line.endswith(".dsc"): + return line.split("_")[-1][:-4] lib.exit_error_cmd(ret, "failed to find package version on OBS by" - " extracting the version from the .dsc filename") + " extracting the version from the file list") def create_package(proj, package): diff --git a/scripts/obs/lib/srcpkg.py b/scripts/obs/lib/srcpkg.py index 159f4ed..1711f70 100644 --- a/scripts/obs/lib/srcpkg.py +++ b/scripts/obs/lib/srcpkg.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright 2022 sysmocom - s.f.m.c. GmbH import os +import pathlib import lib.config import lib.debian import lib.rpm_spec @@ -13,7 +14,7 @@ def checkout_for_feed(project, feed, branch=None): lib.git.checkout(project, f"origin/{branch}") elif feed == "latest": lib.git.checkout_latest_tag(project) - elif feed == "nightly": + elif feed in ["master", "nightly"]: lib.git.checkout_default_branch(project) else: # 2022q1 etc lib.git.checkout(project, f"origin/{feed}") @@ -117,6 +118,18 @@ def write_tarball_version(project, version): f.write(f"{version}\n") +def write_commit_txt(project): + """ Write the current git commit to commit_$commit.txt file, so it gets + uploaded to OBS along with the rest of the source package. This allows + figuring out if the source package is still up-to-date or not for the + master feed. """ + output_path = lib.get_output_path(project) + commit = lib.git.get_head(project) + + print(f"{project}: adding commit_{commit}.txt") + pathlib.Path(f"{output_path}/commit_{commit}.txt").touch() + + def build(project, feed, branch, conflict_version, fetch, gerrit_id=0): lib.git.clone(project, fetch) lib.git.clean(project) @@ -155,5 +168,8 @@ def build(project, feed, branch, conflict_version, fetch, gerrit_id=0): lib.rpm_spec.generate(project, version, epoch) lib.rpm_spec.copy_to_output(project) + if feed == "master": + write_commit_txt(project) + lib.remove_cache_extra_files() return version_epoch diff --git a/scripts/obs/update_obs_project.py b/scripts/obs/update_obs_project.py index 168193d..9bd46ff 100755 --- a/scripts/obs/update_obs_project.py +++ b/scripts/obs/update_obs_project.py @@ -66,13 +66,16 @@ def build_srcpkg_if_needed(proj, feed, branch, pkgs_remote, package, conflict_ve fetch, is_meta_pkg, skip_up_to_date): global srcpkgs_skipped - if feed == "latest": + if feed in ["master", "latest"]: """ Check if we can skip this package by comparing the OBS version with the git remote. """ if is_meta_pkg: latest_version = conflict_version if conflict_version else "1.0.0" else: - latest_version = lib.git.get_latest_tag_remote(package) + if feed == "master": + latest_version = lib.git.get_head_remote(package, branch) + else: + latest_version = lib.git.get_latest_tag_remote(package) if latest_version is None: print(f"{package}: skipping (no git tag found)") @@ -82,7 +85,7 @@ def build_srcpkg_if_needed(proj, feed, branch, pkgs_remote, package, conflict_ve if os.path.basename(package) not in pkgs_remote: print(f"{package}: building source package (not in OBS)") else: - obs_version = lib.osc.get_package_version(proj, package) + obs_version = lib.osc.get_package_version(proj, package, feed) if is_up_to_date(obs_version, latest_version): if skip_up_to_date: print(f"{package}: skipping ({obs_version} is up-to-date)") -- cgit v1.2.3