aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--jobs/osmocom-depcheck.yml72
-rw-r--r--scripts/osmo-depcheck/buildstack.py144
-rw-r--r--scripts/osmo-depcheck/config.py43
-rw-r--r--scripts/osmo-depcheck/dependencies.py114
-rwxr-xr-xscripts/osmo-depcheck/osmo-depcheck.py101
-rw-r--r--scripts/osmo-depcheck/parse.py116
6 files changed, 590 insertions, 0 deletions
diff --git a/jobs/osmocom-depcheck.yml b/jobs/osmocom-depcheck.yml
new file mode 100644
index 0000000..f13d4b7
--- /dev/null
+++ b/jobs/osmocom-depcheck.yml
@@ -0,0 +1,72 @@
+---
+- project:
+ name: Osmocom-depcheck
+ jobs:
+ - Osmocom-depcheck
+
+- job-template:
+ name: 'Osmocom-depcheck'
+ project-type: freestyle
+ defaults: global
+ description: |
+ Verifies that Osmocom programs really build with the dependency
+ versions they claim to support in configure.ac.
+ (Generated by job-builder)
+ node: osmocom-master-debian9
+ parameters:
+ - string:
+ name: PROJECTS
+ description: |
+ Which Osmocom projects and revisions to build, leave
+ empty to default to all projects (!),
+ default revision is "master".
+ Examples: "osmo-hlr", "osmo-hlr:0.2.1 osmo-bts:0.8.1"
+ default: 'osmo-hlr:0.2.1'
+ - string:
+ name: GIT_URL_PREFIX
+ description: |
+ Where to clone the sources from
+ default: 'git://git.osmocom.org/'
+ - bool:
+ name: BUILD
+ description: |
+ Attempt to build the project with the minimum dependency
+ versions found in the configure.ac files. If this is unchecked,
+ this job will only clone the git repositories and parse the
+ configure.ac files.
+ default: true
+ - bool:
+ name: PRINT_OLD_DEPENDS
+ description: |
+ Report dependencies on old releases (printed after the other
+ parsing output, before the build starts)
+ default: false
+ - string:
+ name: BRANCH
+ description: |
+ Branch where the osmo-depcheck.py script gets pulled from.
+ Only modify this if you are hacking on osmo-depcheck.py.
+ default: '*/master'
+ builders:
+ - shell: |
+ # Build the arguments
+ args="$PROJECTS"
+ args="$args -j 5"
+ args="$args -g $PWD/DEPCHECK_GITDIR"
+ args="$args -u $GIT_URL_PREFIX"
+ [ "$BUILD" = "true" ] && args="$args -b"
+ [ "$PRINT_OLD_DEPENDS" = "true" ] && args="$args -o"
+
+ # Run osmo-depcheck
+ mkdir DEPCHECK_GITDIR
+ export PYTHONUNBUFFERED=1
+ scripts/osmo-depcheck/osmo-depcheck.py $args
+ scm:
+ - git:
+ branches:
+ - '$BRANCH'
+ url: git://git.osmocom.org/osmo-ci
+ git-config-name: 'Jenkins Builder'
+ git-config-email: 'jenkins@osmocom.org'
+
+# vim: expandtab tabstop=2 shiftwidth=2
diff --git a/scripts/osmo-depcheck/buildstack.py b/scripts/osmo-depcheck/buildstack.py
new file mode 100644
index 0000000..87210ab
--- /dev/null
+++ b/scripts/osmo-depcheck/buildstack.py
@@ -0,0 +1,144 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2018 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+
+import atexit
+import collections
+import sys
+import os
+import shutil
+import subprocess
+import tempfile
+
+
+def next_buildable(depends, done):
+ """ Find the next program that can be built, because it has all
+ dependencies satisfied. Initially this would be libosmocore, as it has
+ no dependencies, then the only library that depends on libosmocore and
+ so on.
+
+ :param depends: return value of dependencies.generate()
+ :param done: ordered dict of programs that would already have been
+ built at this point.
+ Example: {"lib-a": "0.11.0", "lib-b": "0.5.0"}
+ """
+ # Iterate over dependencies
+ for program, data in depends.items():
+ # Skip what's already done
+ if program in done:
+ continue
+
+ # Check for missing dependencies
+ depends_done = True
+ for depend in data["depends"]:
+ if depend not in done:
+ depends_done = False
+ break
+
+ # All dependencies satisfied: we have a winner!
+ if depends_done:
+ return program, data["version"]
+
+ # Impossible to build the dependency tree
+ print_dict(done)
+ print("ERROR: can't figure out how to build the rest!")
+ sys.exit(1)
+
+
+def generate(depends):
+ """ Generate an ordered dictionary with the right build order.
+
+ :param depends: return value of dependencies.generate()
+ :returns: an ordered dict like the following:
+ {"libosmocore": "0.11.0",
+ "libosmo-abis": "0.5.0",
+ "osmo-bts": "master"} """
+ # Iterate over dependencies
+ ret = collections.OrderedDict()
+ count = len(depends.keys())
+ while len(ret) != count:
+ # Continue with the one without unsatisfied dependencies
+ program, version = next_buildable(depends, ret)
+ ret[program] = version
+ return ret
+
+
+def print_dict(stack):
+ """ Print the whole build stack.
+ :param stack: return value from generate() above """
+ print("Build order:")
+ for program, version in stack.items():
+ print(" * " + program + ":" + version)
+
+
+def temp_install_folder():
+ """ Generate a temporary installation folder
+
+ It will be used as configure prefix, so when running 'make install',
+ the files will get copied in there instead of "/usr/local/". The folder
+ will get removed when the script has finished.
+
+ :returns: the path to the temporary folder """
+ ret = tempfile.mkdtemp(prefix="depcheck_")
+ atexit.register(shutil.rmtree, ret)
+ print("Temporary install folder: " + ret)
+ return ret
+
+
+def set_environment(jobs, tempdir):
+ """ Configure the environment variables before running configure, make etc.
+
+ :param jobs: parallel build jobs (for make)
+ :param tempdir: temporary installation dir (see temp_install_folder())
+ """
+ # Add tempdir to PKG_CONFIG_PATH and LD_LIBRARY_PATH
+ extend = {"PKG_CONFIG_PATH": tempdir + "/lib/pkgconfig",
+ "LD_LIBRARY_PATH": tempdir + "/lib"}
+ for env_var, folder in extend.items():
+ old = os.environ[env_var] if env_var in os.environ else ""
+ os.environ[env_var] = old + ":" + folder
+
+ # Set JOBS for make
+ os.environ["JOBS"] = str(jobs)
+
+
+def build(gitdir, jobs, stack):
+ """ Build one program with all its dependencies.
+
+ :param gitdir: folder to which the sources will be cloned
+ :param jobs: parallel build jobs (for make)
+ :param stack: the build stack as returned by generate() above
+
+ The dependencies.clone() function has already downloaded missing
+ sources and checked out the right version tags. So in this function we
+ can directly enter the source folder and run the build commands.
+
+ Notes about the usage of 'make clean' and 'make distclean':
+ * Without 'make clean' we might have files in the build directory with
+ a different prefix hardcoded (e.g. from a previous run of
+ osmo-depcheck):
+ <https://lists.gnu.org/archive/html/libtool/2006-12/msg00011.html>
+ * 'make distclean' gets used to remove everything that mentioned the
+ prefix set by osmo-depcheck. That way the user won't have it set
+ anymore in case they decide to compile the code again manually from
+ the source folder. """
+ # Prepare the install folder and environment
+ tempdir = temp_install_folder()
+ unitdir = tempdir + "/lib/systemd/system/"
+ set_environment(jobs, tempdir)
+
+ # Iterate over stack
+ for program, version in stack.items():
+ print("Building " + program + ":" + version)
+ os.chdir(gitdir + "/" + program)
+
+ # Run the build commands
+ commands = [["autoreconf", "-fi"],
+ ["./configure", "--prefix", tempdir,
+ "--with-systemdsystemunitdir=" + unitdir],
+ ["make", "clean"],
+ ["make"],
+ ["make", "install"],
+ ["make", "distclean"]]
+ for command in commands:
+ print("+ " + " ".join(command))
+ subprocess.run(command, check=True)
diff --git a/scripts/osmo-depcheck/config.py b/scripts/osmo-depcheck/config.py
new file mode 100644
index 0000000..3e993bf
--- /dev/null
+++ b/scripts/osmo-depcheck/config.py
@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2018 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+
+# Where to clone sources from (with trailing slash)
+git_url_prefix = "git://git.osmocom.org/"
+
+# Default projects to build when none are specified on the command line
+projects = ("osmo-bts",
+ "osmo-pcu",
+ "osmo-hlr",
+ "osmo-mgw",
+ "osmo-msc",
+ "osmo-sgsn",
+ "osmo-ggsn")
+
+# Libraries coming from Osmocom repositories (glob patterns)
+# All other libraries (e.g. libsystemd) are ignored by this script, even if
+# they are mentioned with PKG_CHECK_MODULES in configure.ac.
+relevant_library_patterns = ("libasn1c",
+ "libgtp",
+ "libosmo*")
+
+
+# Library locations in the git repositories
+# Libraries that have the same name as the git repository don't need to be
+# listed here. Left: repository name, right: libraries
+repos = {"libosmocore": ("libosmocodec",
+ "libosmocoding",
+ "libosmoctrl",
+ "libosmogb",
+ "libosmogsm",
+ "libosmosim",
+ "libosmovty"),
+ "libosmo-abis": ("libosmoabis",
+ "libosmotrau"),
+ "libosmo-sccp": ("libosmo-mtp",
+ "libosmo-sigtran",
+ "libosmo-xua"),
+ "osmo-ggsn": ("libgtp"),
+ "osmo-hlr": ("libosmo-gsup-client"),
+ "osmo-iuh": ("libosmo-ranap"),
+ "osmo-mgw": ("libosmo-mgcp-client",
+ "libosmo-legacy-mgcp")}
diff --git a/scripts/osmo-depcheck/dependencies.py b/scripts/osmo-depcheck/dependencies.py
new file mode 100644
index 0000000..78cf4a0
--- /dev/null
+++ b/scripts/osmo-depcheck/dependencies.py
@@ -0,0 +1,114 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2018 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+
+import collections
+import os
+import subprocess
+import sys
+
+# Same folder
+import parse
+
+
+def git_clone(gitdir, prefix, repository, version):
+ """ Clone a missing git repository and checkout a specific version tag.
+
+ :param gitdir: folder to which the sources will be cloned
+ :param prefix: git url prefix (e.g. "git://git.osmocom.org/")
+ :param repository: Osmocom git repository name (e.g. "libosmo-abis")
+ :param version: "master" or a version tag like "0.11.0" """
+ # Clone when needed
+ if not os.path.exists(gitdir + "/" + repository):
+ url = prefix + repository
+ print("Cloning git repo: " + url)
+ try:
+ subprocess.run(["git", "-C", gitdir, "clone", "-q", url],
+ check=True)
+ except subprocess.CalledProcessError:
+ print("NOTE: if '" + repository + "' is part of a git repository"
+ " with a different name, please add it to the mapping in"
+ " 'config.py' and try again.")
+ sys.exit(1)
+
+ # Checkout the version tag
+ subprocess.run(["git", "-C", gitdir + "/" + repository, "checkout",
+ version, "-q"], check=True)
+
+
+def generate(gitdir, prefix, initial, rev):
+ """ Generate the dependency graph of an Osmocom program by cloning the git
+ repository, parsing the "configure.ac" file, and recursing.
+
+ :param gitdir: folder to which the sources will be cloned
+ :param prefix: git url prefix (e.g. "git://git.osmocom.org/")
+ :param initial: the first program to look at (e.g. "osmo-bts")
+ :param rev: the git revision to check out ("master", "0.1.0", ...)
+ :returns: a dictionary like the following:
+ {"osmo-bts": {"version": "master",
+ "depends": {"libosmocore": "0.11.0",
+ "libosmo-abis": "0.5.0"}},
+ "libosmocore": {"version": "0.11.0",
+ "depends": {}},
+ "libosmo-abis": {"version": "0.5.0",
+ "depends": {"libosmocore": "0.11.0"}} """
+ # Iterate over stack
+ stack = collections.OrderedDict({initial: rev})
+ ret = collections.OrderedDict()
+ while len(stack):
+ # Pop program from stack
+ program, version = next(iter(stack.items()))
+ del stack[program]
+
+ # Skip when already parsed
+ if program in ret:
+ continue
+
+ # Add the programs dependencies to the stack
+ print("Looking at " + program + ":" + version)
+ git_clone(gitdir, prefix, program, version)
+ depends = parse.configure_ac(gitdir, program)
+ stack.update(depends)
+
+ # Add the program to the ret
+ ret[program] = {"version": version, "depends": depends}
+
+ return ret
+
+
+def print_dict(depends):
+ """ Print the whole dependency graph.
+ :param depends: return value from generate() above """
+ print("Dependency graph:")
+
+ for program, data in depends.items():
+ version = data["version"]
+ depends = data["depends"]
+ print(" * " + program + ":" + version + " depends: " + str(depends))
+
+
+def git_latest_tag(gitdir, repository):
+ """ Get the last release string by asking git for the latest tag.
+
+ :param gitdir: folder to which the sources will be cloned
+ :param repository: Osmocom git repository name (e.g. "libosmo-abis")
+ :returns: the latest git tag (e.g. "1.0.2") """
+ dir = gitdir + "/" + repository
+ complete = subprocess.run(["git", "-C", dir, "describe", "--abbrev=0",
+ "master"], check=True, stdout=subprocess.PIPE)
+ return complete.stdout.decode().rstrip()
+
+
+def print_old(gitdir, depends):
+ """ Print dependencies tied to an old release tag
+
+ :param gitdir: folder to which the sources will be cloned
+ :param depends: return value from generate() above """
+ print("Dependencies on old releases:")
+
+ for program, data in depends.items():
+ for depend, version in data["depends"].items():
+ latest = git_latest_tag(gitdir, depend)
+ if latest == version:
+ continue
+ print(" * " + program + ":" + data["version"] + " -> " +
+ depend + ":" + version + " (latest: " + latest + ")")
diff --git a/scripts/osmo-depcheck/osmo-depcheck.py b/scripts/osmo-depcheck/osmo-depcheck.py
new file mode 100755
index 0000000..92c0ce6
--- /dev/null
+++ b/scripts/osmo-depcheck/osmo-depcheck.py
@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2018 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+
+import argparse
+import os
+import sys
+
+# Same folder
+import config
+import dependencies
+import buildstack
+
+
+def parse_arguments():
+ # Create argparser
+ description = ("This script verifies that Osmocom programs really build"
+ " with the dependency versions they claim to support in"
+ " configure.ac. In order to do that, it clones the"
+ " dependency repositories if they don't exist in gitdir"
+ " already, and checks out the minimum version tag. This"
+ " happens recursively for their dependencies as well.")
+ parser = argparse.ArgumentParser(description=description)
+
+ # Git sources folder
+ gitdir_default = os.path.expanduser("~") + "/code"
+ parser.add_argument("-g", "--gitdir", default=gitdir_default,
+ help="folder to which the sources will be cloned"
+ " (default: " + gitdir_default + ")")
+
+ # Build switch
+ parser.add_argument("-b", "--build", action="store_true",
+ help="don't only parse the dependencies, but also try"
+ " to build the program")
+
+ # Build switch
+ parser.add_argument("-o", "--old", action="store_true",
+ help="report dependencies on old releases")
+
+ # Job count
+ parser.add_argument("-j", "--jobs", type=int,
+ help="parallel build jobs (for make)")
+
+ # Git URL prefix
+ parser.add_argument("-u", "--git-url-prefix", dest="prefix",
+ default=config.git_url_prefix,
+ help="where to clone the sources from (default: " +
+ config.git_url_prefix + ")")
+
+ # Projects
+ parser.add_argument("projects_revs", nargs="*", default=config.projects,
+ help="which Osmocom projects to look at"
+ " (e.g. 'osmo-hlr:0.2.1', 'osmo-bts', defaults to"
+ " all projects defined in config.py, default"
+ " revision is 'master')",
+ metavar="project[:revision]")
+
+ # Gitdir must exist
+ ret = parser.parse_args()
+ if not os.path.exists(ret.gitdir):
+ print("ERROR: gitdir does not exist: " + ret.gitdir)
+ sys.exit(1)
+ return ret
+
+
+def main():
+ # Iterate over projects
+ args = parse_arguments()
+ for project_rev in args.projects_revs:
+ # Split the git revision from the project name
+ project = project_rev
+ rev = "master"
+ if ":" in project_rev:
+ project, rev = project_rev.split(":", 1)
+
+ # Clone and parse the repositories
+ depends = dependencies.generate(args.gitdir, args.prefix, project, rev)
+ print("---")
+ dependencies.print_dict(depends)
+ stack = buildstack.generate(depends)
+ print("---")
+ buildstack.print_dict(stack)
+
+ # Old versions
+ if args.old:
+ print("---")
+ dependencies.print_old(args.gitdir, depends)
+
+ # Build
+ if args.build:
+ print("---")
+ buildstack.build(args.gitdir, args.jobs, stack)
+
+ # Success
+ print("---")
+ print("Success for " + project + ":" + rev + "!")
+ print("---")
+
+
+if __name__ == '__main__':
+ main()
diff --git a/scripts/osmo-depcheck/parse.py b/scripts/osmo-depcheck/parse.py
new file mode 100644
index 0000000..c6297d6
--- /dev/null
+++ b/scripts/osmo-depcheck/parse.py
@@ -0,0 +1,116 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2018 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+
+import sys
+import fnmatch
+
+# Same folder
+import config
+
+
+def error(line_i, message):
+ """ Print a configure.ac error message with the line number.
+ :param line_i: the zero based line counter """
+ print("ERROR: configure.ac line " + str(line_i+1) + ": " + message)
+ sys.exit(1)
+
+
+def repository(library, version):
+ """ Find the git repository that contains a certain library. Based on the
+ information in config.py.
+
+ :param library: the name as referenced in the PKG_CHECK_MODULES
+ statement. For example: "libosmoabis"
+ :param version: for example "0.5.0"
+ :returns: the repository name, e.g. "libosmo-abis" """
+ for repo, libraries in config.repos.items():
+ if library in libraries:
+ print(" * " + library + ":" + version + " (part of " + repo + ")")
+ return repo
+
+ print(" * " + library + ":" + version)
+ return library
+
+
+def library_is_relevant(library):
+ """ :returns: True when we would build the library in question from source,
+ False otherwise. """
+ for pattern in config.relevant_library_patterns:
+ if fnmatch.fnmatch(library, pattern):
+ return True
+ return False
+
+
+def parse_condition(line):
+ """ Find the PKG_CHECK_MODULES conditions in any line from a configure.ac.
+
+ Example lines:
+ PKG_CHECK_MODULES(LIBOSMOCORE, libosmocore >= 0.10.0)
+ PKG_CHECK_MODULES(LIBSYSTEMD, libsystemd)
+
+ :returns: * None when there's no condition in that line
+ * a string like "libosmocore >= 0.1.0" """
+ # Only look at PKG_CHECK_MODULES lines
+ if "PKG_CHECK_MODULES" not in line:
+ return
+
+ # Extract the condition
+ ret = line.split(",")[1].split(")")[0].strip()
+
+ # Only look at Osmocom libraries
+ library = ret.split(" ")[0]
+ if library_is_relevant(library):
+ return ret
+
+
+def library_version(line_i, condition):
+ """ Get the library and version strings from a condition.
+ :param line_i: the zero based line counter
+ :param condition: a condition like "libosmocore >= 0.1.0" """
+ # Split by space and remove empty list elements
+ split = list(filter(None, condition.split(" ")))
+ if len(split) != 3:
+ error(line_i, "invalid condition format, expected something"
+ " like 'libosmocore >= 0.10.0' but got: '" +
+ condition + "'")
+ library, operator, version = split
+
+ # Right operator
+ if operator == ">=":
+ return (library, version)
+
+ # Wrong operator
+ error(line_i, "invalid operator, expected '>=' but got: '" +
+ operator + "'")
+
+
+def configure_ac(gitdir, repo):
+ """ Parse the PKG_CHECK_MODULES statements of a configure.ac file.
+
+ :param gitdir: parent folder of all locally cloned git repositories
+ :param repo: the repository to look at (e.g. "osmo-bts")
+ :returns: a dictionary like the following:
+ {"libosmocore": "0.11.0",
+ "libosmo-abis": "0.5.0"} """
+ # Read configure.ac
+ path = gitdir + "/" + repo + "/configure.ac"
+ with open(path) as handle:
+ lines = handle.readlines()
+
+ # Parse the file into ret
+ ret = {}
+ for i in range(0, len(lines)):
+ # Parse the line
+ condition = parse_condition(lines[i])
+ if not condition:
+ continue
+ (library, version) = library_version(i, condition)
+
+ # Add to ret (with duplicate check)
+ repo_dependency = repository(library, version)
+ if repo_dependency in ret and version != ret[repo_dependency]:
+ error(i, "found multiple PKG_CHECK_MODULES statements for " +
+ repo_dependency + ".git, and they have different"
+ " versions!")
+ ret[repo_dependency] = version
+ return ret