aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorOliver Smith <osmith@sysmocom.de>2018-09-21 10:29:51 +0200
committerOliver Smith <osmith@sysmocom.de>2018-09-21 14:59:14 +0200
commita99a4ef541af13f93eae4eeb0852f97537886c24 (patch)
tree2c41cc0e80c29686b2e12dd6e380846f3d525f87
parent85c2effd89f2fc83be3a1a7e178c73908976e167 (diff)
osmo-depcheck: don't use /tmp, better git codeosmith/dependency-check
* replace --gitdir with --workdir and give it a new folder structure: * git/$repo: downloaded source code * build/$repo: files created during the build process * install/: installation prefix * adjust the jenkins job to use --workdir * fetch --tags when source exists already * readable error message for failed git checkout Change-Id: I06589277b9d54a2af177451cfab2ca1a658b4058 Relates: OS#2642
-rw-r--r--jobs/osmocom-depcheck.yml4
-rw-r--r--scripts/osmo-depcheck/buildstack.py45
-rw-r--r--scripts/osmo-depcheck/dependencies.py68
-rwxr-xr-xscripts/osmo-depcheck/osmo-depcheck.py41
-rw-r--r--scripts/osmo-depcheck/parse.py6
5 files changed, 96 insertions, 68 deletions
diff --git a/jobs/osmocom-depcheck.yml b/jobs/osmocom-depcheck.yml
index f13d4b7..fd2432e 100644
--- a/jobs/osmocom-depcheck.yml
+++ b/jobs/osmocom-depcheck.yml
@@ -52,13 +52,13 @@
# Build the arguments
args="$PROJECTS"
args="$args -j 5"
- args="$args -g $PWD/DEPCHECK_GITDIR"
+ args="$args -w $PWD/DEPCHECK_WORKDIR"
args="$args -u $GIT_URL_PREFIX"
[ "$BUILD" = "true" ] && args="$args -b"
[ "$PRINT_OLD_DEPENDS" = "true" ] && args="$args -o"
# Run osmo-depcheck
- mkdir DEPCHECK_GITDIR
+ mkdir DEPCHECK_WORKDIR
export PYTHONUNBUFFERED=1
scripts/osmo-depcheck/osmo-depcheck.py $args
scm:
diff --git a/scripts/osmo-depcheck/buildstack.py b/scripts/osmo-depcheck/buildstack.py
index 87210ab..0a9a011 100644
--- a/scripts/osmo-depcheck/buildstack.py
+++ b/scripts/osmo-depcheck/buildstack.py
@@ -70,29 +70,15 @@ def print_dict(stack):
print(" * " + program + ":" + version)
-def temp_install_folder():
- """ Generate a temporary installation folder
-
- It will be used as configure prefix, so when running 'make install',
- the files will get copied in there instead of "/usr/local/". The folder
- will get removed when the script has finished.
-
- :returns: the path to the temporary folder """
- ret = tempfile.mkdtemp(prefix="depcheck_")
- atexit.register(shutil.rmtree, ret)
- print("Temporary install folder: " + ret)
- return ret
-
-
-def set_environment(jobs, tempdir):
+def set_environment(jobs, prefix):
""" Configure the environment variables before running configure, make etc.
:param jobs: parallel build jobs (for make)
- :param tempdir: temporary installation dir (see temp_install_folder())
+ :param prefix: installation folder
"""
- # Add tempdir to PKG_CONFIG_PATH and LD_LIBRARY_PATH
- extend = {"PKG_CONFIG_PATH": tempdir + "/lib/pkgconfig",
- "LD_LIBRARY_PATH": tempdir + "/lib"}
+ # Add prefix to PKG_CONFIG_PATH and LD_LIBRARY_PATH
+ extend = {"PKG_CONFIG_PATH": prefix + "/lib/pkgconfig",
+ "LD_LIBRARY_PATH": prefix + "/lib"}
for env_var, folder in extend.items():
old = os.environ[env_var] if env_var in os.environ else ""
os.environ[env_var] = old + ":" + folder
@@ -101,10 +87,10 @@ def set_environment(jobs, tempdir):
os.environ["JOBS"] = str(jobs)
-def build(gitdir, jobs, stack):
+def build(workdir, jobs, stack):
""" Build one program with all its dependencies.
- :param gitdir: folder to which the sources will be cloned
+ :param workdir: path to where all data (git, build, install) is stored
:param jobs: parallel build jobs (for make)
:param stack: the build stack as returned by generate() above
@@ -122,18 +108,23 @@ def build(gitdir, jobs, stack):
anymore in case they decide to compile the code again manually from
the source folder. """
# Prepare the install folder and environment
- tempdir = temp_install_folder()
- unitdir = tempdir + "/lib/systemd/system/"
- set_environment(jobs, tempdir)
+ prefix = workdir + "/install"
+ unitdir = prefix + "/lib/systemd/system/"
+ set_environment(jobs, prefix)
# Iterate over stack
for program, version in stack.items():
print("Building " + program + ":" + version)
- os.chdir(gitdir + "/" + program)
+
+ # Create and enter the build folder
+ builddir = workdir + "/build/" + program
+ os.mkdir(builddir)
+ os.chdir(builddir)
# Run the build commands
- commands = [["autoreconf", "-fi"],
- ["./configure", "--prefix", tempdir,
+ gitdir = workdir + "/git/" + program
+ commands = [["autoreconf", "-fi", gitdir],
+ [gitdir + "/configure", "--prefix", prefix,
"--with-systemdsystemunitdir=" + unitdir],
["make", "clean"],
["make"],
diff --git a/scripts/osmo-depcheck/dependencies.py b/scripts/osmo-depcheck/dependencies.py
index 78cf4a0..9b5187d 100644
--- a/scripts/osmo-depcheck/dependencies.py
+++ b/scripts/osmo-depcheck/dependencies.py
@@ -10,37 +10,55 @@ import sys
import parse
-def git_clone(gitdir, prefix, repository, version):
+def git_clone(workdir, prefix, cache_git_fetch, repository, version):
""" Clone a missing git repository and checkout a specific version tag.
- :param gitdir: folder to which the sources will be cloned
+ :param workdir: path to where all data (git, build, install) is stored
:param prefix: git url prefix (e.g. "git://git.osmocom.org/")
+ :param cache_git_fetch: list of repositories that have already been
+ fetched in this run of osmo-depcheck
:param repository: Osmocom git repository name (e.g. "libosmo-abis")
:param version: "master" or a version tag like "0.11.0" """
- # Clone when needed
- if not os.path.exists(gitdir + "/" + repository):
- url = prefix + repository
- print("Cloning git repo: " + url)
- try:
- subprocess.run(["git", "-C", gitdir, "clone", "-q", url],
+ repodir = workdir + "/git/" + repository
+ if repository not in cache_git_fetch:
+ if os.path.exists(repodir):
+ # Fetch tags for existing source
+ print("Fetching tags...")
+ subprocess.run(["git", "-C", repodir, "fetch", "--tags", "-q"],
check=True)
- except subprocess.CalledProcessError:
- print("NOTE: if '" + repository + "' is part of a git repository"
- " with a different name, please add it to the mapping in"
- " 'config.py' and try again.")
- sys.exit(1)
+ else:
+ # Clone the source
+ url = prefix + repository
+ print("Cloning git repo: " + url)
+ try:
+ subprocess.run(["git", "-C", workdir + "/git", "clone", "-q",
+ url], check=True)
+ except subprocess.CalledProcessError:
+ print("NOTE: if '" + repository + "' is part of a git"
+ " repository with a different name, please add it to the"
+ " mapping in 'config.py' and try again.")
+ sys.exit(1)
+
+ # Only fetch the same repository once per session
+ cache_git_fetch.append(repository)
# Checkout the version tag
- subprocess.run(["git", "-C", gitdir + "/" + repository, "checkout",
- version, "-q"], check=True)
+ try:
+ subprocess.run(["git", "-C", repodir, "checkout", version, "-q"],
+ check=True)
+ except subprocess.CalledProcessError:
+ print("ERROR: git checkout failed! Invalid version specified?")
+ sys.exit(1)
-def generate(gitdir, prefix, initial, rev):
+def generate(workdir, prefix, cache_git_fetch, initial, rev):
""" Generate the dependency graph of an Osmocom program by cloning the git
repository, parsing the "configure.ac" file, and recursing.
- :param gitdir: folder to which the sources will be cloned
+ :param workdir: path to where all data (git, build, install) is stored
:param prefix: git url prefix (e.g. "git://git.osmocom.org/")
+ :param cache_git_fetch: list of repositories that have already been
+ fetched in this run of osmo-depcheck
:param initial: the first program to look at (e.g. "osmo-bts")
:param rev: the git revision to check out ("master", "0.1.0", ...)
:returns: a dictionary like the following:
@@ -65,8 +83,8 @@ def generate(gitdir, prefix, initial, rev):
# Add the programs dependencies to the stack
print("Looking at " + program + ":" + version)
- git_clone(gitdir, prefix, program, version)
- depends = parse.configure_ac(gitdir, program)
+ git_clone(workdir, prefix, cache_git_fetch, program, version)
+ depends = parse.configure_ac(workdir, program)
stack.update(depends)
# Add the program to the ret
@@ -86,28 +104,28 @@ def print_dict(depends):
print(" * " + program + ":" + version + " depends: " + str(depends))
-def git_latest_tag(gitdir, repository):
+def git_latest_tag(workdir, repository):
""" Get the last release string by asking git for the latest tag.
- :param gitdir: folder to which the sources will be cloned
+ :param workdir: path to where all data (git, build, install) is stored
:param repository: Osmocom git repository name (e.g. "libosmo-abis")
:returns: the latest git tag (e.g. "1.0.2") """
- dir = gitdir + "/" + repository
+ dir = workdir + "/git/" + repository
complete = subprocess.run(["git", "-C", dir, "describe", "--abbrev=0",
"master"], check=True, stdout=subprocess.PIPE)
return complete.stdout.decode().rstrip()
-def print_old(gitdir, depends):
+def print_old(workdir, depends):
""" Print dependencies tied to an old release tag
- :param gitdir: folder to which the sources will be cloned
+ :param workdir: path to where all data (git, build, install) is stored
:param depends: return value from generate() above """
print("Dependencies on old releases:")
for program, data in depends.items():
for depend, version in data["depends"].items():
- latest = git_latest_tag(gitdir, depend)
+ latest = git_latest_tag(workdir, depend)
if latest == version:
continue
print(" * " + program + ":" + data["version"] + " -> " +
diff --git a/scripts/osmo-depcheck/osmo-depcheck.py b/scripts/osmo-depcheck/osmo-depcheck.py
index 92c0ce6..7f701d6 100755
--- a/scripts/osmo-depcheck/osmo-depcheck.py
+++ b/scripts/osmo-depcheck/osmo-depcheck.py
@@ -4,6 +4,7 @@
import argparse
import os
+import shutil
import sys
# Same folder
@@ -17,16 +18,16 @@ def parse_arguments():
description = ("This script verifies that Osmocom programs really build"
" with the dependency versions they claim to support in"
" configure.ac. In order to do that, it clones the"
- " dependency repositories if they don't exist in gitdir"
+ " dependency repositories if they don't exist in workdir"
" already, and checks out the minimum version tag. This"
" happens recursively for their dependencies as well.")
parser = argparse.ArgumentParser(description=description)
# Git sources folder
- gitdir_default = os.path.expanduser("~") + "/code"
- parser.add_argument("-g", "--gitdir", default=gitdir_default,
+ workdir_default = os.path.expanduser("~") + "/osmo-depcheck-work"
+ parser.add_argument("-w", "--workdir", default=workdir_default,
help="folder to which the sources will be cloned"
- " (default: " + gitdir_default + ")")
+ " (default: " + workdir_default + ")")
# Build switch
parser.add_argument("-b", "--build", action="store_true",
@@ -55,17 +56,33 @@ def parse_arguments():
" revision is 'master')",
metavar="project[:revision]")
- # Gitdir must exist
+ # Workdir must exist
ret = parser.parse_args()
- if not os.path.exists(ret.gitdir):
- print("ERROR: gitdir does not exist: " + ret.gitdir)
+ if not os.path.exists(ret.workdir):
+ print("ERROR: workdir does not exist: " + ret.workdir)
sys.exit(1)
return ret
+def workdir_prepare(workdir):
+ """ Delete old binaries and create the subfolders in workdir
+ :param workdir: path to where all data is stored """
+ # Delete folders with binaries from previous runs
+ for subfolder in ("build", "install"):
+ full = workdir + "/" + subfolder
+ if os.path.exists(full):
+ shutil.rmtree(full)
+
+ # Create all subfolders
+ for subfolder in ("build", "install", "git"):
+ os.makedirs(workdir + "/" + subfolder, exist_ok=True)
+
+
def main():
- # Iterate over projects
args = parse_arguments()
+
+ # Iterate over projects
+ cache_git_fetch = []
for project_rev in args.projects_revs:
# Split the git revision from the project name
project = project_rev
@@ -74,7 +91,9 @@ def main():
project, rev = project_rev.split(":", 1)
# Clone and parse the repositories
- depends = dependencies.generate(args.gitdir, args.prefix, project, rev)
+ workdir_prepare(args.workdir)
+ depends = dependencies.generate(args.workdir, args.prefix,
+ cache_git_fetch, project, rev)
print("---")
dependencies.print_dict(depends)
stack = buildstack.generate(depends)
@@ -84,12 +103,12 @@ def main():
# Old versions
if args.old:
print("---")
- dependencies.print_old(args.gitdir, depends)
+ dependencies.print_old(args.workdir, depends)
# Build
if args.build:
print("---")
- buildstack.build(args.gitdir, args.jobs, stack)
+ buildstack.build(args.workdir, args.jobs, stack)
# Success
print("---")
diff --git a/scripts/osmo-depcheck/parse.py b/scripts/osmo-depcheck/parse.py
index c6297d6..06f932e 100644
--- a/scripts/osmo-depcheck/parse.py
+++ b/scripts/osmo-depcheck/parse.py
@@ -84,16 +84,16 @@ def library_version(line_i, condition):
operator + "'")
-def configure_ac(gitdir, repo):
+def configure_ac(workdir, repo):
""" Parse the PKG_CHECK_MODULES statements of a configure.ac file.
- :param gitdir: parent folder of all locally cloned git repositories
+ :param workdir: path to where all data (git, build, install) is stored
:param repo: the repository to look at (e.g. "osmo-bts")
:returns: a dictionary like the following:
{"libosmocore": "0.11.0",
"libosmo-abis": "0.5.0"} """
# Read configure.ac
- path = gitdir + "/" + repo + "/configure.ac"
+ path = workdir + "/git/" + repo + "/configure.ac"
with open(path) as handle:
lines = handle.readlines()