From a99a4ef541af13f93eae4eeb0852f97537886c24 Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Fri, 21 Sep 2018 10:29:51 +0200 Subject: osmo-depcheck: don't use /tmp, better git code * replace --gitdir with --workdir and give it a new folder structure: * git/$repo: downloaded source code * build/$repo: files created during the build process * install/: installation prefix * adjust the jenkins job to use --workdir * fetch --tags when source exists already * readable error message for failed git checkout Change-Id: I06589277b9d54a2af177451cfab2ca1a658b4058 Relates: OS#2642 --- scripts/osmo-depcheck/buildstack.py | 45 +++++++++------------- scripts/osmo-depcheck/dependencies.py | 68 +++++++++++++++++++++------------- scripts/osmo-depcheck/osmo-depcheck.py | 41 ++++++++++++++------ scripts/osmo-depcheck/parse.py | 6 +-- 4 files changed, 94 insertions(+), 66 deletions(-) (limited to 'scripts/osmo-depcheck') diff --git a/scripts/osmo-depcheck/buildstack.py b/scripts/osmo-depcheck/buildstack.py index 87210ab..0a9a011 100644 --- a/scripts/osmo-depcheck/buildstack.py +++ b/scripts/osmo-depcheck/buildstack.py @@ -70,29 +70,15 @@ def print_dict(stack): print(" * " + program + ":" + version) -def temp_install_folder(): - """ Generate a temporary installation folder - - It will be used as configure prefix, so when running 'make install', - the files will get copied in there instead of "/usr/local/". The folder - will get removed when the script has finished. - - :returns: the path to the temporary folder """ - ret = tempfile.mkdtemp(prefix="depcheck_") - atexit.register(shutil.rmtree, ret) - print("Temporary install folder: " + ret) - return ret - - -def set_environment(jobs, tempdir): +def set_environment(jobs, prefix): """ Configure the environment variables before running configure, make etc. :param jobs: parallel build jobs (for make) - :param tempdir: temporary installation dir (see temp_install_folder()) + :param prefix: installation folder """ - # Add tempdir to PKG_CONFIG_PATH and LD_LIBRARY_PATH - extend = {"PKG_CONFIG_PATH": tempdir + "/lib/pkgconfig", - "LD_LIBRARY_PATH": tempdir + "/lib"} + # Add prefix to PKG_CONFIG_PATH and LD_LIBRARY_PATH + extend = {"PKG_CONFIG_PATH": prefix + "/lib/pkgconfig", + "LD_LIBRARY_PATH": prefix + "/lib"} for env_var, folder in extend.items(): old = os.environ[env_var] if env_var in os.environ else "" os.environ[env_var] = old + ":" + folder @@ -101,10 +87,10 @@ def set_environment(jobs, tempdir): os.environ["JOBS"] = str(jobs) -def build(gitdir, jobs, stack): +def build(workdir, jobs, stack): """ Build one program with all its dependencies. - :param gitdir: folder to which the sources will be cloned + :param workdir: path to where all data (git, build, install) is stored :param jobs: parallel build jobs (for make) :param stack: the build stack as returned by generate() above @@ -122,18 +108,23 @@ def build(gitdir, jobs, stack): anymore in case they decide to compile the code again manually from the source folder. """ # Prepare the install folder and environment - tempdir = temp_install_folder() - unitdir = tempdir + "/lib/systemd/system/" - set_environment(jobs, tempdir) + prefix = workdir + "/install" + unitdir = prefix + "/lib/systemd/system/" + set_environment(jobs, prefix) # Iterate over stack for program, version in stack.items(): print("Building " + program + ":" + version) - os.chdir(gitdir + "/" + program) + + # Create and enter the build folder + builddir = workdir + "/build/" + program + os.mkdir(builddir) + os.chdir(builddir) # Run the build commands - commands = [["autoreconf", "-fi"], - ["./configure", "--prefix", tempdir, + gitdir = workdir + "/git/" + program + commands = [["autoreconf", "-fi", gitdir], + [gitdir + "/configure", "--prefix", prefix, "--with-systemdsystemunitdir=" + unitdir], ["make", "clean"], ["make"], diff --git a/scripts/osmo-depcheck/dependencies.py b/scripts/osmo-depcheck/dependencies.py index 78cf4a0..9b5187d 100644 --- a/scripts/osmo-depcheck/dependencies.py +++ b/scripts/osmo-depcheck/dependencies.py @@ -10,37 +10,55 @@ import sys import parse -def git_clone(gitdir, prefix, repository, version): +def git_clone(workdir, prefix, cache_git_fetch, repository, version): """ Clone a missing git repository and checkout a specific version tag. - :param gitdir: folder to which the sources will be cloned + :param workdir: path to where all data (git, build, install) is stored :param prefix: git url prefix (e.g. "git://git.osmocom.org/") + :param cache_git_fetch: list of repositories that have already been + fetched in this run of osmo-depcheck :param repository: Osmocom git repository name (e.g. "libosmo-abis") :param version: "master" or a version tag like "0.11.0" """ - # Clone when needed - if not os.path.exists(gitdir + "/" + repository): - url = prefix + repository - print("Cloning git repo: " + url) - try: - subprocess.run(["git", "-C", gitdir, "clone", "-q", url], + repodir = workdir + "/git/" + repository + if repository not in cache_git_fetch: + if os.path.exists(repodir): + # Fetch tags for existing source + print("Fetching tags...") + subprocess.run(["git", "-C", repodir, "fetch", "--tags", "-q"], check=True) - except subprocess.CalledProcessError: - print("NOTE: if '" + repository + "' is part of a git repository" - " with a different name, please add it to the mapping in" - " 'config.py' and try again.") - sys.exit(1) + else: + # Clone the source + url = prefix + repository + print("Cloning git repo: " + url) + try: + subprocess.run(["git", "-C", workdir + "/git", "clone", "-q", + url], check=True) + except subprocess.CalledProcessError: + print("NOTE: if '" + repository + "' is part of a git" + " repository with a different name, please add it to the" + " mapping in 'config.py' and try again.") + sys.exit(1) + + # Only fetch the same repository once per session + cache_git_fetch.append(repository) # Checkout the version tag - subprocess.run(["git", "-C", gitdir + "/" + repository, "checkout", - version, "-q"], check=True) + try: + subprocess.run(["git", "-C", repodir, "checkout", version, "-q"], + check=True) + except subprocess.CalledProcessError: + print("ERROR: git checkout failed! Invalid version specified?") + sys.exit(1) -def generate(gitdir, prefix, initial, rev): +def generate(workdir, prefix, cache_git_fetch, initial, rev): """ Generate the dependency graph of an Osmocom program by cloning the git repository, parsing the "configure.ac" file, and recursing. - :param gitdir: folder to which the sources will be cloned + :param workdir: path to where all data (git, build, install) is stored :param prefix: git url prefix (e.g. "git://git.osmocom.org/") + :param cache_git_fetch: list of repositories that have already been + fetched in this run of osmo-depcheck :param initial: the first program to look at (e.g. "osmo-bts") :param rev: the git revision to check out ("master", "0.1.0", ...) :returns: a dictionary like the following: @@ -65,8 +83,8 @@ def generate(gitdir, prefix, initial, rev): # Add the programs dependencies to the stack print("Looking at " + program + ":" + version) - git_clone(gitdir, prefix, program, version) - depends = parse.configure_ac(gitdir, program) + git_clone(workdir, prefix, cache_git_fetch, program, version) + depends = parse.configure_ac(workdir, program) stack.update(depends) # Add the program to the ret @@ -86,28 +104,28 @@ def print_dict(depends): print(" * " + program + ":" + version + " depends: " + str(depends)) -def git_latest_tag(gitdir, repository): +def git_latest_tag(workdir, repository): """ Get the last release string by asking git for the latest tag. - :param gitdir: folder to which the sources will be cloned + :param workdir: path to where all data (git, build, install) is stored :param repository: Osmocom git repository name (e.g. "libosmo-abis") :returns: the latest git tag (e.g. "1.0.2") """ - dir = gitdir + "/" + repository + dir = workdir + "/git/" + repository complete = subprocess.run(["git", "-C", dir, "describe", "--abbrev=0", "master"], check=True, stdout=subprocess.PIPE) return complete.stdout.decode().rstrip() -def print_old(gitdir, depends): +def print_old(workdir, depends): """ Print dependencies tied to an old release tag - :param gitdir: folder to which the sources will be cloned + :param workdir: path to where all data (git, build, install) is stored :param depends: return value from generate() above """ print("Dependencies on old releases:") for program, data in depends.items(): for depend, version in data["depends"].items(): - latest = git_latest_tag(gitdir, depend) + latest = git_latest_tag(workdir, depend) if latest == version: continue print(" * " + program + ":" + data["version"] + " -> " + diff --git a/scripts/osmo-depcheck/osmo-depcheck.py b/scripts/osmo-depcheck/osmo-depcheck.py index 92c0ce6..7f701d6 100755 --- a/scripts/osmo-depcheck/osmo-depcheck.py +++ b/scripts/osmo-depcheck/osmo-depcheck.py @@ -4,6 +4,7 @@ import argparse import os +import shutil import sys # Same folder @@ -17,16 +18,16 @@ def parse_arguments(): description = ("This script verifies that Osmocom programs really build" " with the dependency versions they claim to support in" " configure.ac. In order to do that, it clones the" - " dependency repositories if they don't exist in gitdir" + " dependency repositories if they don't exist in workdir" " already, and checks out the minimum version tag. This" " happens recursively for their dependencies as well.") parser = argparse.ArgumentParser(description=description) # Git sources folder - gitdir_default = os.path.expanduser("~") + "/code" - parser.add_argument("-g", "--gitdir", default=gitdir_default, + workdir_default = os.path.expanduser("~") + "/osmo-depcheck-work" + parser.add_argument("-w", "--workdir", default=workdir_default, help="folder to which the sources will be cloned" - " (default: " + gitdir_default + ")") + " (default: " + workdir_default + ")") # Build switch parser.add_argument("-b", "--build", action="store_true", @@ -55,17 +56,33 @@ def parse_arguments(): " revision is 'master')", metavar="project[:revision]") - # Gitdir must exist + # Workdir must exist ret = parser.parse_args() - if not os.path.exists(ret.gitdir): - print("ERROR: gitdir does not exist: " + ret.gitdir) + if not os.path.exists(ret.workdir): + print("ERROR: workdir does not exist: " + ret.workdir) sys.exit(1) return ret +def workdir_prepare(workdir): + """ Delete old binaries and create the subfolders in workdir + :param workdir: path to where all data is stored """ + # Delete folders with binaries from previous runs + for subfolder in ("build", "install"): + full = workdir + "/" + subfolder + if os.path.exists(full): + shutil.rmtree(full) + + # Create all subfolders + for subfolder in ("build", "install", "git"): + os.makedirs(workdir + "/" + subfolder, exist_ok=True) + + def main(): - # Iterate over projects args = parse_arguments() + + # Iterate over projects + cache_git_fetch = [] for project_rev in args.projects_revs: # Split the git revision from the project name project = project_rev @@ -74,7 +91,9 @@ def main(): project, rev = project_rev.split(":", 1) # Clone and parse the repositories - depends = dependencies.generate(args.gitdir, args.prefix, project, rev) + workdir_prepare(args.workdir) + depends = dependencies.generate(args.workdir, args.prefix, + cache_git_fetch, project, rev) print("---") dependencies.print_dict(depends) stack = buildstack.generate(depends) @@ -84,12 +103,12 @@ def main(): # Old versions if args.old: print("---") - dependencies.print_old(args.gitdir, depends) + dependencies.print_old(args.workdir, depends) # Build if args.build: print("---") - buildstack.build(args.gitdir, args.jobs, stack) + buildstack.build(args.workdir, args.jobs, stack) # Success print("---") diff --git a/scripts/osmo-depcheck/parse.py b/scripts/osmo-depcheck/parse.py index c6297d6..06f932e 100644 --- a/scripts/osmo-depcheck/parse.py +++ b/scripts/osmo-depcheck/parse.py @@ -84,16 +84,16 @@ def library_version(line_i, condition): operator + "'") -def configure_ac(gitdir, repo): +def configure_ac(workdir, repo): """ Parse the PKG_CHECK_MODULES statements of a configure.ac file. - :param gitdir: parent folder of all locally cloned git repositories + :param workdir: path to where all data (git, build, install) is stored :param repo: the repository to look at (e.g. "osmo-bts") :returns: a dictionary like the following: {"libosmocore": "0.11.0", "libosmo-abis": "0.5.0"} """ # Read configure.ac - path = gitdir + "/" + repo + "/configure.ac" + path = workdir + "/git/" + repo + "/configure.ac" with open(path) as handle: lines = handle.readlines() -- cgit v1.2.3