jobs/gerrit-verifications: write summary comment

Instead of having multiple mails from jenkins per submitted gerrit
patch, only have one at the end of the pipeline that writes a summary
with all relevant links and votes +1/-1 for verified.

This also resolves the problem we had for a long time, that the link for
the build job didn't directly point to the relevant console output.
Instead it pointed to a matrix job that didn't have a direct link to the
relevant job.

The summary looks like this:

1 failed:
  [build] https://jenkins.osmocom.org/jenkins/job/gerrit-osmo-bsc-nat-build/a1=default,a2=default,a3=default,a4=default,label=osmocom-gerrit-debian9/11/consoleFull

3 passed:
  [rpm] https://jenkins.osmocom.org/jenkins/job/gerrit-binpkgs-rpm/5/consoleFull
  [deb] https://jenkins.osmocom.org/jenkins/job/gerrit-binpkgs-deb/5/consoleFull
  [lint] https://jenkins.osmocom.org/jenkins/job/gerrit-lint/11/consoleFull

Build Failed

Related: OS#2385
Change-Id: Idcab969e1b5ca4e0f1383bee8f36f2d1aac4f624

4 files changed, 280 insertions, 1 deletions

diff --git a/jobs/gerrit-pipeline-result.yml b/jobs/gerrit-pipeline-result.yml
new file mode 100644
index 0000000..ac54cee
--- /dev/null
+++ b/jobs/gerrit-pipeline-result.yml
@@ -0,0 +1,68 @@
+# This job runs at the end of the pipeline in gerrit-verififactions.yml, to
+# post a list of failed/successful job links to gerrit and to vote +V/-V.
+
+- project:
+    name: gerrit-pipeline-result
+    jobs:
+      - 'gerrit-pipeline-result'
+
+- job:
+    name: 'gerrit-pipeline-result'
+    project-type: freestyle
+    node: osmocom-gerrit-debian10 || osmocom-gerrit-debian11
+    retry-count: 3 # scm checkout
+    properties:
+      - build-discarder:
+          days-to-keep: 30
+          num-to-keep: 120
+          artifact-days-to-keep: -1
+          artifact-num-to-keep: -1
+    description: |
+      Result job of CI for patches sent to <a href="https://gerrit.osmocom.org">gerrit</a>.
+      </br></br>
+      Related issue: <a href="https://osmocom.org/issues/2385">OS#2385</a>
+
+    parameters:
+      - string:
+          name: BRANCH_CI
+          description: |
+                osmo-ci.git branch
+          default: 'master'
+      - string:
+          name: GERRIT_BRANCH
+          description: set by gerrit verification pipeline job
+      - string:
+          name: GERRIT_HOST
+          description: set by gerrit verification pipeline job
+      - string:
+          name: GERRIT_PATCHSET_REVISION
+          description: set by gerrit verification pipeline job
+      - string:
+          name: GERRIT_PORT
+          description: set by gerrit verification pipeline job
+      - string:
+          name: GERRIT_REFSPEC
+          description: set by gerrit verification pipeline job
+      - string:
+          name: PIPELINE_BUILD_URL
+          description: set by gerrit verification pipeline job
+
+    scm:
+      - git:
+         url: 'https://gerrit.osmocom.org/osmo-ci'
+         credentials-id: d5eda5e9-b59d-44ba-88d2-43473cb6e42d
+         branches:
+           - '$BRANCH_CI'
+         wipe-workspace: true
+
+    builders:
+      - shell: 'cd scripts/jenkins-gerrit && ./pipeline_summary_send.sh'
+
+    wrappers:
+    - ansicolor:
+        colormap: xterm
+    - ssh-agent-credentials:
+        users:
+        - d5eda5e9-b59d-44ba-88d2-43473cb6e42d
+
+# vim: expandtab tabstop=2 shiftwidth=2
diff --git a/jobs/gerrit-verifications.yml b/jobs/gerrit-verifications.yml
index 5235525..0e64ff3 100644
--- a/jobs/gerrit-verifications.yml
+++ b/jobs/gerrit-verifications.yml
@@ -458,6 +458,19 @@
               echo "PIPELINE_LINT_PASSED=${{env.PIPELINE_LINT_PASSED}}"
               echo "PIPELINE_DEB_PASSED=${{env.PIPELINE_DEB_PASSED}}"
               echo "PIPELINE_RPM_PASSED=${{env.PIPELINE_RPM_PASSED}}"
+
+              // Run the result job to get successful/failed links and add a
+              // comment + vote to gerrit
+              script {{
+               build job: 'gerrit-pipeline-result', parameters: [
+                 string(name: "GERRIT_BRANCH", value: "${{env.GERRIT_BRANCH}}"),
+                 string(name: "GERRIT_HOST", value: "${{env.GERRIT_HOST}}"),
+                 string(name: "GERRIT_PATCHSET_REVISION", value: "${{env.GERRIT_PATCHSET_REVISION}}"),
+                 string(name: "GERRIT_PORT", value: "${{env.GERRIT_PORT}}"),
+                 string(name: "GERRIT_REFSPEC", value: "${{env.GERRIT_REFSPEC}}"),
+                 string(name: "PIPELINE_BUILD_URL", value: "${{env.BUILD_URL}}"),
+               ]
+              }}
             }}
           }}
           // The end result is success if all started jobs were successful,
@@ -509,7 +522,7 @@
               failed: false
               unstable: false
               notbuilt: false
-          silent: false
+          silent: true  # comment + vote is done in gerrit-pipeline-result.yml
           escape-quotes: false
           no-name-and-email: false
           trigger-for-unreviewed-patches: true
diff --git a/scripts/jenkins-gerrit/pipeline_summary.py b/scripts/jenkins-gerrit/pipeline_summary.py
new file mode 100755
index 0000000..c12ad64
--- /dev/null
+++ b/scripts/jenkins-gerrit/pipeline_summary.py
@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+import argparse
+import io
+import json
+import re
+import urllib.request
+
+jenkins_url = "https://jenkins.osmocom.org"
+re_start_build = re.compile("Starting building: gerrit-[a-zA-Z-_]* #[0-9]*")
+re_result = re.compile("^PIPELINE_[A-Z]*_PASSED=[01]$")
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="Get a summary of failed / successful builds from the CI"
+                    " pipeline we run for patches submitted to gerrit.")
+    parser.add_argument("build_url",
+                        help="$BUILD_URL of the pipeline job, e.g."
+                             " https://jenkins.osmocom.org/jenkins/job/gerrit-osmo-bsc-nat/17/")
+    parser.add_argument("-o", "--output", help="output json file")
+    return parser.parse_args()
+
+
+def stage_from_job_name(job_name):
+    if job_name == "gerrit-pipeline-result":
+        # The job that runs this script. Don't include it in the summary.
+        return None
+    if job_name == "gerrit-lint":
+        return "lint"
+    if job_name == "gerrit-binpkgs-deb":
+        return "deb"
+    if job_name == "gerrit-binpkgs-rpm":
+        return "rpm"
+    if job_name.endswith("-build"):
+        return "build"
+    assert False, f"couldn't figure out stage from job_name: {job_name}"
+
+
+def parse_pipeline(build_url):
+    """ Parse started jobs and result from the pipeline log.
+       :returns: a dict that looks like:
+                 {"build": {"name": "gerrit-osmo-bsc-nat-build", id=7,
+                            "passed": True, "url": "https://..."},
+                  "lint": {...},
+                  "deb": {...},
+                  "rpm: {...}} """
+    global re_start_build
+    global re_result
+    global jenkins_url
+    ret = {}
+
+    url = f"{build_url}/consoleText"
+    with urllib.request.urlopen(url) as response:
+        for line in io.TextIOWrapper(response, encoding='utf-8'):
+            # Parse start build lines
+            for match in re_start_build.findall(line):
+                job_name = match.split(" ")[2]
+                job_id = int(match.split(" ")[3].replace("#", ""))
+                job_url = f"{jenkins_url}/jenkins/job/{job_name}/{job_id}"
+                stage = stage_from_job_name(job_name)
+                if stage:
+                    ret[stage] = {"url": job_url, "name": job_name, "id": job_id}
+
+            # Parse result lines
+            if re_result.match(line):
+                stage = line.split("_")[1].lower()
+                passed = line.split("=")[1].rstrip() == "1"
+                ret[stage]["passed"] = passed
+
+    return ret
+
+
+def parse_build_matrix(job):
+    """ Parse started jobs and result from the matrix of the build job. Usually
+        it is only one job, but for some projects we build for multiple arches
+        (x86_64, arm) or build multiple times with different configure flags.
+        :param job: "build" dict from parse_pipeline()
+        :returns: a list of jobs in the matrix, looks like:
+                  [{"passed": True, "url": "https://..."}, ...]
+    """
+    global jenkins_url
+
+    ret = []
+    url = f"{job['url']}/consoleFull"
+    with urllib.request.urlopen(url) as response:
+        for line in io.TextIOWrapper(response, encoding='utf-8'):
+            if " completed with result " in line:
+                url = line.split("<a href='", 1)[1].split("'", 1)[0]
+                url = f"{jenkins_url}{url}{job['id']}"
+                result = line.split(" completed with result ")[1].rstrip()
+                passed = result == "SUCCESS"
+                ret += [{"passed": passed, "url": url}]
+    return ret
+
+
+def jobs_for_summary(pipeline, build_matrix):
+    """ Sort the jobs from pipeline and build matrix into passed/failed lists.
+        :returns: a dict that looks like:
+                  {"passed": [{"stage": "build", "url": "https://..."}, ...],
+                   "failed": [...]} """
+    ret = {"passed": [], "failed": []}
+
+    # Build errors are most interesting, display them first
+    for job in build_matrix:
+        category = "passed" if job["passed"] else "failed"
+        ret[category] += [{"stage": "build", "url": job["url"]}]
+
+    # Hide the build matrix job (we show the jobs started by it instead), as
+    # long as there is at least one failed started job when the matrix failed
+    matrix_failed = "build" in pipeline and not pipeline["build"]["passed"]
+    show_build_matrix_job = matrix_failed and not ret["failed"]
+
+    # Add jobs from the pipeline
+    for stage, job in pipeline.items():
+        if stage == "build" and not show_build_matrix_job:
+            continue
+        category = "passed" if job["passed"] else "failed"
+        ret[category] += [{"stage": stage, "url": job["url"]}]
+
+    return ret
+
+
+def get_jobs_list_str(jobs):
+    ret = ""
+    for job in jobs:
+        ret += f"  [{job['stage']}] {job['url']}/consoleFull\n"
+    return ret
+
+
+def get_pipeline_summary(build_url):
+    """ Generate a summary of failed and successful builds for gerrit.
+        :returns: a dict that is expected by gerrit's set-review api, e.g.
+                  {"tag": "jenkins",
+                   "message": "...",
+                   "labels": {"Code-Review": -1}} """
+    summary = ""
+    pipeline = parse_pipeline(build_url)
+
+    build_matrix = []
+    if "build" in pipeline:
+        build_matrix = parse_build_matrix(pipeline["build"])
+
+    jobs = jobs_for_summary(pipeline, build_matrix)
+
+    if jobs["failed"]:
+        summary += f"{len(jobs['failed'])} failed:\n"
+        summary += get_jobs_list_str(jobs["failed"])
+        summary += "\n"
+
+    summary += f"{len(jobs['passed'])} passed:\n"
+    summary += get_jobs_list_str(jobs["passed"])
+
+    if "lint" in pipeline and not pipeline["lint"]["passed"]:
+        summary += "\n"
+        summary += "Please fix the linting errors. More information:\n"
+        summary += "https://osmocom.org/projects/cellular-infrastructure/wiki/Linting\n"
+
+    summary += "\n"
+    if jobs["failed"]:
+        summary += "Build Failed\n"
+        vote = -1
+    else:
+        summary += "Build Successful\n"
+        vote = 1
+
+    # Reference:
+    # https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#set-review
+    return {"tag": "jenkins",
+            "message": summary,
+            "labels": {"Verified": vote}}
+
+
+def main():
+    args = parse_args()
+    summary = get_pipeline_summary(args.build_url)
+
+    print(summary["message"])
+
+    if args.output:
+        with open(args.output, "w") as handle:
+            json.dump(summary, handle, indent=4)
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/jenkins-gerrit/pipeline_summary_send.sh b/scripts/jenkins-gerrit/pipeline_summary_send.sh
new file mode 100755
index 0000000..d52ad9f
--- /dev/null
+++ b/scripts/jenkins-gerrit/pipeline_summary_send.sh
@@ -0,0 +1,13 @@
+#!/bin/sh -ex
+
+./pipeline_summary.py "$PIPELINE_BUILD_URL" -o gerrit_report.json
+
+ssh \
+	-p "$GERRIT_PORT" \
+	-l jenkins \
+	"$GERRIT_HOST" \
+		gerrit \
+			review \
+			"$GERRIT_PATCHSET_REVISION" \
+			--json \
+			< gerrit_report.json