ci: add support for integration tests

Replace use of the deprecated "only" keyword with the much more flexible
"rules". Introduce support for spawning a dynamic child pipeline to run
tests against packages that were built in CI.

The child pipeline initially allows for packages to define what tests
and for which devices to run. Initially, this supports the following
kinds of tests:

1. Install tests
"pmbootstrap install" is run with the newly built package in the local
repo.

2. Upgrade tests
Where a chroot for the device is created, the new package made available
via a local repo, and then "apk upgrade -a" is run inside the chroot.
This allows for validating the .post-upgrade script in the context of a
real rootfs.

This is currently limited to packages that are automatically pulled in
as part of a default installation. This is because handling non-default
packages requires either spawning jobs per-package, or assuming that
multiple packages updated in the same MR can all be installed at once
(often not the case). In the end this can't be handled properly without
a dependency graph solver, or by imposing some strong restrictions.
Hence it is deemed out of scope for this initial implementation.

Signed-off-by: Caleb Connolly <caleb@connolly.tech>

.ci/generate-test-pipeline.sh

@@ -0,0 +1,16 @@
+#!/bin/sh -e
+# Copyright 2024 Caleb Connolly
+# SPDX-License-Identifier: GPL-3.0-or-later
+# Description: check pkgver/pkgrel bumps, amount of changed pkgs etc
+# Options: native
+# Use 'native' because it requires git commit history.
+# https://postmarketos.org/pmb-ci
+
+if [ "$(id -u)" = 0 ]; then
+	set -x
+	apk -q add python3 git
+	# Keep environment variables
+	exec su -p "${TESTUSER:-build}" -c "sh -e $0"
+fi
+
+.ci/lib/generate_test_pipeline.py

.ci/lib/common.py

@@ -22,9 +22,8 @@ def get_pmaports_dir():
     return ret
 
 
-def run_git(parameters, check=True, stderr=None):
-    """ Run git in the pmaports dir and return the output """
-    cmd = ["git", "-C", get_pmaports_dir()] + parameters
+def run_cmd(cmd, check=True, stderr=None):
+    """ Run a command and return the output """
     try:
         return subprocess.check_output(cmd, stderr=stderr).decode()
     except subprocess.CalledProcessError:
@@ -33,6 +32,12 @@ def run_git(parameters, check=True, stderr=None):
         return None
 
 
+def run_git(parameters, check=True, stderr=None):
+    """ Run git in the pmaports dir and return the output """
+    cmd = ["git", "-C", get_pmaports_dir()] + parameters
+    return run_cmd(cmd, check, stderr)
+
+
 def add_upstream_git_remote():
     """ Add a remote pointing to postmarketOS/pmaports. """
     run_git(["remote", "add", "upstream",
@@ -85,17 +90,21 @@ def get_upstream_branch():
     return ret
 
 
-def get_changed_files(removed=True):
+def get_changed_files(removed=True, print_info=True):
     """ Get all changed files and print them, as well as the branch and the
         commit that was used for the diff.
         :param removed: also return removed files (default: True)
         :returns: set of changed files
     """
+    def log_info(message):
+        if print_info:
+            print(message)
+
     branch_upstream = f"upstream/{get_upstream_branch()}"
     commit_head = run_git(["rev-parse", "HEAD"])[:-1]
     commit_upstream = run_git(["rev-parse", branch_upstream])[:-1]
-    print("commit HEAD: " + commit_head)
-    print(f"commit {branch_upstream}: f{commit_upstream}")
+    log_info("commit HEAD: " + commit_head)
+    log_info(f"commit {branch_upstream}: f{commit_upstream}")
 
     # Check if we are HEAD on the upstream branch
     if commit_head == commit_upstream:
@@ -104,11 +113,11 @@ def get_changed_files(removed=True):
     else:
         # otherwise compare with latest common ancestor
         commit = run_git(["merge-base", branch_upstream, "HEAD"])[:-1]
-    print("comparing HEAD with: " + commit)
+    log_info("comparing HEAD with: " + commit)
 
     # Changed files
     ret = set()
-    print("changed file(s):")
+    log_info("changed file(s):")
     for file in run_git(["diff", "--name-only", commit, "HEAD"]).splitlines():
         message = "  " + file
         if not os.path.exists(file):
@@ -117,7 +126,7 @@ def get_changed_files(removed=True):
                 ret.add(file)
         else:
             ret.add(file)
-        print(message)
+        log_info(message)
     return ret
 
 
@@ -155,9 +164,9 @@ def get_changed_packages_sanity_check(count):
     sys.exit(1)
 
 
-def get_changed_packages():
+def get_changed_packages_paths(print_info=False):
     ret = set()
-    for file in get_changed_files():
+    for file in get_changed_files(print_info=print_info):
         dirname, filename = os.path.split(file)
 
         # Skip files:
@@ -187,11 +196,15 @@ def get_changed_packages():
         elif not os.path.exists(file):
             continue  # APKBUILD was deleted
 
-        ret.add(os.path.basename(dirname))
+        ret.add(dirname)
 
     return ret
 
 
+def get_changed_packages():
+    return set(map(lambda p: os.path.basename(p), iter(get_changed_packages_paths(print_info=True))))
+
+
 def get_changed_kernels():
     ret = []
     for pkgname in get_changed_packages():

.ci/lib/copy-in-packages.sh

@@ -0,0 +1,15 @@
+#!/bin/sh -ex
+# Copies packages from previous build artifacts into the
+# local binary repo
+
+ARCH="$1"
+
+if [ -z "$ARCH" ]; then
+	echo "Usage: $0 <arch>"
+	exit 1
+fi
+
+mkdir -p /home/pmos/.local/var/pmbootstrap/packages/edge/"$ARCH"
+cp packages/edge/"$ARCH"/*.apk /home/pmos/.local/var/pmbootstrap/packages/edge/"$ARCH"
+chown -R 12345:12345 /home/pmos/.local/var/pmbootstrap/packages/edge/"$ARCH"
+sudo -u pmos -- pmbootstrap index

.ci/lib/generate_test_pipeline.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+# Generate a gitlab CI yaml file for the integration tests pipeline
+# Configures the parallel:matrix variables for the install/upgrade jobs
+# based on the union of all the "test-upgrade" configs in the affected APKBUILDs.
+# And the boot matrix based on the "test-boot" config.
+
+# use the matrix variables in the needs: section to sequence the 3 jobs, otherwise
+# spawn a pipeline for each device (yikes).
+
+import os
+import sys
+import glob
+import json
+
+import common
+
+# Get modified packages
+common.add_upstream_git_remote()
+pkgs = common.get_changed_packages_paths()
+
+devices = {}
+
+
+# We support using "main" as a shortcut for main devices
+def get_device_variants(devices):
+    _devices = []
+    for d in devices:
+        if d == "main":
+            _devices += list(map(lambda x: os.path.basename(x).split("-", 1)[-1],
+                             glob.iglob(os.path.join(common.get_pmaports_dir(),
+                                                     "device", "main", "device-*"))))
+        else:
+            _devices.append(d)
+
+    return _devices
+
+
+# YIKES
+def get_device_arch(device):
+    return common.run_cmd(["sh", "-c",
+                           f"""source {glob.glob(os.path.join(common.get_pmaports_dir(),
+                               'device', '*', f'device-{device}'))[0]}/deviceinfo; echo $deviceinfo_arch"""]).strip()
+
+
+# Add a device that we should test upgrading pkg on
+def add_devices(pkg, devices_upgrade, devices_boot):
+    devices_upgrade = get_device_variants(devices_upgrade)
+    devices_boot = get_device_variants(devices_boot)
+    print(f"Devices for {pkg}: {devices_upgrade} {devices_boot}")
+    for d in set(devices_upgrade + devices_boot):
+        if d not in devices.keys():
+            devices[d] = {"upgrade": False, "boot": False, "arch": get_device_arch(d)}
+
+        if d in devices_boot:
+            devices[d]["boot"] = True
+        if d in devices_upgrade:
+            devices[d]["upgrade"] = True
+
+
+def parse_device_list(key):
+    cfg = line.split(":")[-1]
+    if cfg == f"# {key}:":
+        print(f"Invalid test-upgrade config in {apkbuild}")
+        sys.exit(1)
+    return cfg.strip().split()
+
+
+# Get the union of all the test-upgrade configs
+for pkg in pkgs:
+    apkbuild = os.path.join(pkg, "APKBUILD")
+    with open(apkbuild) as f:
+        # ALWAYS test on qemu
+        upgrade_devices = ["qemu-aarch64", "qemu-amd64"]
+        boot_devices = []
+        for line in f:
+            if line != "\n" and line[0] != "#":
+                break
+            if line.startswith("# test-upgrade"):
+                upgrade_devices += parse_device_list("test-upgrade")
+            if line.startswith("# test-boot"):
+                boot_devices += parse_device_list("test-boot")
+        add_devices(os.path.basename(pkg), upgrade_devices, boot_devices)
+
+print(json.dumps(devices, indent=4))
+
+# Generate the yaml
+template = open(os.path.join(os.path.dirname(__file__), "test-pipeline.template.yml")).read()
+matrix = ""
+for (device, cfg) in devices.items():
+    # FIXME: riscv64 is not well supported..
+    if cfg["arch"] == "riscv64":
+        continue
+    boot = "true" if cfg["boot"] and "qemu" in device else "false"
+    upgrade = "true" if cfg["upgrade"] else "false"
+    print(f"Adding {device} to matrix, upgrade test: {upgrade}, boot test: {boot}")
+    matrix += f"""      - DEVICE: {device}
+        UPGRADE_TEST: \"{upgrade}\"
+        BOOT_TEST: \"{boot}\"
+        BUILD_ARCH_JOB: build-{cfg["arch"]}
+"""
+
+variables = f"  PACKAGES: {' '.join(map(lambda x: os.path.basename(x), pkgs))}\n"
+
+f = open("test-pipeline.yml", "w")
+f.write(template.replace("{{ matrix }}", matrix)
+                .replace("{{ variables }}", variables))
+f.close()

.ci/lib/test-pipeline.template.yml

@@ -0,0 +1,123 @@
+# TEMPLATE pipeline for integration tests
+image: alpine:latest
+after_script:
+  - .ci/lib/move_logs.sh $CI_PROJECT_DIR
+
+variables:
+  PYTHONUNBUFFERED: 1
+  # This includes a hack to install the packages that were built in a previous
+  # pipeline (with different signing keys). As for some reason even though we
+  # replace the signing keys in pmbootstrap with the ones from the previous job
+  # BEFORE creating the chroot, apk still rejects the packages :(
+  PMBOOTSTRAP_URL: https://gitlab.com/sdm845-mainline/pmbootstrap.git
+  PMBOOTSTRAP_TAG: caleb/allow-untrusted-hack
+  # defines PACKAGES
+{{ variables }}
+
+# Variables DEVICE, UPGRADE_TEST, and BOOT_TEST are set in the
+# parallel matrix below.
+
+stages:
+  - install
+  - upgrade
+  - boot
+
+workflow:
+  name: "Testing for $PACKAGES"
+  # Default rules for all jobs
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "parent_pipeline"
+
+install:
+  # tags:
+  #   - kvm
+  stage: install
+  # So we can fetch the packages that were built in the parent pipeline
+  # and install them in the chroot
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: $BUILD_ARCH_JOB
+  before_script:
+    - apk add curl sudo
+    # Avoid using the local pmaports checkout so we don't build packages (and sidestep annoying permissions issues)
+    - (cd /tmp ; curl -fsSL "https://gitlab.com/postmarketOS/ci-common/-/raw/master/install_pmbootstrap.sh" | sh)
+    # Use the same keys as the build job so we can generate an index
+    - mkdir -p /home/pmos/.local/var/pmbootstrap/config_abuild/
+    - mv packages/config_abuild/* /home/pmos/.local/var/pmbootstrap/config_abuild/
+    - chown -R 12345:12345 /home/pmos/.local/var/pmbootstrap/config_abuild
+    - sudo -u pmos -- pmbootstrap config device ${DEVICE}
+    # Use the console UI so we pull in some additional system packages
+    - sudo -u pmos -- pmbootstrap config ui console
+    # For QEMU (ignored for other devices)
+    - sudo -u pmos -- pmbootstrap config kernel virt
+  script:
+    - ARCH=${BUILD_ARCH_JOB##build-}
+    # For now we only support packages that are pulled in automatically for the device.
+    # Introducing support for testing packages that must be explicitly installed requires
+    # having a job per package and doing dependency/conflict resolution.
+    - .ci/lib/copy-in-packages.sh $ARCH
+    - sudo -u pmos -- pmbootstrap install --no-image --password 147147
+  artifacts:
+    when: on_failure
+    paths:
+      - log.txt
+      - pmbootstrap.cfg
+    expire_in: 1 week
+  parallel:
+    matrix:
+{{ matrix }}
+
+upgrade:
+  # tags:
+  #   - kvm
+  stage: upgrade
+  # So we can fetch the packages that were built in the parent pipeline
+  # and install them in the chroot
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: $BUILD_ARCH_JOB
+  rules:
+    - if: $UPGRADE_TEST == "true"
+  before_script:
+    - apk add curl sudo
+    # Avoid using the local pmaports checkout so we don't build packages (and sidestep annoying permissions issues)
+    - (cd /tmp ; curl -fsSL "https://gitlab.com/postmarketOS/ci-common/-/raw/master/install_pmbootstrap.sh" | sh)
+    # Use the same keys as the build job so we can generate an index
+    - mkdir -p /home/pmos/.local/var/pmbootstrap/config_abuild/
+    - mv packages/config_abuild/* /home/pmos/.local/var/pmbootstrap/config_abuild/
+    - chown -R 12345:12345 /home/pmos/.local/var/pmbootstrap/config_abuild
+    - sudo -u pmos -- pmbootstrap -y zap -a
+    - sudo -u pmos -- pmbootstrap config device ${DEVICE}
+    # Use the console UI so we pull in some additional system packages
+    - sudo -u pmos -- pmbootstrap config ui console
+  script:
+    - ARCH=${BUILD_ARCH_JOB##build-}
+    # For now we only support packages that are pulled in automatically for the device.
+    # Introducing support for testing packages that must be explicitly installed requires
+    # having a job per package and doing dependency/conflict resolution.
+    - sudo -u pmos -- pmbootstrap install --no-image --password 147147
+    - .ci/lib/copy-in-packages.sh $ARCH
+    - sudo -u pmos -- pmbootstrap chroot -r -- apk upgrade -a --allow-untrusted
+  artifacts:
+    when: on_failure
+    paths:
+      - log.txt
+      - pmbootstrap.cfg
+    expire_in: 1 week
+  parallel:
+    matrix:
+{{ matrix }}
+
+boot:
+  # tags:
+  #   - kvm
+  stage: boot
+  rules:
+    - if: $BOOT_TEST == "true"
+  script:
+    - echo boot
+  parallel:
+    matrix:
+{{ matrix }}
+
+

.gitlab-ci.yml

@@ -7,30 +7,20 @@ after_script:
 stages:
   - lint
   - build
+  - test
 
-# defaults for "only"
-# We need to run the CI jobs in a "merge request specific context", if CI is
-# running in a merge request. Otherwise the environment variable that holds the
-# merge request ID is not available. This means, we must set the "only"
-# variable accordingly - and if we only do it for one job, all other jobs will
-# not get executed. So have the defaults here, and use them in all jobs that
-# should run on both the master branch, and in merge requests.
-# https://docs.gitlab.com/ee/ci/merge_request_pipelines/index.html#excluding-certain-jobs
-.only-default: &only-default
-  only:
-    - master
-    - merge_requests
-    - tags
+workflow:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_PIPELINE_SOURCE == "push"
 
 # device documentation
 wiki:
   stage: lint
-  <<: *only-default
-  only:
-    refs:
-      - merge_requests
-    changes:
-      - device/*/device-*/*
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+        - device/*/device-*/*
   script:
     - .ci/lib/gitlab_prepare_ci.sh
     - .ci/wiki.sh
@@ -38,10 +28,8 @@ wiki:
 # testcases linting
 flake8:
   stage: lint
-  <<: *only-default
-  only:
-    refs:
-      - merge_requests
+  rules:
+  - if: $CI_PIPELINE_SOURCE == "merge_request_event"
     changes:
       - .ci/*
       - .ci/*/*
@@ -54,7 +42,6 @@ flake8:
 # shellcheck and various grep checks
 shellcheck-grep:
   stage: lint
-  <<: *only-default
   image: alpine:edge
   script:
     - .ci/lib/gitlab_prepare_ci.sh
@@ -64,7 +51,6 @@ shellcheck-grep:
 
 editor-config:
   stage: lint
-  <<: *only-default
   image: alpine:edge
   script:
     - .ci/lib/gitlab_prepare_ci.sh
@@ -73,7 +59,6 @@ editor-config:
 # aports checks (generic)
 pytest-commits:
   stage: lint
-  <<: *only-default
   script:
     - .ci/lib/gitlab_prepare_ci.sh
     - .ci/pytest.sh
@@ -92,17 +77,15 @@ aport-lint:
   script:
     - .ci/lib/gitlab_prepare_ci.sh
     - .ci/apkbuild-lint.sh
-  only:
-    - merge_requests
+  rules:
+  - if: $CI_PIPELINE_SOURCE == "merge_request_event"
   allow_failure: true
 
 # kernel kconfig check
 kernel-kconfig:
   stage: lint
-  only:
-    refs:
-      - master
-      - merge_requests
+  rules:
+  - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
     changes:
       - device/*/linux-*/config-*
       - main/linux-*/config-*
@@ -114,8 +97,8 @@ kernel-kconfig:
 # (Checks for "Allow commits from members who can merge to the target branch")
 mr-settings:
   stage: lint
-  only:
-    - merge_requests
+  rules:
+  - if: $CI_PIPELINE_SOURCE == "merge_request_event"
   before_script:
     - apk -q add python3
   script:
@@ -125,11 +108,14 @@ mr-settings:
 # build changed aports
 .build:
   stage: build
-  <<: *only-default
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
   before_script:
     - .ci/lib/gitlab_prepare_ci.sh
   after_script:
     - cp -r /home/pmos/.local/var/pmbootstrap/packages/ packages/ || true
+    - cp -r /home/pmos/.local/var/pmbootstrap/config_abuild packages/ || true
   artifacts:
     expire_in: 1 week
     paths:
@@ -165,3 +151,32 @@ build-riscv64:
   extends: .build
   script:
     - .ci/build-riscv64.sh
+
+# Generate the test pipeline
+generate-test-pipeline:
+  stage: test
+  # Only test in MRs or when pushing to a stable release branch
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    - if: $CI_COMMIT_REF_PROTECTED == 'true' && $CI_COMMIT_BRANCH != '$CI_DEFAULT_BRANCH'
+  before_script:
+    - .ci/lib/gitlab_prepare_ci.sh
+  script:
+    - .ci/generate-test-pipeline.sh
+  artifacts:
+    paths:
+      - test-pipeline.yml
+
+Run tests:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    - if: $CI_COMMIT_REF_PROTECTED == 'true' && $CI_COMMIT_BRANCH != '$CI_DEFAULT_BRANCH'
+  needs:
+    - job: generate-test-pipeline
+      artifacts: true
+  trigger:
+    include:
+      - artifact: test-pipeline.yml
+        job: generate-test-pipeline
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID

main/postmarketos-base/APKBUILD

@@ -1,6 +1,8 @@
+# PMAPORTS-CI
+# test-upgrade: main
 pkgname=postmarketos-base
 pkgver=32
-pkgrel=1
+pkgrel=2
 pkgdesc="Meta package for minimal postmarketOS base"
 url="https://postmarketos.org"
 arch="noarch"

main/postmarketos-initramfs/APKBUILD

@@ -1,7 +1,9 @@
 # Maintainer: Oliver Smith <ollieparanoid@postmarketos.org>
 # Co-Maintainer: Clayton Craft <clayton@craftyguy.net>
+# PMAPORTS-CI
+# test-upgrade: main oneplus-enchilada lenovo-21bx
 pkgname=postmarketos-initramfs
-pkgver=2.4.0
+pkgver=2.5.0
 pkgrel=0
 pkgdesc="Base files for the postmarketOS initramfs / initramfs-extra"
 url="https://postmarketos.org"