Add script to commit repodata/

Let's match the securedrop-apt-prod process by generating metadata at
commit-time instead of doing it on the server.

The publish script takes care to generate reproducible output by
fixing the mtime of all the RPMs and telling `createrepo_c` what the
time should be.

CI verifies the generated metadata is up to date and fully reproducible
using the `--reproduce` flag.

Author: legoktm

.gitattributes

@@ -1,2 +1,4 @@
 *.deb filter=lfs diff=lfs merge=lfs -text
 *.rpm filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text

.github/workflows/ci.yml

@@ -3,6 +3,10 @@ name: CI
 
 on: [push, pull_request]
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
   tests:
     runs-on: ubuntu-latest
@@ -33,3 +37,22 @@ jobs:
       - name: Output the hashes of all rpm artifacts without their signatures
         run: |
           ./scripts/check.py --check-unsigned --all
+
+  metadata:
+    runs-on: ubuntu-latest
+    container: debian:bookworm
+    steps:
+      - name: Install dependencies
+        run: |
+          apt-get update && apt-get install --yes python3 git git-lfs createrepo-c
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          lfs: true
+          fetch-depth: 0
+      - name: Check repository metadata is up-to-date
+        run: |
+          git config --global --add safe.directory '*'
+          ./tools/publish-real --reproduce
+          git status
+          git diff --exit-code

tools/publish

@@ -0,0 +1,6 @@
+#!/bin/bash
+# Pull the latest image
+podman pull debian:bookworm
+# Mount the git repo to /srv, install necessary packages and run the publish script
+podman run --rm -it -v $(git rev-parse --show-toplevel):/srv:Z debian:bookworm \
+    bash -c "apt-get update && apt-get install -y python3 createrepo-c && /srv/tools/publish-real $@"

tools/publish-real

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""
+Script for generating yum repository metadata in a reproducible manner.
+
+Files are copied into public/ and metadata is generated there. All RPMs
+have their mtime fixed to a specific timestamp, so the generated XML/SQLite
+files will be reproducible.
+"""
+
+import os
+import shutil
+import subprocess
+import sys
+import time
+from pathlib import Path
+import xml.etree.ElementTree as ET
+
+
+def fetch_reproduce_timestamp(public: Path) -> int:
+    repomd = next(public.glob("workstation/dom0/*/repodata/repomd.xml"))
+    tree = ET.parse(repomd)
+    root = tree.getroot()
+    revision = root.find(
+        "repo:revision", {"repo": "http://linux.duke.edu/metadata/repo"}
+    )
+    print(f"Will use a timestamp of {revision.text} (from {repomd})")
+    return int(revision.text)
+
+
+def main():
+    root = Path(__file__).parent.parent
+    public = root / "public"
+    workstation = root / "workstation"
+    if "--reproduce" in sys.argv:
+        try:
+            timestamp = fetch_reproduce_timestamp(public)
+        except Exception as err:
+            raise RuntimeError("Failed to fetch timestamp from repomd.xml") from err
+    else:
+        # Use the current time
+        timestamp = int(time.time())
+    # Reset public, copy the workstation/ tree into it
+    print("Creating public/ (from scratch)")
+    if public.exists():
+        shutil.rmtree(public)
+    public.mkdir()
+    shutil.copytree(workstation, public / "workstation")
+    for rpm in public.glob("**/*.rpm"):
+        os.utime(rpm, (timestamp, timestamp))
+    # Folders are public/workstation/dom0/fXX, run createrepo_c in each one
+    for folder in public.glob("*/*/*/"):
+        if not folder.is_dir():
+            continue
+        print(f"Generating metadata for {folder}")
+        # The <revision> and <timestamp> fields are set to the current UNIX time
+        # unless we explicitly override them. Use our fixed time to ensure it's
+        # consistent regardless of how long this command takes to run.
+        subprocess.check_call(
+            [
+                "createrepo_c",
+                "--revision",
+                str(timestamp),
+                "--set-timestamp-to-revision",
+                str(folder),
+            ]
+        )
+    print("Done!")
+
+
+if __name__ == "__main__":
+    main()