buildchain, tox: rework cross-platform dependency handling

Instead of having both a `buildchain/requirements-Linux.txt` file and
`buildchain/requirements-Darwin.txt`, rendered from
`buildchain/requirements.in` and as such being almost identical (modulo
the platform-specific `pyinotify` and `macfsevents` modules), making
updates to the requirement lock files cumbersome (need to be executed on
all platforms), this commit creates a file containing a pinned version
of these two dependencies including a (manually added) platform
constraint, which is then used as a constraint file for `pip-compile` to
do its job (hence the `-c platform-requirements.txt` line in
`requirements.in`). After `pip-compile` generates a temporary lockfile
(cfr. code in `tox.ini`), we parse out the `pyinotify` and/or
`macfsevents` dependency, and append the constraint file to the result.

Using this approach we end up with a single
`buildchain/requirements.txt` which has all cross-platform `doit` (and
other) dependencies from `buildchain/requirements.in`, and
(platform-constrained) mentions of `pyinotify` and `macfsevents`.

The versions of the latter are pinned in `platform-requirements.txt`, so
these won't be upgraded when running `tox -e pip-compile`. If need be,
they can be bumped manually.

See: #2986 (comment)

buildchain/platform-requirements.txt

@@ -0,0 +1,7 @@
+# Note: these come from 'buildchain/platform-requirements.txt'
+macfsevents==0.8.1; sys_platform=="darwin" \
+    --hash=sha256:1324b66b356051de662ba87d84f73ada062acd42b047ed1246e60a449f833e10 \
+     # via doit
+pyinotify==0.9.6; sys_platform=="linux" \
+    --hash=sha256:9c998a5d7606ca835065cdabc013ae6c66eb9ea76a00a1e3bc6e0cfe2b4f71f4 \
+     # via doit

buildchain/requirements.in

@@ -1,3 +1,4 @@
+-c platform-requirements.txt
 doit ~= 0.32.0
 docker ~= 4.1.0
 PyYAML ~= 5.3.1

buildchain/requirements.txt

@@ -0,0 +1,67 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    tox -e pip-compile
+#
+certifi==2020.12.5 \
+    --hash=sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c \
+    --hash=sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830 \
+    # via requests
+chardet==4.0.0 \
+    --hash=sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa \
+    --hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5 \
+    # via requests
+cloudpickle==1.6.0 \
+    --hash=sha256:3a32d0eb0bc6f4d0c57fbc4f3e3780f7a81e6fee0fa935072884d58ae8e1cc7c \
+    --hash=sha256:9bc994f9e9447593bd0a45371f0e7ac7333710fcf64a4eb9834bf149f4ef2f32 \
+    # via doit
+docker==4.1.0 \
+    --hash=sha256:6e06c5e70ba4fad73e35f00c55a895a448398f3ada7faae072e2bb01348bafc1 \
+    --hash=sha256:8f93775b8bdae3a2df6bc9a5312cce564cade58d6555f2c2570165a1270cd8a7 \
+    # via -r /home/nicolas/Projects/metalk8s/buildchain/requirements.in
+doit==0.32.0 \
+    --hash=sha256:323d0411e50a0babb945e2a639aa6e633fbb84a9f98086df0c880e739237750c \
+    # via -r /home/nicolas/Projects/metalk8s/buildchain/requirements.in
+idna==2.10 \
+    --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \
+    --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 \
+    # via requests
+pyyaml==5.3.1 \
+    --hash=sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97 \
+    --hash=sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76 \
+    --hash=sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2 \
+    --hash=sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e \
+    --hash=sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648 \
+    --hash=sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf \
+    --hash=sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f \
+    --hash=sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2 \
+    --hash=sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee \
+    --hash=sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a \
+    --hash=sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d \
+    --hash=sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c \
+    --hash=sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a \
+    # via -r /home/nicolas/Projects/metalk8s/buildchain/requirements.in
+requests==2.25.1 \
+    --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \
+    --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e \
+    # via docker
+six==1.15.0 \
+    --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
+    --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
+    # via docker, websocket-client
+urllib3==1.26.2 \
+    --hash=sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08 \
+    --hash=sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473 \
+    # via requests
+websocket-client==0.57.0 \
+    --hash=sha256:0fc45c961324d79c781bab301359d5a1b00b13ad1b10415a4780229ef71a5549 \
+    --hash=sha256:d735b91d6d1692a6a181f2a8c9e0238e5f6373356f561bb9dc4c7af36f452010 \
+    # via docker
+# Note: these come from 'buildchain/platform-requirements.txt'
+macfsevents==0.8.1; sys_platform=="darwin" \
+    --hash=sha256:1324b66b356051de662ba87d84f73ada062acd42b047ed1246e60a449f833e10 \
+     # via doit
+pyinotify==0.9.6; sys_platform=="linux" \
+    --hash=sha256:9c998a5d7606ca835065cdabc013ae6c66eb9ea76a00a1e3bc6e0cfe2b4f71f4 \
+     # via doit

doit.sh

@@ -9,7 +9,7 @@ BUILDCHAIN=buildchain
 # Location of the virtual environment for the buildchain.
 BUILDENV="${BUILDCHAIN}/.venv"
 # requirements.txt for the buildchain.
-REQUIREMENTS="${BUILDCHAIN}/requirements-$OSTYPE.txt"
+REQUIREMENTS="${BUILDCHAIN}/requirements.txt"
 # Dummy file to keep track of when the virtual environment was installed.
 WITNESS_FILE="${BUILDENV}/installed.tstamp"
 # File containing environment variables.

tox.ini

@@ -21,12 +21,23 @@ deps =
 setenv =
     CUSTOM_COMPILE_COMMAND = tox -e pip-compile
 commands =
-    bash -c 'pip-compile \
-        --emit-index-url --emit-trusted-host --annotate --generate-hashes \
-        --allow-unsafe \
-        {posargs:--upgrade} \
-        -o "{toxinidir}/buildchain/requirements-$(uname -s).txt" \
-        "{toxinidir}/buildchain/requirements.in"'
+    bash -c ' \
+        pip-compile \
+            --emit-index-url --emit-trusted-host --annotate --generate-hashes \
+            --allow-unsafe \
+            {posargs:--upgrade} \
+            -o "{toxinidir}/buildchain/requirements.tmp" \
+            "{toxinidir}/buildchain/requirements.in" \
+        && \
+        ( \
+            gawk "/^[^ ]/\{e=1\}; /^pyinotify/\{e=0\}; /^macfsevents/\{e=0\}; e\{print\}" \
+                < "{toxinidir}/buildchain/requirements.tmp" \
+            && \
+            cat "{toxinidir}/buildchain/platform-requirements.txt" \
+        ) > "{toxinidir}/buildchain/requirements.txt" \
+        && \
+        rm -f "{toxinidir}/buildchain/requirements.tmp" \
+    '
     pip-compile \
         --emit-index-url --emit-trusted-host --annotate --generate-hashes \
         --allow-unsafe \
@@ -91,7 +102,7 @@ description =
 # Set Python version to Python3 as shipped with RHEL/CentOS 7
 basepython = python3.6
 deps =
-    -r{toxinidir}/buildchain/requirements-{env:OSTYPE}.txt
+    -r{toxinidir}/buildchain/requirements.txt
     -r{toxinidir}/buildchain/requirements-dev.txt
 commands =
     bash -c " \