Merge pull request #214438 from agbrooks/master

dockerTools.buildImage: Handle base images w/ duplicate rootfs diffs

nixos/tests/docker-tools.nix

@@ -1,6 +1,52 @@
 # this test creates a simple GNU image with docker tools and sees if it executes
 
-import ./make-test-python.nix ({ pkgs, ... }: {
+import ./make-test-python.nix ({ pkgs, ... }:
+let
+  # nixpkgs#214434: dockerTools.buildImage fails to unpack base images
+  # containing duplicate layers when those duplicate tarballs
+  # appear under the manifest's 'Layers'. Docker can generate images
+  # like this even though dockerTools does not.
+  repeatedLayerTestImage =
+    let
+      # Rootfs diffs for layers 1 and 2 are identical (and empty)
+      layer1 = pkgs.dockerTools.buildImage {  name = "empty";  };
+      layer2 = layer1.overrideAttrs (_: { fromImage = layer1; });
+      repeatedRootfsDiffs = pkgs.runCommandNoCC "image-with-links.tar" {
+        nativeBuildInputs = [pkgs.jq];
+      } ''
+        mkdir contents
+        tar -xf "${layer2}" -C contents
+        cd contents
+        first_rootfs=$(jq -r '.[0].Layers[0]' manifest.json)
+        second_rootfs=$(jq -r '.[0].Layers[1]' manifest.json)
+        target_rootfs=$(sha256sum "$first_rootfs" | cut -d' ' -f 1).tar
+
+        # Replace duplicated rootfs diffs with symlinks to one tarball
+        chmod -R ug+w .
+        mv "$first_rootfs" "$target_rootfs"
+        rm "$second_rootfs"
+        ln -s "../$target_rootfs" "$first_rootfs"
+        ln -s "../$target_rootfs" "$second_rootfs"
+
+        # Update manifest's layers to use the symlinks' target
+        cat manifest.json | \
+        jq ".[0].Layers[0] = \"$target_rootfs\"" |
+        jq ".[0].Layers[1] = \"$target_rootfs\"" > manifest.json.new
+        mv manifest.json.new manifest.json
+
+        tar --sort=name --hard-dereference -cf $out .
+        '';
+    in pkgs.dockerTools.buildImage {
+      fromImage = repeatedRootfsDiffs;
+      name = "repeated-layer-test";
+      tag = "latest";
+      copyToRoot = pkgs.bash;
+      # A runAsRoot script is required to force previous layers to be unpacked
+      runAsRoot = ''
+        echo 'runAsRoot has run.'
+      '';
+    };
+in {
   name = "docker-tools";
   meta = with pkgs.lib.maintainers; {
     maintainers = [ lnl7 roberth ];
@@ -221,6 +267,12 @@ import ./make-test-python.nix ({ pkgs, ... }: {
             "docker run --rm ${examples.layersUnpackOrder.imageName} cat /layer-order"
         )
 
+    with subtest("Ensure repeated base layers handled by buildImage"):
+        docker.succeed(
+            "docker load --input='${repeatedLayerTestImage}'",
+            "docker run --rm ${repeatedLayerTestImage.imageName} /bin/bash -c 'exit 0'"
+        )
+
     with subtest("Ensure environment variables are correctly inherited"):
         docker.succeed(
             "docker load --input='${examples.environmentVariables}'"

pkgs/build-support/docker/default.nix

@@ -229,6 +229,15 @@ rec {
           mount /dev/${vmTools.hd} disk
           cd disk
 
+          function dedup() {
+            declare -A seen
+            while read ln; do
+              if [[ -z "''${seen["$ln"]:-}" ]]; then
+                echo "$ln"; seen["$ln"]=1
+              fi
+            done
+          }
+
           if [[ -n "$fromImage" ]]; then
             echo "Unpacking base image..."
             mkdir image
@@ -245,7 +254,8 @@ rec {
               parentID="$(cat "image/manifest.json" | jq -r '.[0].Config | rtrimstr(".json")')"
             fi
 
-            cat ./image/manifest.json  | jq -r '.[0].Layers | .[]' > layer-list
+            # In case of repeated layers, unpack only the last occurrence of each
+            cat ./image/manifest.json  | jq -r '.[0].Layers | .[]' | tac | dedup | tac > layer-list
           else
             touch layer-list
           fi