Add Docker smoke tests (#764)

exercism · Aug 22, 2023 · b94fac8 · b94fac8
1 parent a4fcc8a
commit b94fac8
Show file tree

Hide file tree

Showing 15 changed files with 445 additions and 0 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -9,3 +9,7 @@ dev.Dockerfile
 .gitignore
 .gitattributes
 .dockerignore
+/bin/
+!/bin/run.sh
+!/bin/prepare.sh
+test/
diff --git a/.github/workflows/ci.js.yml b/.github/workflows/ci.js.yml
@@ -43,3 +43,21 @@ jobs:
 
       - name: Build the test-runner (using Node ${{ matrix.node-version }})
         run: bin/test.sh
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1
+        with:
+          install: true
+
+      - name: Build Docker image and store in cache
+        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825
+        with:
+          context: .
+          push: false
+          load: true
+          tags: exercism/javascript-test-runner
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Run Tests in Docker
+        run: bin/run-tests-in-docker.sh
diff --git a/.github/workflows/pr.ci.js.yml b/.github/workflows/pr.ci.js.yml
@@ -45,3 +45,21 @@ jobs:
 
       - name: Build the test-runner (using Node ${{ matrix.node-version }})
         run: bin/test.sh
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1
+        with:
+          install: true
+
+      - name: Build Docker image and store in cache
+        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825
+        with:
+          context: .
+          push: false
+          load: true
+          tags: exercism/javascript-test-runner
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Run Tests in Docker
+        run: bin/run-tests-in-docker.sh
diff --git a/bin/run-tests-in-docker.sh b/bin/run-tests-in-docker.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env sh
+
+# Synopsis:
+# Test the test runner Docker image by running it against a predefined set of 
+# solutions with an expected output.
+# The test runner Docker image is built automatically.
+
+# Output:
+# Outputs the diff of the expected test results against the actual test results
+# generated by the test runner Docker image.
+
+# Example:
+# ./bin/run-tests-in-docker.sh
+
+# Stop executing when a command returns a non-zero return code
+set -e
+
+# Build the Docker image
+docker build --rm -t exercism/javascript-test-runner .
+
+# Run the Docker image using the settings mimicking the production environment
+docker run \
+    --rm \
+    --network none \
+    --read-only \
+    --mount type=bind,src="${PWD}/test/fixtures",dst=/opt/test-runner/test/fixtures \
+    --mount type=tmpfs,dst=/tmp \
+    --volume "${PWD}/bin/run-tests.sh:/opt/test-runner/bin/run-tests.sh" \
+    --workdir /opt/test-runner \
+    --entrypoint /opt/test-runner/bin/run-tests.sh \
+    exercism/javascript-test-runner
diff --git a/bin/run-tests.sh b/bin/run-tests.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env sh
+
+# Synopsis:
+# Test the test runner by running it against a predefined set of solutions 
+# with an expected output.
+
+# Output:
+# Outputs the diff of the expected test results against the actual test results
+# generated by the test runner.
+
+# Example:
+# ./bin/run-tests.sh
+
+exit_code=0
+
+# We need to copy the fixtures to a temp directory as the user
+# running within the Docker container does not have permissions 
+# to run the sed command on the fixtures directory
+fixtures_dir="test/fixtures"
+tmp_fixtures_dir="/tmp/test/fixtures"
+rm -rf "${tmp_fixtures_dir}"
+mkdir -p "${tmp_fixtures_dir}"
+cp -R ${fixtures_dir}/* "${tmp_fixtures_dir}"
+
+# Iterate over all test directories
+for test_file in $(find "${tmp_fixtures_dir}" -name '*.spec.js'); do
+    slug=$(echo "${test_file:${#tmp_fixtures_dir}+1}" | cut -d / -f 1)
+    test_dir=$(dirname "${test_file}")
+    test_dir_name=$(basename "${test_dir}")
+    test_dir_path=$(realpath "${test_dir}")
+    results_file_path="${test_dir_path}/results.json"
+    expected_results_file_path="${test_dir_path}/expected_results.json"
+
+    # Make sure there is no existing node_modules directory
+    rm -rf "${test_dir_path}/node_modules"
+
+    bin/run.sh "${slug}" "${test_dir_path}" "${test_dir_path}"
+
+    echo "${slug}/${test_dir_name}: comparing results.json to expected_results.json"
+    diff "${results_file_path}" "${expected_results_file_path}"
+
+    if [ $? -ne 0 ]; then
+        exit_code=1
+    fi
+done
+
+exit ${exit_code}
diff --git a/test/fixtures/clock/pass/expected_results.json b/test/fixtures/clock/pass/expected_results.json
@@ -0,0 +1,13 @@
+{
+  "status": "pass",
+  "tests": [
+    {
+      "name": "Clock > Creating a new clock with an initial time > on the hour",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(new Clock(8).toString()).toEqual('08:00');"
+    }
+  ],
+  "version": 3
+}
diff --git a/test/fixtures/lasagna/exemplar/expected_results.json b/test/fixtures/lasagna/exemplar/expected_results.json
@@ -0,0 +1,46 @@
+{
+  "status": "pass",
+  "tests": [
+    {
+      "name": "EXPECTED_MINUTES_IN_OVEN > constant is defined correctly",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(EXPECTED_MINUTES_IN_OVEN).toBe(40);",
+      "task_id": 1
+    },
+    {
+      "name": "remainingMinutesInOven > calculates the remaining time",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(remainingMinutesInOven(25)).toBe(15);\nexpect(remainingMinutesInOven(5)).toBe(35);\nexpect(remainingMinutesInOven(39)).toBe(1);",
+      "task_id": 2
+    },
+    {
+      "name": "remainingMinutesInOven > works correctly for the edge cases",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(remainingMinutesInOven(40)).toBe(0);\nexpect(remainingMinutesInOven(0)).toBe(40);",
+      "task_id": 2
+    },
+    {
+      "name": "preparationTimeInMinutes > calculates the preparation time",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(preparationTimeInMinutes(1)).toBe(2);\nexpect(preparationTimeInMinutes(2)).toBe(4);\nexpect(preparationTimeInMinutes(8)).toBe(16);",
+      "task_id": 3
+    },
+    {
+      "name": "totalTimeInMinutes > calculates the total cooking time",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(totalTimeInMinutes(1, 5)).toBe(7);\nexpect(totalTimeInMinutes(4, 15)).toBe(23);\nexpect(totalTimeInMinutes(1, 35)).toBe(37);",
+      "task_id": 4
+    }
+  ],
+  "version": 3
+}
diff --git a/test/fixtures/poetry-club-door-policy/pass/expected_results.json b/test/fixtures/poetry-club-door-policy/pass/expected_results.json
@@ -0,0 +1,76 @@
+{
+  "status": "pass",
+  "tests": [
+    {
+      "name": "strings > front door > it outputs a character per line",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "const key = ShireGuard.recite();\nexpect(key.length).toBe(SHIRE_HORSE.length);"
+    },
+    {
+      "name": "strings > front door > it outputs takes the first characters",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "const key = ShireGuard.recite();\nexpect(key.toUpperCase()).toBe(SHIRE_HORSE.acrostic.toUpperCase());"
+    },
+    {
+      "name": "strings > front door > it generates the correct password",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "ShireGuard.assert();"
+    },
+    {
+      "name": "strings > front door > frontDoorPassword(SUMMER)",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": null
+    },
+    {
+      "name": "strings > front door > frontDoorPassword(SOPHIA)",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": null
+    },
+    {
+      "name": "strings > front door > frontDoorPassword(CODE)",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": null
+    },
+    {
+      "name": "strings > back door > it outputs a character per line",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "const key = ShireGuard.recite();\nexpect(key.length).toBe(SHIRE_HORSE.length);"
+    },
+    {
+      "name": "strings > back door > it outputs takes the first characters",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "const key = ShireGuard.recite();\nexpect(key.toUpperCase()).toBe(SHIRE_HORSE.telestich.toUpperCase());"
+    },
+    {
+      "name": "strings > back door > it generates the correct password",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "ShireGuard.assert();"
+    },
+    {
+      "name": "strings > back door > backDoorGuard(WORK)",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": null
+    }
+  ],
+  "version": 3
+}
diff --git a/test/fixtures/pythagorean-triplet/exemplar/expected_results.json b/test/fixtures/pythagorean-triplet/exemplar/expected_results.json
@@ -0,0 +1,69 @@
+{
+  "status": "pass",
+  "tests": [
+    {
+      "name": "Triplet > triplets whose sum is 12",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(tripletsWithSum(12)).toEqual([[3, 4, 5]]);"
+    },
+    {
+      "name": "Triplet > triplets whose sum is 108",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(tripletsWithSum(108)).toEqual([[27, 36, 45]]);"
+    },
+    {
+      "name": "Triplet > triplets whose sum is 1000",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(tripletsWithSum(1000)).toEqual([[200, 375, 425]]);"
+    },
+    {
+      "name": "Triplet > no matching triplets for 1001",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(tripletsWithSum(1001)).toEqual([]);"
+    },
+    {
+      "name": "Triplet > returns all matching triplets",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(tripletsWithSum(90)).toEqual([\n      [9, 40, 41],\n      [15, 36, 39],\n    ]);"
+    },
+    {
+      "name": "Triplet > several matching triplets",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(tripletsWithSum(840)).toEqual([\n      [40, 399, 401],\n      [56, 390, 394],\n      [105, 360, 375],\n      [120, 350, 370],\n      [140, 336, 364],\n      [168, 315, 357],\n      [210, 280, 350],\n      [240, 252, 348],\n    ]);"
+    },
+    {
+      "name": "Triplet > returns triplets with no factor smaller than minimum factor",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(tripletsWithSum(90, { minFactor: 10 })).toEqual([[15, 36, 39]]);"
+    },
+    {
+      "name": "Triplet > returns triplets with no factor larger than maximum factor",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(tripletsWithSum(840, { maxFactor: 349 })).toEqual([[240, 252, 348]]);"
+    },
+    {
+      "name": "Triplet > returns triplets with factors in range",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(tripletsWithSum(840, { maxFactor: 352, minFactor: 150 })).toEqual([\n      [210, 280, 350],\n      [240, 252, 348],\n    ]);"
+    }
+  ],
+  "version": 3
+}
diff --git a/test/fixtures/two-fer/error/malformed_tests/expected_results.json b/test/fixtures/two-fer/error/malformed_tests/expected_results.json
@@ -0,0 +1,6 @@
+{
+  "status": "error",
+  "message": "SyntaxError: <solution>/two-fer.spec.js: Unexpected token (2:0)\n\n    \u001b[0m \u001b[90m 1 |\u001b[39m describe(\u001b[32m'twoFer()'\u001b[39m\u001b[33m,\u001b[39m t(\u001b[32m'another name given'\u001b[39m\u001b[33m,\u001b[39m () \u001b[33m=>\u001b[39m {\u001b[0m\n    \u001b[0m\u001b[31m\u001b[1m>\u001b[22m\u001b[39m\u001b[90m 2 |\u001b[39m\u001b[0m\n    \u001b[0m \u001b[90m   |\u001b[39m \u001b[31m\u001b[1m^\u001b[22m\u001b[39m\u001b[0m\n",
+  "tests": [],
+  "version": 3
+}
diff --git a/test/fixtures/two-fer/error/syntax/expected_results.json b/test/fixtures/two-fer/error/syntax/expected_results.json
@@ -0,0 +1,6 @@
+{
+  "status": "error",
+  "message": "SyntaxError: <solution>/two-fer.js: Unexpected keyword 'const'. (1:13)\n\n    \u001b[0m\u001b[31m\u001b[1m>\u001b[22m\u001b[39m\u001b[90m 1 |\u001b[39m \u001b[36mexport\u001b[39m \u001b[36mconst\u001b[39m \u001b[36mconst\u001b[39m () \u001b[33m=>\u001b[39m { (\u001b[0m\n    \u001b[0m \u001b[90m   |\u001b[39m              \u001b[31m\u001b[1m^\u001b[22m\u001b[39m\u001b[0m\n    \u001b[0m \u001b[90m 2 |\u001b[39m\u001b[0m\n",
+  "tests": [],
+  "version": 3
+}
diff --git a/test/fixtures/two-fer/fail/empty/expected_results.json b/test/fixtures/two-fer/fail/empty/expected_results.json
@@ -0,0 +1,27 @@
+{
+  "status": "fail",
+  "tests": [
+    {
+      "name": "twoFer() > no name given",
+      "status": "fail",
+      "message": "TypeError: (0 , _twoFer.twoFer) is not a function",
+      "output": null,
+      "test_code": "expect(twoFer()).toEqual('One for you, one for me.');"
+    },
+    {
+      "name": "twoFer() > a name given",
+      "status": "fail",
+      "message": "TypeError: (0 , _twoFer.twoFer) is not a function",
+      "output": null,
+      "test_code": "const name = 'Alice';\nexpect(twoFer(name)).toEqual('One for Alice, one for me.');"
+    },
+    {
+      "name": "twoFer() > another name given",
+      "status": "fail",
+      "message": "TypeError: (0 , _twoFer.twoFer) is not a function",
+      "output": null,
+      "test_code": "const name = 'Bob';\nexpect(twoFer(name)).toEqual('One for Bob, one for me.');"
+    }
+  ],
+  "version": 3
+}