feat: add entrypoint

.dockerignore

@@ -1,3 +1,6 @@
 Dockerfile
 .git/
+/output.*
+/entrypoint
+/fixtures.car
 /output.*

.github/actions/extract-fixtures/action.yml

@@ -3,9 +3,13 @@ name: 'Gateway Conformance Tests: Extract Fixtures'
 description: 'Extracts the fixtures from the Gateway Conformance Tests'
 inputs:
   output:
-    description: 'The path where the fixtures will be saved'
+    description: 'The path where the test fixtures should be extracted.'
     required: true
     default: 'fixtures'
+  merged:
+    description: 'Whether the fixtures should be merged into a single CAR file.'
+    required: false
+    default: 'false'
 runs:
   using: 'composite'
   steps:
@@ -15,8 +19,9 @@ runs:
       uses: pl-strflt/docker-container-action@v1
       env:
         OUTPUT: ${{ inputs.output }}
+        MERGED: ${{ inputs.merged }}
       with:
         repository: ${{ steps.github.outputs.action_repository }}
         ref: ${{ steps.github.outputs.action_ref }}
         dockerfile: Dockerfile
-        args: extract-fixtures "$OUTPUT"
+        args: extract-fixtures --directory="$OUTPUT" --merged="$MERGED"

.github/actions/test/action.yml

@@ -1,39 +1,46 @@
-# action.yml
-name: 'Gateway Conformance Tests'
-description: 'Runs the Gateway Conformance Tests'
+name: "Gateway Conformance Tests"
+description: "Runs the Gateway Conformance Tests"
 inputs:
   gateway-url:
-    description: 'The URL of the Gateway to test'
+    description: "The URL of the IPFS Gateway implementation to be tested."
     required: true
   json:
-    description: 'The path where the JSON report will be saved'
+    description: "The path where the JSON test report should be generated."
     required: true
-    default: 'report.json'
+    default: "report.json"
   xml:
-    description: 'The path where the jUnit XML report will be saved'
+    description: "The path where the JUnit XML test report should be generated."
     required: false
   html:
-    description: 'The path where the HTML report will be saved'
+    description: "The path where the one-page HTML test report should be generated."
     required: false
   markdown:
-    description: 'The path where the Markdown report will be saved'
+    description: "The path where the summary Markdown test report should be generated."
+    required: false
+  specs:
+    description: "A comma-separated list of specs to be tested. Accepts a spec (test only this spec), a +spec (test also this immature spec), or a -spec (do not test this mature spec)."
+    required: false
+    default: ""
+  args:
+    description: "[DANGER] The `args` input allows you to pass custom, free-text arguments directly to the Go test command that the tool employs to execute tests."
     required: false
 runs:
-  using: 'composite'
+  using: "composite"
   steps:
     - id: github
       uses: pl-strflt/docker-container-action/.github/actions/github@v1
     - name: Run the test
       uses: pl-strflt/docker-container-action@v1
       env:
-        GATEWAY_URL: ${{ inputs.gateway-url }}
+        URL: ${{ inputs.gateway-url }}
         JSON: ${{ inputs.json }}
+        SPECS: ${{ inputs.specs }}
       with:
         repository: ${{ steps.github.outputs.action_repository }}
         ref: ${{ steps.github.outputs.action_ref }}
         dockerfile: Dockerfile
         opts: --network=host
-        args: test "$GATEWAY_URL" "$JSON"
+        args: test --url="$URL" --json="$JSON" --specs="$SPECS" -- ${{ inputs.args }}
     - name: Create the XML
       if: (inputs.xml || inputs.html || inputs.markdown) && (failure() || success())
       uses: pl-strflt/gotest-json-to-junit-xml@v1

.github/workflows/test.yml

@@ -34,6 +34,8 @@ jobs:
           xml: output.xml
           html: output.html
           markdown: output.md
+          specs: -subdomain-gateway
+          args: -skip TestGatewayCar
       - name: Set summary
         if: (failure() || success())
         run: cat ./output.md >> $GITHUB_STEP_SUMMARY

.gitignore

@@ -2,6 +2,8 @@
 /output.xml
 /fixtures.car
 /merge-fixtures
+/entrypoint
+/gateway-conformance
 
 # Logs
 logs

Dockerfile

@@ -1,13 +1,11 @@
-FROM golang:1.19.1-buster
+FROM golang:1.20-alpine
 WORKDIR /app
-
-RUN go install gotest.tools/gotestsum@v1.9.0
+ENV GATEWAY_CONFORMANCE_HOME=/app
 
 COPY ./go.mod ./go.sum ./
 RUN go mod download
 
 COPY . .
-RUN go build -o /merge-fixtures ./tooling/cmd/merge_fixtures.go
+RUN go build -o ./gateway-conformance ./cmd/gateway-conformance
 
-COPY entrypoint.sh /entrypoint.sh
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/app/gateway-conformance"]

Makefile

@@ -3,24 +3,27 @@ provision-cargateway: ./fixtures.car
 	car -c ./fixtures.car &
 
 test-cargateway: provision-cargateway
-	GATEWAY_URL=http://127.0.0.1:8040 make test
+	GATEWAY_URL=http://127.0.0.1:8040 make _test
 
 provision-kubo:
 	find ./fixtures -name '*.car' -exec ipfs dag import {} \;
 
 test-kubo: provision-kubo
-	GATEWAY_URL=http://127.0.0.1:8080 make test
-
-merge-fixtures:
-	go build -o merge-fixtures ./tooling/cmd/merge_fixtures.go
+	GATEWAY_URL=http://127.0.0.1:8080 make _test
 
 # tools
-fixtures.car: merge-fixtures
-	./merge-fixtures ./fixtures.car
+fixtures.car: gateway-conformance
+	./gateway-conformance extract-fixtures --merged=true --dir=.
+
+gateway-conformance:
+	go build -o ./gateway-conformance ./cmd/gateway-conformance
+
+_test: fixtures.car gateway-conformance
+	./gateway-conformance test --json output.json --gateway-url ${GATEWAY_URL}
 
-test: fixtures.car
-	# go install gotest.tools/gotestsum@latest
-	- gotestsum --jsonfile output.json ./tests
+test-docker: fixtures.car gateway-conformance
+	docker build -t gateway-conformance .
+	docker run --rm -v "${PWD}:/workspace" -w "/workspace" --network=host gateway-conformance test
 
 output.xml: test-kubo
 	docker run --rm -v "${PWD}:/workspace" -w "/workspace" --entrypoint "/bin/bash" ghcr.io/pl-strflt/saxon:v1 -c """
@@ -31,4 +34,4 @@ output.html: output.xml
 	docker run --rm -v "${PWD}:/workspace" -w "/workspace" ghcr.io/pl-strflt/saxon:v1 -s:output.xml -xsl:/etc/junit-noframes-saxon.xsl -o:output.html
 	open ./output.html
 
-.PHONY: merge-fixtures
+.PHONY: gateway-conformance

README.md

@@ -1,46 +1,151 @@
-# Gateway Conformance Testing Approach Proposal
+# gateway-conformance
 
-## Key Concepts
+`gateway-conformance` is a tool designed to test if an IPFS Gateway implementation complies with the IPFS Gateway Specification correctly. The tool is distributed as a Docker image, as well as a GitHub Action(s).
 
-- Built with Go for stability & ease of building
-- TBD
+## Table of Contents
 
-## Issues / Open Questions
+- [Commands](#commands)
+  - [test](#test)
+    - [Inputs](#inputs)
+    - [Usage](#usage)
+  - [extract-fixtures](##extract-fixtures)
+    - [Inputs](#inputs-1)
+    - [Usage](#usage-1)
+- [Examples](#examples)
+- [FAQ](#faq)
+- [In Development](#in-development)
 
-- How to deal with subdomains & configuration (t0114 for example)?
-  - Some test relies on querying URLs like `http://$CIDv1.ipfs.example.com/`. While `http://$CIDv1.ipfs.localhost/` works by default, do we need / want to test with `.example.com`?
+## Commands
+
+### test
+
+The `test` command is the main command of the tool. It is used to test a given IPFS Gateway implementation against the IPFS Gateway Specification.
+
+#### Inputs
+
+| Input | Availability | Description | Default |
+|---|---|---|---|
+| gateway-url | Both | The URL of the IPFS Gateway implementation to be tested. | http://localhost:8080 |
+| json | Both | The path where the JSON test report should be generated. | `./report.json` |
+| xml | GitHub Action | The path where the JUnit XML test report should be generated. | `./report.xml` |
+| html | GitHub Action | The path where the one-page HTML test report should be generated. | `./report.html` |
+| markdown | GitHub Action | The path where the summary Markdown test report should be generated. | `./report.md` |
+| specs | Both | A comma-separated list of specs to be tested. Accepts a spec (test only this spec), a +spec (test also this immature spec), or a -spec (do not test this mature spec). | Mature specs only |
+| args | Both | [DANGER] The `args` input allows you to pass custom, free-text arguments directly to the Go test command that the tool employs to execute tests. | N/A |
+
+##### Specs
+
+By default, only mature specs (reliable, stable, or permanent) will be tested if this input is not provided. You can specify particular specs without any prefixes (e.g., subdomain-gateway, trustless-gateway, path-gateway) to test exclusively those, irrespective of their maturity status.
+
+To selectively enable or disable specs based on their maturity, use the "+" and "-" prefixes. Adding a "+" prefix (e.g., +subdomain-gateway) means that the spec should be included in the test, in addition to the mature specs. Conversely, using a "-" prefix (e.g., -subdomain-gateway) means that the spec should be excluded from the test, even if it is mature.
+
+If you provide a list containing both prefixed and unprefixed specs, the prefixed specs will be ignored. It is advisable to use either prefixed or unprefixed specs, but not both. However, you can include specs with both "+" and "-" prefixes in the same list.
+
+##### Args
+
+This input should be used sparingly and with caution, as it involves interacting with the underlying internal processes, which may be subject to changes. It is recommended to use the `args` input only when you have a deep understanding of the tool's inner workings and need to fine-tune the testing process. Users should be mindful of the potential risks associated with using this input.
+
+#### Usage
+
+##### GitHub Action
+
+```yaml
+- name: Run gateway-conformance tests
+  uses: ipfs/gateway-conformance/.github/actions/test@v1
+  with:
+    gateway-url: http://localhost:8080
+    specs: +subdomain-gateway,-path-gateway
+    json: report.json
+    xml: report.xml
+    markdown: report.md
+    html: report.html
+    args: -timeout 30m
+```
+
+##### Docker
+
+```bash
+docker run --network host -v "${PWD}:/workspace" -w "/workspace" ghcr.io/ipfs/gateway-conformance test --gateway-url http://localhost:8080 --json report.json --specs +subdomain-gateway,-path-gateway -- -timeout 30m
+```
+
+### extract-fixtures
+
+The `extract-fixtures` command is used to extract the test fixtures from the `gateway-conformance` tool.
 
-## Usage
+#### Inputs
 
-### Retrieve fixtures
+| Input | Availability | Description | Default |
+|---|---|---|---|
+| output | Both | The path where the test fixtures should be extracted. | `./fixtures` |
+| merged | Both | Whether the fixtures should be merged into a single CAR file. | `false` |
+
+#### Usage
+
+##### GitHub Action
+
+```yaml
+- name: Extract gateway-conformance fixtures
+  uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v1
+  with:
+    output: fixtures
+    merged: false
+```
+
+##### Docker
 
 ```bash
-docker run -v "${PWD}:/workspace" -w "/workspace" ghcr.io/ipfs/gateway-conformance extract-fixtures [OUTPUT_DIR]
+docker run -v "${PWD}:/workspace" -w "/workspace" ghcr.io/ipfs/gateway-conformance extract-fixtures --output fixtures --merged false
 ```
 
-### Run tests
+## Examples
+
+The examples are going to use `gateway-conformance` as a wrapper over `docker run -v "${PWD}:/workspace" -w "/workspace" ghcr.io/ipfs/gateway-conformance` for simplicity.
+
+### Testing only mature specs
 
 ```bash
-docker run --network host -v "${PWD}:/workspace" -w "/workspace" ghcr.io/ipfs/gateway-conformance test [gateway-url] [OUTPUT_JSON]
+gateway-conformance test
 ```
 
-### Generate a XML report
+### Testing specific specs, regardless of their maturity level
 
 ```bash
-docker run --rm -v "${PWD}:/workspace" -w "/workspace" --entrypoint "/bin/bash" ghcr.io/pl-strflt/saxon:v1 -c """
-  java -jar /opt/SaxonHE11-5J/saxon-he-11.5.jar -s:<(jq -s '.' [OUTPUT_JSON]) -xsl:/etc/gotest.xsl -o:[OUTPUT_XML]
-"""
+gateway-conformance test --specs subdomain-gateway,path-gateway
+```
+
+### Testing mature specs and additionally enabling specific specs
 
+```bash
+gateway-conformance test --specs +subdomain-gateway
 ```
 
-### Generate a HTML report
+### Testing mature specs, while disabling specific specs
 
 ```bash
-	docker run --rm -v "${PWD}:/workspace" -w "/workspace" ghcr.io/pl-strflt/saxon:v1 -s:[OUTPUT_XML] -xsl:/etc/junit-noframes-saxon.xsl -o:[OUTPUT_HTML]
+gateway-conformance test --specs -subdomain-gateway
 ```
 
-### Generate a single car file for testing
+### Extracting the test fixtures
 
 ```bash
-docker run --network host -w "/workspace" -v "${PWD}:/workspace" ghcr.io/ipfs/gateway-conformance merge-fixtures /workspace/[output-car]
+gateway-conformance extract-fixtures
 ```
+
+### Extracting the test fixtures into a single CAR file
+
+```bash
+gateway-conformance extract-fixtures --merged true
+```
+
+## FAQ
+
+### How to generate XML, HTML and Markdown reports when using the tool as a Docker container?
+
+The tool can generate XML, HTML and Markdown reports when used as a GitHub Action. However, when using the tool as a Docker container, you can generate these reports by using the [`saxon` Docker image](https://github.com/pl-strflt/saxon). You can draw inspiration from the [gotest-json-to-junit-xml](https://github.com/pl-strflt/gotest-json-to-junit-xml) and the [junit-xml-to-html](https://github.com/pl-strflt/junit-xml-to-html) GitHub Actions.
+
+Please let us know if you would like to see this feature implemented directly in the Docker image distribution.
+
+## In Development
+
+- How to deal with subdomains & configuration (t0114 for example)?
+ - Some test relies on querying URLs like `http://$CIDv1.ipfs.example.com/`. While `http://$CIDv1.ipfs.localhost/` works by default, do we need / want to test with `.example.com`?