From 9ba8fffd67bd170fae0a091823358db65f70620f Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 16 Nov 2024 09:17:11 -0500 Subject: [PATCH] Multi-platform builds, delete branch/test tags, and remove GAR [minor] (#53) --- .github/workflows/build-push.yml | 43 ++++++---------- .github/workflows/delete.yml | 50 +++++++++++++++++++ .github/workflows/lint-test-build.yml | 2 +- ci/delete-tags.sh | 44 +++++++++++++++++ ci/docker-compose.yml | 14 +++--- ci/fixtures/pandoc/input.md | 24 +++++++++ ci/fixtures/pandoc/output.tex | 23 +++++++++ ci/test.sh | 17 +++++++ examples/cache-warmer/Dockerfile | 2 +- examples/coverpage/Dockerfile | 14 +++--- examples/curl/Dockerfile | 2 +- examples/ffmpeg/Dockerfile | 2 +- examples/fits/Dockerfile | 2 +- examples/imagemagick/Dockerfile | 2 +- examples/libreoffice/Dockerfile | 2 +- examples/mergepdf/Dockerfile | 2 +- examples/ocrpdf/Dockerfile | 2 +- examples/pandoc/Dockerfile | 31 ++++-------- examples/pandoc/scyllaridae.yml | 48 +++++++++++++++++- examples/parry/Dockerfile | 2 +- examples/tesseract/Dockerfile | 2 +- examples/whisper/Dockerfile | 2 +- internal/config/server.go | 70 +++++++++++++++++++++++++++ internal/config/server_test.go | 1 + 24 files changed, 326 insertions(+), 77 deletions(-) create mode 100644 .github/workflows/delete.yml create mode 100755 ci/delete-tags.sh create mode 100644 ci/fixtures/pandoc/input.md create mode 100644 ci/fixtures/pandoc/output.tex diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 1a45668..16869de 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -34,6 +34,12 @@ jobs: echo "sha=$SHA" >> $GITHUB_OUTPUT id: extract_tag + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Setup docker build shell: bash run: |- @@ -49,22 +55,6 @@ jobs: echo "context=$DIR" >> $GITHUB_OUTPUT id: setup - - id: 'auth' - name: 'Authenticate to Google Cloud' - uses: 'google-github-actions/auth@v2' - with: - workload_identity_provider: ${{ secrets.GCLOUD_OIDC_POOL }} - create_credentials_file: true - service_account: ${{ secrets.GSA }} - token_format: 'access_token' - - - name: GAR login - uses: 'docker/login-action@v3' - with: - registry: 'us-docker.pkg.dev' - username: 'oauth2accesstoken' - password: '${{ steps.auth.outputs.access_token }}' - - name: Docker Hub Login uses: docker/login-action@v3 with: @@ -72,25 +62,22 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and push GAR - uses: docker/build-push-action@v5 - with: - context: ${{steps.setup.outputs.context}} - build-args: | - TAG=${{steps.extract_tag.outputs.tag}} - DOCKER_REPOSITORY=us-docker.pkg.dev/${{ secrets.GCLOUD_PROJECT }}/public - push: true - tags: | - us-docker.pkg.dev/${{ secrets.GCLOUD_PROJECT }}/public/${{steps.setup.outputs.image}}:${{steps.extract_tag.outputs.tag}} - us-docker.pkg.dev/${{ secrets.GCLOUD_PROJECT }}/public/${{steps.setup.outputs.image}}:${{steps.extract_tag.outputs.tag}}-${{steps.extract_tag.outputs.sha}} + - name: Determine platforms + id: determine-platforms + run: | + if [[ "${{steps.setup.outputs.image}}" == "scyllaridae-whisper" ]]; then + echo "DOCKER_PLATFORMS=linux/amd64" >> $GITHUB_ENV + else + echo "DOCKER_PLATFORMS=linux/amd64,linux/arm64" >> $GITHUB_ENV + fi - name: Build and push dockerhub uses: docker/build-push-action@v5 with: context: ${{steps.setup.outputs.context}} + platforms: ${{ env.DOCKER_PLATFORMS }} build-args: | TAG=${{steps.extract_tag.outputs.tag}} - DOCKER_REPOSITORY=lehighlts push: true tags: | lehighlts/${{steps.setup.outputs.image}}:${{steps.extract_tag.outputs.tag}} diff --git a/.github/workflows/delete.yml b/.github/workflows/delete.yml new file mode 100644 index 0000000..6a5a62a --- /dev/null +++ b/.github/workflows/delete.yml @@ -0,0 +1,50 @@ +name: Delete Docker Tags with Wildcard + +on: + delete: + branches: + - "**" + workflow_dispatch: + inputs: + tag_pattern: + description: 'Wildcard pattern for Docker tags to delete' + required: true + default: "^branch-name.*" + +jobs: + find-images: + name: Find docker images needing deleted + runs-on: ubuntu-latest + outputs: + dockerFiles: ${{ steps.images.outputs.images }} + steps: + - uses: actions/checkout@v4 + + - name: Find docker files + id: images + run: | + dockerFiles=$(find examples -name Dockerfile | jq -c --raw-input --slurp 'split("\n")| .[0:-1]') + images=$(echo "$dockerFiles" | jq -r '.[]' | \ + awk -F'/' '{print "scyllaridae-" $2}' | \ + jq -R -s '. | split("\n") | .[0:-1] | . + ["scyllaridae"]') + echo "images=$images" >> $GITHUB_OUTPUT + + delete-docker-tags: + needs: [find-images] + runs-on: ubuntu-latest + strategy: + matrix: + image: ${{ fromJson(needs.find-images.outputs.images )}} + max-parallel: 1 # go easy on the dockerhub API + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Delete Docker Tags + run: ./ci/delete-tags.sh + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} + DOCKER_REGISTRY: "lehighlts" + DOCKER_IMAGE: ${{ matrix.image }} + TAG_PATTERN: "^${{ github.event_name == 'workflow_dispatch' && github.event.inputs.tag_pattern || github.ref_name }}.*" diff --git a/.github/workflows/lint-test-build.yml b/.github/workflows/lint-test-build.yml index c338c0c..01b1660 100644 --- a/.github/workflows/lint-test-build.yml +++ b/.github/workflows/lint-test-build.yml @@ -122,5 +122,5 @@ jobs: id: images run: ./ci/run.sh env: - DOCKER_REPOSITORY_BASE: us-docker.pkg.dev/${{ secrets.GCLOUD_PROJECT }}/public/scyllaridae + DOCKER_REPOSITORY_BASE: lehighlts/scyllaridae DOCKER_TAG: ${{steps.extract_tag.outputs.tag}} diff --git a/ci/delete-tags.sh b/ci/delete-tags.sh new file mode 100755 index 0000000..d117540 --- /dev/null +++ b/ci/delete-tags.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +set -eou pipefail + +if [ "$TAG_PATTERN" = "" ] || [ "$TAG_PATTERN" = "^.*" ] || [ "$TAG_PATTERN" = "^main.*" ]; then + echo "Not mass deleting" + exit 1 +fi + +echo "Deleting tags matching pattern '$TAG_PATTERN' in repository ${DOCKER_IMAGE}" + +curl -s -o response.json -u "${DOCKER_USERNAME}:${DOCKER_ACCESS_TOKEN}" "https://hub.docker.com/v2/repositories/${DOCKER_REPOSITORY}/${DOCKER_IMAGE}/tags?page_size=100" +TAGS=$(jq -r '.results[].name' response.json) +if [ -z "$TAGS" ] || [ "$TAGS" = "null" ]; then + echo "No tags found or failed to retrieve tags." + exit 1 +fi + +curl -s -o token.json -XPOST \ + -H "Content-Type: application/json" \ + -d '{"username": "'"${DOCKER_USERNAME}"'", "password": "'"${DOCKER_ACCESS_TOKEN}"'"}' \ + "https://hub.docker.com/v2/users/login" + +TOKEN=$(jq -r .token token.json) +if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then + echo "Unable to auth to dockerhub." + exit 1 +fi + +for TAG in $TAGS; do + if [[ "$TAG" =~ $TAG_PATTERN ]]; then + echo "Deleting tag $TAG" + RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE \ + -H "Authorization: bearer ${TOKEN}" \ + "https://hub.docker.com/v2/repositories/${DOCKER_REPOSITORY}/${DOCKER_IMAGE}/tags/$TAG/") + if [ "$RESPONSE" -eq 204 ]; then + echo "Tag $TAG deleted successfully." + else + echo "Failed to delete tag $TAG. Response code: $RESPONSE" + fi + fi +done + +rm response.json token.json diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml index 480e4c1..408ecb2 100644 --- a/ci/docker-compose.yml +++ b/ci/docker-compose.yml @@ -4,20 +4,22 @@ services: fits: image: islandora/fits:main crayfits: - image: ${DOCKER_REPOSITORY_BASE}-fits:${DOCKER_TAG} + image: lehighlts/scyllaridae-fits:${DOCKER_TAG} ffmpeg: - image: ${DOCKER_REPOSITORY_BASE}-ffmpeg:${DOCKER_TAG} + image: lehighlts/scyllaridae-ffmpeg:${DOCKER_TAG} imagemagick: - image: ${DOCKER_REPOSITORY_BASE}-imagemagick:${DOCKER_TAG} + image: lehighlts/scyllaridae-imagemagick:${DOCKER_TAG} tesseract: - image: ${DOCKER_REPOSITORY_BASE}-tesseract:${DOCKER_TAG} + image: lehighlts/scyllaridae-tesseract:${DOCKER_TAG} whisper: - image: ${DOCKER_REPOSITORY_BASE}-whisper:${DOCKER_TAG} + image: lehighlts/scyllaridae-whisper:${DOCKER_TAG} + pandoc: + image: lehighlts/scyllaridae-pandoc:${DOCKER_TAG} test: image: alpine command: sleep 300 environment: - DOCKER_REPOSITORY_BASE: ${DOCKER_REPOSITORY_BASE} DOCKER_TAG: ${DOCKER_TAG} volumes: - ./test.sh:/test.sh + - ./fixtures:/fixtures diff --git a/ci/fixtures/pandoc/input.md b/ci/fixtures/pandoc/input.md new file mode 100644 index 0000000..af73214 --- /dev/null +++ b/ci/fixtures/pandoc/input.md @@ -0,0 +1,24 @@ +# Title + +## Subtitle + +This is a **bold** text and an *italic* text. + +Here is a bullet list: +- Item 1 +- Item 2 + +And a numbered list: +1. First +2. Second + +Here is an equation: +\[ +E = mc^2 +\] + +A table: + +| Column 1 | Column 2 | +|----------|----------| +| Data 1 | Data 2 | diff --git a/ci/fixtures/pandoc/output.tex b/ci/fixtures/pandoc/output.tex new file mode 100644 index 0000000..4c18f82 --- /dev/null +++ b/ci/fixtures/pandoc/output.tex @@ -0,0 +1,23 @@ +\section{Title}\label{title} + +\subsection{Subtitle}\label{subtitle} + +This is a \textbf{bold} text and an \emph{italic} text. + +Here is a bullet list: - Item 1 - Item 2 + +And a numbered list: 1. First 2. Second + +Here is an equation: {[} E = mc\^{}2 {]} + +A table: + +\begin{longtable}[]{@{}ll@{}} +\toprule\noalign{} +Column 1 & Column 2 \\ +\midrule\noalign{} +\endhead +\bottomrule\noalign{} +\endlastfoot +Data 1 & Data 2 \\ +\end{longtable} diff --git a/ci/test.sh b/ci/test.sh index 0b985b0..a78b11d 100755 --- a/ci/test.sh +++ b/ci/test.sh @@ -16,6 +16,7 @@ SERVICES=( "crayfits" "ffmpeg" "whisper" + "pandoc" ) for SERVICE in "${SERVICES[@]}"; do URL="http://$SERVICE:8080/" @@ -67,6 +68,22 @@ for SERVICE in "${SERVICES[@]}"; do grep "ask not what your country can do for you" vtt.txt || exit 1 echo "VTT as expected" rm vtt.txt + elif [ "$SERVICE" == "pandoc" ]; then + curl -o result.tex \ + -H "Content-Type: text/markdown" \ + -H "Accept: application/x-latex" \ + --data-binary "@/fixtures/pandoc/input.md" \ + "$URL" + + if diff -u result.tex "fixtures/pandoc/output.tex" > diff_output.txt; then + echo "Test Passed: Output matches expected." + else + echo "Test Failed: Differences found." + cat diff_output.txt + exit 1 + fi + + else echo "Unknown service" exit 1 diff --git a/examples/cache-warmer/Dockerfile b/examples/cache-warmer/Dockerfile index 8830407..3a4d1d6 100644 --- a/examples/cache-warmer/Dockerfile +++ b/examples/cache-warmer/Dockerfile @@ -1,5 +1,5 @@ ARG TAG=main -ARG DOCKER_REPOSITORY=local +ARG DOCKER_REPOSITORY=lehighlts FROM ${DOCKER_REPOSITORY}/scyllaridae:${TAG} RUN apk update && \ diff --git a/examples/coverpage/Dockerfile b/examples/coverpage/Dockerfile index 5a92de6..31cb330 100644 --- a/examples/coverpage/Dockerfile +++ b/examples/coverpage/Dockerfile @@ -1,11 +1,11 @@ ARG TAG=main -ARG DOCKER_REPOSITORY=local +ARG DOCKER_REPOSITORY=lehighlts FROM ${DOCKER_REPOSITORY}/scyllaridae-pandoc:${TAG} -RUN apk add --no-cache \ - bash==5.2.21-r0 \ - curl==8.5.0-r0 \ - ghostscript==10.04.0-r0 \ - jq==1.7.1-r0 - +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + curl=8.5.0-2ubuntu10.4 \ + ghostscript=10.02.1~dfsg1-0ubuntu7.4 \ + jq=1.7.1-3build1 \ + && rm -rf /var/lib/apt/lists/* COPY . /app diff --git a/examples/curl/Dockerfile b/examples/curl/Dockerfile index c6cd993..c99eb76 100644 --- a/examples/curl/Dockerfile +++ b/examples/curl/Dockerfile @@ -1,5 +1,5 @@ ARG TAG=main -ARG DOCKER_REPOSITORY=local +ARG DOCKER_REPOSITORY=lehighlts FROM ${DOCKER_REPOSITORY}/scyllaridae:${TAG} COPY scyllaridae.yml /app/scyllaridae.yml diff --git a/examples/ffmpeg/Dockerfile b/examples/ffmpeg/Dockerfile index 36237fb..ad62e22 100644 --- a/examples/ffmpeg/Dockerfile +++ b/examples/ffmpeg/Dockerfile @@ -1,5 +1,5 @@ ARG TAG=main -ARG DOCKER_REPOSITORY=local +ARG DOCKER_REPOSITORY=lehighlts FROM ${DOCKER_REPOSITORY}/scyllaridae:${TAG} RUN apk update && \ diff --git a/examples/fits/Dockerfile b/examples/fits/Dockerfile index c6cd993..c99eb76 100644 --- a/examples/fits/Dockerfile +++ b/examples/fits/Dockerfile @@ -1,5 +1,5 @@ ARG TAG=main -ARG DOCKER_REPOSITORY=local +ARG DOCKER_REPOSITORY=lehighlts FROM ${DOCKER_REPOSITORY}/scyllaridae:${TAG} COPY scyllaridae.yml /app/scyllaridae.yml diff --git a/examples/imagemagick/Dockerfile b/examples/imagemagick/Dockerfile index 0149482..e0bda4f 100644 --- a/examples/imagemagick/Dockerfile +++ b/examples/imagemagick/Dockerfile @@ -1,5 +1,5 @@ ARG TAG=main -ARG DOCKER_REPOSITORY=local +ARG DOCKER_REPOSITORY=lehighlts FROM islandora/imagemagick:alpine-3.20.2-imagemagick-7.1.1.36-r0 AS imagemagick FROM ${DOCKER_REPOSITORY}/scyllaridae:${TAG} diff --git a/examples/libreoffice/Dockerfile b/examples/libreoffice/Dockerfile index 0ac6158..cd51b46 100644 --- a/examples/libreoffice/Dockerfile +++ b/examples/libreoffice/Dockerfile @@ -1,5 +1,5 @@ ARG TAG=main -ARG DOCKER_REPOSITORY=local +ARG DOCKER_REPOSITORY=lehighlts FROM ${DOCKER_REPOSITORY}/scyllaridae:${TAG} AS scyllaridae COPY scyllaridae.yml /app/ diff --git a/examples/mergepdf/Dockerfile b/examples/mergepdf/Dockerfile index c71b84b..8f26059 100644 --- a/examples/mergepdf/Dockerfile +++ b/examples/mergepdf/Dockerfile @@ -1,5 +1,5 @@ ARG TAG=main -ARG DOCKER_REPOSITORY=local +ARG DOCKER_REPOSITORY=lehighlts FROM islandora/leptonica:alpine-3.20.2-leptonica-1.84.1-r0 AS leptonica FROM ${DOCKER_REPOSITORY}/scyllaridae-imagemagick:${TAG} AS scyllaridae diff --git a/examples/ocrpdf/Dockerfile b/examples/ocrpdf/Dockerfile index edd437b..1f75fbf 100644 --- a/examples/ocrpdf/Dockerfile +++ b/examples/ocrpdf/Dockerfile @@ -1,5 +1,5 @@ ARG TAG=main -ARG DOCKER_REPOSITORY=local +ARG DOCKER_REPOSITORY=lehighlts FROM ${DOCKER_REPOSITORY}/scyllaridae-imagemagick:${TAG} AS scyllaridae RUN apk update && \ diff --git a/examples/pandoc/Dockerfile b/examples/pandoc/Dockerfile index 4a08e1f..c5b5993 100644 --- a/examples/pandoc/Dockerfile +++ b/examples/pandoc/Dockerfile @@ -1,34 +1,21 @@ ARG TAG=main -ARG DOCKER_REPOSITORY=local +ARG DOCKER_REPOSITORY=lehighlts FROM ${DOCKER_REPOSITORY}/scyllaridae:${TAG} AS scyllaridae -FROM pandoc/latex:3.2.0 AS pandoc - -SHELL ["/bin/ash", "-o", "pipefail", "-c"] - -ENV GOSU_VERSION 1.17 -RUN apk add --no-cache --virtual .gosu-deps \ - ca-certificates==20240226-r0 \ - dpkg==1.22.1-r0 \ - gnupg==2.4.4-r0 && \ - dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" && \ - wget -q -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch" && \ - wget -q -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc" && \ - GNUPGHOME="$(mktemp -d)" && \ - export GNUPGHOME && \ - gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 && \ - gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu && \ - gpgconf --kill all && \ - rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc && \ - apk del --no-network .gosu-deps && \ - chmod +x /usr/local/bin/gosu +FROM pandoc/latex:3.5.0-ubuntu AS pandoc +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + bash=5.2.21-2ubuntu4 \ + gosu=1.17-1ubuntu0.24.04.2 \ + && rm -rf /var/lib/apt/lists/* WORKDIR /app COPY --from=scyllaridae /app/scyllaridae /app/scyllaridae COPY --from=scyllaridae /app/docker-entrypoint.sh /app/docker-entrypoint.sh COPY scyllaridae.yml . -RUN adduser -S -G nobody scyllaridae +RUN groupadd -r nobody && \ + useradd -r -g nobody scyllaridae ENTRYPOINT ["/bin/bash"] CMD ["/app/docker-entrypoint.sh"] diff --git a/examples/pandoc/scyllaridae.yml b/examples/pandoc/scyllaridae.yml index b55d57f..afc9786 100644 --- a/examples/pandoc/scyllaridae.yml +++ b/examples/pandoc/scyllaridae.yml @@ -1,10 +1,54 @@ +# list from https://pandoc.org/MANUAL.html#general-options allowedMimeTypes: + - "application/x-bibtex" + - "application/x-biblatex" + - "application/xml" + - "text/x-commonmark" + - "text/x-commonmark+extensions" + - "text/x-creole" + - "application/vnd.citationstyles.csl+json" - "text/csv" + - "text/tab-separated-values" + - "text/x-djot" + - "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + - "application/docbook+xml" + - "application/xml-dokuwiki" + - "application/vnd.endnote+xml" + - "application/epub+zip" + - "application/x-fictionbook+xml" + - "text/x-gfm" + - "text/x-haddock" + - "text/html" + - "application/x-ipynb+json" + - "application/jats+xml" + - "text/x-jira" + - "application/json" + - "application/x-latex" + - "text/markdown" + - "text/markdown+mmd" + - "text/markdown+phpextra" + - "text/markdown+strict" + - "text/x-mediawiki" + - "application/x-troff-man" + - "text/x-muse" + - "application/vnd.haskell.native" + - "application/vnd.oasis.opendocument.text" + - "application/x-opml+xml" + - "text/x-org" + - "application/x-research-info-systems" + - "application/rtf" + - "text/x-rst" + - "text/x-txt2tags" + - "text/x-textile" + - "text/x-tikiwiki" + - "text/x-twiki" + - "application/x-typst" + - "text/x-vimwiki" cmdByMimeType: default: cmd: /usr/local/bin/pandoc args: - "-f" - - "%source-mime-ext" + - "%source-mime-pandoc" - "-t" - - "%destination-mime-ext" + - "%destination-mime-pandoc" diff --git a/examples/parry/Dockerfile b/examples/parry/Dockerfile index c6cd993..c99eb76 100644 --- a/examples/parry/Dockerfile +++ b/examples/parry/Dockerfile @@ -1,5 +1,5 @@ ARG TAG=main -ARG DOCKER_REPOSITORY=local +ARG DOCKER_REPOSITORY=lehighlts FROM ${DOCKER_REPOSITORY}/scyllaridae:${TAG} COPY scyllaridae.yml /app/scyllaridae.yml diff --git a/examples/tesseract/Dockerfile b/examples/tesseract/Dockerfile index 3811632..43d527a 100644 --- a/examples/tesseract/Dockerfile +++ b/examples/tesseract/Dockerfile @@ -1,5 +1,5 @@ ARG TAG=main -ARG DOCKER_REPOSITORY=local +ARG DOCKER_REPOSITORY=lehighlts FROM islandora/leptonica:alpine-3.20.2-leptonica-1.84.1-r0 AS leptonica FROM ${DOCKER_REPOSITORY}/scyllaridae:${TAG} diff --git a/examples/whisper/Dockerfile b/examples/whisper/Dockerfile index 4d10d46..3c25c38 100644 --- a/examples/whisper/Dockerfile +++ b/examples/whisper/Dockerfile @@ -1,5 +1,5 @@ ARG TAG=main -ARG DOCKER_REPOSITORY=local +ARG DOCKER_REPOSITORY=lehighlts FROM ${DOCKER_REPOSITORY}/scyllaridae:${TAG} AS scyllaridae FROM ghcr.io/ggerganov/whisper.cpp:main-cuda diff --git a/internal/config/server.go b/internal/config/server.go index 6962db6..0cdb4c9 100644 --- a/internal/config/server.go +++ b/internal/config/server.go @@ -184,6 +184,19 @@ func BuildExecCommand(message api.Payload, c *ServerConfig) (*exec.Cmd, error) { a = fmt.Sprintf("%s:-", a) } args = append(args, a) + } else if a == "%source-mime-pandoc" { + a, err := MimeToPandoc(message.Attachment.Content.SourceMimeType) + if err != nil { + return nil, fmt.Errorf("unknown mime extension: %s", message.Attachment.Content.SourceMimeType) + } + + args = append(args, a) + } else if a == "%destination-mime-pandoc" { + a, err := MimeToPandoc(message.Attachment.Content.DestinationMimeType) + if err != nil { + return nil, fmt.Errorf("unknown mime extension: %s", message.Attachment.Content.DestinationMimeType) + } + args = append(args, a) } else if a == "%target" { args = append(args, message.Target) @@ -249,6 +262,8 @@ func GetMimeTypeExtension(mimeType string) (string, error) { "audio/x-realaudio": "ra", "audio/midi": "mid", "audio/x-wav": "wav", + + "text/markdown": "md", } cleanMimeType := strings.TrimSpace(strings.ToLower(mimeType)) if ext, ok := mimeToExtension[cleanMimeType]; ok { @@ -310,3 +325,58 @@ func (c *ServerConfig) GetFileStream(r *http.Request, message api.Payload, auth return sourceResp.Body, http.StatusOK, nil } + +func MimeToPandoc(mimeType string) (string, error) { + mapping := map[string]string{ + "text/x-bibtex": "bibtex", + "text/x-biblatex": "biblatex", + "application/xml": "bits", + "text/x-commonmark": "commonmark", + "text/x-commonmark+extensions": "commonmark_x", + "text/x-creole": "creole", + "application/vnd.citationstyles.csl+json": "csljson", + "text/csv": "csv", + "text/tab-separated-values": "tsv", + "text/x-djot": "djot", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx", + "application/docbook+xml": "docbook", + "application/xml-dokuwiki": "dokuwiki", + "application/vnd.endnote+xml": "endnotexml", + "application/epub+zip": "epub", + "application/x-fictionbook+xml": "fb2", + "text/x-gfm": "gfm", + "text/x-haddock": "haddock", + "text/html": "html", + "application/x-ipynb+json": "ipynb", + "application/jats+xml": "jats", + "text/x-jira": "jira", + "application/json": "json", + "application/x-latex": "latex", + "text/markdown": "markdown", + "text/markdown+mmd": "markdown_mmd", + "text/markdown+phpextra": "markdown_phpextra", + "text/markdown+strict": "markdown_strict", + "text/x-mediawiki": "mediawiki", + "application/x-troff-man": "man", + "text/x-muse": "muse", + "application/vnd.haskell.native": "native", + "application/vnd.oasis.opendocument.text": "odt", + "application/x-opml+xml": "opml", + "text/x-org": "org", + "application/x-research-info-systems": "ris", + "application/rtf": "rtf", + "text/x-rst": "rst", + "text/x-txt2tags": "t2t", + "text/x-textile": "textile", + "text/x-tikiwiki": "tikiwiki", + "text/x-twiki": "twiki", + "application/x-typst": "typst", + "text/x-vimwiki": "vimwiki", + } + + pandoc, ok := mapping[mimeType] + if !ok { + return GetMimeTypeExtension(mimeType) + } + return pandoc, nil +} diff --git a/internal/config/server_test.go b/internal/config/server_test.go index 7e29bfd..adaf79f 100644 --- a/internal/config/server_test.go +++ b/internal/config/server_test.go @@ -81,6 +81,7 @@ func TestMimeTypes(t *testing.T) { "text/html": "html", "application/pdf": "pdf", "text/csv": "csv", + "text/markdown": "md", } for mimeType, extension := range mimeTypes {