Merge pull request #641 from Mara3l/master

Update documentation pipelines - local preview and only 4 versions

.github/actions/hugo-build-action/action.yaml

@@ -12,10 +12,6 @@ inputs:
     required: false
     description: Environment variable
     default: public
-  keep-master:
-    required: false
-    description: Preview of the master branch
-    default: no
   fetch-from:
     required: false
     description: repo to fetch the versions from
@@ -31,39 +27,22 @@ runs:
       python-version-file: ".python-version"
       cache: 'pip'
       cache-dependency-path: scripts/script-requirements.txt
-  - name: Install Dependencies
-    shell: bash
-    run: |
-      python -m pip install --upgrade pip
-      pip install -r scripts/script-requirements.txt
   - uses: actions/setup-go@v4
     with:
       go-version: '>=1.20.1'
   - uses: actions/setup-node@v3
     with:
       node-version: 18
-  - name: Install dependencies
-    working-directory: ${{ inputs.working-directory }}
-    run: npm ci
-    shell: bash
   - name: "Setup Hugo"
-    uses: peaceiris/actions-hugo@v2
+    uses: peaceiris/actions-hugo@v3
     with:
       extended: true
       hugo-version: '0.110.0'
   - name: "Build"
     env:
-      KEEP_MASTER: ${{ inputs.keep-master }}
-      THIS_BRANCH: ${{ inputs.this-branch }}
       BASE_URL: ${{ inputs.base-url }}
-      HUGO_ENV: production
     shell: bash
     run: |
       cd docs
-      rm -f generate.sh
-      wget https://raw.githubusercontent.com/gooddata/gooddata-python-sdk/master/scripts/generate.sh
-      chmod +x ./generate.sh
-      ./generate.sh ${{ inputs.fetch-from }} master ${{ inputs.keep-master}}
       npm install
-      hugo version
       hugo --minify ${BASE_URL:+--baseURL $BASE_URL}

.github/actions/hugo-build-versioned-action/action.yaml

@@ -1,5 +1,5 @@
 name: Hugo Versioned Build
-description: Builds documentation using Hugo
+description: Builds versioned documentation using Hugo
 inputs:
   base-url:
     required: false
@@ -12,10 +12,6 @@ inputs:
     required: false
     description: Environment variable
     default: production
-  keep-master:
-    required: false
-    description: Preview of the master branch
-    default: no
   fetch-from:
     required: false
     description: repo to fetch the versions from
@@ -24,6 +20,7 @@ inputs:
 runs:
   using: "composite"
   steps:
+    - uses: actions/checkout@v4
     - name: Setup Python
       uses: actions/setup-python@v5
       with:
@@ -35,17 +32,28 @@ runs:
       run: |
         python -m pip install --upgrade pip
         pip install -r scripts/script-requirements.txt
-    - name: Generate Versioned Documentation
+    - uses: actions/setup-go@v4
+      with:
+        go-version: '>=1.20.1'
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 18
+    - name: "Setup Hugo"
+      uses: peaceiris/actions-hugo@v3
+      with:
+        extended: true
+        hugo-version: '0.110.0'
+    - name: "Build"
+      env:
+        THIS_BRANCH: ${{ inputs.this-branch }}
+        BASE_URL: ${{ inputs.base-url }}
+        HUGO_ENV: ${{ inputs.hugo-env }}
       shell: bash
       run: |
         cd docs
         rm -f generate.sh
         wget https://raw.githubusercontent.com/gooddata/gooddata-python-sdk/master/scripts/generate.sh
         chmod +x ./generate.sh
-        ./generate.sh ${{ inputs.fetch-from }} master ${{ inputs.keep-master}}
-    - name: Hugo Build
-      uses: gooddata/gooddata-python-sdk/.github/actions/hugo-build-action@master
-      with:
-        hugo-env: ${{ inputs.hugo-env }}
-        working-directory: ${{ inputs.working-directory }}
-        base-url: ${{ inputs.base-url }}
+        ./generate.sh ${{ inputs.fetch-from }} master
+        npm install
+        hugo --minify ${BASE_URL:+--baseURL $BASE_URL}

.github/workflows/netlify-deploy-preview.yaml

@@ -28,12 +28,10 @@ jobs:
         run: |
           rsync --delete -av PR-tmp/docs/content/en/ docs/content/en/
           rm -rf PR-tmp
-      - name: Generate Versioned Docs
+      - name: Generate Docs
         uses: gooddata/gooddata-python-sdk/.github/actions/hugo-build-action@master
         with:
           base-url: https://preview-${{ env.GITHUB_PR_NUMBER }}--${{ env.NETLIFY_SITE_NAME }}.netlify.app
-          keep-master: keep_master
-          fetch-from: upstream
       - name: Publish
         uses: netlify/actions/cli@master
         with:

.github/workflows/netlify-deploy.yaml

@@ -11,7 +11,7 @@ jobs:
         with:
           submodules: recursive
       - name: Hugo Build
-        uses: gooddata/gooddata-python-sdk/.github/actions/hugo-build-action@master
+        uses: gooddata/gooddata-python-sdk/.github/actions/hugo-build-versioned-action@master
         with:
           base-url: https://www.gooddata.com/docs/python-sdk
       - name: Publish

docs/config/production/params.toml

@@ -27,43 +27,3 @@ url = "/1.16/"
 version = "1.15"
 dirpath = "1.15"
 url = "/1.15/"
-[[versions]]
-version = "1.14"
-dirpath = "1.14"
-url = "/1.14/"
-[[versions]]
-version = "1.13"
-dirpath = "1.13"
-url = "/1.13/"
-[[versions]]
-version = "1.12"
-dirpath = "1.12"
-url = "/1.12/"
-[[versions]]
-version = "1.11"
-dirpath = "1.11"
-url = "/1.11/"
-[[versions]]
-version = "1.10"
-dirpath = "1.10"
-url = "/1.10/"
-[[versions]]
-version = "1.9"
-dirpath = "1.9"
-url = "/1.9/"
-[[versions]]
-version = "1.8"
-dirpath = "1.8"
-url = "/1.8/"
-[[versions]]
-version = "1.7"
-dirpath = "1.7"
-url = "/1.7/"
-[[versions]]
-version = "1.6"
-dirpath = "1.6"
-url = "/1.6/"
-[[versions]]
-version = "1.5"
-dirpath = "1.5"
-url = "/1.5/"

scripts/bump_doc_dependencies.py

@@ -35,6 +35,9 @@ def bump_toml(file_path: Path, version: list[int]):
     new_version_data["url"] = "/latest/"
     versions.insert(1, new_version_data)
 
+    # Pop the last version
+    versions.pop()
+
     with open(file_path, "w") as file:
         file.write(tomlkit.dumps(data))
 

scripts/generate.sh

@@ -8,8 +8,8 @@ content_dir=versioned_docs
 remote_name=${1:-origin}
 # target branch where changes will be applied (master, rel/0.7, ...)
 target_branch=${2:-master}
-# if set to "keep_master", the "docs" from master will be preserved (only for preview!)
-keep_master=$3
+# Number of the versions to persist. Older ones are accesible only from GitHub.
+num_versions=${4:-4}
 
 echo "Validating target branch '$target_branch'"
 case "$target_branch" in
@@ -42,22 +42,30 @@ fi
 
 git fetch "$remote_name"
 
+latest_branches=()
+# Get all relevant rel/* branches, sort them, and pick the latest num_versions
+while IFS= read -r vers; do
+    latest_branches+=("$remote_name/rel/$vers")
+done < <(git branch -rl "$remote_name/rel/*" | sed 's|.*/rel/||' | sort -t. -k1,1n -k2,2n | tail -n"$num_versions")
+
+# Add special branches to the array (only rel/dev and master for now)
+special_branches=("$remote_name/rel/dev")
+
+branches_to_process=("${latest_branches[@]}" "${special_branches[@]}")
+
+echo "Branches to be processed:"
+for branch in "${branches_to_process[@]}"; do
+    echo "$branch"
+done
+
 # For every release branch, copy the docs to conetentDir/x.y
-for branch in "$remote_name/master" $(git branch -rl "$remote_name/rel/*") ; do
+for branch in "${branches_to_process[@]}" ; do
     target_section=${branch#"$remote_name"/}
     target_section=${target_section#rel/}
     target_section=${target_section%.*}
-    if [ "$target_section" == "master" ] ; then
-        # handle master branch specially, all contents is copied, not just docs
-        target_section=""
-        # number of path segments to throw away by tar: content/en => 2
-        strip_count=2
-        src_section=""
-    else
-        # number of path segments to throw away by tar: content/en/x.y => 3
-        strip_count=3
-        src_section=docs
-    fi
+    # number of path segments to throw away by tar: content/en/x.y => 3
+    strip_count=3
+    src_section=docs
     if [ "$target_section" == "$current_section" ] ; then
         # copy the current docs to proper section
         echo "Getting data from workdir for $branch"
@@ -73,30 +81,21 @@ for branch in "$remote_name/master" $(git branch -rl "$remote_name/rel/*") ; do
     if git cat-file -e $API_GEN_FILE; then
         echo "$API_GEN_FILE exists."
         echo "Generating API ref..."
-        if [ "$target_section" == "" ] ; then
-            echo "Skipping master api ref"
+        if git ls-tree --name-only "$branch" | grep -q "^api_spec.toml$"; then
+            git checkout "$branch" -- api_spec.toml
         else
-            directories=$(ls -d ../gooddata-*)
-
-            for dir in $directories; do
-                git checkout "$branch" -- "$dir"
-            done
-            if git ls-tree --name-only "$branch" | grep -q "^api_spec.toml$"; then
-                git checkout "$branch" -- api_spec.toml
-            else
-              echo "removing the API_spec"
-              rm -rf api_spec.toml
-            fi
-            python3 ../scripts/docs/json_builder.py
-            mv -f data.json ./versioned_docs/"$target_section"/
-            python3 ../scripts/docs/python_ref_builder.py api_spec.toml ./versioned_docs/"$target_section"/data.json "$target_section" versioned_docs
+          echo "removing the API_spec"
+          rm -rf api_spec.toml
         fi
+        python3 ../scripts/docs/json_builder.py
+        mv -f data.json ./versioned_docs/"$target_section"/
+        python3 ../scripts/docs/python_ref_builder.py api_spec.toml ./versioned_docs/"$target_section"/data.json "$target_section" versioned_docs
     fi
 done
 
 
 ## Moving the highest version to latest
-highest_version=$(ls -v1 ./versioned_docs/ | grep -E '^[0-9]+.[0-9]+$' | tail -n 1)
+highest_version=$(ls -v1 ./versioned_docs/ | grep -E '^[0-9]+.[0-9]+$' | sort -V | tail -n 1)
 echo "Moving ${highest_version} to /latest"
 mv -f ./versioned_docs/$highest_version ./versioned_docs/latest
 
@@ -105,10 +104,4 @@ sed "s|${highest_version}|latest|g" ./versioned_docs/latest/links.json > temp.js
 
 mv temp.json ./versioned_docs/latest/links.json
 
-if [ "$keep_master" != "keep_master" ] ; then
-    echo "master docs will not be published, removing"
-    rm -rf "${content_dir}/docs"
-fi
 popd
-
-git reset --hard