ci: Fixing the RTS build workflow (#16748)

## Description The problem with the RTS workflow after the induction of AST parsing was that while copying the `node_modules`, Ubuntu doesn't follow the symlink and copy the content. It simply copies the symlink as is. This causes issues for RTS service to start. ## Type of change - Bug fix (non-breaking change which fixes an issue) ## How Has This Been Tested? - Personal repository ## Checklist: - [ ] My code follows the style guidelines of this project - [ ] I have performed a self-review of my own code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [ ] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes
appsmithorg · Sep 14, 2022 · 7580eb9 · 7580eb9
1 parent 16b04f3
commit 7580eb9
Show file tree

Hide file tree

Showing 8 changed files with 43 additions and 49 deletions.
diff --git a/.github/workflows/client-build.yml b/.github/workflows/client-build.yml
@@ -63,7 +63,7 @@ jobs:
 
       # Timestamp will be used to create cache key
       - id: timestamp
-        run: echo "::set-output name=timestamp::$(timestamp +'%Y-%m-%dT%H:%M:%S')"
+        run: echo "::set-output name=timestamp::$(date +'%Y-%m-%dT%H:%M:%S')"
 
       # In case this is second attempt try restoring status of the prior attempt from cache
       - name: Restore the previous run result

diff --git a/.github/workflows/integration-tests-command.yml b/.github/workflows/integration-tests-command.yml
@@ -83,7 +83,7 @@ jobs:
 
       # Timestamp will be used to create cache key
       - id: timestamp
-        run: echo "::set-output name=timestamp::$(timestamp +'%Y-%m-%dT%H:%M:%S')"
+        run: echo "::set-output name=timestamp::$(date +'%Y-%m-%dT%H:%M:%S')"
 
       # In case this is second attempt try restoring status of the prior attempt from cache
       - name: Restore the previous run result
@@ -477,7 +477,7 @@ jobs:
 
       # Timestamp will be used to create cache key
       - id: timestamp
-        run: echo "::set-output name=timestamp::$(timestamp +'%Y-%m-%dT%H:%M:%S')"
+        run: echo "::set-output name=timestamp::$(date +'%Y-%m-%dT%H:%M:%S')"
 
       # In case this is second attempt try restoring status of the prior attempt from cache
       - name: Restore the previous run result

diff --git a/.github/workflows/perf-test.yml b/.github/workflows/perf-test.yml
@@ -60,7 +60,7 @@ jobs:
 
       # Timestamp will be used to create cache key
       - id: timestamp
-        run: echo "::set-output name=timestamp::$(timestamp +'%Y-%m-%dT%H:%M:%S')"
+        run: echo "::set-output name=timestamp::$(date +'%Y-%m-%dT%H:%M:%S')"
 
       # In case this is second attempt try restoring status of the prior attempt from cache
       - name: Restore the previous run result

diff --git a/.github/workflows/rts-build.yml b/.github/workflows/rts-build.yml
@@ -62,7 +62,7 @@ jobs:
 
       # Timestamp will be used to create cache key
       - id: timestamp
-        run: echo "::set-output name=timestamp::$(timestamp +'%Y-%m-%dT%H:%M:%S')"
+        run: echo "::set-output name=timestamp::$(date +'%Y-%m-%dT%H:%M:%S')"
 
       # In case this is second attempt try restoring status of the prior attempt from cache
       - name: Restore the previous run result
@@ -129,7 +129,7 @@ jobs:
         if: steps.run_result.outputs.run_result != 'success'
         run: |
           echo 'export const VERSION = "${{ steps.vars.outputs.version }}"' > src/version.js
-          ./build.sh
+          yarn build
 
       # Restore the previous built bundle if present. If not push the newly built into the cache
       - name: Restore the previous bundle
@@ -141,18 +141,18 @@ jobs:
           restore-keys: |
             ${{ github.run_id }}-${{ github.job }}
 
+      # Tar the bundles to speed up the upload & download process
+      - name: Tar the rts bundles
+        run: |
+          ls -al dist/node_modules/@shared/ast
+          tar -cvf rts-dist.tar dist
+
       # Upload the build artifacts and dependencies so that it can be used by the test & deploy job in other workflows
       - name: Upload rts build bundle
         uses: actions/upload-artifact@v2
         with:
-          name: rts-build
-          path: app/rts/dist/
-
-      - name: Upload RTS dependencies bundle
-        uses: actions/upload-artifact@v2
-        with:
-          name: rts-build-deps
-          path: app/rts/node_modules/
+          name: rts-dist
+          path: app/rts/rts-dist.tar
 
       # Set status = success
       - run: echo "::set-output name=run_result::success" > ~/run_result
diff --git a/.github/workflows/server-build.yml b/.github/workflows/server-build.yml
@@ -69,7 +69,7 @@ jobs:
 
       # Timestamp will be used to create cache key
       - id: timestamp
-        run: echo "::set-output name=timestamp::$(timestamp +'%Y-%m-%dT%H:%M:%S')"
+        run: echo "::set-output name=timestamp::$(date +'%Y-%m-%dT%H:%M:%S')"
 
       # In case this is second attempt try restoring status of the prior attempt from cache
       - name: Restore the previous run result

diff --git a/.github/workflows/test-build-docker-image.yml b/.github/workflows/test-build-docker-image.yml
@@ -54,7 +54,7 @@ jobs:
 
       # Timestamp will be used to create cache key
       - id: timestamp
-        run: echo "::set-output name=timestamp::$(timestamp +'%Y-%m-%dT%H:%M:%S')"
+        run: echo "::set-output name=timestamp::$(date +'%Y-%m-%dT%H:%M:%S')"
 
       # In case this is second attempt try restoring status of the prior attempt from cache
       - name: Restore the previous run result
@@ -203,7 +203,7 @@ jobs:
 
       # Timestamp will be used to create cache key
       - id: timestamp
-        run: echo "::set-output name=timestamp::$(timestamp +'%Y-%m-%dT%H:%M:%S')"
+        run: echo "::set-output name=timestamp::$(date +'%Y-%m-%dT%H:%M:%S')"
 
       # In case this is second attempt try restoring status of the prior attempt from cache
       - name: Restore the previous run result
@@ -318,7 +318,7 @@ jobs:
 
       # Timestamp will be used to create cache key
       - id: timestamp
-        run: echo "::set-output name=timestamp::$(timestamp +'%Y-%m-%dT%H:%M:%S')"
+        run: echo "::set-output name=timestamp::$(date +'%Y-%m-%dT%H:%M:%S')"
 
       # In case this is second attempt try restoring status of the prior attempt from cache
       - name: Restore the previous run result
@@ -366,8 +366,7 @@ jobs:
         if: steps.run_result.outputs.run_result != 'success'
         run: |
           echo 'export const VERSION = "${{ steps.vars.outputs.version }}"' > src/version.js
-          ./build.sh
-          ls -l dist
+          yarn build
 
       # Restore the previous built bundle if present. If not push the newly built into the cache
       - name: Restore the previous bundle
@@ -379,28 +378,22 @@ jobs:
           restore-keys: |
             ${{ github.run_id }}-${{ github.job }}
 
-      # Restore the previous built bundle if present. If not push the newly built into the cache
-      - name: Restore the previous bundle
-        uses: actions/cache@v2
-        with:
-          path: |
-            app/rts/node_modules/
-          key: ${{ github.run_id }}-${{ github.job }}-${{ steps.timestamp.outputs.timestamp }}
-          restore-keys: |
-            ${{ github.run_id }}-${{ github.job }}
+      # Tar the bundles to speed up the upload & download process
+      - name: Tar the rts bundles
+        run: |
+          ls -al dist/node_modules/@shared/ast
+          tar -cvf rts-dist.tar dist
 
       # Upload the build artifact so that it can be used by the test & deploy job in the workflow
       - name: Upload server build bundle
         uses: actions/upload-artifact@v2
         with:
-          name: rts-build
-          path: app/rts/dist/
+          name: rts-dist
+          path: app/rts/rts-dist.tar
+
+      # Set status = success
+      - run: echo "::set-output name=run_result::success" > ~/run_result
 
-      - name: Upload RTS dependencies bundle
-        uses: actions/upload-artifact@v2
-        with:
-          name: rts-build-deps
-          path: app/rts/node_modules/
   perf-test:
     needs: [buildClient, buildServer, buildRts]
 
@@ -439,7 +432,7 @@ jobs:
 
       # Timestamp will be used to create cache key
       - id: timestamp
-        run: echo "::set-output name=timestamp::$(timestamp +'%Y-%m-%dT%H:%M:%S')"
+        run: echo "::set-output name=timestamp::$(date +'%Y-%m-%dT%H:%M:%S')"
 
       # In case this is second attempt try restoring status of the prior attempt from cache
       - name: Restore the previous run result
@@ -665,7 +658,7 @@ jobs:
 
       # Timestamp will be used to create cache key
       - id: timestamp
-        run: echo "::set-output name=timestamp::$(timestamp +'%Y-%m-%dT%H:%M:%S')"
+        run: echo "::set-output name=timestamp::$(date +'%Y-%m-%dT%H:%M:%S')"
 
       # In case this is second attempt try restoring status of the prior attempt from cache
       - name: Restore the previous run result
@@ -1072,7 +1065,7 @@ jobs:
 
       # Timestamp will be used to create cache key
       - id: timestamp
-        run: echo "::set-output name=timestamp::$(timestamp +'%Y-%m-%dT%H:%M:%S')"
+        run: echo "::set-output name=timestamp::$(date +'%Y-%m-%dT%H:%M:%S')"
 
       # In case this is second attempt try restoring status of the prior attempt from cache
       - name: Restore the previous run result
@@ -1540,14 +1533,14 @@ jobs:
       - name: Download the rts build artifact
         uses: actions/download-artifact@v2
         with:
-          name: rts-build
+          name: rts-dist
           path: app/rts/dist
 
-      - name: Download the rts build artifact
-        uses: actions/download-artifact@v2
-        with:
-          name: rts-build-deps
-          path: app/rts/node_modules/
+      - name: Untar the rts folder
+        run: |
+          tar -xvf app/rts/dist/rts-dist.tar -C app/rts/
+          echo "Cleaning up the tar files"
+          rm app/rts/dist/rts-dist.tar
 
       # Here, the GITHUB_REF is of type /refs/head/<branch_name>. We extract branch_name from this by removing the
       # first 11 characters. This can be used to build images for several branches
@@ -1656,7 +1649,6 @@ jobs:
           tags: |
             ${{ secrets.DOCKER_HUB_ORGANIZATION }}/appsmith-server:${{ github.sha }}
             ${{ secrets.DOCKER_HUB_ORGANIZATION }}/appsmith-server:nightly
-
       # Build release Docker image and push to Docker Hub
       - name: Push RTS release image to Docker Hub
         if: success() && github.ref == 'refs/heads/release'

diff --git a/app/rts/build.sh b/app/rts/build.sh
@@ -5,5 +5,7 @@ set -o errexit
 cd "$(dirname "$0")"
 yarn install --frozen-lockfile
 npx tsc && npx tsc-alias
-# Copying node_modules directory into dist as rts server requires node_modules to run server build properly. This was previously being done in dockerfile which was copying the symlinks to image rather than the whole directory of shared modules (e.g. AST)
-cp -r node_modules ./dist
+# Copying node_modules directory into dist as rts server requires node_modules to run server build properly. 
+# This was previously being done in dockerfile which was copying the symlinks to image rather than the whole directory of shared modules (e.g. AST)
+# Also, we copy node_modules with -L flag in order to follow the symlinks for @shared folder and copy the contents instead of just the symlink
+cp -rL node_modules ./dist
diff --git a/depot.json b/depot.json
@@ -1 +1 @@
-{"id":"ts81441f91"}
+{ "id": "ts81441f91" }