feat(run): add Cloud Run + Filestore sample (#3288)

* initial package.json * basic express with test * add testing dependencies * add nonexistant path test * refactor test to use chai expect as assert * add chai to package.json * hide console.log output from test output * label console output * semi-colons ;;;; * add redirects to mount dir * writeFile function * test for new file write * return list of files and links * add test for files * add Dockerfile and wrapper script * add header * eslint --fix * add rate limit to app * add testing workflows * dont store value for path * add missing #! * refactor to use serve-index * eslint * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Update run/filesystem/package.json Co-authored-by: Averi Kitsch <akitsch@google.com> * Update run/filesystem/package.json Co-authored-by: Averi Kitsch <akitsch@google.com> * remove fs from package.json * typo * move port var down * change filename format * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * refactor: redirect all to mnt path * eslint * additional comments * separate function for index * eslint * fix hrefs * move file prefix var * period * rename wrapper script * update wrapper * comments and sample description * cross-site scripting * add try catch * better error response * sanitize res * use node:20-slim * rm unused dep * arrow functions and consts * eslint * no mutations * unused var * remove unneeded async * lint * add sigterm handling * fix sigterm handling * formatting * e2e test cloudbuild config * parallel delete resources * fix cleanup step * add waitfor * break cleanup into 3 steps * clean up AR Repo build * split build config into setup and cleanup * e2e test boilerplate * remove unused subs from cleanup yaml * update waitFor in cleanup yaml * recombine build yamls * lint * split cloudbuild config * split tests to system.test.js * lint * system.test add auth * add before and after to e2e test * add endpoint test to e2e * update package.json * headers * typo * add to system e2e tests * kokoro config * remove cfg * typo * remove server.close * refactor system.test.js to match other samples * add test for generated txt to e2e * add comment re: rate limit * remove allow unauthenticated * lint: unused var * add unit test for file generation * clarify rate limit comment * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * add wait -n to run script * use fs/promises * write file before generating html --------- Co-authored-by: Patti Shin <pattishin@users.noreply.github.com> Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com> Co-authored-by: Karl Weinmeister <11586922+kweinmeister@users.noreply.github.com> Co-authored-by: Averi Kitsch <akitsch@google.com>
GoogleCloudPlatform · Jul 25, 2023 · bdda289 · bdda289
1 parent 735eea7
commit bdda289
Show file tree

Hide file tree

Showing 10 changed files with 553 additions and 0 deletions.
diff --git a/.github/workflows/run-filesystem.yaml b/.github/workflows/run-filesystem.yaml
@@ -0,0 +1,62 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: run-filesystem
+on:
+  push:
+    branches:
+    - main
+    paths:
+    - 'run/filesystem/**'
+    - '.github/workflows/run-filesystem.yaml'
+  pull_request:
+    paths:
+    - 'run/filesystem/**'
+    - '.github/workflows/run-filesystem.yaml'
+  pull_request_target:
+    types: [labeled]
+    paths:
+    - 'run/filesystem/**'
+    - '.github/workflows/run-filesystem.yaml'
+  schedule:
+  - cron:  '0 0 * * 0'
+jobs:
+  test:
+    # Ref: https://github.com/google-github-actions/auth#usage
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    if: github.event.action != 'labeled' || github.event.label.name == 'actions:force-run'
+    uses: ./.github/workflows/test.yaml
+    with:
+      name: 'run-filesystem'
+      path: 'run/filesystem'
+  remove_label:
+    # Ref: https://github.com/google-github-actions/auth#usage
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    if: |
+      github.event.action == 'labeled' &&
+      github.event.label.name == 'actions:force-run' &&
+      always()
+    uses: ./.github/workflows/remove-label.yaml
+  flakybot:
+    # Ref: https://github.com/google-github-actions/auth#usage
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    if: github.event_name == 'schedule' && always() # always() submits logs even if tests fail
+    uses: ./.github/workflows/flakybot.yaml
+    needs: [test]
diff --git a/.github/workflows/utils/workflows.json b/.github/workflows/utils/workflows.json
@@ -85,6 +85,7 @@
   "monitoring/snippets",
   "opencensus",
   "retail",
+  "run/filesystem",
   "scheduler",
   "secret-manager",
   "service-directory/snippets",

diff --git a/run/filesystem/Dockerfile b/run/filesystem/Dockerfile
@@ -0,0 +1,51 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# [START cloudrun_fs_dockerfile]
+
+# Use the official Node.js image.
+# https://hub.docker.com/_/node
+FROM node:20-slim
+
+# Install system dependencies
+RUN apt-get update -y && apt-get install -y \
+    tini \
+    nfs-common \
+	libtool \ 
+    && apt-get clean
+
+# Set fallback mount directory
+ENV MNT_DIR /mnt/nfs/filestore
+
+# Copy local code to the container image.
+ENV APP_HOME /app
+WORKDIR $APP_HOME
+COPY package*.json ./
+
+# Install production dependencies.
+RUN npm install --only=production
+
+# Copy local code to the container image.
+COPY . ./
+
+# Ensure the script is executable
+RUN chmod +x /app/run.sh
+
+# Use tini to manage zombie processes and signal forwarding
+# https://github.com/krallin/tini
+ENTRYPOINT ["/usr/bin/tini", "--"]
+
+# Pass the wrapper script as arguments to tini
+CMD ["/app/run.sh"]
+# [END cloudrun_fs_dockerfile]
diff --git a/run/filesystem/index.js b/run/filesystem/index.js
@@ -0,0 +1,106 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+/**
+ * Express webapp that generates files to a mounted NFS  when deployed to Cloud Run.
+ *
+ * See https://cloud.google.com/run/docs/tutorials/network-filesystems-filestore before running the code snippet.
+ */
+
+const express = require('express');
+const rateLimit = require('express-rate-limit');
+const fs = require('fs/promises');
+const app = express();
+const mntDir = process.env.MNT_DIR || '/mnt/nfs/filestore';
+const port = parseInt(process.env.PORT) || 8080;
+const limit = rateLimit({
+  // Use of rate limit to fullfill CodeQL rule js/missing-rate-limiting
+  // HTTP request handlers should not perform expensive operations such as
+  // accessing the file system. Setting rate limit to maximum 100 requests
+  // per 15 minute window.
+  windowMs: 15 * 60 * 1000,
+  max: 100,
+  message: 'Rate limit exceeded',
+  headers: true,
+});
+
+app.use(limit);
+app.use(mntDir, express.static(mntDir));
+
+app.listen(port, () => {
+  console.log(`Listening on port ${port}`);
+});
+
+app.get(mntDir, async (req, res) => {
+  // Have all requests to mount directory generate a new file on the filesystem.
+  try {
+    writeFile(mntDir);
+    const html = await generateIndex(mntDir);
+    // Respond with html with list of files on the filesystem.
+    res.send(html);
+  } catch (error) {
+    console.error(error);
+    res
+      .status(500)
+      .send(
+        'Something went wrong when writing to the filesystem. Refresh the page to try again.'
+      );
+  }
+});
+
+app.all('*', (req, res) => {
+  // Redirect all requests to the mount directory
+  res.redirect(mntDir);
+});
+
+const writeFile = async (path, filePrefix = 'test') => {
+  // Write a test file to the provided path.
+  const date = new Date();
+  const formattedDate = date.toString().split(' ').slice(0, 5).join('-');
+  const filename = `${filePrefix}-${formattedDate}.txt`;
+  const contents = `This test file was created on ${formattedDate}.\n`;
+
+  try {
+    const newFile = fs.writeFile(`${path}/${filename}`, contents);
+    await newFile;
+  } catch (error) {
+    console.error(error);
+  }
+};
+
+const generateIndex = async mntDir => {
+  // Return html for page with a list of files on the mounted filesystem.
+  try {
+    const header =
+      '<html><body>A new file is generated each time this page is reloaded.<p>Files created on filesystem:<p>';
+    const footer = '</body></html>';
+    // Get list of files on mounted filesystem.
+    const existingFiles = await fs.readdir(mntDir);
+    // Insert each file into html content
+    const htmlBody = existingFiles.map(fileName => {
+      const sanitized = encodeURIComponent(fileName);
+      return `<a href="${mntDir}/${sanitized}">${decodeURIComponent(
+        sanitized
+      )}</a><br>`;
+    });
+    return header + htmlBody.join(' ') + footer;
+  } catch (error) {
+    console.log(error);
+  }
+};
+
+process.on('SIGTERM', () => {
+  console.log('Received SIGTERM signal. Exiting.');
+});
+
+module.exports = app;
diff --git a/run/filesystem/package.json b/run/filesystem/package.json
@@ -0,0 +1,30 @@
+{
+  "name": "filesystem",
+  "description": "Demonstrate accessing a mounted network filesystem from a Cloud Run service.",
+  "version": "1.0.0",
+  "main": "index.js",
+  "scripts": {
+    "start": "node index.js",
+    "test": "c8 mocha test/index.test.js --exit",
+    "system-test": "c8 mocha test/system.test.js --timeout=360000 --exit"
+  },
+  "engines": {
+    "node": ">=12.0.0"
+  },
+  "author": "Google LLC",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "express": "^4.18.2",
+    "express-rate-limit": "^6.7.0"
+  },
+  "devDependencies": {
+    "c8": "^7.14.0",
+    "chai": "^4.3.7",
+    "chai-http": "^4.4.0",
+    "google-auth-library": "^8.0.0",
+    "got": "^11.0.0",
+    "mocha": "^10.2.0",
+    "mock-fs": "^5.2.0",
+    "supertest": "^6.3.3"
+  }
+}
diff --git a/run/filesystem/run.sh b/run/filesystem/run.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# [START cloudrun_fs_script]
+#!/usr/bin/env bash
+set -eo pipefail
+
+# Create mount directory for service.
+mkdir -p $MNT_DIR
+
+echo "Mounting Cloud Filestore."
+mount -o nolock $FILESTORE_IP_ADDRESS:/$FILE_SHARE_NAME $MNT_DIR
+echo "Mounting completed."
+
+# Start the application
+node index.js
+
+# Exit immediately when one of the background processes terminate.
+wait -n
+# [END cloudrun_fs_script]
diff --git a/run/filesystem/test/e2e_test_cleanup.yaml b/run/filesystem/test/e2e_test_cleanup.yaml
@@ -0,0 +1,38 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+steps:
+- id: 'Delete Cloud Run service'
+  name: 'gcr.io/cloud-builders/gcloud:latest'
+  entrypoint: /bin/bash
+  waitFor: ['-']
+  args:
+  - '-c'
+  - |
+    gcloud run services delete $_RUN_SERVICE --region $_REGION --quiet
+
+- id: 'Delete Artifact Registry build'
+  name: 'gcr.io/cloud-builders/gcloud:latest'
+  entrypoint: /bin/bash
+  waitFor: ['-']
+  args:
+  - '-c'
+  - |
+    gcloud artifacts docker images delete \
+    us-central1-docker.pkg.dev/$PROJECT_ID/cloud-run-source-deploy/$_RUN_SERVICE:latest --quiet
+
+substitutions:
+  _REGION: us-central1
+  _RUN_SERVICE: filesystem-app
+
diff --git a/run/filesystem/test/e2e_test_setup.yaml b/run/filesystem/test/e2e_test_setup.yaml
@@ -0,0 +1,34 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+steps:
+- id: 'Build and deploy to Cloud Run'
+  name: 'gcr.io/cloud-builders/gcloud:latest'
+  entrypoint: /bin/bash
+  args:
+  - '-c'
+  - |
+    gcloud run deploy $_RUN_SERVICE --source . \
+    --region $_REGION \
+    --vpc-connector $_CONNECTOR_NAME \
+    --execution-environment gen2 \
+    --update-env-vars FILESTORE_IP_ADDRESS=$_FILESTORE_IP_ADDRESS,FILE_SHARE_NAME=$_SHARE_NAME
+
+substitutions:
+  _REGION: us-central1
+  _FILESTORE_IP_ADDRESS: 10.103.89.66 # Existing long-standing resource
+  _SHARE_NAME: filestoresamples # Existing long-standing resource
+  _CONNECTOR_NAME: run-filesystem-e2e-test # Existing long-standing resource
+  _RUN_SERVICE: filesystem-app
+