Create a setup script to run Tsunami with callback server & python pl…

…ugins.

PiperOrigin-RevId: 635828583
Change-Id: I4598e3e9a2c3d50885be19306f66af9192bb7de9

README.md

@@ -60,6 +60,14 @@ To quickly get started with Tsunami scans,
  1.  Print example Tsunami command for scanning `127.0.0.1` using the previously
      generated artifacts.
 
+#### Advanced Configuration with Callback Server and Python Language Server
+
+ 1.  execute the following command:
+
+     ```
+     bash -c "$(curl -sfL https://raw.githubusercontent.com/google/tsunami-security-scanner/master/quick_start_advanced.sh)"
+     ```
+
 ### Docker install
 
 1.  start a vulnerable application that can be identified by Tsunami, e.g. an
@@ -90,7 +98,7 @@ To quickly get started with Tsunami scans,
     docker exec -it tsunami bash
     ```
 
-    ## Contributing
+## Contributing
 
 Read how to [contribute to Tsunami](docs/contributing.md).
 

plugin_server/py/common/net/http/http_header_fields.py

@@ -93,7 +93,6 @@ class HttpHeaderFields(enum.Enum):
   CROSS_ORIGIN_EMBEDDER_POLICY = "Cross-Origin-Embedder-Policy"
   CROSS_ORIGIN_EMBEDDER_POLICY_REPORT_ONLY = "Cross-Origin-Embedder-Policy-Report-Only"
   CROSS_ORIGIN_OPENER_POLICY = "Cross-Origin-Opener-Policy"
-  external code
   ETAG = "ETag"
   EXPIRES = "Expires"
   LAST_MODIFIED = "Last-Modified"
@@ -157,7 +156,6 @@ class HttpHeaderFields(enum.Enum):
   SEC_CH_PREFERS_COLOR_SCHEME = "Sec-CH-Prefers-Color-Scheme"
   ACCEPT_CH = "Accept-CH"
   CRITICAL_CH = "Critical-CH"
-  external code
   SEC_CH_UA = "Sec-CH-UA"
   SEC_CH_UA_ARCH = "Sec-CH-UA-Arch"
   SEC_CH_UA_MODEL = "Sec-CH-UA-Model"

plugin_server/py/plugin/payload/payload_utility.py

@@ -24,7 +24,8 @@ def get_parsed_payload() -> list[pg.PayloadDefinition]:
           be used.
   """
   payload_str = Path(_PATH).read_text()
-  payload_dict = yaml.safe_load(payload_str)
+  yaml_parser = yaml.YAML(typ='safe', pure=True)
+  payload_dict = yaml_parser.load(payload_str)
   payload_library = json_format.ParseDict(payload_dict, pg.PayloadLibrary())
   return _validate_payloads([p for p in payload_library.payloads])
 

plugin_server/py/requirements.txt

@@ -0,0 +1,17 @@
+absl-py==2.1.0
+aenum==3.1.15
+certifi==2024.2.2
+charset-normalizer==3.3.2
+glog==0.3.1
+grpcio==1.63.0
+grpcio-health-checking==1.63.0
+grpcio-reflection==1.63.0
+grpcio-tools==1.63.0
+idna==3.7
+protobuf==5.26.1
+python-gflags==3.1.2
+requests==2.31.0
+ruamel.yaml==0.18.6
+ruamel.yaml.clib==0.2.8
+six==1.16.0
+urllib3==2.2.1

quick_start_advanced.sh

@@ -0,0 +1,129 @@
+#!/bin/bash
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+WD="${HOME}/tsunami"
+REPOS="${WD}/repos"
+PLUGINS="${WD}/plugins"
+PROTOC="${WD}/protoc"
+
+mkdir -p "${REPOS}"
+mkdir -p "${PLUGINS}"
+mkdir -p "${PROTOC}"
+
+# Clone repos.
+pushd "${REPOS}" >/dev/null
+
+printf "\nFetching source code for Tsunami scanner ...\n"
+if [[ ! -d "tsunami-security-scanner" ]] ; then
+  git clone https://github.com/google/tsunami-security-scanner
+else
+  pushd "tsunami-security-scanner" >/dev/null
+  git pull origin master
+  popd >/dev/null
+fi
+printf "\nFetching source code for Tsunami scanner plugins ...\n"
+if [[ ! -d "tsunami-security-scanner-plugins" ]] ; then
+  git clone https://github.com/google/tsunami-security-scanner-plugins
+else
+  pushd "tsunami-security-scanner-plugins" >/dev/null
+  git pull origin master
+  popd >/dev/null
+fi
+printf "\nFetching source code for Tsunami scanner callback server ...\n"
+if [[ ! -d "tsunami-security-scanner-callback-server" ]] ; then
+  git clone https://github.com/google/tsunami-security-scanner-callback-server
+else
+  pushd "tsunami-security-scanner-callback-server" >/dev/null
+  git pull origin master
+  popd >/dev/null
+fi
+popd >/dev/null
+
+# Build all plugins.
+pushd "${REPOS}/tsunami-security-scanner-plugins/google" >/dev/null
+printf "\nBuilding all Google plugins ...\n"
+./build_all.sh
+cp build/plugins/*.jar "${PLUGINS}"
+mkdir -p "${REPOS}/tsunami-security-scanner/plugin_server/py/py_plugins"
+# Exclude the python example plugin.
+find . -type f -name '*.py' -not -iname '*example_py_vuln_detector*' -exec cp '{}' '${REPOS}/tsunami-security-scanner/plugin_server/py/py_plugins/{}' ';'
+popd >/dev/null
+cd
+
+# Build the callback server.
+pushd "${REPOS}/tsunami-security-scanner-callback-server" >/dev/null
+printf "\nBuilding Tsunami callback server ...\n"
+./gradlew shadowJar
+TCS_JAR=$(find "${REPOS}/tsunami-security-scanner-callback-server" -name 'tcs-main-*-cli.jar')
+TCS_JAR_FILENAME=$(basename -- "${TCS_JAR}")
+cp "${TCS_JAR}" "${WD}"
+cp "${REPOS}/tsunami-security-scanner-callback-server/tcs_config.yaml" "${WD}"
+popd >/dev/null
+cd
+
+# Build the scanner.
+pushd "${REPOS}/tsunami-security-scanner" >/dev/null
+printf "\nBuilding Tsunami scanner jar file ...\n"
+./gradlew shadowJar
+JAR=$(find "${REPOS}/tsunami-security-scanner" -name 'tsunami-main-*-cli.jar')
+JAR_FILENAME=$(basename -- "${JAR}")
+cp "${JAR}" "${WD}"
+cp "${REPOS}/tsunami-security-scanner/tsunami_tcs.yaml" "${WD}"
+popd >/dev/null
+
+# Install python libs and generate python proto targets.
+pushd "${REPOS}/tsunami-security-scanner/plugin_server/py" >/dev/null
+# sudo apt install python3.11-venv
+python3 -m venv .
+source bin/activate
+pip install -r requirements.txt
+popd >/dev/null
+
+pushd "${REPOS}/tsunami-security-scanner/proto" >/dev/null
+PROTO_OUT="${REPOS}/tsunami-security-scanner/plugin_server/py"
+for proto in `ls *.proto`; do
+  python -m grpc_tools.protoc -I. --python_out=${PROTO_OUT}/. --grpc_python_out=${PROTO_OUT}/. "${proto}"
+done
+popd >/dev/null
+
+pushd "${REPOS}/tsunami-security-scanner-callback-server/proto" >/dev/null
+python -m grpc_tools.protoc -I. --python_out=${PROTO_OUT}/. --grpc_python_out=${PROTO_OUT}/. "polling.proto"
+popd >/dev/null
+
+printf "\nBuild successful, execute the following command to start the callback server\n"
+printf "\ncd ${WD} && \\\\\n"
+printf "java -cp \"${TCS_JAR_FILENAME}\" \\\\\n"
+printf "  com.google.tsunami.callbackserver.main.TcsMain \\\\\n"
+printf "  --custom-config=tcs_config.yaml \\\\\n"
+
+printf "\nBuild successful, execute the following command to start the pythan language server\n"
+printf "\ncd ${REPOS}/tsunami-security-scanner/plugin_server/py && \\\\\n"
+printf "\npython3 -m venv . && source bin/activate && \\\\\n"
+printf "java -cp \"${TCS_JAR_FILENAME}\" \\\\\n"
+printf "  com.google.tsunami.callbackserver.main.TcsMain \\\\\n"
+printf "  --custom-config=tcs_config.yaml \\\\\n"
+
+printf "\nBuild successful, execute the following command to scan 127.0.0.1:\n"
+printf "\ncd ${WD} && \\\\\n"
+printf "python3 plugin_server.py \\\\\n"
+printf "  --port=34567 \\\\\n"
+printf "  --trust_all_ssl_cert=true \\\\\n"
+printf "  --timeout_seconds=180 \\\\\n"
+printf "  --callback_address=127.0.0.1 \\\\\n"
+printf "  --callback_port=8881 \\\\\n"
+printf "  --polling_uri=http://127.0.0.1:8080  \\\\\n"
+printf "  --plugin-server-ports=34567  \\\\\n"

tsunami_tcs.yaml

@@ -0,0 +1,5 @@
+plugin:
+  callbackserver:
+    callback_address: "127.0.0.1"  # Running callback server locally
+    callback_port: 8881            # Make sure to match with ones configured in tcs_config.yaml
+    polling_uri: "http://127.0.0.1:8880"