diff --git a/README.md b/README.md index e16b3d9b..b8389515 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,14 @@ To quickly get started with Tsunami scans, 1. Print example Tsunami command for scanning `127.0.0.1` using the previously generated artifacts. +#### Advanced Configuration with Callback Server and Python Language Server + + 1. execute the following command: + + ``` + bash -c "$(curl -sfL https://raw.githubusercontent.com/google/tsunami-security-scanner/master/quick_start_advanced.sh)" + ``` + ### Docker install 1. start a vulnerable application that can be identified by Tsunami, e.g. an @@ -90,7 +98,7 @@ To quickly get started with Tsunami scans, docker exec -it tsunami bash ``` - ## Contributing +## Contributing Read how to [contribute to Tsunami](docs/contributing.md). diff --git a/plugin_server/py/common/net/http/http_header_fields.py b/plugin_server/py/common/net/http/http_header_fields.py index 2130deaf..a04a27ba 100644 --- a/plugin_server/py/common/net/http/http_header_fields.py +++ b/plugin_server/py/common/net/http/http_header_fields.py @@ -93,7 +93,6 @@ class HttpHeaderFields(enum.Enum): CROSS_ORIGIN_EMBEDDER_POLICY = "Cross-Origin-Embedder-Policy" CROSS_ORIGIN_EMBEDDER_POLICY_REPORT_ONLY = "Cross-Origin-Embedder-Policy-Report-Only" CROSS_ORIGIN_OPENER_POLICY = "Cross-Origin-Opener-Policy" - external code ETAG = "ETag" EXPIRES = "Expires" LAST_MODIFIED = "Last-Modified" @@ -157,7 +156,6 @@ class HttpHeaderFields(enum.Enum): SEC_CH_PREFERS_COLOR_SCHEME = "Sec-CH-Prefers-Color-Scheme" ACCEPT_CH = "Accept-CH" CRITICAL_CH = "Critical-CH" - external code SEC_CH_UA = "Sec-CH-UA" SEC_CH_UA_ARCH = "Sec-CH-UA-Arch" SEC_CH_UA_MODEL = "Sec-CH-UA-Model" diff --git a/plugin_server/py/plugin/payload/payload_utility.py b/plugin_server/py/plugin/payload/payload_utility.py index 0cd9f529..99ba8f7e 100644 --- a/plugin_server/py/plugin/payload/payload_utility.py +++ b/plugin_server/py/plugin/payload/payload_utility.py @@ -24,7 +24,8 @@ def get_parsed_payload() -> list[pg.PayloadDefinition]: be used. """ payload_str = Path(_PATH).read_text() - payload_dict = yaml.safe_load(payload_str) + yaml_parser = yaml.YAML(typ='safe', pure=True) + payload_dict = yaml_parser.load(payload_str) payload_library = json_format.ParseDict(payload_dict, pg.PayloadLibrary()) return _validate_payloads([p for p in payload_library.payloads]) diff --git a/plugin_server/py/requirements.txt b/plugin_server/py/requirements.txt new file mode 100644 index 00000000..f91b4e3a --- /dev/null +++ b/plugin_server/py/requirements.txt @@ -0,0 +1,17 @@ +absl-py==2.1.0 +aenum==3.1.15 +certifi==2024.2.2 +charset-normalizer==3.3.2 +glog==0.3.1 +grpcio==1.63.0 +grpcio-health-checking==1.63.0 +grpcio-reflection==1.63.0 +grpcio-tools==1.63.0 +idna==3.7 +protobuf==5.26.1 +python-gflags==3.1.2 +requests==2.31.0 +ruamel.yaml==0.18.6 +ruamel.yaml.clib==0.2.8 +six==1.16.0 +urllib3==2.2.1 diff --git a/quick_start_advanced.sh b/quick_start_advanced.sh new file mode 100755 index 00000000..13087824 --- /dev/null +++ b/quick_start_advanced.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +WD="${HOME}/tsunami" +REPOS="${WD}/repos" +PLUGINS="${WD}/plugins" +PROTOC="${WD}/protoc" + +mkdir -p "${REPOS}" +mkdir -p "${PLUGINS}" +mkdir -p "${PROTOC}" + +# Clone repos. +pushd "${REPOS}" >/dev/null + +printf "\nFetching source code for Tsunami scanner ...\n" +if [[ ! -d "tsunami-security-scanner" ]] ; then + git clone https://github.com/google/tsunami-security-scanner +else + pushd "tsunami-security-scanner" >/dev/null + git pull origin master + popd >/dev/null +fi +printf "\nFetching source code for Tsunami scanner plugins ...\n" +if [[ ! -d "tsunami-security-scanner-plugins" ]] ; then + git clone https://github.com/google/tsunami-security-scanner-plugins +else + pushd "tsunami-security-scanner-plugins" >/dev/null + git pull origin master + popd >/dev/null +fi +printf "\nFetching source code for Tsunami scanner callback server ...\n" +if [[ ! -d "tsunami-security-scanner-callback-server" ]] ; then + git clone https://github.com/google/tsunami-security-scanner-callback-server +else + pushd "tsunami-security-scanner-callback-server" >/dev/null + git pull origin master + popd >/dev/null +fi +popd >/dev/null + +# Build all plugins. +pushd "${REPOS}/tsunami-security-scanner-plugins/google" >/dev/null +printf "\nBuilding all Google plugins ...\n" +./build_all.sh +cp build/plugins/*.jar "${PLUGINS}" +mkdir -p "${REPOS}/tsunami-security-scanner/plugin_server/py/py_plugins" +# Exclude the python example plugin. +find . -type f -name '*.py' -not -iname '*example_py_vuln_detector*' -exec cp '{}' '${REPOS}/tsunami-security-scanner/plugin_server/py/py_plugins/{}' ';' +popd >/dev/null +cd + +# Build the callback server. +pushd "${REPOS}/tsunami-security-scanner-callback-server" >/dev/null +printf "\nBuilding Tsunami callback server ...\n" +./gradlew shadowJar +TCS_JAR=$(find "${REPOS}/tsunami-security-scanner-callback-server" -name 'tcs-main-*-cli.jar') +TCS_JAR_FILENAME=$(basename -- "${TCS_JAR}") +cp "${TCS_JAR}" "${WD}" +cp "${REPOS}/tsunami-security-scanner-callback-server/tcs_config.yaml" "${WD}" +popd >/dev/null +cd + +# Build the scanner. +pushd "${REPOS}/tsunami-security-scanner" >/dev/null +printf "\nBuilding Tsunami scanner jar file ...\n" +./gradlew shadowJar +JAR=$(find "${REPOS}/tsunami-security-scanner" -name 'tsunami-main-*-cli.jar') +JAR_FILENAME=$(basename -- "${JAR}") +cp "${JAR}" "${WD}" +cp "${REPOS}/tsunami-security-scanner/tsunami_tcs.yaml" "${WD}" +popd >/dev/null + +# Install python libs and generate python proto targets. +pushd "${REPOS}/tsunami-security-scanner/plugin_server/py" >/dev/null +# sudo apt install python3.11-venv +python3 -m venv . +source bin/activate +pip install -r requirements.txt +popd >/dev/null + +pushd "${REPOS}/tsunami-security-scanner/proto" >/dev/null +PROTO_OUT="${REPOS}/tsunami-security-scanner/plugin_server/py" +for proto in `ls *.proto`; do + python -m grpc_tools.protoc -I. --python_out=${PROTO_OUT}/. --grpc_python_out=${PROTO_OUT}/. "${proto}" +done +popd >/dev/null + +pushd "${REPOS}/tsunami-security-scanner-callback-server/proto" >/dev/null +python -m grpc_tools.protoc -I. --python_out=${PROTO_OUT}/. --grpc_python_out=${PROTO_OUT}/. "polling.proto" +popd >/dev/null + +printf "\nBuild successful, execute the following command to start the callback server\n" +printf "\ncd ${WD} && \\\\\n" +printf "java -cp \"${TCS_JAR_FILENAME}\" \\\\\n" +printf " com.google.tsunami.callbackserver.main.TcsMain \\\\\n" +printf " --custom-config=tcs_config.yaml \\\\\n" + +printf "\nBuild successful, execute the following command to start the pythan language server\n" +printf "\ncd ${REPOS}/tsunami-security-scanner/plugin_server/py && \\\\\n" +printf "\npython3 -m venv . && source bin/activate && \\\\\n" +printf "java -cp \"${TCS_JAR_FILENAME}\" \\\\\n" +printf " com.google.tsunami.callbackserver.main.TcsMain \\\\\n" +printf " --custom-config=tcs_config.yaml \\\\\n" + +printf "\nBuild successful, execute the following command to scan 127.0.0.1:\n" +printf "\ncd ${WD} && \\\\\n" +printf "python3 plugin_server.py \\\\\n" +printf " --port=34567 \\\\\n" +printf " --trust_all_ssl_cert=true \\\\\n" +printf " --timeout_seconds=180 \\\\\n" +printf " --callback_address=127.0.0.1 \\\\\n" +printf " --callback_port=8881 \\\\\n" +printf " --polling_uri=http://127.0.0.1:8080 \\\\\n" +printf " --plugin-server-ports=34567 \\\\\n" diff --git a/tsunami_tcs.yaml b/tsunami_tcs.yaml new file mode 100644 index 00000000..3ab34083 --- /dev/null +++ b/tsunami_tcs.yaml @@ -0,0 +1,5 @@ +plugin: + callbackserver: + callback_address: "127.0.0.1" # Running callback server locally + callback_port: 8881 # Make sure to match with ones configured in tcs_config.yaml + polling_uri: "http://127.0.0.1:8880"