diff --git a/build.gradle b/build.gradle index 4b24472b..97abca5e 100644 --- a/build.gradle +++ b/build.gradle @@ -32,7 +32,7 @@ subprojects { googleCloudStorageVersion = '1.103.1' googleHttpClientVersion = '1.44.1' guavaVersion = '28.2-jre' - guiceVersion = '4.2.3' + guiceVersion = '6.0.0' grpcVersion = '1.60.0' gsonVersion = '2.8.6' jaxbVersion = '2.3.1' @@ -47,7 +47,7 @@ subprojects { protocVersion = protobufVersion snakeyamlVersion = '1.26' junitVersion = '4.13' - mockitoVersion = '2.28.2' + mockitoVersion = '5.12.0' truthVersion = '1.4.0' tcsVersion = '0.0.1' diff --git a/common/src/main/java/com/google/tsunami/common/server/LanguageServerCommand.java b/common/src/main/java/com/google/tsunami/common/server/LanguageServerCommand.java index 5ada887f..1f19e120 100644 --- a/common/src/main/java/com/google/tsunami/common/server/LanguageServerCommand.java +++ b/common/src/main/java/com/google/tsunami/common/server/LanguageServerCommand.java @@ -17,12 +17,14 @@ import com.google.auto.value.AutoValue; import java.time.Duration; +import javax.annotation.Nullable; /** Command to spawn a language server and associated command lines. */ @AutoValue public abstract class LanguageServerCommand { public static LanguageServerCommand create( String serverCommand, + @Nullable String serverAddress, String port, String logId, String outputDir, @@ -33,6 +35,7 @@ public static LanguageServerCommand create( String pollingUri) { return new AutoValue_LanguageServerCommand( serverCommand, + serverAddress, port, logId, outputDir, @@ -45,6 +48,8 @@ public static LanguageServerCommand create( public abstract String serverCommand(); + public abstract String serverAddress(); + public abstract String port(); public abstract String logId(); diff --git a/main/src/main/java/com/google/tsunami/main/cli/LanguageServerOptions.java b/main/src/main/java/com/google/tsunami/main/cli/LanguageServerOptions.java index 989cf4ab..e6a16e0d 100644 --- a/main/src/main/java/com/google/tsunami/main/cli/LanguageServerOptions.java +++ b/main/src/main/java/com/google/tsunami/main/cli/LanguageServerOptions.java @@ -41,6 +41,16 @@ public final class LanguageServerOptions implements CliOption { + " chosen.") public List pluginServerPorts; + @Parameter( + names = "--python-plugin-server-address", + description = "The address for python language server.") + public String pythonPluginServerAddress; + + @Parameter( + names = "--python-plugin-server-port", + description = "The port of the python plugin server to open connection with.") + public Integer pythonPluginServerPort; + @Override public void validate() { if (pluginServerFilenames != null || pluginServerPorts != null) { @@ -81,5 +91,15 @@ public void validate() { pathCounts, portCounts)); } } + + if (pythonPluginServerAddress != null) { + if (!(pythonPluginServerPort <= NetworkEndpointUtils.MAX_PORT_NUMBER + && pythonPluginServerPort > 0)) { + throw new ParameterException( + String.format( + "Python plugin server port out of range. Expected [0, %s], actual %s.", + NetworkEndpointUtils.MAX_PORT_NUMBER, pythonPluginServerPort)); + } + } } } diff --git a/main/src/main/java/com/google/tsunami/main/cli/TsunamiCli.java b/main/src/main/java/com/google/tsunami/main/cli/TsunamiCli.java index 2980370b..cc80bd0c 100644 --- a/main/src/main/java/com/google/tsunami/main/cli/TsunamiCli.java +++ b/main/src/main/java/com/google/tsunami/main/cli/TsunamiCli.java @@ -24,6 +24,7 @@ import com.beust.jcommander.ParameterException; import com.google.common.base.Splitter; import com.google.common.base.Stopwatch; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; @@ -183,6 +184,8 @@ private ImmutableList extractPluginServerArgs( Boolean trustAllSslCertCli = extractCliTrustAllSslCert(args); var paths = extractCliPluginServerArgs(args, "--plugin-server-paths="); var ports = extractCliPluginServerArgs(args, "--plugin-server-ports="); + var pythonServerAddress = extractPythonPluginServerAddress(args); + var pythonServerPort = extractPythonPluginServerPort(args); if (paths.size() != ports.size()) { throw new ParameterException( String.format( @@ -190,7 +193,7 @@ private ImmutableList extractPluginServerArgs( + " Paths: %s. Ports: %s.", paths.size(), ports.size())); } - if (paths.size() == 0) { + if (paths.isEmpty() && Strings.isNullOrEmpty(pythonServerAddress)) { return ImmutableList.of(); } if (tsunamiConfig.getRawConfigData().isEmpty()) { @@ -198,9 +201,24 @@ private ImmutableList extractPluginServerArgs( commands.add( LanguageServerCommand.create( paths.get(i), + "", ports.get(i), + logId, extractOutputDir(args), + trustAllSslCertCli != null && trustAllSslCertCli.booleanValue(), + Duration.ZERO, + "", + 0, + "")); + } + if (!Strings.isNullOrEmpty(pythonServerAddress)) { + commands.add( + LanguageServerCommand.create( + "", + pythonServerAddress, + pythonServerPort, logId, + extractOutputDir(args), trustAllSslCertCli != null && trustAllSslCertCli.booleanValue(), Duration.ZERO, "", @@ -219,6 +237,7 @@ private ImmutableList extractPluginServerArgs( commands.add( LanguageServerCommand.create( paths.get(i), + "", ports.get(i), logId, extractOutputDir(args), @@ -230,16 +249,32 @@ private ImmutableList extractPluginServerArgs( (Integer) ((Map) callbackConfig).get("callback_port"), (String) ((Map) callbackConfig).get("polling_uri"))); } + if (!Strings.isNullOrEmpty(pythonServerAddress)) { + commands.add( + LanguageServerCommand.create( + "", + pythonServerAddress, + pythonServerPort, + logId, + extractOutputDir(args), + trustAllSslCertCli == null + ? trustAllSslCertConfig + : trustAllSslCertCli.booleanValue(), + Duration.ofSeconds((int) ((Map) httpClientConfig).get("connect_timeout_seconds")), + (String) ((Map) callbackConfig).get("callback_address"), + (Integer) ((Map) callbackConfig).get("callback_port"), + (String) ((Map) callbackConfig).get("polling_uri"))); + } return ImmutableList.copyOf(commands); } } @Nullable private Boolean extractCliTrustAllSslCert(String[] args) { - for (int i = 0; i < args.length; ++i) { - if (args[i].startsWith("--http-client-trust-all-certificates")) { - if (args[i].contains("=")) { - return Boolean.valueOf(Iterables.get(Splitter.on('=').split(args[i]), 1)); + for (String arg : args) { + if (arg.startsWith("--http-client-trust-all-certificates")) { + if (arg.contains("=")) { + return Boolean.valueOf(Iterables.get(Splitter.on('=').split(arg), 1)); } else { return true; } @@ -248,10 +283,38 @@ private Boolean extractCliTrustAllSslCert(String[] args) { return null; } + @Nullable + private String extractPythonPluginServerAddress(String[] args) { + for (String arg : args) { + if (arg.startsWith("--python-plugin-server-address")) { + if (arg.contains("=")) { + return Iterables.get(Splitter.on('=').split(arg), 1); + } else { + return null; + } + } + } + return null; + } + + @Nullable + private String extractPythonPluginServerPort(String[] args) { + for (String arg : args) { + if (arg.startsWith("--python-plugin-server-port")) { + if (arg.contains("=")) { + return Iterables.get(Splitter.on('=').split(arg), 1); + } else { + return null; + } + } + } + return null; + } + private String extractOutputDir(String[] args) { - for (int i = 0; i < args.length; ++i) { - if (args[i].startsWith("--scan-results-local-output-filename=")) { - String filename = Iterables.get(Splitter.on('=').split(args[i]), 1) + ": "; + for (String arg : args) { + if (arg.startsWith("--scan-results-local-output-filename=")) { + String filename = Iterables.get(Splitter.on('=').split(arg), 1) + ": "; return Path.of(filename).getParent().toString(); } } @@ -259,9 +322,9 @@ private String extractOutputDir(String[] args) { } private ImmutableList extractCliPluginServerArgs(String[] args, String flag) { - for (int i = 0; i < args.length; ++i) { - if (args[i].startsWith(flag)) { - var count = Iterables.get(Splitter.on('=').split(args[i]), 1); + for (String arg : args) { + if (arg.startsWith(flag)) { + var count = Iterables.get(Splitter.on('=').split(arg), 1); return ImmutableList.copyOf(Splitter.on(',').split(count)); } } @@ -270,9 +333,9 @@ private ImmutableList extractCliPluginServerArgs(String[] args, String f private String extractLogId(String[] args) { // TODO(b/171405612): Use the Flag class instead of manual parsing. - for (int i = 0; i < args.length; ++i) { - if (args[i].startsWith("--log-id=")) { - return Iterables.get(Splitter.on('=').split(args[i]), 1) + ": "; + for (String arg : args) { + if (arg.startsWith("--log-id=")) { + return Iterables.get(Splitter.on('=').split(arg), 1) + ": "; } } return ""; diff --git a/main/src/main/java/com/google/tsunami/main/cli/server/RemoteServerLoader.java b/main/src/main/java/com/google/tsunami/main/cli/server/RemoteServerLoader.java index fa4c67eb..175da5f5 100644 --- a/main/src/main/java/com/google/tsunami/main/cli/server/RemoteServerLoader.java +++ b/main/src/main/java/com/google/tsunami/main/cli/server/RemoteServerLoader.java @@ -19,6 +19,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static java.lang.annotation.RetentionPolicy.RUNTIME; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.flogger.GoogleLogger; import com.google.tsunami.common.command.CommandExecutor; @@ -46,6 +47,8 @@ public class RemoteServerLoader { public ImmutableList runServerProcesses() { logger.atInfo().log("Starting language server processes (if any)..."); return commands.stream() + // Filter out commands that don't need server start up + .filter(command -> !Strings.isNullOrEmpty(command.serverCommand())) .map( command -> runProcess( diff --git a/main/src/test/java/com/google/tsunami/main/cli/LanguageServerOptionsTest.java b/main/src/test/java/com/google/tsunami/main/cli/LanguageServerOptionsTest.java index da2f4b91..8244f995 100644 --- a/main/src/test/java/com/google/tsunami/main/cli/LanguageServerOptionsTest.java +++ b/main/src/test/java/com/google/tsunami/main/cli/LanguageServerOptionsTest.java @@ -59,4 +59,19 @@ public void validate_whenPortNumberOutOfRange_throwsParameterException() { ParameterException.class, options::validate); } + + @Test + public void validate_whenPythonPluginServerPortNumberOutOfRange_throwsParameterException() { + LanguageServerOptions options = new LanguageServerOptions(); + options.pythonPluginServerAddress = "127.0.0.1"; + options.pythonPluginServerPort = -1; + + assertThrows( + "Python plugin server port out of range. Expected [0, " + + NetworkEndpointUtils.MAX_PORT_NUMBER + + "]" + + ", actual -1", + ParameterException.class, + options::validate); + } } diff --git a/main/src/test/java/com/google/tsunami/main/cli/server/RemoteServerLoaderTest.java b/main/src/test/java/com/google/tsunami/main/cli/server/RemoteServerLoaderTest.java index d67a279c..dcd06ec3 100644 --- a/main/src/test/java/com/google/tsunami/main/cli/server/RemoteServerLoaderTest.java +++ b/main/src/test/java/com/google/tsunami/main/cli/server/RemoteServerLoaderTest.java @@ -34,6 +34,7 @@ public void runServerProcess_whenPathExistsAndNormalPort_returnsValidProcessList ImmutableList.of( LanguageServerCommand.create( "/bin/sh", + "", "34567", "34", "/output-here", @@ -50,6 +51,29 @@ public void runServerProcess_whenPathExistsAndNormalPort_returnsValidProcessList assertThat(processList).hasSize(1); assertThat(processList.get(0)).isNotNull(); } + + @Test + public void runServerProcess_whenServerAddressExistsAndNormalPort_returnsEmptyProcessList() { + ImmutableList commands = + ImmutableList.of( + LanguageServerCommand.create( + "", + "127.0.0.1", + "34567", + "34", + "/output-here", + false, + Duration.ofSeconds(10), + "157.34.0.2", + 8080, + "157.34.0.2:8881")); + + RemoteServerLoader loader = + Guice.createInjector(new RemoteServerLoaderModule(commands)) + .getInstance(RemoteServerLoader.class); + var processList = loader.runServerProcesses(); + assertThat(processList).isEmpty(); + } } diff --git a/plugin/src/main/java/com/google/tsunami/plugin/RemoteVulnDetectorLoadingModule.java b/plugin/src/main/java/com/google/tsunami/plugin/RemoteVulnDetectorLoadingModule.java index b2c6da47..9c5c891b 100644 --- a/plugin/src/main/java/com/google/tsunami/plugin/RemoteVulnDetectorLoadingModule.java +++ b/plugin/src/main/java/com/google/tsunami/plugin/RemoteVulnDetectorLoadingModule.java @@ -22,6 +22,7 @@ import com.google.auto.value.AutoAnnotation; import com.google.auto.value.AutoBuilder; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.inject.AbstractModule; import com.google.inject.multibindings.MapBinder; @@ -74,12 +75,21 @@ private ImmutableList getLanguageServerChannels( ImmutableList commands) { return commands.stream() .map( - command -> + command -> { + if (Strings.isNullOrEmpty(command.serverCommand())) { + return NettyChannelBuilder.forTarget( + String.format("%s:%s", command.serverAddress(), command.port())) + .negotiationType(NegotiationType.PLAINTEXT) + .maxInboundMessageSize(MAX_MESSAGE_SIZE) + .build(); + } else { // TODO(b/289462738): Support IPv6 loopback (::1) interface - NettyChannelBuilder.forTarget("127.0.0.1:" + command.port()) + return NettyChannelBuilder.forTarget("127.0.0.1:" + command.port()) .negotiationType(NegotiationType.PLAINTEXT) .maxInboundMessageSize(MAX_MESSAGE_SIZE) - .build()) + .build(); + } + }) .collect(toImmutableList()); } diff --git a/plugin/src/test/java/com/google/tsunami/plugin/RemoteVulnDetectorLoadingModuleTest.java b/plugin/src/test/java/com/google/tsunami/plugin/RemoteVulnDetectorLoadingModuleTest.java index dbae5997..245dc6c5 100644 --- a/plugin/src/test/java/com/google/tsunami/plugin/RemoteVulnDetectorLoadingModuleTest.java +++ b/plugin/src/test/java/com/google/tsunami/plugin/RemoteVulnDetectorLoadingModuleTest.java @@ -55,6 +55,7 @@ public void configure_always_loadsAllRemotePlugins() { var path0 = LanguageServerCommand.create( generateServerName(), + "", "34567", "193", "/output/here", @@ -66,6 +67,7 @@ public void configure_always_loadsAllRemotePlugins() { var path1 = LanguageServerCommand.create( generateServerName(), + "", "34566", "193", "/output/now", @@ -74,10 +76,23 @@ public void configure_always_loadsAllRemotePlugins() { "157.34.0.2", 8080, "157.34.0.2:8881"); + var server0 = + LanguageServerCommand.create( + "", + "127.0.0.1", + "34567", + "193", + "/output/here", + false, + Duration.ofSeconds(10), + "157.34.0.2", + 8080, + "157.34.0.2:8881"); Map remotePlugins = - Guice.createInjector(new RemoteVulnDetectorLoadingModule(ImmutableList.of(path0, path1))) + Guice.createInjector( + new RemoteVulnDetectorLoadingModule(ImmutableList.of(path0, path1, server0))) .getInstance(PLUGIN_BINDING_KEY); - assertThat(remotePlugins).hasSize(2); + assertThat(remotePlugins).hasSize(3); } } diff --git a/plugin_server/py/plugin_server.py b/plugin_server/py/plugin_server.py index 6182f7b9..93e1fc21 100644 --- a/plugin_server/py/plugin_server.py +++ b/plugin_server/py/plugin_server.py @@ -152,6 +152,9 @@ def _configure_plugin_service(server): cls(http_client, payload_generator) for cls in tsunami_plugin.VulnDetector.__subclasses__() ] + logging.info('Configured %d python plugin:', len(plugins)) + for plugin in plugins: + logging.info('\t%s', plugin.GetPluginDefinition().info.name) servicer = plugin_service.PluginServiceServicer( py_plugins=plugins, max_workers=_THREADS.value ) diff --git a/quick_start_advanced.sh b/quick_start_advanced.sh index 13087824..0e043c12 100755 --- a/quick_start_advanced.sh +++ b/quick_start_advanced.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -53,16 +53,21 @@ else fi popd >/dev/null -# Build all plugins. +# Build all google plugins. pushd "${REPOS}/tsunami-security-scanner-plugins/google" >/dev/null printf "\nBuilding all Google plugins ...\n" ./build_all.sh cp build/plugins/*.jar "${PLUGINS}" -mkdir -p "${REPOS}/tsunami-security-scanner/plugin_server/py/py_plugins" +popd >/dev/null + +# Copy over python plugins. # Exclude the python example plugin. -find . -type f -name '*.py' -not -iname '*example_py_vuln_detector*' -exec cp '{}' '${REPOS}/tsunami-security-scanner/plugin_server/py/py_plugins/{}' ';' +pushd "${REPOS}/tsunami-security-scanner-plugins/py_plugins/" >/dev/null +mkdir -p "${REPOS}/tsunami-security-scanner/plugin_server/py/py_plugins" +for py_plugin in `find * -type f -name '*.py' -not -iname '*example_py_vuln_detector*' -not -iname '*_test.py'`; do + cp $py_plugin /usr/local/google/home/anniemao/tsunami/repos/tsunami-security-scanner/plugin_server/py/py_plugins/`basename $py_plugin` +done popd >/dev/null -cd # Build the callback server. pushd "${REPOS}/tsunami-security-scanner-callback-server" >/dev/null @@ -73,7 +78,6 @@ TCS_JAR_FILENAME=$(basename -- "${TCS_JAR}") cp "${TCS_JAR}" "${WD}" cp "${REPOS}/tsunami-security-scanner-callback-server/tcs_config.yaml" "${WD}" popd >/dev/null -cd # Build the scanner. pushd "${REPOS}/tsunami-security-scanner" >/dev/null @@ -108,22 +112,26 @@ printf "\nBuild successful, execute the following command to start the callback printf "\ncd ${WD} && \\\\\n" printf "java -cp \"${TCS_JAR_FILENAME}\" \\\\\n" printf " com.google.tsunami.callbackserver.main.TcsMain \\\\\n" -printf " --custom-config=tcs_config.yaml \\\\\n" +printf " --custom-config=tcs_config.yaml\n" printf "\nBuild successful, execute the following command to start the pythan language server\n" printf "\ncd ${REPOS}/tsunami-security-scanner/plugin_server/py && \\\\\n" -printf "\npython3 -m venv . && source bin/activate && \\\\\n" -printf "java -cp \"${TCS_JAR_FILENAME}\" \\\\\n" -printf " com.google.tsunami.callbackserver.main.TcsMain \\\\\n" -printf " --custom-config=tcs_config.yaml \\\\\n" - -printf "\nBuild successful, execute the following command to scan 127.0.0.1:\n" -printf "\ncd ${WD} && \\\\\n" +printf "python3 -m venv . && source bin/activate && \\\\\n" printf "python3 plugin_server.py \\\\\n" printf " --port=34567 \\\\\n" printf " --trust_all_ssl_cert=true \\\\\n" printf " --timeout_seconds=180 \\\\\n" printf " --callback_address=127.0.0.1 \\\\\n" printf " --callback_port=8881 \\\\\n" -printf " --polling_uri=http://127.0.0.1:8080 \\\\\n" -printf " --plugin-server-ports=34567 \\\\\n" +printf " --polling_uri=http://127.0.0.1:8880\n" + +printf "\nBuild successful, execute the following command to scan 127.0.0.1:\n" +printf "\ncd ${WD} && \\\\\n" +printf "java -cp \"${JAR_FILENAME}:${WD}/plugins/*\" \\\\\n" +printf " -Dtsunami-config.location=${WD}/tsunami_tcs.yaml \\\\\n" +printf " com.google.tsunami.main.cli.TsunamiCli \\\\\n" +printf " --ip-v4-target=127.0.0.1 \\\\\n" +printf " --scan-results-local-output-format=JSON \\\\\n" +printf " --scan-results-local-output-filename=/tmp/tsunami-output.json \\\\\n" +printf " --python-plugin-server-address=127.0.0.1 \\\\\n" +printf " --python-plugin-server-ports=34567 \n"