From 0188cfec453d4ef3cd0e1b55a6bf4ae95c17a208 Mon Sep 17 00:00:00 2001 From: Simon Bennetts Date: Fri, 6 Dec 2024 15:18:28 +0000 Subject: [PATCH] Client: Change spider to access unvisited URLs Signed-off-by: Simon Bennetts --- .../client/ExtensionClientIntegration.java | 24 +++++-- .../addon/client/spider/ClientSpider.java | 36 +++++++++- .../ExtensionClientIntegrationUnitTest.java | 34 +++++++--- .../client/spider/ClientSpiderUnitTest.java | 68 +++++++++++++++++-- 4 files changed, 139 insertions(+), 23 deletions(-) diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/ExtensionClientIntegration.java b/addOns/client/src/main/java/org/zaproxy/addon/client/ExtensionClientIntegration.java index 38fc4b25221..b947287b329 100644 --- a/addOns/client/src/main/java/org/zaproxy/addon/client/ExtensionClientIntegration.java +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/ExtensionClientIntegration.java @@ -89,7 +89,6 @@ public class ExtensionClientIntegration extends ExtensionAdaptor { ExtensionSelenium.class); private ClientMap clientTree; - private ClientMapPanel clientMapPanel; private ClientDetailsPanel clientDetailsPanel; private ClientHistoryPanel clientHistoryPanel; @@ -113,9 +112,7 @@ public ExtensionClientIntegration() { } @Override - public void hook(ExtensionHook extensionHook) { - super.hook(extensionHook); - + public void init() { clientHistoryTableModel = new ClientHistoryTableModel(); clientTree = new ClientMap( @@ -123,6 +120,11 @@ public void hook(ExtensionHook extensionHook) { new ClientSideDetails( Constant.messages.getString("client.tree.title"), null), this.getModel().getSession())); + } + + @Override + public void hook(ExtensionHook extensionHook) { + super.hook(extensionHook); scanController = new ClientPassiveScanController(); this.api = new ClientIntegrationAPI(this); @@ -377,8 +379,12 @@ public void unload() { .getExtension(ExtensionSelenium.class); extSelenium.deregisterBrowserHook(redirectScript); } - ZAP.getEventBus().unregisterPublisher(clientTree); - ZAP.getEventBus().unregisterConsumer(eventConsumer); + if (clientTree != null) { + ZAP.getEventBus().unregisterPublisher(clientTree); + } + if (eventConsumer != null) { + ZAP.getEventBus().unregisterConsumer(eventConsumer); + } } @Override @@ -399,6 +405,10 @@ public ClientNode getOrAddClientNode(String url, boolean visited, boolean storag return this.clientTree.getOrAddNode(url, visited, storage); } + public ClientNode getClientNode(String url, boolean visited, boolean storage) { + return this.clientTree.getNode(url, visited, storage); + } + public void clientNodeSelected(ClientNode node) { getClientDetailsPanel().setClientNode(node); } @@ -548,7 +558,7 @@ public int runSpider(String url) { */ public int runSpider(String url, ClientOptions options) { synchronized (spiders) { - ClientSpider cs = new ClientSpider(url, options, spiders.size()); + ClientSpider cs = new ClientSpider(this, url, options, spiders.size()); spiders.add(cs); cs.start(); return spiders.indexOf(cs); diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java index f39136d252e..b306e710293 100644 --- a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java @@ -36,7 +36,9 @@ import org.parosproxy.paros.control.Control; import org.parosproxy.paros.view.View; import org.zaproxy.addon.client.ClientMap; +import org.zaproxy.addon.client.ClientNode; import org.zaproxy.addon.client.ClientOptions; +import org.zaproxy.addon.client.ExtensionClientIntegration; import org.zaproxy.zap.ZAP; import org.zaproxy.zap.eventBus.Event; import org.zaproxy.zap.eventBus.EventConsumer; @@ -72,6 +74,7 @@ public class ClientSpider implements EventConsumer { private ClientOptions options; private String targetUrl; + private ExtensionClientIntegration extClient; private ExtensionSelenium extSelenium; private List webDriverPool = new ArrayList<>(); @@ -88,7 +91,9 @@ public class ClientSpider implements EventConsumer { private int tasksDoneCount; private int tasksTotalCount; - public ClientSpider(String targetUrl, ClientOptions options, int id) { + public ClientSpider( + ExtensionClientIntegration extClient, String targetUrl, ClientOptions options, int id) { + this.extClient = extClient; this.targetUrl = targetUrl; this.options = options; this.id = id; @@ -111,7 +116,36 @@ public void start() { new ClientSpiderThreadFactory( "ZAP-ClientSpiderThreadPool-" + id + "-thread-")); + List unvisitedUrls = getUnvisitedUrls(); + addTask(targetUrl, options.getInitialLoadTimeInSecs()); + + // Add all of the known but unvisited URLs otherwise these will get ignored + unvisitedUrls.forEach(url -> addTask(url, options.getInitialLoadTimeInSecs())); + } + + private List getUnvisitedUrls() { + List urls = new ArrayList<>(); + ClientNode targetNode = extClient.getClientNode(targetUrl, false, false); + if (targetUrl.endsWith("/") && targetNode != null) { + // Start up one level as "/" will be a leaf node + getUnvisitedUrls(targetNode.getParent(), urls); + } + + return urls; + } + + private void getUnvisitedUrls(ClientNode node, List urls) { + String nodeUrl = node.getUserObject().getUrl(); + if (nodeUrl.startsWith(targetUrl) + && !(nodeUrl.length() == targetUrl.length()) + && !node.isStorage() + && !node.getUserObject().isVisited()) { + urls.add(nodeUrl); + } + for (int i = 0; i < node.getChildCount(); i++) { + getUnvisitedUrls(node.getChildAt(i), urls); + } } public synchronized WebDriver getWebDriver() { diff --git a/addOns/client/src/test/java/org/zaproxy/addon/client/ExtensionClientIntegrationUnitTest.java b/addOns/client/src/test/java/org/zaproxy/addon/client/ExtensionClientIntegrationUnitTest.java index cd6d38eec62..89248e60f48 100644 --- a/addOns/client/src/test/java/org/zaproxy/addon/client/ExtensionClientIntegrationUnitTest.java +++ b/addOns/client/src/test/java/org/zaproxy/addon/client/ExtensionClientIntegrationUnitTest.java @@ -35,21 +35,25 @@ import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import org.junit.jupiter.api.Test; import org.openqa.selenium.WebDriver; +import org.parosproxy.paros.Constant; import org.parosproxy.paros.control.Control; import org.parosproxy.paros.extension.ExtensionLoader; import org.parosproxy.paros.model.Model; +import org.parosproxy.paros.model.Session; import org.zaproxy.addon.client.spider.ClientSpider; import org.zaproxy.zap.extension.selenium.Browser; import org.zaproxy.zap.extension.selenium.ExtensionSelenium; import org.zaproxy.zap.extension.selenium.internal.FirefoxProfileManager; +import org.zaproxy.zap.utils.I18N; import org.zaproxy.zap.utils.ZapXmlConfiguration; class ExtensionClientIntegrationUnitTest { @Test - void shouldCreatFirefoxPrefFile() throws IOException { + void shouldCreateFirefoxPrefFile() throws IOException { // Given ExtensionLoader extensionLoader = mock(ExtensionLoader.class); Control.initSingletonForTesting(mock(Model.class), extensionLoader); @@ -127,23 +131,33 @@ void shouldAddZapProfileToFirefoxPrefIniFile() throws IOException { @Test void shouldStartSpider() throws IOException { // Given + Constant.messages = new I18N(Locale.ENGLISH); ExtensionLoader extensionLoader = mock(ExtensionLoader.class); - Control.initSingletonForTesting(mock(Model.class), extensionLoader); + Model model = mock(Model.class); + Session session = mock(Session.class); + when(model.getSession()).thenReturn(session); + Control.initSingletonForTesting(model, extensionLoader); ExtensionSelenium extSel = mock(ExtensionSelenium.class); when(extensionLoader.getExtension(ExtensionSelenium.class)).thenReturn(extSel); given(extSel.getProxiedBrowser(anyString(), anyString())).willReturn(mock(WebDriver.class)); ExtensionClientIntegration extClient = new ExtensionClientIntegration(); + extClient.initModel(model); + extClient.init(); ClientOptions options = new ClientOptions(); options.load(new ZapXmlConfiguration()); options.setThreadCount(1); - // When - int spiderId = extClient.runSpider("https://www.example.com", options); - ClientSpider spider = extClient.getSpider(spiderId); - boolean isRunning = spider.isRunning(); - spider.stop(); - - // Then - assertEquals(true, isRunning); + try { + // When + int spiderId = extClient.runSpider("https://www.example.com", options); + ClientSpider spider = extClient.getSpider(spiderId); + boolean isRunning = spider.isRunning(); + spider.stop(); + + // Then + assertEquals(true, isRunning); + } finally { + extClient.unload(); + } } } diff --git a/addOns/client/src/test/java/org/zaproxy/addon/client/spider/ClientSpiderUnitTest.java b/addOns/client/src/test/java/org/zaproxy/addon/client/spider/ClientSpiderUnitTest.java index 6a051a559c7..4bdc1ef759f 100644 --- a/addOns/client/src/test/java/org/zaproxy/addon/client/spider/ClientSpiderUnitTest.java +++ b/addOns/client/src/test/java/org/zaproxy/addon/client/spider/ClientSpiderUnitTest.java @@ -48,6 +48,7 @@ import org.zaproxy.addon.client.ClientNode; import org.zaproxy.addon.client.ClientOptions; import org.zaproxy.addon.client.ClientSideDetails; +import org.zaproxy.addon.client.ExtensionClientIntegration; import org.zaproxy.zap.ZAP; import org.zaproxy.zap.extension.selenium.ExtensionSelenium; import org.zaproxy.zap.utils.ZapXmlConfiguration; @@ -55,6 +56,7 @@ class ClientSpiderUnitTest { private ExtensionSelenium extSel; + private ExtensionClientIntegration extClient; private ClientOptions clientOptions; private ClientMap map; private WebDriver wd; @@ -62,6 +64,7 @@ class ClientSpiderUnitTest { @BeforeEach void setUp() { Control.initSingletonForTesting(Model.getSingleton(), mock(ExtensionLoader.class)); + extClient = mock(ExtensionClientIntegration.class); extSel = mock(ExtensionSelenium.class); when(Control.getSingleton().getExtensionLoader().getExtension(ExtensionSelenium.class)) .thenReturn(extSel); @@ -82,7 +85,8 @@ void tearDown() { @Test void shouldRequestInScopeUrls() { // Given - ClientSpider spider = new ClientSpider("https://www.example.com/", clientOptions, 1); + ClientSpider spider = + new ClientSpider(extClient, "https://www.example.com/", clientOptions, 1); Options options = mock(Options.class); Timeouts timeouts = mock(Timeouts.class, withSettings().defaultAnswer(CALLS_REAL_METHODS)); when(wd.manage()).thenReturn(options); @@ -117,7 +121,8 @@ void shouldRequestInScopeUrls() { @Test void shouldIgnoreRequestAfterStopped() { // Given - ClientSpider spider = new ClientSpider("https://www.example.com/", clientOptions, 1); + ClientSpider spider = + new ClientSpider(extClient, "https://www.example.com/", clientOptions, 1); Options options = mock(Options.class); Timeouts timeouts = mock(Timeouts.class, withSettings().defaultAnswer(CALLS_REAL_METHODS)); when(wd.manage()).thenReturn(options); @@ -140,7 +145,8 @@ void shouldIgnoreRequestAfterStopped() { @Test void shouldStartPauseResumeStopSpider() { // Given - ClientSpider spider = new ClientSpider("https://www.example.com", clientOptions, 1); + ClientSpider spider = + new ClientSpider(extClient, "https://www.example.com", clientOptions, 1); SpiderStatus statusPostStart; SpiderStatus statusPostPause; SpiderStatus statusPostResume; @@ -178,7 +184,8 @@ void shouldStartPauseResumeStopSpider() { void shouldIgnoreUrlsTooDeep() { // Given clientOptions.setMaxDepth(5); - ClientSpider spider = new ClientSpider("https://www.example.com/", clientOptions, 1); + ClientSpider spider = + new ClientSpider(extClient, "https://www.example.com/", clientOptions, 1); Options options = mock(Options.class); Timeouts timeouts = mock(Timeouts.class, withSettings().defaultAnswer(CALLS_REAL_METHODS)); when(wd.manage()).thenReturn(options); @@ -220,7 +227,8 @@ void shouldIgnoreUrlsTooDeep() { void shouldIgnoreUrlsTooWide() { // Given clientOptions.setMaxChildren(4); - ClientSpider spider = new ClientSpider("https://www.example.com/", clientOptions, 1); + ClientSpider spider = + new ClientSpider(extClient, "https://www.example.com/", clientOptions, 1); Options options = mock(Options.class); Timeouts timeouts = mock(Timeouts.class, withSettings().defaultAnswer(CALLS_REAL_METHODS)); when(wd.manage()).thenReturn(options); @@ -258,6 +266,56 @@ void shouldIgnoreUrlsTooWide() { assertThat(l6Node, is(notNullValue())); } + @Test + void shouldVisitKnownUnvisitedUrls() { + // Given + ClientSpider spider = + new ClientSpider(extClient, "https://www.example.com/", clientOptions, 1); + Options options = mock(Options.class); + Timeouts timeouts = mock(Timeouts.class, withSettings().defaultAnswer(CALLS_REAL_METHODS)); + when(wd.manage()).thenReturn(options); + when(options.timeouts()).thenReturn(timeouts); + ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); + + ClientNode exampleTopNode = getClientNode("https://www.example.com"); + ClientNode exampleSlashNode = getClientNode("https://www.example.com/"); + ClientNode exampleTest1Node = getClientNode("https://www.example.com/test#1"); + ClientNode exampleTest2Node = getClientNode("https://www.example.com/test#2"); + ClientNode exampleVisitedNode = getClientNode("https://www.example.com/visited"); + exampleVisitedNode.getUserObject().setVisited(true); + exampleTopNode.add(exampleSlashNode); + exampleTopNode.add(exampleTest1Node); + exampleTopNode.add(exampleTest2Node); + exampleTopNode.add(exampleVisitedNode); + when(extClient.getClientNode("https://www.example.com/", false, false)) + .thenReturn(exampleSlashNode); + + // When + spider.start(); + + try { + Thread.sleep(200); + } catch (InterruptedException e) { + // Ignore + } + spider.stop(); + + // Then + verify(wd, atLeastOnce()).get(argument.capture()); + + List values = argument.getAllValues(); + assertThat( + values, + contains( + "https://www.example.com/", + "https://www.example.com/test#1", + "https://www.example.com/test#2")); + } + + private ClientNode getClientNode(String url) { + return new ClientNode(new ClientSideDetails(url, url, false, false), false); + } + class SpiderStatus { private boolean running; private boolean paused;