Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Client: Change spider to access unvisited URLs #5996

Merged
merged 1 commit into from
Dec 10, 2024
Merged

Client: Change spider to access unvisited URLs #5996

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 17 additions & 7 deletions addOns/client/src/main/java/org/zaproxy/addon/client/ExtensionClientIntegration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,6 @@ public class ExtensionClientIntegration extends ExtensionAdaptor {
ExtensionSelenium.class);

private ClientMap clientTree;

private ClientMapPanel clientMapPanel;
private ClientDetailsPanel clientDetailsPanel;
private ClientHistoryPanel clientHistoryPanel;
Expand All @@ -113,16 +112,19 @@ public ExtensionClientIntegration() {
}

@Override
public void hook(ExtensionHook extensionHook) {
super.hook(extensionHook);

public void init() {
clientHistoryTableModel = new ClientHistoryTableModel();
clientTree =
new ClientMap(
new ClientNode(
new ClientSideDetails(
Constant.messages.getString("client.tree.title"), null),
this.getModel().getSession()));
}

@Override
public void hook(ExtensionHook extensionHook) {
super.hook(extensionHook);

scanController = new ClientPassiveScanController();
this.api = new ClientIntegrationAPI(this);
Expand Down Expand Up @@ -377,8 +379,12 @@ public void unload() {
.getExtension(ExtensionSelenium.class);
extSelenium.deregisterBrowserHook(redirectScript);
}
ZAP.getEventBus().unregisterPublisher(clientTree);
ZAP.getEventBus().unregisterConsumer(eventConsumer);
if (clientTree != null) {
ZAP.getEventBus().unregisterPublisher(clientTree);
}
if (eventConsumer != null) {
ZAP.getEventBus().unregisterConsumer(eventConsumer);
}
}

@Override
Expand All @@ -399,6 +405,10 @@ public ClientNode getOrAddClientNode(String url, boolean visited, boolean storag
return this.clientTree.getOrAddNode(url, visited, storage);
}

public ClientNode getClientNode(String url, boolean visited, boolean storage) {
return this.clientTree.getNode(url, visited, storage);
}

public void clientNodeSelected(ClientNode node) {
getClientDetailsPanel().setClientNode(node);
}
Expand Down Expand Up @@ -548,7 +558,7 @@ public int runSpider(String url) {
*/
public int runSpider(String url, ClientOptions options) {
synchronized (spiders) {
ClientSpider cs = new ClientSpider(url, options, spiders.size());
ClientSpider cs = new ClientSpider(this, url, options, spiders.size());
spiders.add(cs);
cs.start();
return spiders.indexOf(cs);
Expand Down
36 changes: 35 additions & 1 deletion addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,9 @@
import org.parosproxy.paros.control.Control;
import org.parosproxy.paros.view.View;
import org.zaproxy.addon.client.ClientMap;
import org.zaproxy.addon.client.ClientNode;
import org.zaproxy.addon.client.ClientOptions;
import org.zaproxy.addon.client.ExtensionClientIntegration;
import org.zaproxy.zap.ZAP;
import org.zaproxy.zap.eventBus.Event;
import org.zaproxy.zap.eventBus.EventConsumer;
Expand Down Expand Up @@ -72,6 +74,7 @@ public class ClientSpider implements EventConsumer {
private ClientOptions options;

private String targetUrl;
private ExtensionClientIntegration extClient;
private ExtensionSelenium extSelenium;

private List<WebDriver> webDriverPool = new ArrayList<>();
Expand All @@ -88,7 +91,9 @@ public class ClientSpider implements EventConsumer {
private int tasksDoneCount;
private int tasksTotalCount;

public ClientSpider(String targetUrl, ClientOptions options, int id) {
public ClientSpider(
ExtensionClientIntegration extClient, String targetUrl, ClientOptions options, int id) {
this.extClient = extClient;
this.targetUrl = targetUrl;
this.options = options;
this.id = id;
Expand All @@ -111,7 +116,36 @@ public void start() {
new ClientSpiderThreadFactory(
"ZAP-ClientSpiderThreadPool-" + id + "-thread-"));

List<String> unvisitedUrls = getUnvisitedUrls();

addTask(targetUrl, options.getInitialLoadTimeInSecs());

// Add all of the known but unvisited URLs otherwise these will get ignored
unvisitedUrls.forEach(url -> addTask(url, options.getInitialLoadTimeInSecs()));
kingthorin marked this conversation as resolved.
Show resolved Hide resolved
}

private List<String> getUnvisitedUrls() {
List<String> urls = new ArrayList<>();
ClientNode targetNode = extClient.getClientNode(targetUrl, false, false);
if (targetUrl.endsWith("/") && targetNode != null) {
// Start up one level as "/" will be a leaf node
getUnvisitedUrls(targetNode.getParent(), urls);
}

return urls;
}

private void getUnvisitedUrls(ClientNode node, List<String> urls) {
String nodeUrl = node.getUserObject().getUrl();
if (nodeUrl.startsWith(targetUrl)
&& !(nodeUrl.length() == targetUrl.length())
&& !node.isStorage()
&& !node.getUserObject().isVisited()) {
urls.add(nodeUrl);
}
for (int i = 0; i < node.getChildCount(); i++) {
getUnvisitedUrls(node.getChildAt(i), urls);
}
}

public synchronized WebDriver getWebDriver() {
Expand Down
34 changes: 24 additions & 10 deletions addOns/client/src/test/java/org/zaproxy/addon/client/ExtensionClientIntegrationUnitTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,21 +35,25 @@
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import org.junit.jupiter.api.Test;
import org.openqa.selenium.WebDriver;
import org.parosproxy.paros.Constant;
import org.parosproxy.paros.control.Control;
import org.parosproxy.paros.extension.ExtensionLoader;
import org.parosproxy.paros.model.Model;
import org.parosproxy.paros.model.Session;
import org.zaproxy.addon.client.spider.ClientSpider;
import org.zaproxy.zap.extension.selenium.Browser;
import org.zaproxy.zap.extension.selenium.ExtensionSelenium;
import org.zaproxy.zap.extension.selenium.internal.FirefoxProfileManager;
import org.zaproxy.zap.utils.I18N;
import org.zaproxy.zap.utils.ZapXmlConfiguration;

class ExtensionClientIntegrationUnitTest {

@Test
void shouldCreatFirefoxPrefFile() throws IOException {
void shouldCreateFirefoxPrefFile() throws IOException {
// Given
ExtensionLoader extensionLoader = mock(ExtensionLoader.class);
Control.initSingletonForTesting(mock(Model.class), extensionLoader);
Expand Down Expand Up @@ -127,23 +131,33 @@ void shouldAddZapProfileToFirefoxPrefIniFile() throws IOException {
@Test
void shouldStartSpider() throws IOException {
// Given
Constant.messages = new I18N(Locale.ENGLISH);
ExtensionLoader extensionLoader = mock(ExtensionLoader.class);
Control.initSingletonForTesting(mock(Model.class), extensionLoader);
Model model = mock(Model.class);
Session session = mock(Session.class);
when(model.getSession()).thenReturn(session);
Control.initSingletonForTesting(model, extensionLoader);
ExtensionSelenium extSel = mock(ExtensionSelenium.class);
when(extensionLoader.getExtension(ExtensionSelenium.class)).thenReturn(extSel);
given(extSel.getProxiedBrowser(anyString(), anyString())).willReturn(mock(WebDriver.class));
ExtensionClientIntegration extClient = new ExtensionClientIntegration();
extClient.initModel(model);
extClient.init();
ClientOptions options = new ClientOptions();
options.load(new ZapXmlConfiguration());
options.setThreadCount(1);

// When
int spiderId = extClient.runSpider("https://www.example.com", options);
ClientSpider spider = extClient.getSpider(spiderId);
boolean isRunning = spider.isRunning();
spider.stop();

// Then
assertEquals(true, isRunning);
try {
// When
int spiderId = extClient.runSpider("https://www.example.com", options);
ClientSpider spider = extClient.getSpider(spiderId);
boolean isRunning = spider.isRunning();
spider.stop();

// Then
assertEquals(true, isRunning);
} finally {
extClient.unload();
}
}
}
68 changes: 63 additions & 5 deletions addOns/client/src/test/java/org/zaproxy/addon/client/spider/ClientSpiderUnitTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,20 +48,23 @@
import org.zaproxy.addon.client.ClientNode;
import org.zaproxy.addon.client.ClientOptions;
import org.zaproxy.addon.client.ClientSideDetails;
import org.zaproxy.addon.client.ExtensionClientIntegration;
import org.zaproxy.zap.ZAP;
import org.zaproxy.zap.extension.selenium.ExtensionSelenium;
import org.zaproxy.zap.utils.ZapXmlConfiguration;

class ClientSpiderUnitTest {

private ExtensionSelenium extSel;
private ExtensionClientIntegration extClient;
private ClientOptions clientOptions;
private ClientMap map;
private WebDriver wd;

@BeforeEach
void setUp() {
Control.initSingletonForTesting(Model.getSingleton(), mock(ExtensionLoader.class));
extClient = mock(ExtensionClientIntegration.class);
extSel = mock(ExtensionSelenium.class);
when(Control.getSingleton().getExtensionLoader().getExtension(ExtensionSelenium.class))
.thenReturn(extSel);
Expand All @@ -82,7 +85,8 @@ void tearDown() {
@Test
void shouldRequestInScopeUrls() {
// Given
ClientSpider spider = new ClientSpider("https://www.example.com/", clientOptions, 1);
ClientSpider spider =
new ClientSpider(extClient, "https://www.example.com/", clientOptions, 1);
Options options = mock(Options.class);
Timeouts timeouts = mock(Timeouts.class, withSettings().defaultAnswer(CALLS_REAL_METHODS));
when(wd.manage()).thenReturn(options);
Expand Down Expand Up @@ -117,7 +121,8 @@ void shouldRequestInScopeUrls() {
@Test
void shouldIgnoreRequestAfterStopped() {
// Given
ClientSpider spider = new ClientSpider("https://www.example.com/", clientOptions, 1);
ClientSpider spider =
new ClientSpider(extClient, "https://www.example.com/", clientOptions, 1);
Options options = mock(Options.class);
Timeouts timeouts = mock(Timeouts.class, withSettings().defaultAnswer(CALLS_REAL_METHODS));
when(wd.manage()).thenReturn(options);
Expand All @@ -140,7 +145,8 @@ void shouldIgnoreRequestAfterStopped() {
@Test
void shouldStartPauseResumeStopSpider() {
// Given
ClientSpider spider = new ClientSpider("https://www.example.com", clientOptions, 1);
ClientSpider spider =
new ClientSpider(extClient, "https://www.example.com", clientOptions, 1);
SpiderStatus statusPostStart;
SpiderStatus statusPostPause;
SpiderStatus statusPostResume;
Expand Down Expand Up @@ -178,7 +184,8 @@ void shouldStartPauseResumeStopSpider() {
void shouldIgnoreUrlsTooDeep() {
// Given
clientOptions.setMaxDepth(5);
ClientSpider spider = new ClientSpider("https://www.example.com/", clientOptions, 1);
ClientSpider spider =
new ClientSpider(extClient, "https://www.example.com/", clientOptions, 1);
Options options = mock(Options.class);
Timeouts timeouts = mock(Timeouts.class, withSettings().defaultAnswer(CALLS_REAL_METHODS));
when(wd.manage()).thenReturn(options);
Expand Down Expand Up @@ -220,7 +227,8 @@ void shouldIgnoreUrlsTooDeep() {
void shouldIgnoreUrlsTooWide() {
// Given
clientOptions.setMaxChildren(4);
ClientSpider spider = new ClientSpider("https://www.example.com/", clientOptions, 1);
ClientSpider spider =
new ClientSpider(extClient, "https://www.example.com/", clientOptions, 1);
Options options = mock(Options.class);
Timeouts timeouts = mock(Timeouts.class, withSettings().defaultAnswer(CALLS_REAL_METHODS));
when(wd.manage()).thenReturn(options);
Expand Down Expand Up @@ -258,6 +266,56 @@ void shouldIgnoreUrlsTooWide() {
assertThat(l6Node, is(notNullValue()));
}

@Test
void shouldVisitKnownUnvisitedUrls() {
// Given
ClientSpider spider =
new ClientSpider(extClient, "https://www.example.com/", clientOptions, 1);
Options options = mock(Options.class);
Timeouts timeouts = mock(Timeouts.class, withSettings().defaultAnswer(CALLS_REAL_METHODS));
when(wd.manage()).thenReturn(options);
when(options.timeouts()).thenReturn(timeouts);
ArgumentCaptor<String> argument = ArgumentCaptor.forClass(String.class);

ClientNode exampleTopNode = getClientNode("https://www.example.com");
ClientNode exampleSlashNode = getClientNode("https://www.example.com/");
ClientNode exampleTest1Node = getClientNode("https://www.example.com/test#1");
ClientNode exampleTest2Node = getClientNode("https://www.example.com/test#2");
ClientNode exampleVisitedNode = getClientNode("https://www.example.com/visited");
exampleVisitedNode.getUserObject().setVisited(true);
exampleTopNode.add(exampleSlashNode);
exampleTopNode.add(exampleTest1Node);
exampleTopNode.add(exampleTest2Node);
exampleTopNode.add(exampleVisitedNode);
when(extClient.getClientNode("https://www.example.com/", false, false))
.thenReturn(exampleSlashNode);

// When
spider.start();

try {
Thread.sleep(200);
} catch (InterruptedException e) {
// Ignore
}
spider.stop();

// Then
verify(wd, atLeastOnce()).get(argument.capture());

List<String> values = argument.getAllValues();
assertThat(
values,
contains(
"https://www.example.com/",
"https://www.example.com/test#1",
"https://www.example.com/test#2"));
}

private ClientNode getClientNode(String url) {
return new ClientNode(new ClientSideDetails(url, url, false, false), false);
}

class SpiderStatus {
private boolean running;
private boolean paused;
Expand Down
Loading