Merge pull request #39 from vincenzocaputo/develop

Develop

CHANGELOG.md

@@ -4,6 +4,23 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.14.0] - 2023-03
+### Added
+- New popup main page reporting the number of indicators for each type found in the current visiting webpage
+- Each button on the popup main page can be clicked to display indicators of the corresponding type
+- Defanged indicators detection and collection from the current visiting webpage
+- Twitter, Leakpeek, DNS History, BreachDirectory, Phishunt, CleanTalk resources
+
+### Changed
+- Improved support for defanged indicators
+- Removed badge text when the indicators count is 0
+- Removed ThreatCrowd (service no more available)
+
+### Fixed
+- Tools list loading at startup
+- Indicators counting when a page is first loaded in a new tab
+- Acceptance of IP addresses with a 0 at the beginning of one or more octects
+
 ## [0.13.1] - 2023-03-06
 ### Fixed
 - Bug: the number in the badge text is not being updated when a new page is loaded

manifest.json

@@ -1,20 +1,15 @@
 {
 	"manifest_version": 2,
 	"name": "FoxyRecon",
-	"version": "0.13.1",
+	"version": "0.14.0",
 	"description": "A Firefox add-on for OSINT investigations",
 
 	"icons": {
 		"16": "assets/icons/foxyrecon-icon-16.png",
 		"48": "assets/icons/foxyrecon-icon-48.png"
 	},
 
-    "web_accessible_resources": [
-        "assets/icons/foxyrecon-icon-32.png",
-        "assets/icons/settings.png"
-    ],
-
-    "content_security_policy": "default-src 'self'; img-src 'self' data:",
+    "content_security_policy": "default-src 'self'; img-src 'self' data:; script-src 'self'; style-src 'self'",
 
     "background": {
         "scripts": [
@@ -46,7 +41,9 @@
                 "https://www.eurodns.com/whois-search",
                 "https://www.virustotal.com/gui/home/url/",
                 "https://app.any.run/submissions/",
-                "https://leakcheck.io/"
+                "https://leakcheck.io/",
+                "https://leakpeek.com/",
+                "https://breachdirectory.org/"
             ],
             "js": ["src/content_scripts/autofill.js"]
         }

src/background.js

@@ -28,44 +28,61 @@ loadToolsList(function(ts) {
  * @param {toolsList} list of available tools
  */
 function createToolsMenu(toolsList) {
-    for (i=0; i<toolsList.length; i++){
-        let tool = toolsList[i];
-        // Create menu entry
-        browser.contextMenus.create({
-            id: i.toString(), // Incremental ID
-            title: tool["name"], // Tool name
-            contexts: ["selection"], // Show menu on selected text
-            /*icons: {
-                16: browser.runtime.getURL(tool["icon"]),
-            },*/
-            visible: true,
-        });
+    if(toolsList) {
+        for (i=0; i<toolsList.length; i++){
+            let tool = toolsList[i];
+            // Create menu entry
+            browser.contextMenus.create({
+                id: i.toString(), // Incremental ID
+                title: tool["name"], // Tool name
+                contexts: ["selection"], // Show menu on selected text
+                /*icons: {
+                    16: browser.runtime.getURL(tool["icon"]),
+                },*/
+                visible: true,
+            });
+        }
     }
 }
 
+/**
+ * Harvest and collect the indicators present in the current webpage. Save the list in the local storage.
+ */
 function catchIndicators(e) {
     browser.tabs.query({active:true, lastFocusedWindow: true}).then(tabs => {    
-        let activeTab = tabs[0].id;    
-        // Send a message to the content script    
-        browser.tabs.sendMessage(activeTab, "catch");    
-        let token = 1;    
-        browser.runtime.onMessage.addListener(function(message) {    
-            if(token) {              
-                // No indicators found. Show a message    
-                if(message['indicators'] == "[]") {    
-                    browser.browserAction.setBadgeText({text: "0"});
-                } else {                        
-                    const indicatorsList = JSON.parse(message['indicators']);    
-                    browser.browserAction.setBadgeText({text: indicatorsList.length.toString()});    
-                }    
-            }    
-            // Consume token    
-            token = 0;    
-        })    
-    },     
-    error => {    
-        browser.browserAction.setBadgeText({text: "0"});
-        console.error("Error: "+error)
+        let activeTab = tabs[0].id;
+        console.log(activeTab);
+        // Send a message to the content script
+        browser.tabs.sendMessage(activeTab, "catch")
+                    .then((response) => {
+                        console.log(response);
+                    })
+                    .catch((error) => {
+                        browser.browserAction.setBadgeText({text: ""});
+                        localStorage.setItem("catched_indicators", "[]");
+                    });
+        let token = 1;
+        browser.runtime.onMessage.addListener(function(message) {
+            if(token) {
+                const indicatorsListJson = message['indicators'];
+                // Save the indicators list in the local storage
+                localStorage.setItem("catched_indicators", indicatorsListJson);
+                // No indicators found. Show a message
+                if(indicatorsListJson == "[]") {
+                    browser.browserAction.setBadgeText({text: ""});
+                } else {
+                    const indicatorsList = JSON.parse(indicatorsListJson);
+                    browser.browserAction.setBadgeText({text: indicatorsList.length.toString()});
+                }
+            } 
+            // Consume token
+            token = 0;
+        })
+    },
+    error => {
+        browser.browserAction.setBadgeText({text: ""});
+        localStorage.setItem("catched_indicators", "[]");
+        console.error("Error: "+error);
     });
 }
 /**
@@ -74,7 +91,6 @@ function catchIndicators(e) {
 browser.tabs.onActivated.addListener(catchIndicators);
 browser.tabs.onUpdated.addListener(catchIndicators);
 browser.tabs.onCreated.addListener(catchIndicators);
-document.addEventListener("readystatechange", catchIndicators);
 
 /**
  * Updates context menu making visible only the tools which are compatible with the selected string

src/content_scripts/autofill.js

@@ -1,84 +1,86 @@
 
 let indicator = "";
 // Send a message to background script in order to retrieve the indicator saved in the local storage
-browser.runtime.sendMessage({
-    id: 1,
-    msg: ""
-}).then((resp)=>{
-    indicator = resp.msg;
-    // Get the query to find submit button
-    query = resp.query;
-    if(query) {
-        // Check if auto-submit is enabled
-        submit = resp.submit;
+window.addEventListener("load", function() {
+    browser.runtime.sendMessage({
+        id: 1,
+        msg: ""
+    }).then((resp)=>{
+        indicator = resp.msg;
+        // Get the query to find submit button
+        query = resp.query;
+        if(query) {
+            // Check if auto-submit is enabled
+            submit = resp.submit;
 
-        let current_url = window.location.href;
-        if(current_url.includes("urlscan")) {
-            // Get input field
-            inputNode = document.getElementById("url");
-            document.getElementById(query).click();
-            inputNode.value = indicator;
-                setTimeout(() => {
-                    document.getElementById("submitbtn_text").click();
-                }, 1000);
-        } else if(current_url.includes("virustotal")) {
-            window.addEventListener('load', function () {
-                    // Get input field
-                    inputNode = document.querySelector('home-view').shadowRoot.querySelector('#urlSearchInput');
-                    inputNode.value = indicator;
-                setTimeout(() => {
-                    // "touch" the input field
-                    inputNode.dispatchEvent(new Event('input'));
-                }, 100);
-                setTimeout(() => {
-                    // Fill
-                    if(submit === "true" && query === "VT") {
-                        // after 100ms press "enter"
-                        document.querySelector('home-view').shadowRoot.querySelector('#searchUrlForm').dispatchEvent(new Event("submit"));
-                    }
-                }, 100);
-            })
-        } else if(current_url.includes("centralops")) {
-            // Fill the input field
-            document.getElementById("addr").value = indicator;
-
-            // Select checkboxes
-            document.getElementById("dom_whois").checked = true;
-            document.getElementById("net_whois").checked = true;
-            document.getElementById("dom_dns").checked = true;
+            let current_url = window.location.href;
+            if(current_url.includes("urlscan")) {
+                // Get input field
+                inputNode = document.getElementById("url");
+                document.getElementById(query).click();
+                inputNode.value = indicator;
+                    setTimeout(() => {
+                        document.getElementById("submitbtn_text").click();
+                    }, 1000);
+            } else if(current_url.includes("virustotal")) {
+                window.addEventListener('load', function () {
+                        // Get input field
+                        inputNode = document.querySelector('home-view').shadowRoot.querySelector('#urlSearchInput');
+                        inputNode.value = indicator;
+                    setTimeout(() => {
+                        // "touch" the input field
+                        inputNode.dispatchEvent(new Event('input'));
+                    }, 100);
+                    setTimeout(() => {
+                        // Fill
+                        if(submit === "true" && query === "VT") {
+                            // after 100ms press "enter"
+                            document.querySelector('home-view').shadowRoot.querySelector('#searchUrlForm').dispatchEvent(new Event("submit"));
+                        }
+                    }, 100);
+                })
+            } else if(current_url.includes("centralops")) {
+                // Fill the input field
+                document.getElementById("addr").value = indicator;
 
-            if(submit === "true") {
-                document.querySelector(query).click();
-            }
+                // Select checkboxes
+                document.getElementById("dom_whois").checked = true;
+                document.getElementById("net_whois").checked = true;
+                document.getElementById("dom_dns").checked = true;
 
-        } else if(current_url.includes("eurodns")) {
-            let inputNode = document.getElementsByTagName("textarea")[0];
+                if(submit === "true") {
+                    document.querySelector(query).click();
+                }
 
-            inputNode.value = indicator;
-            if(submit === "true") {
-                document.querySelector(query).click();
-            }
+            } else if(current_url.includes("eurodns")) {
+                let inputNode = document.getElementsByTagName("textarea")[0];
 
-        } else if(current_url.includes("any.run")) {
-            document.querySelector("#history-filterBtn").click();
-            document.querySelector("#hashSearch").value = indicator;
-            document.querySelector(query).click();
-        } else {
-            var inputNodes = document.getElementsByTagName("input");
-            console.log(inputNodes);
-            // Get only text or email input nodes
-            for(i=0; i<inputNodes.length; i++){
-                if(inputNodes[i].type === "text" || inputNodes[i].type === "email" || inputNodes[i].type === "url"){
-                    // Fill the input field
-                    inputNodes[i].value = indicator;
+                inputNode.value = indicator;
+                if(submit === "true") {
+                    document.querySelector(query).click();
                 }
-            }
-            if(submit === "true") {
+
+            } else if(current_url.includes("any.run")) {
+                document.querySelector("#history-filterBtn").click();
+                document.querySelector("#hashSearch").value = indicator;
                 document.querySelector(query).click();
+            } else {
+                var inputNodes = document.getElementsByTagName("input");
+                console.log(inputNodes);
+                // Get only text or email input nodes
+                for(i=0; i<inputNodes.length; i++){
+                    if(inputNodes[i].type === "text" || inputNodes[i].type === "email" || inputNodes[i].type === "url"){
+                        // Fill the input field
+                        inputNodes[i].value = indicator;
+                    }
+                }
+                if(submit === "true") {
+                    document.querySelector(query).click();
+                }
             }
         }
-    }
 
-},(error)=>{
-    console.error(error);
-});   
+    },(error)=>{
+        console.error(error);
+    });   
+});

src/content_scripts/catch.js

@@ -1,39 +1,48 @@
 const regexes = {
-    'domain': new RegExp(/((?!-)[_A-Za-z0-9-]{1,63}(?<!-)\.)+[A-Za-z]{2,6}/,'g'),
-    'ip': new RegExp(/((?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))*/,'g'),
-    'url': new RegExp(/(?:http[s]?):\/\/((?:www\.)?[-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6})\b(?:[-a-zA-Z0-9@:%_\+.~#?&//=]*)/,'g'),
+    'domain': new RegExp(/((?!-)[_A-Za-z0-9-]{1,63}(?<!-)(?:(\[\.\]|\.)))+[A-Za-z]{2,6}/,'g'),
+    'ip': new RegExp(/(?!0)((2[0-4][0-9]|25[0-5]|1[0-9][0-9]|[1-9][0-9]|\d)(?:(\[\.\]|\.))){3}(2[0-4][0-9]|25[0-5]|1[0-9][0-9]|[1-9][0-9]|\d)/,'g'),
+    'url': new RegExp(/(?:h(xx|XX|tt)p[s]?):\/\/((?:www(?:(\[\.\]|\.)))?[-a-zA-Z0-9@:%._\+~#=]{2,256}(?:(\[\.\]|\.))[a-z]{2,6})\b(?:[-a-zA-Z0-9@:%_\+.~#?&//=]*)/,'g'),
     'hash': new RegExp(/([a-z0-9]{64})|([a-z0-9]{40})|([a-z0-9]{32})/,'g'), 
-    'email': new RegExp(/[a-z0-9]+(\.[_a-z0-9]+)*@([a-z0-9-]+(\.[a-z0-9-]+)*(\.[a-z]{2,15}))/,'g'),
+    'email': new RegExp(/[a-z0-9]+(\.[_a-z0-9]+)*(\[at\]|@)([a-z0-9-]+((?:(\[\.\]|\.))[a-z0-9-]+)*((?:(\[\.\]|\.))[a-z]{2,15}))/,'g'),
     'cve': new RegExp(/CVE-\d{4}-\d{4,7}/,'g')
 }
 
-browser.runtime.onMessage.addListener(function(message) {
-    if (message === "catch") {
-        let indParser = new IndicatorParser();
-        const bodyContent = document.body.innerText;
-
-        let indicators = [];
-        for(indicatorType of ['domain', 'ip', 'url', 'hash', 'email', 'cve']) {
-            let matches = bodyContent.matchAll(regexes[indicatorType]);
-            let match = matches.next();
-            while(!match.done) {
-                let value = match.value[0];
-                if(value) {
-                    indicators.push({'type': indicatorType, 'value': match.value[0]});
+function catchIndicators() {
+    let indParser = new IndicatorParser();
+    const bodyContent = document.body.innerText;
+
+    let indicators = [];
+    for(indicatorType of ['domain', 'ip', 'url', 'hash', 'email', 'cve']) {
+        let matches = bodyContent.matchAll(regexes[indicatorType]);
+        let match = matches.next();
+        while(!match.done) {
+            let value = match.value[0];
+            if(value) {
+                let [type, tld] = indParser.getIndicatorType(value);
+                if(type == "defanged") {
+                    refangedValue = indParser.refangIndicator(value);
+                    [type, tld] = indParser.getIndicatorType(refangedValue);
                 }
-                match = matches.next();
+                indicators.push({'type': type, 'value': value});
             }
+            match = matches.next();
         }
-        if(indicators) {
-            browser.runtime.sendMessage({
-             "indicators": JSON.stringify(indicators)
-            }).then(message=>{console.log(message)},error=>{console.error(error)});
-        }
+    }
+    if(indicators) {
+        browser.runtime.sendMessage({
+         "indicators": JSON.stringify(indicators)
+        }).then(message=>{console.log(message)},error=>{console.error(error)});
+    }
+}
 
+browser.runtime.onMessage.addListener(function(message) {
+    if (message === "catch") {
+        catchIndicators();
     } else if (message['cmd'] === 'find') {
         if(!window.find(message['indicator'])) {
             window.find(message['indicator'], false, true, false, false, false, false);
         }
     }
 });
 
+window.addEventListener("load", catchIndicators);