From 948b4175cd5758e9249e3130378656bf0bfd7b7a Mon Sep 17 00:00:00 2001 From: ankushduacodes Date: Sat, 31 Oct 2020 22:28:47 +0530 Subject: [PATCH 1/9] remove support for random_attributes, ua_strings, screen_res --- automation/BrowserManager.py | 1 - automation/DeployBrowsers/deploy_firefox.py | 38 +-------------------- automation/default_browser_params.json | 1 - 3 files changed, 1 insertion(+), 39 deletions(-) diff --git a/automation/BrowserManager.py b/automation/BrowserManager.py index 1bb96183b..b8505da1f 100644 --- a/automation/BrowserManager.py +++ b/automation/BrowserManager.py @@ -111,7 +111,6 @@ def launch_browser_manager(self): # make sure browser loads crashed profile self.browser_params['recovery_tar'] = tempdir # don't re-randomize attributes - self.browser_params['random_attributes'] = False crash_recovery = True else: """ diff --git a/automation/DeployBrowsers/deploy_firefox.py b/automation/DeployBrowsers/deploy_firefox.py index 54665e08c..45b6e9cb7 100755 --- a/automation/DeployBrowsers/deploy_firefox.py +++ b/automation/DeployBrowsers/deploy_firefox.py @@ -82,40 +82,6 @@ def deploy_firefox(status_queue, browser_params, manager_params, crash_recovery) ) status_queue.put(("STATUS", "Profile Tar", None)) - if browser_params["random_attributes"] and profile_settings is None: - logger.debug( - "BROWSER %i: Loading random attributes for browser" - % browser_params["browser_id"] - ) - profile_settings = dict() - - # choose a random screen-res from list - resolutions = list() - with open(os.path.join(root_dir, "screen_resolutions.txt"), "r") as f: - for line in f: - resolutions.append(tuple(line.strip().split(","))) - profile_settings["screen_res"] = random.choice(resolutions) - - # set a random user agent from list - ua_strings = list() - with open(os.path.join(root_dir, "user_agent_strings.txt"), "r") as f: - for line in f: - ua_strings.append(line.strip()) - profile_settings["ua_string"] = random.choice(ua_strings) - - # If profile settings still not set - set defaults - if profile_settings is None: - profile_settings = dict() - profile_settings["screen_res"] = DEFAULT_SCREEN_RES - profile_settings["ua_string"] = None - - if profile_settings["ua_string"] is not None: - logger.debug( - "BROWSER %i: Overriding user agent string to '%s'" - % (browser_params["browser_id"], profile_settings["ua_string"]) - ) - fo.set_preference("general.useragent.override", profile_settings["ua_string"]) - display_mode = browser_params["display_mode"] display_pid = None display_port = None @@ -125,7 +91,7 @@ def deploy_firefox(status_queue, browser_params, manager_params, crash_recovery) fo.add_argument("--height={}".format(DEFAULT_SCREEN_RES[1])) if display_mode == "xvfb": try: - display = Display(visible=0, size=profile_settings["screen_res"]) + display = Display(visible=0) display.start() display_pid, display_port = display.pid, display.cmd_param[-1][1:] except EasyProcessError: @@ -226,8 +192,6 @@ def deploy_firefox(status_queue, browser_params, manager_params, crash_recovery) % browser_params["browser_id"] ) - # set window size - driver.set_window_size(*profile_settings["screen_res"]) # Get browser process pid if hasattr(driver, "service") and hasattr(driver.service, "process"): diff --git a/automation/default_browser_params.json b/automation/default_browser_params.json index 3ee729a5e..eeb4ce4b5 100644 --- a/automation/default_browser_params.json +++ b/automation/default_browser_params.json @@ -8,7 +8,6 @@ "save_content": false, "callstack_instrument": false, "dns_instrument": false, - "random_attributes": false, "bot_mitigation": false, "seed_tar": null, "recovery_tar": null, From 7f615bc4e0f39601af62ab5550944f30287e91d6 Mon Sep 17 00:00:00 2001 From: ankushduacodes Date: Sat, 31 Oct 2020 23:34:08 +0530 Subject: [PATCH 2/9] add DEFAULT_SCREEN_SIZE replacing screen_res --- automation/DeployBrowsers/deploy_firefox.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/automation/DeployBrowsers/deploy_firefox.py b/automation/DeployBrowsers/deploy_firefox.py index 45b6e9cb7..0f4884089 100755 --- a/automation/DeployBrowsers/deploy_firefox.py +++ b/automation/DeployBrowsers/deploy_firefox.py @@ -91,7 +91,7 @@ def deploy_firefox(status_queue, browser_params, manager_params, crash_recovery) fo.add_argument("--height={}".format(DEFAULT_SCREEN_RES[1])) if display_mode == "xvfb": try: - display = Display(visible=0) + display = Display(visible=0, size=DEFAULT_SCREEN_RES) display.start() display_pid, display_port = display.pid, display.cmd_param[-1][1:] except EasyProcessError: @@ -192,6 +192,8 @@ def deploy_firefox(status_queue, browser_params, manager_params, crash_recovery) % browser_params["browser_id"] ) + # set window size + driver.set_window_size(*DEFAULT_SCREEN_RES) # Get browser process pid if hasattr(driver, "service") and hasattr(driver.service, "process"): From 4ec583fc9da27d5a5a5cbf7a0eb8ee6957015878 Mon Sep 17 00:00:00 2001 From: ankushduacodes <61025943+ankushduacodes@users.noreply.github.com> Date: Mon, 2 Nov 2020 19:56:35 +0530 Subject: [PATCH 3/9] Update automation/BrowserManager.py Co-authored-by: Stefan Zabka --- automation/BrowserManager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/BrowserManager.py b/automation/BrowserManager.py index b8505da1f..3e8d548d2 100644 --- a/automation/BrowserManager.py +++ b/automation/BrowserManager.py @@ -110,7 +110,7 @@ def launch_browser_manager(self): ) # make sure browser loads crashed profile self.browser_params['recovery_tar'] = tempdir - # don't re-randomize attributes + crash_recovery = True else: """ From b704c62b89c3d6bb93d8752a2df2e1cb91cbc070 Mon Sep 17 00:00:00 2001 From: ankushduacodes <61025943+ankushduacodes@users.noreply.github.com> Date: Mon, 2 Nov 2020 19:56:43 +0530 Subject: [PATCH 4/9] Update automation/default_browser_params.json Co-authored-by: Stefan Zabka --- automation/default_browser_params.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/default_browser_params.json b/automation/default_browser_params.json index f170203a7..6334abc92 100644 --- a/automation/default_browser_params.json +++ b/automation/default_browser_params.json @@ -8,7 +8,7 @@ "save_content": false, "callstack_instrument": false, "dns_instrument": false, - "bot_mitigation": false, + "seed_tar": null, "display_mode": "native", "browser": "firefox", From 5c9b7941eb3ddbb352df32f93011bef27acaedba Mon Sep 17 00:00:00 2001 From: ankushduacodes Date: Mon, 2 Nov 2020 20:57:50 +0530 Subject: [PATCH 5/9] removed random attrs property from json file removed some comments mentioning screen_res and ua_strings --- automation/BrowserManager.py | 1 - automation/default_browser_params.json | 4 ---- 2 files changed, 5 deletions(-) diff --git a/automation/BrowserManager.py b/automation/BrowserManager.py index 3e8d548d2..4dbc48402 100644 --- a/automation/BrowserManager.py +++ b/automation/BrowserManager.py @@ -75,7 +75,6 @@ def __init__(self, manager_params, browser_params) -> None: self.restart_required = False self.current_timeout: Optional[int] = None # timeout of the current command - # dict of additional browser profile settings (e.g. screen_res) self.browser_settings = None self.browser_manager = None # process that controls browser diff --git a/automation/default_browser_params.json b/automation/default_browser_params.json index 6334abc92..82f53784a 100644 --- a/automation/default_browser_params.json +++ b/automation/default_browser_params.json @@ -8,18 +8,14 @@ "save_content": false, "callstack_instrument": false, "dns_instrument": false, - "seed_tar": null, "display_mode": "native", "browser": "firefox", "prefs": {}, "tp_cookies": "always", - "bot_mitigation": false, "profile_archive_dir": null, "recovery_tar": null, - - "random_attributes": false, "donottrack": false, "tracking-protection": false } From a5d7ff50cfcce7704c917321452ab7c0e924b96f Mon Sep 17 00:00:00 2001 From: ankushduacodes Date: Wed, 4 Nov 2020 17:19:59 +0530 Subject: [PATCH 6/9] removed traces of screen_res & ua_string from docs --- automation/Commands/profile_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/Commands/profile_commands.py b/automation/Commands/profile_commands.py index c8b0404cf..e8fd81eef 100644 --- a/automation/Commands/profile_commands.py +++ b/automation/Commands/profile_commands.py @@ -13,7 +13,7 @@ def save_browser_settings(location, browser_settings): """ browser_settings stores additional profile config parameters - e.g. screen_res, plugin sets, user_agent string + e.g. plugin sets """ if browser_settings is not None: # see if the browser_settings file exists, and if so delete From f95a38c8c6406e7f75ce63eabb292c80bd0f123a Mon Sep 17 00:00:00 2001 From: ankushduacodes Date: Wed, 4 Nov 2020 18:40:43 +0530 Subject: [PATCH 7/9] Removed traces of browser settings --- automation/BrowserManager.py | 10 ++---- automation/Commands/command_executor.py | 3 -- automation/Commands/profile_commands.py | 34 --------------------- automation/DeployBrowsers/deploy_firefox.py | 9 +++--- 4 files changed, 6 insertions(+), 50 deletions(-) diff --git a/automation/BrowserManager.py b/automation/BrowserManager.py index 4dbc48402..c2efecbef 100644 --- a/automation/BrowserManager.py +++ b/automation/BrowserManager.py @@ -75,7 +75,6 @@ def __init__(self, manager_params, browser_params) -> None: self.restart_required = False self.current_timeout: Optional[int] = None # timeout of the current command - self.browser_settings = None self.browser_manager = None # process that controls browser self.logger = logging.getLogger("openwpm") @@ -105,7 +104,6 @@ def launch_browser_manager(self): self.browser_params, tempdir, close_webdriver=False, - browser_settings=self.browser_settings ) # make sure browser loads crashed profile self.browser_params['recovery_tar'] = tempdir @@ -164,9 +162,7 @@ def check_queue(launch_status): # 4. Browser launch attempted check_queue(launch_status) # 5. Browser launched - (self.geckodriver_pid, self.browser_settings) = check_queue( - launch_status - ) + (self.geckodriver_pid) = check_queue(launch_status) (driver_profile_path, ready) = check_queue(launch_status) if ready != "READY": @@ -415,7 +411,6 @@ def shutdown_browser(self, during_init: bool, force: bool = False) -> None: self.browser_params, self.browser_params['profile_archive_dir'], close_webdriver=False, - browser_settings=self.browser_settings, compress=True ) """ @@ -438,7 +433,7 @@ def BrowserManager( logger = logging.getLogger("openwpm") try: # Start Xvfb (if necessary), webdriver, and browser - driver, prof_folder, browser_settings = deploy_browser.deploy_browser( + driver, prof_folder = deploy_browser.deploy_browser( status_queue, browser_params, manager_params, crash_recovery ) if prof_folder[-1] != "/": @@ -520,7 +515,6 @@ def BrowserManager( command_executor.execute_command( command, driver, - browser_settings, browser_params, manager_params, extension_socket, diff --git a/automation/Commands/command_executor.py b/automation/Commands/command_executor.py index be447e603..6ae9527d3 100644 --- a/automation/Commands/command_executor.py +++ b/automation/Commands/command_executor.py @@ -17,7 +17,6 @@ def execute_command( command, webdriver, - browser_settings, browser_params, manager_params, extension_socket, @@ -55,7 +54,6 @@ def execute_command( tar_location=command.dump_folder, close_webdriver=command.close_webdriver, webdriver=webdriver, - browser_settings=browser_settings, compress=command.compress, ) @@ -97,7 +95,6 @@ def execute_command( arg_dict = { "command": command, "driver": webdriver, - "browser_settings": browser_settings, "browser_params": browser_params, "manager_params": manager_params, "extension_socket": extension_socket, diff --git a/automation/Commands/profile_commands.py b/automation/Commands/profile_commands.py index e8fd81eef..04f0e223c 100644 --- a/automation/Commands/profile_commands.py +++ b/automation/Commands/profile_commands.py @@ -10,30 +10,6 @@ logger = logging.getLogger("openwpm") -def save_browser_settings(location, browser_settings): - """ - browser_settings stores additional profile config parameters - e.g. plugin sets - """ - if browser_settings is not None: - # see if the browser_settings file exists, and if so delete - if os.path.isfile(location + "browser_settings.p"): - os.remove(location + "browser_settings.p") - - with open(location + "browser_settings.p", "wb") as f: - pickle.dump(browser_settings, f) - - -def load_browser_settings(location): - """ loads the browser settings from a pickled dictionary in """ - try: - with open(location + "browser_settings.p", "rb") as f: - browser_settings = pickle.load(f) - except IOError: - browser_settings = None - return browser_settings - - def dump_profile( browser_profile_folder, manager_params, @@ -41,13 +17,11 @@ def dump_profile( tar_location, close_webdriver, webdriver=None, - browser_settings=None, compress=False, ): """ dumps a browser profile currently stored in to in which both folders are absolute paths. - if exists they are also saved """ logger.debug( "BROWSER %i: Profile dumping is currently unsupported. " @@ -135,10 +109,6 @@ def dump_profile( tar.add(full_path, arcname=item) tar.close() - # save the browser settings - if browser_settings is not None: - save_browser_settings(tar_location, browser_settings) - def load_profile(browser_profile_folder, manager_params, browser_params, tar_location): """ @@ -178,13 +148,9 @@ def load_profile(browser_profile_folder, manager_params, browser_params, tar_loc os.remove(browser_profile_folder + tar_name) logger.debug("BROWSER %i: Tarfile extracted" % browser_params["browser_id"]) - # load the browser settings - browser_settings = load_browser_settings(tar_location) except Exception as ex: logger.critical( "BROWSER %i: Error: %s while attempting to load profile" % (browser_params["browser_id"], str(ex)) ) raise ProfileLoadError("Profile Load not successful") - - return browser_settings diff --git a/automation/DeployBrowsers/deploy_firefox.py b/automation/DeployBrowsers/deploy_firefox.py index 0f4884089..0bc1bcd7c 100755 --- a/automation/DeployBrowsers/deploy_firefox.py +++ b/automation/DeployBrowsers/deploy_firefox.py @@ -57,13 +57,12 @@ def deploy_firefox(status_queue, browser_params, manager_params, crash_recovery) # https://github.com/SeleniumHQ/selenium/issues/2106#issuecomment-320238039 fo = Options() - profile_settings = None # Imported browser settings if browser_params["seed_tar"] and not crash_recovery: logger.info( "BROWSER %i: Loading initial browser profile from: %s" % (browser_params["browser_id"], browser_params["seed_tar"]) ) - profile_settings = load_profile( + load_profile( browser_profile_path, manager_params, browser_params, @@ -74,7 +73,7 @@ def deploy_firefox(status_queue, browser_params, manager_params, crash_recovery) "BROWSER %i: Loading recovered browser profile from: %s" % (browser_params["browser_id"], browser_params["recovery_tar"]) ) - profile_settings = load_profile( + load_profile( browser_profile_path, manager_params, browser_params, @@ -203,6 +202,6 @@ def deploy_firefox(status_queue, browser_params, manager_params, crash_recovery) else: raise RuntimeError("Unable to identify Firefox process ID.") - status_queue.put(("STATUS", "Browser Launched", (int(pid), profile_settings))) + status_queue.put(("STATUS", "Browser Launched", (int(pid)))) - return driver, driver.capabilities["moz:profile"], profile_settings + return driver, driver.capabilities["moz:profile"] From b24484fa25138335e01a1d77553f9dae7dca5634 Mon Sep 17 00:00:00 2001 From: ankushduacodes <61025943+ankushduacodes@users.noreply.github.com> Date: Wed, 4 Nov 2020 19:52:44 +0530 Subject: [PATCH 8/9] Update automation/DeployBrowsers/deploy_firefox.py Co-authored-by: Stefan Zabka --- automation/DeployBrowsers/deploy_firefox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/DeployBrowsers/deploy_firefox.py b/automation/DeployBrowsers/deploy_firefox.py index 0bc1bcd7c..c0dec67ac 100755 --- a/automation/DeployBrowsers/deploy_firefox.py +++ b/automation/DeployBrowsers/deploy_firefox.py @@ -202,6 +202,6 @@ def deploy_firefox(status_queue, browser_params, manager_params, crash_recovery) else: raise RuntimeError("Unable to identify Firefox process ID.") - status_queue.put(("STATUS", "Browser Launched", (int(pid)))) + status_queue.put(("STATUS", "Browser Launched", int(pid))) return driver, driver.capabilities["moz:profile"] From 0414adeb221ade969cdaca41f2c39890aa15eaa0 Mon Sep 17 00:00:00 2001 From: ankushduacodes <61025943+ankushduacodes@users.noreply.github.com> Date: Wed, 4 Nov 2020 19:53:13 +0530 Subject: [PATCH 9/9] Update automation/BrowserManager.py Co-authored-by: Stefan Zabka --- automation/BrowserManager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/BrowserManager.py b/automation/BrowserManager.py index c2efecbef..48066a560 100644 --- a/automation/BrowserManager.py +++ b/automation/BrowserManager.py @@ -162,7 +162,7 @@ def check_queue(launch_status): # 4. Browser launch attempted check_queue(launch_status) # 5. Browser launched - (self.geckodriver_pid) = check_queue(launch_status) + self.geckodriver_pid = check_queue(launch_status) (driver_profile_path, ready) = check_queue(launch_status) if ready != "READY":