Upgrade Ubuntu 22.04 geckodriver/firefox crashes

[chainlaboratory]

Dear Google Colab Team,

Until July 18, 2023, the following code worked perfectly:

from google.colab import drive
drive.mount('/content/drive')

!cp /content/drive/MyDrive/KI/geckodriver /usr/local/bin/geckodriver

!pip install selenium > /dev/null

!pip install pyvirtualdisplay selenium webdriver_manager > /dev/null

!apt-get update

!apt install firefox #> /dev/null

from selenium import webdriver
from webdriver_manager.firefox import GeckoDriverManager
from selenium.webdriver.firefox.service import Service
from selenium.webdriver.firefox.options import Options as FirefoxOptions

#from selenium.webdriver.common.keys import Keys
#from selenium.webdriver.common.by import By

#from selenium.webdriver import ActionChains
#from datetime import datetime, date
#import time
#import os.path

def selenium_firefox_agent(driver_path="/usr/local/bin/geckodriver"):
#def selenium_firefox_agent(driver_path="/bin/geckodriver"):
firefox_driver_path = f'{driver_path}'
user_agent = 'Mozilla'
firefox_options = FirefoxOptions()
firefox_options.add_argument("-headless")
firefox_options.add_argument("-width=1920")
firefox_options.add_argument("-height=1080")
firefox_options.set_preference("general.useragent.override", user_agent)
driver = webdriver.Firefox(service=Service(GeckoDriverManager().install()), options=firefox_options)

return driver

driver = selenium_firefox_agent()

Since the upgrade to Ubuntu 22.04, this code no longer works, and I am receiving the following error message:

in selenium_firefox_agent(driver_path)
22 firefox_options.add_argument("-height=1080")
23 firefox_options.set_preference("general.useragent.override", user_agent)
---> 24 driver = webdriver.Firefox(service=Service(GeckoDriverManager().install()), options=firefox_options)
25
26 #time.sleep(2)

/usr/local/lib/python3.10/dist-packages/selenium/webdriver/firefox/webdriver.py in init(self, options, service, keep_alive)
66 keep_alive=self.keep_alive,
67 )
---> 68 super().init(command_executor=executor, options=self.options)
69
70 self._is_remote = False

/usr/local/lib/python3.10/dist-packages/selenium/webdriver/remote/webdriver.py in init(self, command_executor, keep_alive, file_detector, options)
204 self._authenticator_id = None
205 self.start_client()
--> 206 self.start_session(capabilities)
207
208 def repr(self):

/usr/local/lib/python3.10/dist-packages/selenium/webdriver/remote/webdriver.py in start_session(self, capabilities)
289
290 caps = _create_caps(capabilities)
--> 291 response = self.execute(Command.NEW_SESSION, caps)["value"]
292 self.session_id = response.get("sessionId")
293 self.caps = response.get("capabilities")

/usr/local/lib/python3.10/dist-packages/selenium/webdriver/remote/webdriver.py in execute(self, driver_command, params)
344 response = self.command_executor.execute(driver_command, params)
345 if response:
--> 346 self.error_handler.check_response(response)
347 response["value"] = self._unwrap_value(response.get("value", None))
348 return response

/usr/local/lib/python3.10/dist-packages/selenium/webdriver/remote/errorhandler.py in check_response(self, response)
243 alert_text = value["alert"].get("text")
244 raise exception_class(message, screen, stacktrace, alert_text) # type: ignore[call-arg] # mypy is not smart enough here
--> 245 raise exception_class(message, screen, stacktrace)

WebDriverException: Message: Process unexpectedly closed with status 1

I am using the latest geckodriver version 0.33.0.

Do you have any solutions to resolve this issue?

Best regards,
Daniel Siegmann
dsiegmann@bluewin.ch

[metrizable]

@chainlaboratory Thanks for reporting the issue and thanks for using Colab.

Your issue is likely related to the recent Ubuntu 22.04 upgrade (#3778 ) we performed. Firefox on Ubuntu 22.04, by default, is distributed as a snap package. Firefox can be installed in various other ways, for example, downloading it and ensuring the environment can discover and use it properly.

As an aside, I noticed that the geckodriver that you specify in your example is from your Google Drive, which may or may not be compatible with Ubuntu 22.04.

Although YMMV, riffing on the example code you provided, I was able to successfully install Firefox and geckodriver, and pull data from web pages with the following:

%%shell
wget 'https://download.mozilla.org/?product=firefox-latest&os=linux64&lang=en-US' -O firefox.tar.bz2
tar -xjf firefox.tar.bz2
mv firefox /opt/
ln -s /opt/firefox/firefox /usr/local/bin/firefox

wget 'https://github.com/mozilla/geckodriver/releases/download/v0.33.0/geckodriver-v0.33.0-linux64.tar.gz'
tar -xzf geckodriver*.tar.gz
chmod +x geckodriver
mv geckodriver /usr/local/bin/

apt-get install libxtst6 libdbus-glib-1-2

pip install selenium webdriver_manager

import os

# Add firefox-related .so files to library path
os.environ['LD_LIBRARY_PATH'] += ':/opt/firefox'

from selenium import webdriver
from webdriver_manager.firefox import GeckoDriverManager
from selenium.webdriver.firefox.service import Service
from selenium.webdriver.firefox.options import Options

opts = Options()
opts.binary_location = '/usr/local/bin/firefox'
opts.add_argument('--headless')

service = Service(executable_path=GeckoDriverManager().install())
driver = webdriver.Firefox(service=service, options=opts)

driver.get('http://www.python.org')
print(driver.title)

[chainlaboratory]

many thanks

[chainlaboratory]

Many thanks, great work Von: Eric Johnson ***@***.***> Gesendet: Donnerstag, 20. Juli 2023 19:40 An: googlecolab/colabtools ***@***.***> Cc: Daniel Siegmann ***@***.***>; Mention ***@***.***> Betreff: Re: [googlecolab/colabtools] Upgrade Ubuntu 22.04 geckodriver/firefox crashes (Issue #3861) Closed #3861 <#3861> as completed. — Reply to this email directly, view it on GitHub <#3861 (comment)> , or unsubscribe <https://github.com/notifications/unsubscribe-auth/AJOAT3DAI2QSZTN3CUAQDADXRFUNPANCNFSM6AAAAAA2RJPEJA> . You are receiving this because you were mentioned.Message ID: ***@***.***>

[Looker1981]

i have problem with my script help me plz

pytest /usr/local/bin/ciaobot/ciao.py --indirizzo "https://www.test.it" --profilo "ciao1"
========================================================== test session starts ==========================================================
platform linux -- Python 3.10.12, pytest-8.0.2, pluggy-1.4.0
rootdir: /usr/local/bin/ciaobot
plugins: devtools-0.12.2, anyio-4.3.0
collected 1 item

../usr/local/bin/ciao/ciao.py F [100%]

=============================================================== FAILURES ================================================================
______________________________________________________________ test_fbpost ______________________________________________________________

params = {'filelista': None, 'gruppo': None, 'indirizzo': 'https://www.test.it', 'messaggio': None, ...}

def test_fbpost(params):
  profile_path = '/root/.mozilla/firefox-esr/' + params['profilo']
  options = Options()
  options.add_argument('-profile')
  options.add_argument(profile_path)
  options.add_argument("--width=1665")
  options.add_argument("--height=1720")
  options.set_preference('permissions.default.image', 2)
  options.set_preference('layout.css.devPixelsPerPx', '0.6')

  service = Service('/usr/local/bin/geckodriver', log_path='/dev/null')

driver = Firefox(service=service, options=options)

/usr/local/bin/ciaobot/ciao.py:34:

---

/usr/local/lib/python3.10/dist-packages/selenium/webdriver/firefox/webdriver.py:69: in init
super().init(command_executor=executor, options=options)
/usr/local/lib/python3.10/dist-packages/selenium/webdriver/remote/webdriver.py:208: in init
self.start_session(capabilities)
/usr/local/lib/python3.10/dist-packages/selenium/webdriver/remote/webdriver.py:292: in start_session
response = self.execute(Command.NEW_SESSION, caps)["value"]
/usr/local/lib/python3.10/dist-packages/selenium/webdriver/remote/webdriver.py:347: in execute
self.error_handler.check_response(response)

---

self = <selenium.webdriver.remote.errorhandler.ErrorHandler object at 0x7f1232ca8490>
response = {'status': 500, 'value': '{"value":{"error":"unknown error","message":"Process unexpectedly closed with status 1","stacktrace":""}}'}

def check_response(self, response: Dict[str, Any]) -> None:
    """Checks that a JSON response from the WebDriver does not have an
    error.

    :Args:
     - response - The JSON response from the WebDriver server as a dictionary
       object.

    :Raises: If the response contains an error message.
    """
    status = response.get("status", None)
    if not status or status == ErrorCode.SUCCESS:
        return
    value = None
    message = response.get("message", "")
    screen: str = response.get("screen", "")
    stacktrace = None
    if isinstance(status, int):
        value_json = response.get("value", None)
        if value_json and isinstance(value_json, str):
            import json

            try:
                value = json.loads(value_json)
                if len(value) == 1:
                    value = value["value"]
                status = value.get("error", None)
                if not status:
                    status = value.get("status", ErrorCode.UNKNOWN_ERROR)
                    message = value.get("value") or value.get("message")
                    if not isinstance(message, str):
                        value = message
                        message = message.get("message")
                else:
                    message = value.get("message", None)
            except ValueError:
                pass

    exception_class: Type[WebDriverException]
    e = ErrorCode()
    error_codes = [item for item in dir(e) if not item.startswith("__")]
    for error_code in error_codes:
        error_info = getattr(ErrorCode, error_code)
        if isinstance(error_info, list) and status in error_info:
            exception_class = getattr(ExceptionMapping, error_code, WebDriverException)
            break
    else:
        exception_class = WebDriverException

    if not value:
        value = response["value"]
    if isinstance(value, str):
        raise exception_class(value)
    if message == "" and "message" in value:
        message = value["message"]

    screen = None  # type: ignore[assignment]
    if "screen" in value:
        screen = value["screen"]

    stacktrace = None
    st_value = value.get("stackTrace") or value.get("stacktrace")
    if st_value:
        if isinstance(st_value, str):
            stacktrace = st_value.split("\n")
        else:
            stacktrace = []
            try:
                for frame in st_value:
                    line = frame.get("lineNumber", "")
                    file = frame.get("fileName", "<anonymous>")
                    if line:
                        file = f"{file}:{line}"
                    meth = frame.get("methodName", "<anonymous>")
                    if "className" in frame:
                        meth = f"{frame['className']}.{meth}"
                    msg = "    at %s (%s)"
                    msg = msg % (meth, file)
                    stacktrace.append(msg)
            except TypeError:
                pass
    if exception_class == UnexpectedAlertPresentException:
        alert_text = None
        if "data" in value:
            alert_text = value["data"].get("text")
        elif "alert" in value:
            alert_text = value["alert"].get("text")
        raise exception_class(message, screen, stacktrace, alert_text)  # type: ignore[call-arg]  # mypy is not smart enough here

  raise exception_class(message, screen, stacktrace)

E selenium.common.exceptions.WebDriverException: Message: Process unexpectedly closed with status 1

/usr/local/lib/python3.10/dist-packages/selenium/webdriver/remote/errorhandler.py:229: WebDriverException
======================================================== short test summary info ========================================================
FAILED ../usr/local/bin/ciaobot/ciao.py::test_fbpost - selenium.common.exceptions.WebDriverException: Message: Process unexpectedly closed with status 1
========================================================== 1 failed in 10.21s ===========================================================