Cherrypy ssl error #354

DogAteMyCode · 2020-12-27T18:40:28Z

It is a simple cherrypy server, however after the server running fine for 1-2 hours I get this error and it becomes unresponsive. How can I fix this

StackOverflow
I added this to stack overflow if you prefer to see it there

[27/Dec/2020:09:46:29] ENGINE Error in HTTPServer.serve
Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/cheroot/server.py", line 1810, in serve
    self._connections.run(self.expiration_interval)
  File "/usr/local/lib/python3.9/site-packages/cheroot/connections.py", line 201, in run
    self._run(expiration_interval)
  File "/usr/local/lib/python3.9/site-packages/cheroot/connections.py", line 218, in _run
    new_conn = self._from_server_socket(self.server.socket)
  File "/usr/local/lib/python3.9/site-packages/cheroot/connections.py", line 271, in _from_server_socket
    s, ssl_env = self.server.ssl_adapter.wrap(s)
  File "/usr/local/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap
    s = self.context.wrap_socket(
  File "/usr/local/Cellar/python@3.9/3.9.0_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/ssl.py", line 500, in wrap_socket
    return self.sslsocket_class._create(
  File "/usr/local/Cellar/python@3.9/3.9.0_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/ssl.py", line 1040, in _create
    self.do_handshake()
  File "/usr/local/Cellar/python@3.9/3.9.0_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/ssl.py", line 1309, in do_handshake
    self._sslobj.do_handshake()
ssl.SSLError: [SSL: UNEXPECTED_RECORD] unexpected record (_ssl.c:1122)

This is the server code

import cherrypy
import json
import sys
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import threading

WEBSITEURL = "http"
STORAGEDIR = sys.argv[0].replace("__main__.py", "") + '/'


def sendMessage(msgToSend, toWhom, decay):
    """
    Keeps trying to send message until success, decay defines an expiry after a certain amount of attempts



    :param msgToSend: What to send
    :param toWhom: Who will recieve the message
    :param decay: What amount of retries on failure
    :return: Nothing
    """
    for i in range(decay):
        result = order(msgToSend, toWhom)
        if result:
            break


def order(msgToSend, toWhom):
    """
    Send Email from email@gmail.com



    :param msgToSend: Message to send
    :param toWhom: Email to send message to
    :return: True if success, nothing on failure
    """
    from_address = "email@gmail.com"
    to_address = f"{toWhom}"
    # Create message container - the correct MIME type is multipart/alternative.
    msg = MIMEMultipart('alternative')
    msg['Subject'] = "Pedido"
    msg['From'] = from_address
    msg['To'] = to_address
    msg['Bcc'] = "email@gmail.com"
    # Create the message (HTML).
    html = f"""\
    {msgToSend}
    """
    # Record the MIME type - text/html.
    part1 = MIMEText(html, 'html')
    # Credentials
    username = 'email@gmail.com'
    password = 'app sign in'
    server = smtplib.SMTP('smtp.gmail.com', 587)
    server.ehlo()
    server.starttls()
    server.login(username, password)
    server.sendmail(from_address, to_address, msg.as_string())
    server.quit()
    return True


def smartMessage(msgToSend, toWhom, decay=100):
    x = threading.Thread(target=sendMessage, args=(msgToSend, toWhom, decay))
    x.start()
    x.join()


class Webpage:
    """
    Website class
    """

    @cherrypy.expose
    def products(self):
        """
        Products page
        :return:Products Page
        """
        response = ""
        with open(STORAGEDIR + "productos.csv", "r") as f:
            text = f.read()
            response = text.replace('\n', ';')
        return response

    @cherrypy.expose
    @cherrypy.tools.json_in()
    def api(self):
        """
        User
        Login
        API
        :return:Api Page
        """
        data = cherrypy.request.json
        try:
            if data["method"] == "verify":
                with open(STORAGEDIR + "Users.json", "r+") as json_file:
                    users = json.load(json_file)
                    if data["email"] in users:
                        user = users[data["email"]]
                        if data["password"] == user["password"]:
                            return "True"
                raise cherrypy.HTTPError(403)
        except KeyError:
            return "missing parameters"
        try:
            if data["method"] == "signup":
                users = {}
                with open(STORAGEDIR + "USERS.json", "r+") as json_file:
                    users = json.load(json_file)
                    if data["email"] in users:
                        raise cherrypy.HTTPError(403)
                    else:
                        users[data["email"]] = data
                        with open(STORAGEDIR + "USERS.json", "w+") as f:
                            json.dump(users, f)
                            return "True"

        except KeyError:
            return "missing parameters"
        return "test"

    @cherrypy.expose
    @cherrypy.tools.json_in()
    def order(self):
        data = cherrypy.request.json
        email = data["email"]
        password = data["password"]
        items = data["order"]
        with open(STORAGEDIR + "Users.json", "r+") as json_file:
            users = json.load(json_file)
            if email in users:
                user = users[email]
                if password == user["password"]:
                    smartMessage(items,email,decay=50)
                    return "True"
            return "False"

def runserver():
    """
    Cherry Py starting
    """

    cherrypy.tree.mount(Webpage(), '/', config={
        '/': {
            'tools.staticdir.on': True,
            'tools.staticdir.dir': "/Users/User/Documents/Webpage",
            'tools.staticdir.index': 'index.html',
            'error_page.404': "/Users/User/Documents/Webpage/404.html",
            'error_page.403': "/Users/User/Documents/Webpage/404.html"
        }
    })

    cherrypy.config.update({
        'server.socket_port': 443,
        'server.socket_host': '0.0.0.0',
        'server.ssl_module': 'builtin',
        'server.ssl_certificate': '/Users/user/letsencrypt/config/live/website.com/cert.pem',
        'server.ssl_certificate_chain': '/Users/user/letsencrypt/config/live/website.com/fullchain.pem',
        'server.ssl_private_key': '/Users/user/letsencrypt/config/live/website.com/privkey.pem'
    })

    try:
        cherrypy.engine.start()
        cherrypy.engine.block()
    except KeyboardInterrupt:
        cherrypy.engine.stop()

runserver()

This issue happens after 1-2 hours of running the server, you get this error and it becomes unresponsive. What can I do to fix this. Is it a Cherrypy error? or am I doing something wrong?

Cheroot version: 8.5.1
CherryPy version: 18.6.0
Python version: 3.9
OS: macOS Catalina
Browser: all

Other information (e.g. detailed explanation, stacktraces, related issues, suggestions how to fix, links for us to have context, e.g. stackoverflow, gitter, etc.)
StackOverflow question

webknjaz · 2020-12-27T19:05:54Z

Cheroot version: X.X.X installed with cherrypy with pip

So which is it? Use pip show to get this info.

CherryPy version: 16.8

And have you tried the latest version?

DogAteMyCode · 2020-12-27T20:34:20Z

Cheroot version: X.X.X installed with cherrypy with pip
it is version 8.5.1

DogAteMyCode · 2020-12-27T20:35:44Z

And have you tried the latest version?

I am using the latest version I can install with pip

webknjaz · 2020-12-28T11:44:38Z

Sounds like it's something that needs to be solved in Cheroot. So I'm moving this issue over there.
Also, I'm pretty sure that it doesn't crash by itself but this happens after a certain TLS connection appears. It looks like the client-side does something wrong like sending the data over the TLS connection that still hasn't completed the TLS handshake.

P.S. Where did you take your CPython copy? Did you download a DMG from python.org or used some other way of installing it (like pyenv or brew)? Non-official builds may have problems like that. Also, do you know what OpenSSL version it's linked against?

webknjaz · 2020-12-28T11:48:18Z

We need a Cheroot-only reproducer and it'd be useful to have the client code, which seems broken too.

Oh, and you could narrow down what's happening by recording the network traffic with tcpdump or Wireshark. Specifically, before/during the time when it gets stuck.

webknjaz · 2020-12-28T11:52:11Z

Here's one of the mentions of unexpected record on the Internet: https://groups.google.com/g/mailing.openssl.users/c/WAmXHwrExNI.

DogAteMyCode · 2020-12-28T14:07:38Z

Sounds like it's something that needs to be solved in Cheroot. So I'm moving this issue over there.
Also, I'm pretty sure that it doesn't crash by itself but this happens after a certain TLS connection appears. It looks like the client-side does something wrong like sending the data over the TLS connection that still hasn't completed the TLS handshake.

P.S. Where did you take your CPython copy? Did you download a DMG from python.org or used some other way of installing it (like pyenv or brew)? Non-official builds may have problems like that. Also, do you know what OpenSSL version it's linked against?

CPython was installed with the python 3.9 DMG from python.org.
OpenSSL version is 20.0.1

DogAteMyCode · 2020-12-28T14:16:21Z

We need a Cheroot-only reproducer and it'd be useful to have the client code, which seems broken too.

Oh, and you could narrow down what's happening by recording the network traffic with tcpdump or Wireshark. Specifically, before/during the time when it gets stuck.

As for traffic before the crash. This happens with varying IPs just before the error.

40.89.129.90 - - [27/Dec/2020:10:29:05] "POST /GponForm/diag_Form?style/ HTTP/1.1" 404 513 "" "curl/7.3.2" 45.79.236.250 - - [27/Dec/2020:10:29:12] "POST /GponForm/diag_Form?style/ HTTP/1.1" 404 513 "" "curl/7.3.2" 192.46.222.202 - - [27/Dec/2020:10:30:03] "POST /GponForm/diag_Form?style/ HTTP/1.1" 404 513 "" "curl/7.3.2" 45.79.238.87 - - [27/Dec/2020:10:30:57] "POST /GponForm/diag_Form?style/ HTTP/1.1" 404 513 "" "curl/7.3.2" 192.46.219.130 - - [27/Dec/2020:10:31:56] "POST /GponForm/diag_Form?style/ HTTP/1.1" 404 513 "" "curl/7.3.2" 192.46.236.148 - - [27/Dec/2020:10:44:33] "POST /GponForm/diag_Form?style/ HTTP/1.1" 404 513 "" "curl/7.3.2" 139.162.220.139 - - [27/Dec/2020:10:45:44] "POST /GponForm/diag_Form?style/ HTTP/1.1" 404 513 "" "curl/7.3.2" 45.79.238.87 - - [27/Dec/2020:10:47:29] "POST /GponForm/diag_Form?style/ HTTP/1.1" 404 513 "" "curl/7.3.2" 45.79.236.250 - - [27/Dec/2020:10:52:59] "POST /GponForm/diag_Form?style/ HTTP/1.1" 404 513 "" "curl/7.3.2" 139.162.220.139 - - [27/Dec/2020:11:02:12] "POST /GponForm/diag_Form?style/ HTTP/1.1" 404 513 "" "curl/7.3.2" 40.89.170.224 - - [27/Dec/2020:11:06:22] "POST /GponForm/diag_Form?style/ HTTP/1.1" 404 513 "" "curl/7.3.2" 40.89.171.45 - - [27/Dec/2020:11:11:17] "POST /GponForm/diag_Form?style/ HTTP/1.1" 404 513 "" "curl/7.3.2" 192.46.219.130 - - [27/Dec/2020:11:15:42] "POST /GponForm/diag_Form?style/ HTTP/1.1" 404 513 "" "curl/7.3.2" 192.46.236.148 - - [27/Dec/2020:11:16:14] "POST /GponForm/diag_Form?style/ HTTP/1.1" 404 513 "" "curl/7.3.2" 192.46.222.202 - - [27/Dec/2020:11:25:03] "POST /GponForm/diag_Form?style/ HTTP/1.1" 404 513 "" "curl/7.3.2" 139.162.220.139 - - [27/Dec/2020:11:28:23] "POST /GponForm/diag_Form?style/ HTTP/1.1" 404 513 "" "curl/7.3.2" 45.79.236.250 - - [27/Dec/2020:11:36:50] "POST /GponForm/diag_Form?style/ HTTP/1.1" 404 513 "" "curl/7.3.2" 192.46.222.202 - - [27/Dec/2020:11:39:21] "POST /GponForm/diag_Form?style/ HTTP/1.1" 404 513 "" "curl/7.3.2" 45.79.238.87 - - [27/Dec/2020:11:40:28] "POST /GponForm/diag_Form?style/ HTTP/1.1" 404 513 "" "curl/7.3.2"

I don't know if this is related somehow though.

However I have searched /GponForm/diag_Form?style/ and it seems to be an attempt to enter my ISP modem by using the exploit found here exploit. Because my ISP modem can be accessed by external users, however the exploit doesn't work on my modem, or won't work because I'm redirecting port 80 to port 443 externally through the modem and google domains configuration.

webknjaz · 2020-12-28T17:12:07Z

OpenSSL version is 20.0.1

There is no such version.

I don't know if this is related somehow though.

These are just HTTP access logs and are probably unrelated. You need to record actual traffic. The problem is happening on the TLS level, not HTTP.

DogAteMyCode · 2020-12-28T18:28:55Z

OpenSSL version is 20.0.1

There is no such version.

I was listing the pyOpenSSL, I misunderstood. OpenSSL version is 1.1.1i > > I don't know if this is related somehow though. > > These are just HTTP access logs and are probably unrelated. You need to record actual traffic. The problem is happening on the TLS level, not HTTP. Ok will try Wireshark then, just what filter should I use, I am still inexperienced in Wireshark.

DogAteMyCode · 2020-12-28T21:49:50Z

CherryPy.pcapng.zip
This is the Wireshark file of the error, the error should have occurred within 100 seconds of the first packet, however just in case I captured the following 1000 packets.

webknjaz · 2020-12-29T02:10:31Z

I was listing the pyOpenSSL, I misunderstood.

That is unrelated. You use a builtin adapter that relies on the ssl module that is present in Python stdlib. That module works with some OpenSSL that is present on your system. pyOpenSSL is a third-party library that also links against OpenSSL but that's not something you use.

I'll try to take a look at your capture but it mostly sounds like it's a bad client. FWIW the solution would be to extend the ignore list in the adapter on Cheroot side, I'm just trying to understand how to reproduce this case and whether it's specific to your combo of the software and if it's reproducible on other platforms.

webknjaz · 2020-12-29T02:23:00Z

I see there's some noise from different protocols like ICMP and also there's TLS traffic from different apps. It's useful to filter by port or something like this (maybe by the process even, but I'm not sure).

Could you clarify, is 34.107.247.156 the IP your app listens to?

DogAteMyCode · 2020-12-29T15:43:49Z

I see there's some noise from different protocols like ICMP and also there's TLS traffic from different apps. It's useful to filter by port or something like this (maybe by the process even, but I'm not sure).

Could you clarify, is 34.107.247.156 the IP your app listens to?

It was at the time, you see I don't have a static IP, I am using google domain's dns.

webknjaz · 2020-12-29T15:48:08Z

It was at the time, you see I don't have a static IP, I am using google domain's dns.

Alright, so this was an IP on the interface the app was listening to. Right? DNS doesn't matter much on the TLS level. I don't fully understand the topology you have, I just want to figure out which TCP sessions are related to the app activity. Also, did you record traffic on the same machine?
Oh, and what's 192.168.1.119? It seems to be the client that caused all of this.

DogAteMyCode · 2020-12-29T17:57:44Z

FWIW the solution would be to extend the ignore list in the adapter on Cheroot side, I'm just trying to understand how to reproduce this case and whether it's specific to your combo of the software and if it's reproducible on other platforms.

And just how would one do that?

webknjaz · 2020-12-29T19:41:50Z

And just how would one do that?

Reproduce the issue? For this we need to understand what sort of client you have at 192.168.1.119.

Extend the ignore list? Just add some substring to a var in the errors module, I guess. I haven't looked at it closely yet. It's important to have a robust reproducer first to properly test that the fix works.

DogAteMyCode · 2021-01-01T01:39:37Z

I did a botch,
runserver() is the function that starts the server.
I surrounded this with

import ssl
try:
    runserver()
except ssl.SSLError:
    import os
    os.system(xterm -c python3.9 python-module-dir &)
    sys.exit()

this just restarts the server in a self closing xterm window, whenever the server crashes it just creates a new instance and closes the old one. However I am still in search of a proper solution
This didn't fix the problem.

webknjaz transferred this issue from cherrypy/cherrypy Dec 28, 2020

webknjaz added bug Something is broken help wanted Somebody help us, please! reproducer: missing This PR or issue lacks code, which reproduce the problem described or clearly understandable STR labels Dec 28, 2020

webknjaz closed this as completed in 9720c31 Mar 31, 2024

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Cherrypy ssl error #354

Cherrypy ssl error #354

DogAteMyCode commented Dec 27, 2020

webknjaz commented Dec 27, 2020

DogAteMyCode commented Dec 27, 2020

DogAteMyCode commented Dec 27, 2020

webknjaz commented Dec 28, 2020

webknjaz commented Dec 28, 2020

webknjaz commented Dec 28, 2020

DogAteMyCode commented Dec 28, 2020

DogAteMyCode commented Dec 28, 2020 •

edited

Loading

webknjaz commented Dec 28, 2020

DogAteMyCode commented Dec 28, 2020 •

edited

Loading

DogAteMyCode commented Dec 28, 2020

webknjaz commented Dec 29, 2020

webknjaz commented Dec 29, 2020

DogAteMyCode commented Dec 29, 2020

webknjaz commented Dec 29, 2020

DogAteMyCode commented Dec 29, 2020 •

edited

Loading

webknjaz commented Dec 29, 2020

DogAteMyCode commented Jan 1, 2021 •

edited

Loading

Cherrypy ssl error #354

Cherrypy ssl error #354

Comments

DogAteMyCode commented Dec 27, 2020

webknjaz commented Dec 27, 2020

DogAteMyCode commented Dec 27, 2020

DogAteMyCode commented Dec 27, 2020

webknjaz commented Dec 28, 2020

webknjaz commented Dec 28, 2020

webknjaz commented Dec 28, 2020

DogAteMyCode commented Dec 28, 2020

DogAteMyCode commented Dec 28, 2020 • edited Loading

webknjaz commented Dec 28, 2020

DogAteMyCode commented Dec 28, 2020 • edited Loading

DogAteMyCode commented Dec 28, 2020

webknjaz commented Dec 29, 2020

webknjaz commented Dec 29, 2020

DogAteMyCode commented Dec 29, 2020

webknjaz commented Dec 29, 2020

DogAteMyCode commented Dec 29, 2020 • edited Loading

webknjaz commented Dec 29, 2020

DogAteMyCode commented Jan 1, 2021 • edited Loading

DogAteMyCode commented Dec 28, 2020 •

edited

Loading

DogAteMyCode commented Dec 28, 2020 •

edited

Loading

DogAteMyCode commented Dec 29, 2020 •

edited

Loading

DogAteMyCode commented Jan 1, 2021 •

edited

Loading