From 67ebcd516d85e5785777c40e86611dbbf2597828 Mon Sep 17 00:00:00 2001 From: magnus Date: Wed, 4 Dec 2024 16:21:47 +0100 Subject: [PATCH] fix: ipv4 address regex --- test_unstructured/cleaners/test_extract.py | 4 ++-- unstructured/nlp/patterns.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test_unstructured/cleaners/test_extract.py b/test_unstructured/cleaners/test_extract.py index 6ca059883c..86ac0e8487 100644 --- a/test_unstructured/cleaners/test_extract.py +++ b/test_unstructured/cleaners/test_extract.py @@ -5,7 +5,7 @@ from unstructured.cleaners import extract EMAIL_META_DATA_INPUT = """from ABC.DEF.local ([ba23::58b5:2236:45g2:88h2]) by - \n ABC.DEF.local ([ba23::58b5:2236:45g2:88h2%25]) with mapi id\ + \n ABC.DEF.local ([68.183.71.12]) with mapi id\ n 32.88.5467.123; Fri, 26 Mar 2021 11:04:09 +1200""" @@ -37,7 +37,7 @@ def test_extract_email_address(): def test_extract_ip_address(): assert extract.extract_ip_address(EMAIL_META_DATA_INPUT) == [ "ba23::58b5:2236:45g2:88h2", - "ba23::58b5:2236:45g2:88h2%25", + "68.183.71.12", ] diff --git a/unstructured/nlp/patterns.py b/unstructured/nlp/patterns.py index e18f067c4a..b3a77f77a8 100644 --- a/unstructured/nlp/patterns.py +++ b/unstructured/nlp/patterns.py @@ -92,9 +92,9 @@ ONE_LINE_BREAK_PARAGRAPH_PATTERN = r"^(?:(?!\.\s*$).)*$" ONE_LINE_BREAK_PARAGRAPH_PATTERN_RE = re.compile(ONE_LINE_BREAK_PARAGRAPH_PATTERN) -# IP Address examples: ba23::58b5:2236:45g2:88h2 or 10.0.2.01 +# IP Address examples: ba23::58b5:2236:45g2:88h2, 10.0.2.01 or 68.183.71.12 IP_ADDRESS_PATTERN = ( - r"[0-9]{1,2}\.[0-9]{1,2}\.[0-9]{1,2}\.[0-9]{1,2}", + r"(?:25[0-5]|2[0-4]\d|1\d{2}|[1-9]?\d)(?:\.(?:25[0-5]|2[0-4]\d|1\d{2}|[1-9]?\d)){3}", "[a-z0-9]{4}::[a-z0-9]{4}:[a-z0-9]{4}:[a-z0-9]{4}:[a-z0-9]{4}%?[0-9]*", ) IP_ADDRESS_PATTERN_RE = re.compile(f"({'|'.join(IP_ADDRESS_PATTERN)})")