TEST: no markdown

Author: rgaudin

src/sotoki/utils/html.py

@@ -4,10 +4,11 @@
 
 import re
 import urllib.parse
+import warnings
 
 import bs4
-import mistune
-from mistune.plugins import plugin_strikethrough, plugin_table, plugin_footnotes
+# import mistune
+# from mistune.plugins import plugin_strikethrough, plugin_table, plugin_footnotes
 from tld import get_fld
 from slugify import slugify
 
@@ -60,6 +61,9 @@ def is_in_code(elem):
 
 REDACTED_STRING = "[redacted]"
 
+# prevent beautifulsoup warning on comments containing only URLs
+warnings.filterwarnings('ignore', category=bs4.MarkupResemblesLocatorWarning)
+
 
 class Rewriter(GlobalMixin):
     redacted_text = "[redacted]"
@@ -94,10 +98,10 @@ def __init__(self):
         )
 
         self.redacted_string = bs4.NavigableString(self.redacted_text)
-        self.markdown = mistune.create_markdown(
-            escape=False,
-            plugins=[plugin_strikethrough, plugin_table, plugin_footnotes],
-        )
+        # self.markdown = mistune.create_markdown(
+        #     escape=False,
+        #     plugins=[plugin_strikethrough, plugin_table, plugin_footnotes],
+        # )
         if self.conf.censor_words_list:
             with open(self.conf.build_dir.joinpath("words.list"), "r") as fh:
                 # this will actually replace occurences of ~strings matching
@@ -146,7 +150,8 @@ def rewrite(self, content: str, to_root: str = "", unwrap: bool = False):
             return ""
 
         try:
-            soup = bs4.BeautifulSoup(self.markdown(content), "lxml")
+            # soup = bs4.BeautifulSoup(self.markdown(content), "lxml")
+            soup = bs4.BeautifulSoup(content, "lxml")
         except Exception as exc:
             logger.error(f"Unable to init soup or markdown for {content}: {exc}")
             return content