Update internet.py

ParisNeo · Jul 29, 2024 · 01d70e6 · 01d70e6
1 parent 199c9e2
commit 01d70e6
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/lollms/internet.py b/lollms/internet.py
@@ -152,7 +152,7 @@ def scrape_and_save(url, file_path:str|Path=None, use_selenium=False, follow_lin
 
     # Save scraped texts to the markdown file
     if file_path:
-        with open(file_path, 'w') as md_file:
+        with open(file_path, 'w', encoding="utf8", errors='ignore') as md_file:
             for text in results['texts']:
                 md_file.write(text + '\n\n')
         print(f"Texts saved to {file_path}")