scrape_websites vs scrape_website

agixt/AGiXT.py

@@ -369,7 +369,7 @@ async def learn_from_websites(
                 role=self.agent_name,
                 message=f"[ACTIVITY_START] Browsing the web... [ACTIVITY_END]",
             )
-        response = await self.agent_interactions.websearch.scrape_website(
+        response = await self.agent_interactions.websearch.scrape_websites(
             user_input=user_input,
             search_depth=scrape_depth,
             summarize_content=summarize_content,

agixt/Interactions.py

@@ -449,8 +449,11 @@ async def run(
         else:
             websearch_timeout = 0
         if browse_links != False:
-            await self.websearch.scrape_website(
-                user_input=user_input, search_depth=websearch_depth
+            await self.websearch.scrape_websites(
+                user_input=user_input,
+                search_depth=websearch_depth,
+                summarize_content=True,
+                conversation_name=conversation_name,
             )
         if websearch:
             if user_input == "":

agixt/Websearch.py

@@ -427,7 +427,7 @@ async def search(self, query: str) -> List[str]:
             self.searx_instance_url = ""
             return await self.search(query=query)
 
-    async def scrape_website(
+    async def scrape_websites(
         self,
         user_input: str = "",
         search_depth: int = 0,
@@ -437,6 +437,8 @@ async def scrape_website(
         # user_input = "I am browsing {url} and collecting data from it to learn more."
         c = Conversations(conversation_name=conversation_name, user=self.user)
         links = re.findall(r"(?P<url>https?://[^\s]+)", user_input)
+        if len(links) < 1:
+            return ""
         scraped_links = []
         if links is not None and len(links) > 0:
             for link in links:
@@ -493,7 +495,7 @@ async def websearch_agent(
         websearch_depth: int = 0,
         websearch_timeout: int = 0,
     ):
-        await self.scrape_website(user_input=user_input, search_depth=websearch_depth)
+        await self.scrape_websites(user_input=user_input, search_depth=websearch_depth)
         try:
             websearch_depth = int(websearch_depth)
         except:

agixt/endpoints/Memory.py

@@ -197,7 +197,7 @@ async def learn_url(
         agent=agent,
         user=user,
         ApiClient=ApiClient,
-    ).scrape_website(
+    ).scrape_websites(
         user_input=f"I am browsing {url.url} and collecting data from it to learn more.",
         search_depth=3,
     )