scrape all pages

dream-aim-deliver · Apr 14, 2024 · fdaf3d3 · fdaf3d3
1 parent 5d5c01a
commit fdaf3d3
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/app/scraper.py b/app/scraper.py
@@ -116,7 +116,7 @@ def scrape(
                     except Exception as e:
                         logger.info("could not register file")
                     last_successful_data = current_data
-                if page >= 2:
+                else:
                     logger.error(f"{job_id}: Error: {response.status_code} - {response.text}")
                     logger.info("No more tweets found for this query. Scraping completed.")