Skip to content

Commit

Permalink
[twitter] "fix" search pagination (#3536, #3534)
Browse files Browse the repository at this point in the history
- properly process instructions
- do not expect a predetermined instruction order
  • Loading branch information
mikf committed Jan 16, 2023
1 parent 4e86aaa commit 9683d79
Showing 1 changed file with 15 additions and 8 deletions.
23 changes: 15 additions & 8 deletions gallery_dl/extractor/twitter.py
Original file line number Diff line number Diff line change
Expand Up @@ -1245,17 +1245,28 @@ def _pagination_legacy(self, endpoint, params):
while True:
data = self._call(endpoint, params)

instr = data["timeline"]["instructions"]
if not instr:
instructions = data["timeline"]["instructions"]
if not instructions:
return

tweets = data["globalObjects"]["tweets"]
users = data["globalObjects"]["users"]
tweet_id = cursor = None
tweet_ids = []
entries = ()

# process instructions
for instr in instructions:
if "addEntries" in instr:
entries = instr["addEntries"]["entries"]
elif "replaceEntry" in instr:
entry = instr["replaceEntry"]["entry"]
if entry["entryId"].startswith("cursor-bottom-"):
cursor = (entry["content"]["operation"]
["cursor"]["value"])

# collect tweet IDs and cursor value
for entry in instr[0]["addEntries"]["entries"]:
for entry in entries:
entry_startswith = entry["entryId"].startswith

if entry_startswith(("tweet-", "sq-I-t-")):
Expand Down Expand Up @@ -1315,11 +1326,7 @@ def _pagination_legacy(self, endpoint, params):
quoted["quoted_by_id_str"] = tweet["id_str"]
yield quoted

# update cursor value
if "replaceEntry" in instr[-1] :
cursor = (instr[-1]["replaceEntry"]["entry"]
["content"]["operation"]["cursor"]["value"])

# stop on empty response
if not cursor or (not tweets and not tweet_id):
return
params["cursor"] = cursor
Expand Down

0 comments on commit 9683d79

Please sign in to comment.