[Twitter] Recursive reply expansion

[gavin-a99]

When using Gallery-DL to archive posts on Twitter only a small fraction of the total replies in a thread are being downloaded.

This behavior seems to partly come from the _expand_tweets function:

def _expand_tweets(self, tweets):
    seen = set()
    for tweet in tweets:
        obj = tweet["legacy"] if "legacy" in tweet else tweet
        cid = obj.get("conversation_id_str")
        if not cid:
            tid = obj["id_str"]
            self.log.warning(
                "Unable to expand %s (no 'conversation_id')", tid)
            continue
        if cid in seen:
            self.log.debug(
                "Skipping expansion of %s (previously seen)", cid)
            continue
        seen.add(cid)
        try:
            yield from self.api.tweet_detail(cid)
        except Exception:
            yield tweet

Adding the conversation_id to seen anywhere, either before or after the tweet is yielded--results in all of the top-level replies to a tweet being added to seen before they are expanded.

[twitter][debug] Skipping expansion of 1812956662950400214 (previously seen)
[twitter][debug] Skipping expansion of 1812956662950400214 (previously seen)
[twitter][debug] Skipping expansion of 1812956662950400214 (previously seen)
[twitter][debug] Skipping expansion of 1812956662950400214 (previously seen)
[twitter][debug] Skipping expansion of 1812956662950400214 (previously seen)
[twitter][debug] Skipping expansion of 1812956662950400214 (previously seen)

Removing seen.add(cid) results in more (but not all) replies being downloaded--but causes the function to loop until "skip": kicks in.

The problem is replies which are deeper than a reply to the main tweet, I.E. a reply to a reply to a reply--do not get downloaded. With the only exception being when author[name] is the same as user[name], and only when its one reply level deep.

I've tested out other configs and have ran into the same issue, butI haven't seen anybody else mention this, so Im wondering if there is a fix within the extarctor config or if it's an API limitation.

My current Twitter config:

{
    "twitter": {
        "#": "======= ARCHIVE =======",
        "archive": "./downloads/twitter.com/{user['name']!l}/archives/media.sqlite3",
        "archive-event": "file",
        "archive-format": "{author['id']}, {tweet_id}, {reply_id}, {retweet_id}, {quote_id}, {num}, {extension}",
        "archive-mode": "file",
        "archive-prefix": "twitter.com, ",
        "archive-pragma": ["journal_mode=WAL", "synchronous=NORMAL"],
        "#": "======= DIRECTORY =======",
        "base-directory": "./downloads/twitter.com/",
        "directory": {
            "": ["{user['name']!l}", "timeline", "{conversation_id}"]
        },
        "#": "======= FILE NAME =======",
        "filename": {
            "locals().get('bitrate', 0) > 0 and count > 1": "{author['name']!l}_{tweet_id}_{date:%Y-%m-%d_%H-%M-%S}_vid-{num}.{extension}",
            "locals().get('bitrate', 0) > 0": "{author['name']!l}_{tweet_id}_{date:%Y-%m-%d_%H-%M-%S}_vid.{extension}",
            "locals().get('bitrate') == 0 and count > 1": "{author['name']!l}_{tweet_id}_{date:%Y-%m-%d_%H-%M-%S}_gif-{num}.{extension}",
            "locals().get('bitrate') == 0": "{author['name']!l}_{tweet_id}_{date:%Y-%m-%d_%H-%M-%S}_gif.{extension}",
            "count > 1": "{author['name']!l}_{tweet_id}_{date:%Y-%m-%d_%H-%M-%S}_img-{num}.{extension}",
            "": "{author['name']!l}_{tweet_id}_{date:%Y-%m-%d_%H-%M-%S}_img.{extension}"
        },
        "#": "======= COOKIES =======",
        "cookies": {
            "auth_token": "***",
            "ct0": "***",
            "lang": "en"
        },
        "#": "======= CONFIGURATION =======",
        "ads": false,
        "cards": true,
        "cards-blacklist": null,
        "conversations": true,
        "csrf": "cookies",
        "expand": true,
        "unavailable": false,
        "include": "all",
        "transform": true,
        "tweet-endpoint": "detail",
        "size": ["orig", "4096x4096", "large", "medium", "small"],
        "logout": true,
        "pinned": true,
        "quoted": true,
        "ratelimit": "wait",
        "locked": "abort",
        "replies": true,
        "retweets": true,
        "timeline": {
            "strategy": "with_replies"
        },
        "text-tweets": true,
        "twitpic": true,
        "unique": false,
        "authors": "timeline",
        "videos": true,
        "syndication": "extended",
        "avatar": {
            "archive": "./downloads/twitter.com/{user['name']!l}/archives/avatar.sqlite3",
            "archive-event": "file",
            "archive-format": "{author['id']}, {author['profile_image'].rsplit('/', 1)[-1].rsplit('.', 1)[0]}, {extension}",
            "archive-mode": "file",
            "archive-prefix": "\fF twitter.com, ",
            "archive-pragma": ["journal_mode=WAL", "synchronous=NORMAL"],
            "directory": {
                "": ["{user['name']!l}", "avatars"]
            },
            "expand": false,
            "filename": "\fF {author['name'].lower()}_avatar_{date:%Y-%m-%d_%H-%M-%S}_{author['profile_image'].rsplit('/', 1)[-1].rsplit('.', 1)[0]}.{extension}"
        },
        "background": {
            "archive": "./downloads/twitter.com/{user['name']!l}/archives/banner.sqlite3",
            "archive-event": "file",
            "archive-format": "{author['id']}, {author['profile_banner'].rsplit('/', 1)[-1].rsplit('.', 1)[0]}",
            "archive-mode": "file",
            "archive-prefix": "\fF twitter.com, ",
            "archive-pragma": ["journal_mode=WAL", "synchronous=NORMAL"],
            "directory": {
                "": ["{user['name']!l}", "banners"]
            },
            "expand": false,
            "filename": "\fF {author['name'].lower()}_banner_{date:%Y-%m-%d_%H-%M-%S}_{author['profile_banner'].rsplit('/', 1)[-1].rsplit('.', 1)[0]}.{extension}"
        },
        "#": "======= POST PROCESSORS =======",
        "postprocessors": [
            {
                "#": "======= WRITE METADATA FOR TEXT-ONLY POSTS =======",
                "name": "metadata",
                "mode": "json",
                "archive": "./downloads/twitter.com/{user['name']!l}/archives/metadata.sqlite3",
                "archive-prefix": "twitter.com, ",
                "archive-format": "{author['id']}, {tweet_id}, {reply_id}, {retweet_id}, {quote_id}",
                "archive-pragma": ["journal_mode=WAL", "synchronous=NORMAL"],
                "filter": "subcategory != 'background' and subcategory != 'avatar'",
                "filename": "{author['name']!l}_{tweet_id}_{date:%Y-%m-%d_%H-%M-%S}.json",
                "event": "post"
            }
        ]
    }
}