From ed21908fda1cb30ecef9823cff6027109152e836 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Fri, 28 Jul 2023 17:07:25 +0200 Subject: [PATCH] initial support for child extractor options Using "parent-category>child-category" as extractor category in a config file allows to set options for a child extractor when it was spawned by that parent. For example "reddit>gfycat" to set gfycat options for when it was found in a reddit post. { "extractor": { "gfycat": { "filename": "regular filename" }, "reddit>gfycat": { "filename": "reddit-specific filename" } } } Note: This does currently not work for most imgur links due to how its extractor hierarchy is structured. --- gallery_dl/extractor/common.py | 26 ++++++++++++++------------ gallery_dl/job.py | 15 +++++++++++++++ 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index fc6b197cdc..6b94112c49 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -45,10 +45,6 @@ class Extractor(): def __init__(self, match): self.log = logging.getLogger(self.category) self.url = match.string - - if self.basecategory: - self.config = self._config_shared - self.config_accumulate = self._config_shared_accumulate self._cfgpath = ("extractor", self.category, self.subcategory) self._parentdir = "" @@ -98,16 +94,22 @@ def config_accumulate(self, key): return config.accumulate(self._cfgpath, key) def _config_shared(self, key, default=None): - return config.interpolate_common(("extractor",), ( - (self.category, self.subcategory), - (self.basecategory, self.subcategory), - ), key, default) + return config.interpolate_common( + ("extractor",), self._cfgpath, key, default) def _config_shared_accumulate(self, key): - values = config.accumulate(self._cfgpath, key) - conf = config.get(("extractor",), self.basecategory) - if conf: - values[:0] = config.accumulate((self.subcategory,), key, conf=conf) + first = True + extr = ("extractor",) + + for path in self._cfgpath: + if first: + first = False + values = config.accumulate(extr + path, key) + else: + conf = config.get(extr, path[0]) + if conf: + values[:0] = config.accumulate( + (self.subcategory,), key, conf=conf) return values def request(self, url, method="GET", session=None, diff --git a/gallery_dl/job.py b/gallery_dl/job.py index 7ecdc39134..2ea8feb721 100644 --- a/gallery_dl/job.py +++ b/gallery_dl/job.py @@ -32,6 +32,21 @@ def __init__(self, extr, parent=None): self.kwdict = {} self.status = 0 + cfgpath = [] + if parent and parent.extractor.category != extr.category: + cat = "{}>{}".format( + parent.extractor.category, extr.category) + cfgpath.append((cat, extr.subcategory)) + cfgpath.append((extr.category, extr.subcategory)) + if extr.basecategory: + if not cfgpath: + cfgpath.append((extr.category, extr.subcategory)) + cfgpath.append((extr.basecategory, extr.subcategory)) + if cfgpath: + extr._cfgpath = cfgpath + extr.config = extr._config_shared + extr.config_accumulate = extr._config_shared_accumulate + actions = extr.config("actions") if actions: from .actions import parse