[postprocessor:exec] improve (#421, #413)

- add 'final' option
- include job status in pp finalization
- improve and extend documentation

docs/configuration.rst

@@ -1388,12 +1388,31 @@ Description Controls whether to wait for a subprocess to finish
 exec.command
 ------------
 =========== =====
-Type        ``list`` of ``strings``
-Example     ``["echo", "{user[account]}", "{id}"]``
+Type        ``string`` or ``list`` of ``strings``
+Example     * ``"convert {} {}.png && rm {}"``
+            * ``["echo", "{user[account]}", "{id}"]``
 Description The command to run.
 
-            Each element of this list is treated as a `format string`_ using
-            the files' metadata.
+            * If this is a ``string``, it will be executed using the system's
+              shell, e.g. ``/bin/sh``. Any ``{}`` will be replaced
+              with the full path of a file or target directory, depending on
+              `exec.final`_
+
+            * If this is a ``list``, the first element specifies the program
+              name and any further elements its arguments.
+              Each element of this list is treated as a `format string`_ using
+              the files' metadata as well as ``{_path}``, ``{_directory}``,
+              and ``{_filename}``.
+=========== =====
+
+exec.final
+----------
+=========== =====
+Type        ``bool``
+Default     ``false``
+Description Controls whether to execute `exec.command`_ for each
+            downloaded file or only once after all files
+            have been downloaded successfully.
 =========== =====
 
 

gallery_dl/job.py

@@ -67,6 +67,9 @@ def run(self):
                       exc.__class__.__name__, exc)
             log.debug("", exc_info=True)
             self.status |= 1
+        except BaseException:
+            self.status |= 1
+            raise
         finally:
             self.handle_finalize()
         return self.status
@@ -255,13 +258,15 @@ def handle_queue(self, url, kwdict):
             self._write_unsupported(url)
 
     def handle_finalize(self):
-        if self.postprocessors:
-            for pp in self.postprocessors:
-                pp.finalize()
+        pathfmt = self.pathfmt
         if self.archive:
             self.archive.close()
-        if self.pathfmt:
+        if pathfmt:
             self.extractor._store_cookies()
+            if self.postprocessors:
+                status = self.status
+                for pp in self.postprocessors:
+                    pp.run_final(pathfmt, status)
 
     def handle_skip(self):
         self.out.skip(self.pathfmt.path)

gallery_dl/postprocessor/common.py

@@ -31,8 +31,8 @@ def run_after(pathfmt):
         """Execute postprocessor after moving a file to its target location"""
 
     @staticmethod
-    def finalize():
-        """Cleanup"""
+    def run_final(pathfmt, status):
+        """Postprocessor finalization after all files have been downloaded"""
 
     def __repr__(self):
         return self.__class__.__name__

gallery_dl/postprocessor/exec.py

@@ -26,27 +26,43 @@ class ExecPP(PostProcessor):
     def __init__(self, pathfmt, options):
         PostProcessor.__init__(self)
         args = options["command"]
+        final = options.get("final", False)
 
         if isinstance(args, str):
+            if final:
+                self._format = self._format_args_directory
+            else:
+                self._format = self._format_args_path
             if "{}" not in args:
                 args += " {}"
             self.args = args
             self.shell = True
-            self._format = self._format_args_string
         else:
+            self._format = self._format_args_list
             self.args = [util.Formatter(arg) for arg in args]
             self.shell = False
-            self._format = self._format_args_list
+
+        if final:
+            self.run_after = PostProcessor.run_after
+        else:
+            self.run_final = PostProcessor.run_final
 
         if options.get("async", False):
             self._exec = self._exec_async
 
     def run_after(self, pathfmt):
         self._exec(self._format(pathfmt))
 
-    def _format_args_string(self, pathfmt):
+    def run_final(self, pathfmt, status):
+        if status == 0:
+            self._exec(self._format(pathfmt))
+
+    def _format_args_path(self, pathfmt):
         return self.args.replace("{}", quote(pathfmt.realpath))
 
+    def _format_args_directory(self, pathfmt):
+        return self.args.replace("{}", quote(pathfmt.realdirectory))
+
     def _format_args_list(self, pathfmt):
         kwdict = pathfmt.kwdict
         kwdict["_directory"] = pathfmt.realdirectory

gallery_dl/postprocessor/zip.py

@@ -59,7 +59,7 @@ def _write_safe(self, pathfmt):
         with zipfile.ZipFile(*self.args) as zfile:
             self._write(pathfmt, zfile)
 
-    def finalize(self):
+    def run_final(self, pathfmt, status):
         if self.zfile:
             self.zfile.close()
 

test/test_postprocessor.py

@@ -327,7 +327,7 @@ def test_zip_write(self):
             self.assertEqual(len(pp.zfile.NameToInfo), 3)
 
         # close file
-        pp.finalize()
+        pp.run_final(self.pathfmt, 0)
 
         # reopen to check persistence
         with zipfile.ZipFile(pp.zfile.filename) as file:
@@ -360,7 +360,7 @@ def side_effect(_, name):
         pp.prepare(self.pathfmt)
         pp.run(self.pathfmt)
 
-        pp.finalize()
+        pp.run_final(self.pathfmt, 0)
 
         self.assertEqual(pp.zfile.write.call_count, 3)
         for call in pp.zfile.write.call_args_list: