WIP: Add pristine-lfs support

Add pristine-lfs support similar to the existing pristine-tar support.
Since pristine-lfs has a Python interface, use that instead of the
command-line interface, but use it in a fail-safe way.

Unlike pristine-tar, pristine-lfs supports looking up tarballs by a
version, so use that instead of picking tarballs manually.

If both pristine-lfs and pristine-tar are enabled, only import tarballs
with pristine-lfs; when checking out, prefer pristine-lfs, but also
do pristine-tar.

Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>

gbp/config.py

@@ -166,6 +166,8 @@ class GbpOptionParser(OptionParser):
                 'pq-from': 'DEBIAN',
                 'prebuild': '',
                 'preexport': '',
+                'pristine-lfs': 'False',
+                'pristine-lfs-commit': 'False',
                 'pristine-tar': 'False',
                 'pristine-tar-commit': 'False',
                 'purge': 'True',
@@ -224,6 +226,12 @@ class GbpOptionParser(OptionParser):
         'commit-msg':
             "Format string for commit message used to commit, "
             "the changelog, default is '%(commit-msg)s'",
+        'pristine-lfs':
+            "Use pristine-lfs to create orig tarball, "
+            "default is '%(pristine-lfs)s'",
+        'pristine-lfs-commit':
+            "When generating a tarball, commit it to the pristine-lfs branch '%(pristine-lfs-commit)s' "
+            "default is '%(pristine-lfs-commit)s'",
         'pristine-tar':
             "Use pristine-tar to create orig tarball, "
             "default is '%(pristine-tar)s'",

gbp/deb/git.py

@@ -22,6 +22,7 @@
 from gbp.command_wrappers import CommandExecFailed
 from gbp.git import GitRepositoryError
 from gbp.deb.pristinetar import DebianPristineTar
+from gbp.deb.pristinelfs import PristineLfs
 from gbp.paths import to_bin
 from gbp.pkg.git import PkgGitRepository
 from gbp.pkg.pkgpolicy import PkgPolicy
@@ -339,6 +340,44 @@ def create_upstream_tarball_via_pristine_tar(self, source, output_dir, comp, ups
                                                                   e))
         return True
 
+    @property
+    def pristine_lfs_branch(self):
+        """
+        The name of the pristine-lfs branch, whether it already exists or
+        not.
+        """
+        return PristineLfs.branch
+
+    def has_pristine_lfs_branch(self):
+        """
+        Whether the repo has a I{pristine-lfs} branch.
+
+        @return: C{True} if the repo has pristine-lfs commits already, C{False}
+            otherwise
+        @rtype: C{Bool}
+        """
+        return True if self.has_branch(self.pristine_lfs_branch) else False
+
+    def create_pristine_lfs_commits(self, sources):
+        """
+        Create pristine-lfs commits for a package with main tarball
+        and (optional) component tarballs based on upstream_tree
+
+        @param soures: C{list} of tarball as I{UpstreamSource}. First one being the main
+                       tarball the other ones additional tarballs.
+        """
+        components = [t.component for t in sources[1:]]
+        all_files = [
+            source.path for source in sources
+        ] + [
+            source.signaturefile for source in sources if source.signaturefile
+        ]
+
+        try:
+            self.pristine_lfs.commit(all_files)
+        except CommandExecFailed as e:
+            raise GitRepositoryError(str(e))
+
     def create_upstream_tarball_via_git_archive(self, source, output_dir, treeish,
                                                 comp, with_submodules, component=None):
         """

gbp/deb/pristinelfs.py

@@ -0,0 +1,86 @@
+# vim: set fileencoding=utf-8 :
+#
+# (C) 2021 Andrej Shadura <andrewsh@debian.org>
+# (C) 2021 Collabora Ltd
+#    This program is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, please see
+#    <http://www.gnu.org/licenses/>
+"""Check in and check out archives from the pristine-lfs branch"""
+
+from __future__ import annotations
+
+import logging
+
+from gbp.command_wrappers import CommandExecFailed
+from gbp.deb import DebianPkgPolicy
+from gbp.git import GitRepository, GitRepositoryError
+
+try:
+    from pristine_lfs.main import do_list, do_verify, do_commit_files, do_checkout
+    from pristine_lfs.errors import CommandFailed, DifferentFilesExist, GitError
+except ImportError:
+    def pristine_lfs_not_found(*args, **kwargs):
+        raise CommandExecFailed("pristine-lfs not installed")
+
+    do_list = do_verify = do_checkout = do_commit_files = pristine_lfs_not_found
+
+    class DifferentFilesExist(Exception):
+        pass
+
+    GitError = DifferentFilesExist
+
+logger = logging.getLogger('pristine-lfs')
+
+class PristineLfs:
+    branch = 'pristine-lfs'
+
+    def __init__(self, repo: GitRepository):
+        self.repo = repo
+
+    def commit(self, files: list[str], quiet: bool = False):
+        """
+        Commit files I{files} to the pristine-lfs branch
+
+        @param files: list of files to commit
+        @type files: C{list}
+        """
+        logger.setLevel(logging.WARNING if quiet else logging.INFO)
+
+        try:
+            ios = [open(f, 'rb') for f in files]
+            do_commit_files(tarballs=ios, branch=self.branch)
+        except (OSError, CommandFailed) as e:
+            raise CommandExecFailed(str(e))
+        except (DifferentFilesExist, GitError) as e:
+            raise GitRepositoryError(str(e))
+
+    def checkout(self, package: str, version: str, output_dir: str, quiet: bool = False):
+        """
+        Check out all orig tarballs for package I{package} of I{version} to
+        I{output_dir}
+
+        @param package: the package to check out the orig tarballs for
+        @type package: C{str}
+        @param version: the version to check out the orig tarballs for
+        @type version: C{str}
+        @param output_dir: the directory to put the tarballs into
+        @type output_dir: C{str}
+        """
+        logger.setLevel(logging.WARNING if quiet else logging.INFO)
+
+        try:
+            do_checkout(package=package, version=version, branch=self.branch, outdir=output_dir)
+        except (OSError, CommandFailed) as e:
+            raise CommandExecFailed(str(e))
+        except GitError as e:
+            raise GitRepositoryError(str(e))

gbp/pkg/git.py

@@ -24,6 +24,7 @@
 from gbp.command_wrappers import (CatenateTarArchive, CatenateZipArchive)
 from gbp.git import GitRepository, GitRepositoryError
 from gbp.deb.pristinetar import DebianPristineTar
+from gbp.deb.pristinelfs import PristineLfs
 
 
 import gbp.log
@@ -35,6 +36,7 @@ class PkgGitRepository(GitRepository):
     def __init__(self, *args, **kwargs):
         super(PkgGitRepository, self).__init__(*args, **kwargs)
         self.pristine_tar = DebianPristineTar(self)
+        self.pristine_lfs = PristineLfs(self)
 
     @staticmethod
     def sanitize_prefix(prefix):

gbp/scripts/buildpackage.py

@@ -385,6 +385,7 @@ def build_parser(name, prefix=None):
     tag_group.add_config_file_option(option_name="debian-tag-msg", dest="debian_tag_msg")
     tag_group.add_config_file_option(option_name="upstream-tag", dest="upstream_tag")
     orig_group.add_config_file_option(option_name="upstream-tree", dest="upstream_tree")
+    orig_group.add_boolean_config_file_option(option_name="pristine-lfs", dest="pristine_lfs")
     orig_group.add_boolean_config_file_option(option_name="pristine-tar", dest="pristine_tar")
     orig_group.add_boolean_config_file_option(option_name="pristine-tar-commit",
                                               dest="pristine_tar_commit")

gbp/scripts/clone.py

@@ -199,6 +199,8 @@ def main(argv):
             branches = [options.debian_branch, options.upstream_branch]
             if options.pristine_tar:
                 branches += [repo.pristine_tar_branch]
+            if options.pristine_lfs:
+                branches += [repo.pristine_lfs_branch]
             gbp.log.debug('Will track branches: %s' % branches)
             for branch in branches:
                 remote = 'origin/%s' % branch

gbp/scripts/export_orig.py

@@ -36,6 +36,7 @@ def prepare_upstream_tarballs(repo, source, options, tarball_dir, output_dir):
     """
     Make sure we have the needed upstream tarballs. The default order is:
     - look in tarball_dir and if found symlink to it
+    - create tarball using pristine-lfs
     - create tarball using pristine-tar
     - create tarball using git-archive
 
@@ -68,7 +69,8 @@ def prepare_upstream_tarballs(repo, source, options, tarball_dir, output_dir):
 
     # Create tarball if missing or forced
     if not du.DebianPkgPolicy.has_origs(orig_files, output_dir) or options.force_create:
-        if not pristine_tar_build_origs(repo, source, output_dir, options):
+        if (not pristine_lfs_checkout_origs(repo, source, output_dir, options) and
+            not pristine_tar_build_origs(repo, source, output_dir, options)):
             git_archive_build_origs(repo, source, output_dir, options)
     maybe_pristine_tar_commit(repo, source, options, output_dir, orig_files)
     pristine_tar_verify_origs(repo, source, options, output_dir, orig_files)
@@ -92,6 +94,26 @@ def pristine_tar_prepare_orig_tree(repo, source, options):
                            "orig tarball via pristine-tar" % tree_name)
 
 
+def pristine_lfs_checkout_origs(repo, source, output_dir, options):
+    """
+    Check out orig tarballs using pristine-lfs
+
+    @returns: C{True} if tarball was built, C{False} otherwise
+    """
+    gbp.log.info(options)
+    if not options.pristine_lfs:
+        return False
+
+    if not repo.has_branch(repo.pristine_lfs_branch):
+        gbp.log.warn('Pristine-lfs branch "%s" not found' %
+                     repo.pristine_lfs_branch)
+
+    try:
+        repo.pristine_lfs.checkout(source.name, source.version, output_dir, quiet=not options.verbose)
+        return True
+    except CommandExecFailed:
+        return False
+
 def pristine_tar_build_origs(repo, source, output_dir, options):
     """
     Build orig tarball using pristine-tar
@@ -297,6 +319,7 @@ def build_parser(name):
     tag_group.add_config_file_option(option_name="upstream-tag", dest="upstream_tag")
     orig_group.add_config_file_option(option_name="upstream-tree", dest="upstream_tree")
     orig_group.add_boolean_config_file_option(option_name="pristine-tar", dest="pristine_tar")
+    orig_group.add_boolean_config_file_option(option_name="pristine-lfs", dest="pristine_lfs")
     orig_group.add_config_file_option(option_name="force-create", dest="force_create",
                                       help="force creation of orig tarball", action="store_true")
     orig_group.add_config_file_option(option_name="tarball-dir", dest="tarball_dir", type="path",

gbp/scripts/import_orig.py

@@ -323,8 +323,10 @@ def set_bare_repo_options(options):
     if options.pristine_tar or options.merge:
         gbp.log.info("Bare repository: setting %s%s options"
                      % (["", " '--no-pristine-tar'"][options.pristine_tar],
+                        ["", " '--no-pristine-lfs'"][options.pristine_lfs],
                         ["", " '--no-merge'"][options.merge]))
         options.pristine_tar = False
+        options.pristine_lfs = False
         options.merge = False
 
 
@@ -378,6 +380,8 @@ def build_parser(name):
                                      dest="upstream_tag")
     import_group.add_config_file_option(option_name="filter",
                                         dest="filters", action="append")
+    import_group.add_boolean_config_file_option(option_name="pristine-lfs",
+                                                dest="pristine_lfs")
     import_group.add_boolean_config_file_option(option_name="pristine-tar",
                                                 dest="pristine_tar")
     import_group.add_boolean_config_file_option(option_name="filter-pristine-tar",
@@ -514,9 +518,12 @@ def main(argv):
                                      create_missing_branch=is_empty,
                                      )
 
-            if options.pristine_tar:
+            if options.pristine_tar or options.pristine_lfs:
                 if pristine_orig:
-                    repo.rrr_branch(repo.pristine_tar_branch)
+                    if options.pristine_lfs:
+                        repo.rrr_branch(repo.pristine_lfs_branch)
+                    elif options.pristine_tar:
+                        repo.rrr_branch(repo.pristine_tar_branch)
                     for source in sources:
                         # Enforce signature file exists with --upstream-signatures=on
                         if options.upstream_signatures.is_on() and not source.signaturefile:
@@ -526,9 +533,12 @@ def main(argv):
                     # For all practical purposes we're interested in pristine_orig's path
                     if pristine_orig != sources[0].path:
                         sources[0]._path = pristine_orig
-                    repo.create_pristine_tar_commits(import_branch, sources)
+                    if options.pristine_lfs:
+                        repo.create_pristine_lfs_commits(sources)
+                    elif options.pristine_tar:
+                        repo.create_pristine_tar_commits(import_branch, sources)
                 else:
-                    gbp.log.warn("'%s' not an archive, skipping pristine-tar" % sources[0].path)
+                    gbp.log.warn("'%s' not an archive, skipping pristine-tar/pristine-lfs" % sources[0].path)
 
             repo.create_tag(name=tag,
                             msg="Upstream version %s" % version,