Skip to content

Commit

Permalink
Build reproducible tarballs from git
Browse files Browse the repository at this point in the history
  • Loading branch information
@Rovanion
Rovanion committed Jun 19, 2023
1 parent f725e61 commit bf4f3f6
Show file tree
Hide file tree
Showing 2 changed files with 44 additions and 18 deletions.
14 changes: 9 additions & 5 deletions easybuild/tools/filetools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2707,11 +2707,15 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
for cmd in cmds:
run.run_cmd(cmd, log_all=True, simple=True, regexp=False, path=repo_name)

# create an archive and delete the git repo directory
if keep_git_dir:
tar_cmd = ['tar', 'cfvz', targetpath, repo_name]
else:
tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name]
# When CentOS 7 is phased out and tar>1.28 is everywhere, replace find-sort-pipe with tar-flag
# '--sort=name' and place LC_ALL in front of tar. Also remove flags --null, --no-recursion, and
# --files-from - from the flags to tar. See https://reproducible-builds.org/docs/archives/
tar_cmd = ['find', repo_name, '-print0', '-path \'*/.git\' -prune' if not keep_git_dir else '', '|',
'LC_ALL=C', 'sort', '--zero-terminated', '|',
'GZIP=--no-name', 'tar', '--create', '--file', targetpath, '--no-recursion',
'--gzip', '--mtime="1970-01-01 00:00Z"', '--owner=0', '--group=0',
'--numeric-owner', '--format=gnu', '--null',
'--no-recursion', '--files-from -']
run.run_cmd(' '.join(tar_cmd), log_all=True, simple=True, regexp=False)

# cleanup (repo_name dir does not exist in dry run mode)
Expand Down
48 changes: 35 additions & 13 deletions test/framework/filetools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2780,41 +2780,57 @@ def run_check():
'url': 'git@github.com:easybuilders',
'tag': 'tag_for_tests',
}
git_repo = {'git_repo': 'git@github.com:easybuilders/testrepository.git'} # Just to make the below shorter
string_args = {
'git_repo': 'git@github.com:easybuilders/testrepository.git',
'test_prefix': self.test_prefix,
}

expected = '\n'.join([
r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"',
r" \(in .*/tmp.*\)",
r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated'
r' | GZIP=--no-name tar --create --file %(test_prefix)s/target/test.tar.gz --no-recursion'
r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu'
r' --null --no-recursion --files-from -"',
r" \(in .*/tmp.*\)",
]) % git_repo
]) % string_args
run_check()

git_config['clone_into'] = 'test123'
expected = '\n'.join([
r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"',
r" \(in .*/tmp.*\)",
r' running command "tar cfvz .*/target/test.tar.gz --exclude .git test123"',
r' running command "find test123 -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated'
r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion'
r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu'
r' --null --no-recursion --files-from -"',
r" \(in .*/tmp.*\)",
]) % git_repo
]) % string_args
run_check()
del git_config['clone_into']

git_config['recursive'] = True
expected = '\n'.join([
r' running command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"',
r" \(in .*/tmp.*\)",
r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated'
r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion'
r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu'
r' --null --no-recursion --files-from -"',
r" \(in .*/tmp.*\)",
]) % git_repo
]) % string_args
run_check()

git_config['keep_git_dir'] = True
expected = '\n'.join([
r' running command "git clone --branch tag_for_tests --recursive %(git_repo)s"',
r" \(in .*/tmp.*\)",
r' running command "tar cfvz .*/target/test.tar.gz testrepository"',
r' running command "find testrepository -print0 | LC_ALL=C sort --zero-terminated | GZIP=--no-name tar'
r' --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion --gzip'
r' --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu --null --no-recursion'
r' --files-from -"',
r" \(in .*/tmp.*\)",
]) % git_repo
]) % string_args
run_check()
del git_config['keep_git_dir']

Expand All @@ -2825,9 +2841,12 @@ def run_check():
r" \(in .*/tmp.*\)",
r' running command "git checkout 8456f86 && git submodule update --init --recursive"',
r" \(in testrepository\)",
r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated'
r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion'
r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu'
r' --null --no-recursion --files-from -"',
r" \(in .*/tmp.*\)",
]) % git_repo
]) % string_args
run_check()

del git_config['recursive']
Expand All @@ -2836,9 +2855,12 @@ def run_check():
r" \(in .*/tmp.*\)",
r' running command "git checkout 8456f86"',
r" \(in testrepository\)",
r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated'
r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion'
r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu'
r' --null --no-recursion --files-from -"',
r" \(in .*/tmp.*\)",
]) % git_repo
]) % string_args
run_check()

# Test with real data.
Expand Down

0 comments on commit bf4f3f6

Please sign in to comment.