Skip to content

Commit

Permalink
add "archiveType" source.json property to set the http_archive's "typ…
Browse files Browse the repository at this point in the history 
…e" value.

Closes #17789.

PiperOrigin-RevId: 517408668
Change-Id: I7f89db0d2587cde3ff9d77c8657d162981cf32dc
  • Loading branch information
@jjjhhhlll @Wyverald
jjjhhhlll authored and Wyverald committed Jul 12, 2023
1 parent 13b0b7a commit 6bb1a1e
Show file tree
Hide file tree
Showing 6 changed files with 83 additions and 203 deletions.
6 changes: 5 additions & 1 deletion site/en/external/registry.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,10 @@ An index registry must follow the format below:
patch file names, and the values are the integrity checksum of
the patch files
* `patch_strip`: Same as the `--strip` argument of Unix `patch`.
* `archive_type`: The archive type of the downloaded file (Same as `type` on `http_archive`).
By default, the archive type is determined from the file extension of the URL. If the file has
no extension, you can explicitly specify one of the following: `"zip"`, `"jar"`, `"war"`, `"aar"`,
`"tar"`, `"tar.gz"`, `"tgz"`, `"tar.xz"`, `"txz"`, `"tar.zst"`, `"tzst"`, `tar.bz2`, `"ar"`, or `"deb"`.
* The type can be changed to use a local path, representing a
`local_repository` repo, with these fields:
* `type`: `local_path`
Expand Down Expand Up @@ -109,4 +113,4 @@ branch of the `my-org` fork, you would set
`--registry=https://raw.githubusercontent.com/my-org/bazel-central-registry/main/`.

Using the `--registry` flag stops the Bazel Central Registry from being used by
default, but you can add it back by adding `--registry=https://bcr.bazel.build`.
default, but you can add it back by adding `--registry=https://bcr.bazel.build`.
9 changes: 9 additions & 0 deletions src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

package com.google.devtools.build.lib.bazel.bzlmod;

import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
Expand Down Expand Up @@ -85,6 +86,14 @@ public ArchiveRepoSpecBuilder setRemotePatchStrip(int remotePatchStrip) {
return this;
}

@CanIgnoreReturnValue
public ArchiveRepoSpecBuilder setArchiveType(String archiveType) {
if (!Strings.isNullOrEmpty(archiveType)) {
attrBuilder.put("type", archiveType);
}
return this;
}

public RepoSpec build() {
return RepoSpec.builder()
.setBzlFile("@bazel_tools//tools/build_defs/repo:http.bzl")
Expand Down
2 changes: 2 additions & 0 deletions src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ private static class SourceJson {
Map<String, String> patches;
int patchStrip;
String path;
String archiveType;
}

/**
Expand Down Expand Up @@ -254,6 +255,7 @@ private RepoSpec createArchiveRepoSpec(
.setStripPrefix(Strings.nullToEmpty(sourceJson.get().stripPrefix))
.setRemotePatches(remotePatches.buildOrThrow())
.setRemotePatchStrip(sourceJson.get().patchStrip)
.setArchiveType(sourceJson.get().archiveType)
.build();
}

Expand Down
29 changes: 29 additions & 0 deletions src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -296,4 +296,33 @@ public void testGetYankedVersion() throws Exception {
Version.parse("1.0"),
"red-pill 1.0 is yanked due to CVE-2000-101, please upgrade to 2.0"));
}

@Test
public void testArchiveWithExplicitType() throws Exception {
server.serve(
"/modules/archive_type/1.0/source.json",
"{",
" \"url\": \"https://mysite.com/thing?format=zip\",",
" \"integrity\": \"sha256-blah\",",
" \"archive_type\": \"zip\"",
"}");
server.start();

Registry registry = registryFactory.getRegistryWithUrl(server.getUrl());
assertThat(
registry.getRepoSpec(
createModuleKey("archive_type", "1.0"),
RepositoryName.create("archive_type_repo"),
reporter))
.isEqualTo(
new ArchiveRepoSpecBuilder()
.setRepoName("archive_type_repo")
.setUrls(ImmutableList.of("https://mysite.com/thing?format=zip"))
.setIntegrity("sha256-blah")
.setStripPrefix("")
.setArchiveType("zip")
.setRemotePatches(ImmutableMap.of())
.setRemotePatchStrip(0)
.build());
}
}
216 changes: 17 additions & 199 deletions src/test/py/bazel/bzlmod/bazel_module_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -450,221 +450,39 @@ def testWorkspaceItselfCanSeeRootModuleMappings(self):
_, _, stderr = self.RunBazel(['build', ':a'], allow_failure=False)
self.assertIn('I LUV U!', '\n'.join(stderr))

def testNativeModuleNameAndVersion(self):
self.main_registry.setModuleBasePath('projects')
projects_dir = self.main_registry.projects
def testArchiveWithArchiveType(self):
# make the archive without the .zip extension
self.main_registry.createCcModule(
'aaa', '1.2', archive_pattern='%s.%s', archive_type='zip'
)

self.ScratchFile(
'MODULE.bazel',
[
'module(name="root",version="0.1")',
'bazel_dep(name="foo",version="1.0")',
'report_ext = use_extension("@foo//:ext.bzl", "report_ext")',
'use_repo(report_ext, "report_repo")',
'bazel_dep(name="bar")',
'local_path_override(module_name="bar",path="bar")',
'bazel_dep(name = "aaa", version = "1.2")',
],
)
self.ScratchFile('WORKSPACE')
self.ScratchFile(
'WORKSPACE.bzlmod', ['local_repository(name="quux",path="quux")']
)
self.ScratchFile(
'BUILD',
[
'load("@foo//:report.bzl", "report")',
'report()',
],
)
# foo: a repo defined by a normal Bazel module. Also hosts the extension
# `report_ext` which generates a repo `report_repo`.
self.main_registry.createLocalPathModule('foo', '1.0', 'foo')
projects_dir.joinpath('foo').mkdir(exist_ok=True)
scratchFile(projects_dir.joinpath('foo', 'WORKSPACE'))
scratchFile(
projects_dir.joinpath('foo', 'BUILD'),
[
'load(":report.bzl", "report")',
'report()',
],
)
scratchFile(
projects_dir.joinpath('foo', 'report.bzl'),
[
'def report():',
' repo = native.repository_name()',
' name = str(native.module_name())',
' version = str(native.module_version())',
' print("@" + repo + " reporting in: " + name + "@" + version)',
' native.filegroup(name="a")',
],
)
scratchFile(
projects_dir.joinpath('foo', 'ext.bzl'),
[
'def _report_repo(rctx):',
' rctx.file("BUILD",',
' "load(\\"@foo//:report.bzl\\", \\"report\\")\\n" +',
' "report()")',
'report_repo = repository_rule(_report_repo)',
'report_ext = module_extension(',
' lambda mctx: report_repo(name="report_repo"))',
],
)
# bar: a repo defined by a Bazel module with a non-registry override
self.ScratchFile('bar/WORKSPACE')
self.ScratchFile(
'bar/MODULE.bazel',
[
'module(name="bar", version="2.0")',
'bazel_dep(name="foo",version="1.0")',
],
)
self.ScratchFile(
'bar/BUILD',
[
'load("@foo//:report.bzl", "report")',
'report()',
],
)
# quux: a repo defined by WORKSPACE
self.ScratchFile('quux/WORKSPACE')
self.ScratchFile(
'quux/BUILD',
[
'load("@foo//:report.bzl", "report")',
'report()',
],
)

_, _, stderr = self.RunBazel(
[
'build',
':a',
'@foo//:a',
'@report_repo//:a',
'@bar//:a',
'@quux//:a',
],
allow_failure=False,
)
stderr = '\n'.join(stderr)
self.assertIn('@@ reporting in: root@0.1', stderr)
self.assertIn('@@foo~1.0 reporting in: foo@1.0', stderr)
self.assertIn(
'@@foo~1.0~report_ext~report_repo reporting in: foo@1.0', stderr
)
self.assertIn('@@bar~override reporting in: bar@2.0', stderr)
self.assertIn('@@quux reporting in: None@None', stderr)

def testWorkspaceToolchainRegistrationWithPlatformsConstraint(self):
"""Regression test for https://github.com/bazelbuild/bazel/issues/17289."""
self.ScratchFile('MODULE.bazel')
self.ScratchFile(
'WORKSPACE', ['register_toolchains("//:my_toolchain_toolchain")']
)
os.remove(self.Path('WORKSPACE.bzlmod'))

self.ScratchFile(
'BUILD.bazel',
[
'load(":defs.bzl", "get_host_os", "my_consumer", "my_toolchain")',
'toolchain_type(name = "my_toolchain_type")',
'my_toolchain(',
' name = "my_toolchain",',
' my_value = "Hello, Bzlmod!",',
')',
'toolchain(',
' name = "my_toolchain_toolchain",',
' toolchain = ":my_toolchain",',
' toolchain_type = ":my_toolchain_type",',
' target_compatible_with = [',
' "@platforms//os:" + get_host_os(),',
' ],',
')',
'my_consumer(',
' name = "my_consumer",',
')',
],
)

self.ScratchFile(
'defs.bzl',
[
(
'load("@local_config_platform//:constraints.bzl",'
' "HOST_CONSTRAINTS")'
),
'def _my_toolchain_impl(ctx):',
' return [',
' platform_common.ToolchainInfo(',
' my_value = ctx.attr.my_value,',
' ),',
' ]',
'my_toolchain = rule(',
' implementation = _my_toolchain_impl,',
' attrs = {',
' "my_value": attr.string(),',
' },',
')',
'def _my_consumer(ctx):',
' my_toolchain_info = ctx.toolchains["//:my_toolchain_type"]',
' out = ctx.actions.declare_file(ctx.attr.name)',
(
' ctx.actions.write(out, "my_value ='
' {}".format(my_toolchain_info.my_value))'
),
' return [DefaultInfo(files = depset([out]))]',
'my_consumer = rule(',
' implementation = _my_consumer,',
' attrs = {},',
' toolchains = ["//:my_toolchain_type"],',
'cc_binary(',
' name = "main",',
' srcs = ["main.cc"],',
' deps = ["@aaa//:lib_aaa"],',
')',
'def get_host_os():',
' for constraint in HOST_CONSTRAINTS:',
' if constraint.startswith("@platforms//os:"):',
' return constraint.removeprefix("@platforms//os:")',
],
)

self.RunBazel([
'build',
'//:my_consumer',
'--toolchain_resolution_debug=//:my_toolchain_type',
])
with open(self.Path('bazel-bin/my_consumer'), 'r') as f:
self.assertEqual(f.read().strip(), 'my_value = Hello, Bzlmod!')

def testModuleExtensionWithRuleError(self):
self.ScratchFile(
'MODULE.bazel',
[
'ext = use_extension("extensions.bzl", "ext")',
'use_repo(ext, "ext")',
],
)
self.ScratchFile('BUILD')
self.ScratchFile(
'extensions.bzl',
'main.cc',
[
'def _rule_impl(ctx):',
' print("RULE CALLED")',
'init_rule = rule(_rule_impl)',
'def ext_impl(module_ctx):',
' init_rule()',
'ext = module_extension(implementation = ext_impl,)',
'#include "aaa.h"',
'int main() {',
' hello_aaa("main function");',
'}',
],
)
exit_code, _, stderr = self.RunBazel(
['build', '--nobuild', '@ext//:all'],
allow_failure=True,
)
self.AssertExitCode(exit_code, 48, stderr)
self.assertIn(
'Error in init_rule: A rule can only be instantiated in a BUILD file, '
'or a macro invoked from a BUILD file',
stderr,
)
_, stdout, _ = self.RunBazel(['run', '//:main'], allow_failure=False)
self.assertIn('main function => aaa@1.2', stdout)


if __name__ == '__main__':
Expand Down
24 changes: 21 additions & 3 deletions src/test/py/bazel/bzlmod/test_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ def __init__(self, name, version):
self.module_dot_bazel = None
self.patches = []
self.patch_strip = 0
self.archive_type = None

def set_source(self, archive_url, strip_prefix=None):
self.archive_url = archive_url
Expand All @@ -81,6 +82,10 @@ def set_patches(self, patches, patch_strip):
self.patch_strip = patch_strip
return self

def set_archive_type(self, archive_type):
self.archive_type = archive_type
return self


class BazelRegistry:
"""A class to help create a Bazel module project from scatch and add it into the registry."""
Expand Down Expand Up @@ -204,9 +209,9 @@ def calc_repo_name_str(dep):
])
return src_dir

def createArchive(self, name, version, src_dir):
def createArchive(self, name, version, src_dir, filename_pattern='%s.%s.zip'):
"""Create an archive with a given source directory."""
zip_path = self.archives.joinpath('%s.%s.zip' % (name, version))
zip_path = self.archives.joinpath(filename_pattern % (name, version))
zip_obj = zipfile.ZipFile(str(zip_path), 'w')
for foldername, _, filenames in os.walk(str(src_dir)):
for filename in filenames:
Expand Down Expand Up @@ -243,6 +248,9 @@ def addModule(self, module):
source['patches'][patch.name] = integrity(read(patch))
shutil.copy(str(patch), str(patch_dir))

if module.archive_type:
source['archive_type'] = module.archive_type

with module_dir.joinpath('source.json').open('w') as f:
json.dump(source, f, indent=4, sort_keys=True)

Expand All @@ -255,17 +263,27 @@ def createCcModule(
patches=None,
patch_strip=0,
extra_module_file_contents=None,
archive_pattern=None,
archive_type=None,
):
"""Generate a cc project and add it as a module into the registry."""
src_dir = self.generateCcSource(
name, version, deps, repo_names, extra_module_file_contents
)
archive = self.createArchive(name, version, src_dir)
if archive_pattern:
archive = self.createArchive(
name, version, src_dir, filename_pattern=archive_pattern
)
else:
archive = self.createArchive(name, version, src_dir)
module = Module(name, version)
module.set_source(archive.resolve().as_uri())
module.set_module_dot_bazel(src_dir.joinpath('MODULE.bazel'))
if patches:
module.set_patches(patches, patch_strip)
if archive_type:
module.set_archive_type(archive_type)

self.addModule(module)
return self

Expand Down

0 comments on commit 6bb1a1e

Please sign in to comment.