Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add platform_allowlist for migrator #1655

Merged
merged 4 commits into from
Apr 18, 2023
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 18 additions & 17 deletions conda_forge_tick/feedstock_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ def populate_feedstock_attributes(
ci_support_files = glob.glob(
os.path.join(feedstock_dir, ".ci_support", "*.yaml"),
)
varient_yamls = []
variant_yamls = []
plat_arch = []
for cbc_path in ci_support_files:
LOGGER.debug("parsing conda-build config: %s", cbc_path)
Expand All @@ -242,7 +242,7 @@ def populate_feedstock_attributes(
break
plat_arch.append((plat, arch))

varient_yamls.append(
variant_yamls.append(
parse_meta_yaml(
meta_yaml,
platform=plat,
Expand All @@ -259,41 +259,41 @@ def populate_feedstock_attributes(
# sometimes the requirements come out to None or [None]
# and this ruins the aggregated meta_yaml / breaks stuff
LOGGER.debug("getting reqs for config: %s", cbc_path)
if "requirements" in varient_yamls[-1]:
varient_yamls[-1]["requirements"] = _clean_req_nones(
varient_yamls[-1]["requirements"],
if "requirements" in variant_yamls[-1]:
variant_yamls[-1]["requirements"] = _clean_req_nones(
variant_yamls[-1]["requirements"],
)
if "outputs" in varient_yamls[-1]:
for iout in range(len(varient_yamls[-1]["outputs"])):
if "requirements" in varient_yamls[-1]["outputs"][iout]:
varient_yamls[-1]["outputs"][iout][
if "outputs" in variant_yamls[-1]:
for iout in range(len(variant_yamls[-1]["outputs"])):
if "requirements" in variant_yamls[-1]["outputs"][iout]:
variant_yamls[-1]["outputs"][iout][
"requirements"
] = _clean_req_nones(
varient_yamls[-1]["outputs"][iout]["requirements"],
variant_yamls[-1]["outputs"][iout]["requirements"],
)

# collapse them down
LOGGER.debug("collapsing reqs for config: %s", cbc_path)
final_cfgs = {}
for plat_arch, varyml in zip(plat_arch, varient_yamls):
for plat_arch, varyml in zip(plat_arch, variant_yamls):
if plat_arch not in final_cfgs:
final_cfgs[plat_arch] = []
final_cfgs[plat_arch].append(varyml)
for k in final_cfgs:
ymls = final_cfgs[k]
final_cfgs[k] = _convert_to_dict(ChainDB(*ymls))
plat_arch = []
varient_yamls = []
variant_yamls = []
for k, v in final_cfgs.items():
plat_arch.append(k)
varient_yamls.append(v)
variant_yamls.append(v)
else:
LOGGER.debug("doing generic parsing")
plat_arch = [("win", "64"), ("osx", "64"), ("linux", "64")]
for k in set(sub_graph["conda-forge.yml"].get("provider", {})):
if "_" in k:
plat_arch.append(tuple(k.split("_")))
varient_yamls = [
variant_yamls = [
parse_meta_yaml(meta_yaml, platform=plat, arch=arch)
for plat, arch in plat_arch
]
Expand All @@ -305,10 +305,11 @@ def populate_feedstock_attributes(
raise

LOGGER.debug("platforms: %s", plat_arch)
sub_graph["platforms"] = ["_".join(k) for k in plat_arch]

# this makes certain that we have consistent ordering
sorted_varient_yamls = [x for _, x in sorted(zip(plat_arch, varient_yamls))]
yaml_dict = ChainDB(*sorted_varient_yamls)
sorted_variant_yamls = [x for _, x in sorted(zip(plat_arch, variant_yamls))]
yaml_dict = ChainDB(*sorted_variant_yamls)
if not yaml_dict:
LOGGER.error(f"Something odd happened when parsing recipe {name}")
sub_graph["parsing_error"] = "make_graph: Could not parse"
Expand All @@ -317,7 +318,7 @@ def populate_feedstock_attributes(
sub_graph["meta_yaml"] = _convert_to_dict(yaml_dict)
meta_yaml = sub_graph["meta_yaml"]

for k, v in zip(plat_arch, varient_yamls):
for k, v in zip(plat_arch, variant_yamls):
plat_arch_name = "_".join(k)
sub_graph[f"{plat_arch_name}_meta_yaml"] = v
_, sub_graph[f"{plat_arch_name}_requirements"], _ = _extract_requirements(v)
Expand Down
31 changes: 24 additions & 7 deletions conda_forge_tick/migrators/migration_yaml.py
Original file line number Diff line number Diff line change
Expand Up @@ -176,10 +176,23 @@ def __init__(
self.max_solver_attempts = max_solver_attempts

def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:
wait_for_migrators = self.loaded_yaml.get("__migrator", {}).get(
"wait_for_migrators",
[],
)
"""
Determine whether migrator needs to be filtered out.

Return value of True means to skip migrator, False means to go ahead.
Calls up the MRO until Migrator.filter, see docstring there (./core.py).
"""
migrator_payload = self.loaded_yaml.get("__migrator", {})
platform_allowlist = migrator_payload.get("platform_allowlist", [])
wait_for_migrators = migrator_payload.get("wait_for_migrators", [])

platform_filtered = False
if platform_allowlist:
# filter out nodes where the intersection between
# attrs.platforms and platform_allowlist is empty
intersection = set(attrs.get("platforms", {})) & set(platform_allowlist)
platform_filtered = not bool(intersection)

need_to_wait = False
if wait_for_migrators:
found_migrators = set()
Expand All @@ -199,9 +212,13 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:
wait_for_migrators,
)

return need_to_wait or super().filter(
attrs=attrs,
not_bad_str_start=not_bad_str_start,
return (
platform_filtered
or need_to_wait
or super().filter(
attrs=attrs,
not_bad_str_start=not_bad_str_start,
)
)

def migrate(
Expand Down
1 change: 1 addition & 0 deletions conda_forge_tick/migrators_types.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ class AttrsTypedDict_(TypedDict, total=False):
package: PackageTypedDict
raw_meta_yaml: str
req: Set[str]
platforms: List[str]
requirements: RequirementsTypedDict
source: SourceTypedDict
test: TestTypedDict
Expand Down