Skip to content

Commit

Permalink
fix s3 logic for new molecules
Browse files Browse the repository at this point in the history
  • Loading branch information
tschaume committed Dec 6, 2024
1 parent ea8c37b commit 0c71a95
Showing 1 changed file with 9 additions and 15 deletions.
24 changes: 9 additions & 15 deletions mp_api/client/core/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -487,26 +487,20 @@ def _query_resource(

if query_s3:
db_version = self.db_version.replace(".", "-")

suffix = (
self.suffix.split("/")[-1]
if "core" not in self.suffix
else self.suffix.split("/")[0]
)
suffix = suffix.replace("_", "-")
if self.suffix == "molecules/summary":
suffix = "molecules"
else:
infix, suffix = self.suffix.split("/", 1)
suffix = infix if suffix == "core" else suffix
suffix = suffix.replace("_", "-")

# Paginate over all entries in the bucket.
# This will have to change for when a subset of entries from
# the DB is needed.
is_tasks = "tasks" in suffix
bucket = (
"materialsproject-build"
if not is_tasks
else "materialsproject-parsed"
)
prefix = (
f"{suffix}" if is_tasks else f"collections/{db_version}/{suffix}"
)
bucket_suffix = "parsed" if is_tasks else "build"
bucket = f"materialsproject-{bucket_suffix}"
prefix = suffix if is_tasks else f"collections/{db_version}/{suffix}"
paginator = self.s3_client.get_paginator("list_objects_v2")
pages = paginator.paginate(Bucket=bucket, Prefix=prefix)

Expand Down

0 comments on commit 0c71a95

Please sign in to comment.