Skip to content

Commit

Permalink
Add uri to projection
Browse files Browse the repository at this point in the history
  • Loading branch information
@Sicheng-Pan
Sicheng Pan authored and Sicheng-Pan committed Oct 14, 2024
1 parent f5b9c8f commit d1f91de
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 6 deletions.
27 changes: 24 additions & 3 deletions chromadb/execution/executor/distributed.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,14 @@ def _doc(metadata: Optional[Metadata]) -> Optional[str]:
return None


def _uri(metadata: Optional[Metadata]) -> Optional[str]:
"""Retrieve the uri (if any) from a Metadata map"""

if metadata and "chroma:uri" in metadata:
return str(metadata["chroma:uri"])
return None


class DistributedExecutor(Executor):
_manager: DistributedSegmentManager

Expand All @@ -66,6 +74,7 @@ def get(self, plan: GetPlan) -> GetResult:
ids = [r["id"] for r in records]
embeddings = None
documents = None
uris = None
metadatas = None
included = list()

Expand All @@ -83,6 +92,10 @@ def get(self, plan: GetPlan) -> GetResult:
documents = [_doc(r["metadata"]) for r in records]
included.append(IncludeEnum.documents)

if plan.projection.uri:
uris = [_uri(r["metadata"]) for r in records]
included.append(IncludeEnum.uris)

if plan.projection.metadata:
metadatas = [_clean_metadata(r["metadata"]) for r in records]
included.append(IncludeEnum.metadatas)
Expand All @@ -92,7 +105,7 @@ def get(self, plan: GetPlan) -> GetResult:
ids=ids,
embeddings=embeddings,
documents=documents, # type: ignore[typeddict-item]
uris=None,
uris=uris, # type: ignore[typeddict-item]
data=None,
metadatas=metadatas, # type: ignore[typeddict-item]
included=included,
Expand Down Expand Up @@ -126,6 +139,7 @@ def knn(self, plan: KNNPlan) -> QueryResult:
ids = [[r["id"] for r in result] for result in knns]
embeddings = None
documents = None
uris = None
metadatas = None
distances = None
included = list()
Expand All @@ -138,7 +152,7 @@ def knn(self, plan: KNNPlan) -> QueryResult:
distances = [[r["distance"] for r in result] for result in knns]
included.append(IncludeEnum.distances)

if plan.projection.document or plan.projection.metadata:
if plan.projection.document or plan.projection.metadata or plan.projection.uri:
merged_ids = list(set([id for result in ids for id in result]))
hydrated_records = self._metadata_segment(
plan.scan.collection
Expand All @@ -160,6 +174,13 @@ def knn(self, plan: KNNPlan) -> QueryResult:
]
included.append(IncludeEnum.documents)

if plan.projection.uri:
uris = [
[_uri(metadata_by_id.get(id, None)) for id in result]
for result in ids
]
included.append(IncludeEnum.uris)

if plan.projection.metadata:
metadatas = [
[_clean_metadata(metadata_by_id.get(id, None)) for id in result]
Expand All @@ -172,7 +193,7 @@ def knn(self, plan: KNNPlan) -> QueryResult:
ids=ids,
embeddings=embeddings, # type: ignore[typeddict-item]
documents=documents, # type: ignore[typeddict-item]
uris=None,
uris=uris, # type: ignore[typeddict-item]
data=None,
metadatas=metadatas, # type: ignore[typeddict-item]
distances=distances,
Expand Down
27 changes: 24 additions & 3 deletions chromadb/execution/executor/local.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,14 @@ def _doc(metadata: Optional[Metadata]) -> Optional[str]:
return None


def _uri(metadata: Optional[Metadata]) -> Optional[str]:
"""Retrieve the uri (if any) from a Metadata map"""

if metadata and "chroma:uri" in metadata:
return str(metadata["chroma:uri"])
return None


class LocalExecutor(Executor):
_manager: LocalSegmentManager

Expand All @@ -63,6 +71,7 @@ def get(self, plan: GetPlan) -> GetResult:
ids = [r["id"] for r in records]
embeddings = None
documents = None
uris = None
metadatas = None
included = list()

Expand All @@ -80,6 +89,10 @@ def get(self, plan: GetPlan) -> GetResult:
documents = [_doc(r["metadata"]) for r in records]
included.append(IncludeEnum.documents)

if plan.projection.uri:
uris = [_uri(r["metadata"]) for r in records]
included.append(IncludeEnum.uris)

if plan.projection.metadata:
metadatas = [_clean_metadata(r["metadata"]) for r in records]
included.append(IncludeEnum.metadatas)
Expand All @@ -89,7 +102,7 @@ def get(self, plan: GetPlan) -> GetResult:
ids=ids,
embeddings=embeddings,
documents=documents, # type: ignore[typeddict-item]
uris=None,
uris=uris, # type: ignore[typeddict-item]
data=None,
metadatas=metadatas, # type: ignore[typeddict-item]
included=included,
Expand Down Expand Up @@ -123,6 +136,7 @@ def knn(self, plan: KNNPlan) -> QueryResult:
ids = [[r["id"] for r in result] for result in knns]
embeddings = None
documents = None
uris = None
metadatas = None
distances = None
included = list()
Expand All @@ -135,7 +149,7 @@ def knn(self, plan: KNNPlan) -> QueryResult:
distances = [[r["distance"] for r in result] for result in knns]
included.append(IncludeEnum.distances)

if plan.projection.document or plan.projection.metadata:
if plan.projection.document or plan.projection.metadata or plan.projection.uri:
merged_ids = list(set([id for result in ids for id in result]))
hydrated_records = self._metadata_segment(
plan.scan.collection
Expand All @@ -157,6 +171,13 @@ def knn(self, plan: KNNPlan) -> QueryResult:
]
included.append(IncludeEnum.documents)

if plan.projection.uri:
uris = [
[_uri(metadata_by_id.get(id, None)) for id in result]
for result in ids
]
included.append(IncludeEnum.uris)

if plan.projection.metadata:
metadatas = [
[_clean_metadata(metadata_by_id.get(id, None)) for id in result]
Expand All @@ -169,7 +190,7 @@ def knn(self, plan: KNNPlan) -> QueryResult:
ids=ids,
embeddings=embeddings, # type: ignore[typeddict-item]
documents=documents, # type: ignore[typeddict-item]
uris=None,
uris=uris, # type: ignore[typeddict-item]
data=None,
metadatas=metadatas, # type: ignore[typeddict-item]
distances=distances,
Expand Down

0 comments on commit d1f91de

Please sign in to comment.