Skip to content

Commit 61523f3

Browse files
fix: Batch Snowflake materialization queries to obey Snowpark 100 feature limit
Signed-off-by: miles.adkins <miles.adkins@snowflake.com>
1 parent 98a24a3 commit 61523f3

File tree

1 file changed

+27
-18
lines changed

1 file changed

+27
-18
lines changed

sdk/python/feast/infra/materialization/snowflake_engine.py

+27-18
Original file line numberDiff line numberDiff line change
@@ -276,27 +276,35 @@ def _materialize_one(
276276
fv_latest_values_sql, feature_view.batch_source.field_mapping
277277
)
278278

279-
fv_to_proto_sql = self.generate_snowflake_materialization_query(
280-
self.repo_config,
281-
fv_latest_mapped_values_sql,
282-
feature_view,
283-
project,
284-
)
285-
286-
if self.repo_config.online_store.type == "snowflake.online":
287-
self.materialize_to_snowflake_online_store(
279+
features_full_list = feature_view.features
280+
feature_batches = [
281+
features_full_list[i : i + 100]
282+
for i in range(0, len(features_full_list), 100)
283+
]
284+
for feature_batch in feature_batches:
285+
286+
fv_to_proto_sql = self.generate_snowflake_materialization_query(
288287
self.repo_config,
289-
fv_to_proto_sql,
288+
fv_latest_mapped_values_sql,
290289
feature_view,
290+
feature_batch,
291291
project,
292292
)
293-
else:
294-
self.materialize_to_external_online_store(
295-
self.repo_config,
296-
fv_to_proto_sql,
297-
feature_view,
298-
tqdm_builder,
299-
)
293+
294+
if self.repo_config.online_store.type == "snowflake.online":
295+
self.materialize_to_snowflake_online_store(
296+
self.repo_config,
297+
fv_to_proto_sql,
298+
feature_view,
299+
project,
300+
)
301+
else:
302+
self.materialize_to_external_online_store(
303+
self.repo_config,
304+
fv_to_proto_sql,
305+
feature_view,
306+
tqdm_builder,
307+
)
300308

301309
return SnowflakeMaterializationJob(
302310
job_id=job_id, status=MaterializationJobStatus.SUCCEEDED
@@ -311,6 +319,7 @@ def generate_snowflake_materialization_query(
311319
repo_config: RepoConfig,
312320
fv_latest_mapped_values_sql: str,
313321
feature_view: Union[BatchFeatureView, FeatureView],
322+
feature_batch: list,
314323
project: str,
315324
) -> str:
316325

@@ -333,7 +342,7 @@ def generate_snowflake_materialization_query(
333342
UDF serialization function.
334343
"""
335344
feature_sql_list = []
336-
for feature in feature_view.features:
345+
for feature in feature_batch:
337346
feature_value_type_name = feature.dtype.to_value_type().name
338347

339348
feature_sql = _convert_value_name_to_snowflake_udf(

0 commit comments

Comments
 (0)