@@ -276,27 +276,35 @@ def _materialize_one(
276
276
fv_latest_values_sql , feature_view .batch_source .field_mapping
277
277
)
278
278
279
- fv_to_proto_sql = self .generate_snowflake_materialization_query (
280
- self .repo_config ,
281
- fv_latest_mapped_values_sql ,
282
- feature_view ,
283
- project ,
284
- )
285
-
286
- if self .repo_config .online_store .type == "snowflake.online" :
287
- self .materialize_to_snowflake_online_store (
279
+ features_full_list = feature_view .features
280
+ feature_batches = [
281
+ features_full_list [i : i + 100 ]
282
+ for i in range (0 , len (features_full_list ), 100 )
283
+ ]
284
+ for feature_batch in feature_batches :
285
+
286
+ fv_to_proto_sql = self .generate_snowflake_materialization_query (
288
287
self .repo_config ,
289
- fv_to_proto_sql ,
288
+ fv_latest_mapped_values_sql ,
290
289
feature_view ,
290
+ feature_batch ,
291
291
project ,
292
292
)
293
- else :
294
- self .materialize_to_external_online_store (
295
- self .repo_config ,
296
- fv_to_proto_sql ,
297
- feature_view ,
298
- tqdm_builder ,
299
- )
293
+
294
+ if self .repo_config .online_store .type == "snowflake.online" :
295
+ self .materialize_to_snowflake_online_store (
296
+ self .repo_config ,
297
+ fv_to_proto_sql ,
298
+ feature_view ,
299
+ project ,
300
+ )
301
+ else :
302
+ self .materialize_to_external_online_store (
303
+ self .repo_config ,
304
+ fv_to_proto_sql ,
305
+ feature_view ,
306
+ tqdm_builder ,
307
+ )
300
308
301
309
return SnowflakeMaterializationJob (
302
310
job_id = job_id , status = MaterializationJobStatus .SUCCEEDED
@@ -311,6 +319,7 @@ def generate_snowflake_materialization_query(
311
319
repo_config : RepoConfig ,
312
320
fv_latest_mapped_values_sql : str ,
313
321
feature_view : Union [BatchFeatureView , FeatureView ],
322
+ feature_batch : list ,
314
323
project : str ,
315
324
) -> str :
316
325
@@ -333,7 +342,7 @@ def generate_snowflake_materialization_query(
333
342
UDF serialization function.
334
343
"""
335
344
feature_sql_list = []
336
- for feature in feature_view . features :
345
+ for feature in feature_batch :
337
346
feature_value_type_name = feature .dtype .to_value_type ().name
338
347
339
348
feature_sql = _convert_value_name_to_snowflake_udf (
0 commit comments