Merge remote-tracking branch 'origin/dev' into opt_chrom

pgscatalog_utils/match/write.py

@@ -29,11 +29,17 @@ def write_scorefiles(matches: pl.LazyFrame, split: bool, dataset: str):
     dominant: pl.LazyFrame
     recessive: pl.LazyFrame
 
+    # collect and cache minimum required columns
+    min_cols: list[str] = ['accession', 'effect_type', 'chr_name', 'ID', 'matched_effect_allele', 'effect_weight']
+    matches: pl.LazyFrame = (matches.select(min_cols)
+                             .collect()
+                             .lazy())
+
     if split:
         chroms: list[str] = matches.select("chr_name").unique().collect().get_column("chr_name").to_list()
         for chrom in chroms:
-            # 1. filter by chromosome & collect to cache!
-            chrom_df: pl.LazyFrame = matches.filter(pl.col('chr_name') == chrom).collect().lazy()
+            # 1. filter by chromosome
+            chrom_df: pl.LazyFrame = matches.filter(pl.col('chr_name') == chrom)
             # 2. split by effect type
             additive, dominant, recessive = _split_effect_type(chrom_df)
 
@@ -44,9 +50,6 @@ def write_scorefiles(matches: pl.LazyFrame, split: bool, dataset: str):
             # 4. pivot and write!
             _write_split(deduped, chrom, dataset)
     else:
-        # collect to cache!
-        matches: pl.LazyFrame = matches.collect().lazy()
-
         # 1. split by effect type
         additive, dominant, recessive = _split_effect_type(matches)