use separate nextflow work dirs for different subgraphs

EBISPOT · Sep 11, 2024 · 1102073 · 1102073
1 parent 7f27d4e
commit 1102073
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/scripts/dataload_codon.py b/scripts/dataload_codon.py
@@ -12,12 +12,12 @@
 for subgraph in config['subgraphs']:
     print(f"===== LOADING SUBGRAPH: {subgraph} =====")
     os.environ['GREBI_SUBGRAPH'] = subgraph
-    res = os.system(f'nextflow {GREBI_HOME}/nextflow/01_create_subgraph.nf -c {GREBI_HOME}/nextflow/codon_nextflow.config')
+    res = os.system(f'NXF_WORK=work_{subgraph} nextflow {GREBI_HOME}/nextflow/01_create_subgraph.nf -c {GREBI_HOME}/nextflow/codon_nextflow.config')
     if res != 0:
         exit(res)
     print(f"===== FINISHED LOADING SUBGRAPH: {subgraph} =====")
 
-res = os.system(f'nextflow {GREBI_HOME}/nextflow/02_create_dbs.nf -c {GREBI_HOME}/nextflow/codon_nextflow.config')
+res = os.system(f'NXF_WORK=work_combined nextflow {GREBI_HOME}/nextflow/02_create_dbs.nf -c {GREBI_HOME}/nextflow/codon_nextflow.config')
 if res != 0:
     exit(res)
 
diff --git a/scripts/dataload_codon.sh b/scripts/dataload_codon.sh
@@ -11,7 +11,7 @@ export PYTHONPATH="/homes/spotbot/.local/lib/python3.6/site-packages:$PYTHONPATH
 cd /hps/nobackup/parkinso/spot/grebi/
 export PYTHONUNBUFFERED=true
 srun -p datamover --time 1:0:0 --mem 8g bash -c "rm -rf /nfs/public/rw/ontoapps/grebi/staging && mkdir /nfs/public/rw/ontoapps/grebi/staging"
-srun --time 3-0:0:0 --mem 8g bash -c "rm -rf work tmp && python3 ${GREBI_HOME}/scripts/dataload_codon.py"
+srun --time 3-0:0:0 --mem 8g bash -c "rm -rf work* tmp && python3 ${GREBI_HOME}/scripts/dataload_codon.py"
 #srun --time 23:0:0 --mem 8g bash -c "python3 ${GREBI_HOME}/scripts/dataload_codon.py"