jobscript-feature_builder

#!/bin/bash

# set a job name
#SBATCH --job-name=reranker
#################

# a file for job output, you can check job progress
#SBATCH --output=reranker_slurm_output_%j.out
#################

# a file for errors
#SBATCH --error=reranker_slurm_output_%j.err
#################

# time needed for job
#SBATCH --time=03:00:00
#################

# gpus per node
#SBATCH --gres=gpu:1
#################

# cpus per job
#SBATCH --cpus-per-task=5
#################

# number of requested nodes
#SBATCH --nodes=1
#################

# memory per node
#SBATCH --mem=8GB
#################

# slurm will send a signal this far out before it kills the job
#SBATCH --signal=USR1@300
#################


# tasks per node
#SBATCH --tasks-per-node=1
#################

module load python/3.7.11
source venv/bin/activate

srun python3 examples/msmarco-doc/helpers/build_train_from_ranking_agask.py --tokenizer_name ./pt-bert-large-msmarco --qrel examples/msmarco-doc/helpers/known_item_qrels.tsv --json_dir examples/msmarco-doc/helpers/agask_outdir --query_collection examples/msmarco-doc/helpers/agask_queries.csv --doc_collection examples/msmarco-doc/helpers/agask_docs.csv