diff --git a/_sources/assess.rst.txt b/_sources/assess.rst.txt index 8e4878e..c508e61 100644 --- a/_sources/assess.rst.txt +++ b/_sources/assess.rst.txt @@ -2,5 +2,4 @@ Assess Module ============= .. automodule:: assess - :members: - \ No newline at end of file + :members: \ No newline at end of file diff --git a/assess.html b/assess.html index 8353e9f..1092fa7 100644 --- a/assess.html +++ b/assess.html @@ -78,35 +78,80 @@

Assess Module

+
+
+assess.check_errs(path: str, logger, savetype=None, examine=None)
+

Check error files and summarise results

+
+ +
+
+assess.extract_keys(filepath: str, logger, savetype=None, examine=None)
+

Extract keys from error/output files, collect into groups and examine a particular type if required.

+
+ +
+
+assess.find_codes(phase: str, workdir: str, groupID: str, check: str, ignore=[])
+

Find project codes for datasets that failed at various stages of the pipeline

+

Checks ‘in-progress’ and ‘complete’ directories for datasets and config files to determine progress of all datasets.

+
+
phasestr

Check config and output files corresponding to a pipeline phase.

+
+
workdirstr

Path to current working directory of the pipeline.

+
+
groupIDstr

Check pipeline for a specific group ID.

+
+
checkstr

File type or specific file required for this phase to be considered complete

+
+
+

ignore : list (str-like)

+
+
redo_pcodeslist (str-like)

List of project codes to re-run for this phase.

+
+
completelist (str-like)

List of project codes considered to be complete for the whole pipeline

+
+
+
+
-assess.get_attribute(env, args, var)
+assess.get_attribute(env: str, args, var: str)

Assemble environment variable or take from passed argument.

+
+
+assess.get_code_from_val(path: str, index: str)
+

Takes some index value from command line and fetches the corresponding project code

+
+
-assess.init_logger(verbose, mode, name)
+assess.init_logger(verbose: int, mode: int, name: str)

Logger object init and configure with formatting

-
verbose(int)

Display level can range from 0-2 for WARNING, INFO and DEBUG.

+
verboseint

Display level can range from 0-2 for WARNING, INFO and DEBUG.

modeint

Unused mode for saving data.

namestr

Name of master script from which logger is defined.

-
-
Returns
-

Logging-type object

-
-
+

logging.Logger

-
-
-assess.levels = [30, 20, 10]
-
+
+
+assess.save_sel(codes: list, groupdir: str, label: str, logger)
+

Save selection of codes to a file with a given repeat label

+
+ +
+
+assess.show_options(option: str, groupdir: str, operation: str, logger)
+

Display current

+
diff --git a/genindex.html b/genindex.html index a2e7a1b..9fa984e 100644 --- a/genindex.html +++ b/genindex.html @@ -81,6 +81,7 @@

Index

A | C | E + | F | G | H | I @@ -109,6 +110,10 @@

A

C

+
+
  • get_code_from_val() (in module assess) +
  • get_group_len() (in module group_run)
  • get_input() (in module pipeline.init) @@ -192,10 +211,6 @@

    K

    L

    -
    • load_from_input_file() (in module pipeline.init)
      • @@ -287,6 +302,12 @@

      R

      S

      +
      • single_run diff --git a/objects.inv b/objects.inv index 47a19e1..774be16 100644 Binary files a/objects.inv and b/objects.inv differ diff --git a/searchindex.js b/searchindex.js index 2cafa72..61c1642 100644 --- a/searchindex.js +++ b/searchindex.js @@ -1 +1 @@ -Search.setIndex({docnames:["assess","assess-overview","compute","examples","execution","index","init","pipeline-overview","scan","serial-process","validate"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":4,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":3,"sphinx.domains.rst":2,"sphinx.domains.std":2,sphinx:56},filenames:["assess.rst","assess-overview.rst","compute.rst","examples.rst","execution.rst","index.rst","init.rst","pipeline-overview.rst","scan.rst","serial-process.rst","validate.rst"],objects:{"":[[0,0,0,"-","assess"],[4,0,0,"-","group_run"],[4,0,0,"-","single_run"]],"pipeline.compute":[[9,0,0,"-","serial_process"]],"pipeline.compute.serial_process":[[9,3,1,"","Converter"],[9,5,1,"","KerchunkDriverFatalError"],[9,1,1,"","init_logger"]],"pipeline.compute.serial_process.Converter":[[9,4,1,"","hdf5_to_zarr"],[9,4,1,"","ncf3_to_zarr"],[9,4,1,"","tiff_to_zarr"]],"pipeline.init":[[6,1,1,"","get_input"],[6,1,1,"","get_removals"],[6,1,1,"","get_updates"],[6,1,1,"","init_config"],[6,1,1,"","init_logger"],[6,1,1,"","load_from_input_file"],[6,1,1,"","make_dirs"],[6,1,1,"","make_filelist"],[6,1,1,"","text_file_to_csv"]],assess:[[0,1,1,"","get_attribute"],[0,1,1,"","init_logger"],[0,2,1,"","levels"]],group_run:[[4,1,1,"","get_attribute"],[4,1,1,"","get_group_len"],[4,1,1,"","init_logger"],[4,1,1,"","main"]],pipeline:[[6,0,0,"-","init"]],single_run:[[4,5,1,"","ExpectTimeoutError"],[4,5,1,"","MissingVariableError"],[4,5,1,"","ProjectCodeError"],[4,1,1,"","get_proj_code"],[4,1,1,"","init_logger"],[4,1,1,"","main"],[4,1,1,"","run_compute"],[4,1,1,"","run_init"],[4,1,1,"","run_scan"],[4,1,1,"","run_validation"]]},objnames:{"0":["py","module","Python module"],"1":["py","function","Python function"],"2":["py","data","Python data"],"3":["py","class","Python class"],"4":["py","method","Python method"],"5":["py","exception","Python exception"]},objtypes:{"0":"py:module","1":"py:function","2":"py:data","3":"py:class","4":"py:method","5":"py:exception"},terms:{"0":[0,4],"1":4,"10":0,"2":0,"20":0,"30":0,"class":9,"final":6,"function":4,"int":0,"return":0,"var":[0,4],The:5,all:9,an:5,ar:5,archiv:5,arg:[0,4,6],argument:[0,4],assembl:[0,4],assess:5,assessor:5,associ:6,attribut:6,base:6,bypass_err:9,can:0,code:4,command:6,comput:[4,5,9],config:6,configur:[0,4,6,9],convers:9,convert:[6,9],correct:[4,5],creat:[5,6],csv:6,current:4,data:0,dataset:[4,5],debug:0,defin:0,differ:5,directori:6,displai:0,driver:9,easi:5,ensur:5,env:[0,4],environ:[0,4],exampl:5,except:[4,9],execut:5,expecttimeouterror:4,fail:9,fals:9,file:[4,5,6,9],format:[0,4,6,9],from:[0,4,5,6],get:[4,6],get_attribut:[0,4],get_group_len:4,get_input:6,get_proj_cod:4,get_remov:6,get_upd:6,given:4,group:4,group_run:4,groupdir:4,hdf5:9,hdf5_to_zarr:9,hdf:5,id:4,implement:4,index:5,individu:4,info:0,init:[0,4,5,6,9],init_config:6,init_logg:[0,4,6,9],initialis:4,input:6,job:4,kei:6,kerchunkdriverfatalerror:9,kwarg:9,level:0,line:6,list:6,load:6,load_from_input_fil:6,log:0,logger:[0,4,6,9],main:[4,6],make:5,make_dir:6,make_filelist:6,master:0,messag:9,metadata:6,missingvariableerror:4,mode:[0,4,6,9],modul:5,multipl:5,name:[0,4,6,9],ncf3_to_zarr:9,netcdf3:9,netcdf:5,nfile:9,none:6,object:[0,4,6,9],output:5,overview:5,packag:5,page:5,pair:6,parallel:[4,5],paramet:4,pass:[0,4],pattern:6,perform:9,pid:4,pipelin:[5,6,9],prefix:6,process:[2,4,5],proj_dir:6,project:[4,6],projectcodeerror:4,python:5,rang:0,read:4,remov:6,repeat_id:4,requir:4,run:4,run_comput:4,run_init:4,run_scan:4,run_valid:4,save:0,sbatch:4,scan:[4,5],script:[0,4,6],search:5,serial:[2,5],serial_process:9,set:[5,6],setup:4,singl:4,single_run:4,slurm:4,sourc:[5,6],specif:6,start:4,step:5,str:0,structur:6,subset:4,take:[0,4],text:6,text_file_to_csv:6,tfile:9,thi:6,tiff:[5,9],tiff_to_zarr:9,tool:5,type:[0,4,9],unus:0,up:6,updat:6,valid:[4,5],valu:6,variabl:[0,4],verbos:[0,4,6,9],warn:0,when:9,which:0,work:[5,6],workdir:4},titles:["Assess Module","Assessor Tool","Initialisation Module","Worked Examples","Pipeline Execution","Welcome to kerchunk-builder\u2019s documentation!","Initialisation Module","Pipeline Overview","Scanner Module","Serial Processor","Validation Module"],titleterms:{advanc:5,assess:0,assessor:1,builder:5,content:5,document:5,exampl:3,execut:4,indic:5,initialis:[2,6],kerchunk:5,modul:[0,2,6,8,10],overview:7,pipelin:[4,7],processor:[2,9],s:5,scanner:8,serial:9,tabl:5,tool:1,valid:10,welcom:5,work:3}}) \ No newline at end of file +Search.setIndex({docnames:["assess","assess-overview","compute","examples","execution","index","init","pipeline-overview","scan","serial-process","validate"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":4,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":3,"sphinx.domains.rst":2,"sphinx.domains.std":2,sphinx:56},filenames:["assess.rst","assess-overview.rst","compute.rst","examples.rst","execution.rst","index.rst","init.rst","pipeline-overview.rst","scan.rst","serial-process.rst","validate.rst"],objects:{"":[[0,0,0,"-","assess"],[4,0,0,"-","group_run"],[4,0,0,"-","single_run"]],"pipeline.compute":[[9,0,0,"-","serial_process"]],"pipeline.compute.serial_process":[[9,2,1,"","Converter"],[9,4,1,"","KerchunkDriverFatalError"],[9,1,1,"","init_logger"]],"pipeline.compute.serial_process.Converter":[[9,3,1,"","hdf5_to_zarr"],[9,3,1,"","ncf3_to_zarr"],[9,3,1,"","tiff_to_zarr"]],"pipeline.init":[[6,1,1,"","get_input"],[6,1,1,"","get_removals"],[6,1,1,"","get_updates"],[6,1,1,"","init_config"],[6,1,1,"","init_logger"],[6,1,1,"","load_from_input_file"],[6,1,1,"","make_dirs"],[6,1,1,"","make_filelist"],[6,1,1,"","text_file_to_csv"]],assess:[[0,1,1,"","check_errs"],[0,1,1,"","extract_keys"],[0,1,1,"","find_codes"],[0,1,1,"","get_attribute"],[0,1,1,"","get_code_from_val"],[0,1,1,"","init_logger"],[0,1,1,"","save_sel"],[0,1,1,"","show_options"]],group_run:[[4,1,1,"","get_attribute"],[4,1,1,"","get_group_len"],[4,1,1,"","init_logger"],[4,1,1,"","main"]],pipeline:[[6,0,0,"-","init"]],single_run:[[4,4,1,"","ExpectTimeoutError"],[4,4,1,"","MissingVariableError"],[4,4,1,"","ProjectCodeError"],[4,1,1,"","get_proj_code"],[4,1,1,"","init_logger"],[4,1,1,"","main"],[4,1,1,"","run_compute"],[4,1,1,"","run_init"],[4,1,1,"","run_scan"],[4,1,1,"","run_validation"]]},objnames:{"0":["py","module","Python module"],"1":["py","function","Python function"],"2":["py","class","Python class"],"3":["py","method","Python method"],"4":["py","exception","Python exception"]},objtypes:{"0":"py:module","1":"py:function","2":"py:class","3":"py:method","4":"py:exception"},terms:{"0":[0,4],"1":4,"2":0,"class":9,"final":6,"function":4,"int":0,"var":[0,4],The:5,all:[0,9],an:5,ar:5,archiv:5,arg:[0,4,6],argument:[0,4],assembl:[0,4],assess:5,assessor:5,associ:6,attribut:6,base:6,bypass_err:9,can:0,check:0,check_err:0,code:[0,4],collect:0,command:[0,6],complet:0,comput:[4,5,9],config:[0,6],configur:[0,4,6,9],consid:0,convers:9,convert:[6,9],correct:[4,5],correspond:0,creat:[5,6],csv:6,current:[0,4],data:0,dataset:[0,4,5],debug:0,defin:0,determin:0,differ:5,directori:[0,6],displai:0,driver:9,easi:5,ensur:5,env:[0,4],environ:[0,4],error:0,examin:0,exampl:5,except:[4,9],execut:5,expecttimeouterror:4,extract:0,extract_kei:0,fail:[0,9],fals:9,fetch:0,file:[0,4,5,6,9],filepath:0,find:0,find_cod:0,format:[0,4,6,9],from:[0,4,5,6],get:[4,6],get_attribut:[0,4],get_code_from_v:0,get_group_len:4,get_input:6,get_proj_cod:4,get_remov:6,get_upd:6,given:[0,4],group:[0,4],group_run:4,groupdir:[0,4],groupid:0,hdf5:9,hdf5_to_zarr:9,hdf:5,id:[0,4],ignor:0,implement:4,index:[0,5],individu:4,info:0,init:[0,4,5,6,9],init_config:6,init_logg:[0,4,6,9],initialis:4,input:6,job:4,kei:[0,6],kerchunkdriverfatalerror:9,kwarg:9,label:0,level:0,like:0,line:[0,6],list:[0,6],load:6,load_from_input_fil:6,log:0,logger:[0,4,6,9],main:[4,6],make:5,make_dir:6,make_filelist:6,master:0,messag:9,metadata:6,missingvariableerror:4,mode:[0,4,6,9],modul:5,multipl:5,name:[0,4,6,9],ncf3_to_zarr:9,netcdf3:9,netcdf:5,nfile:9,none:[0,6],object:[0,4,6,9],oper:0,option:0,output:[0,5],overview:5,packag:5,page:5,pair:6,parallel:[4,5],paramet:4,particular:0,pass:[0,4],path:0,pattern:6,perform:9,phase:0,pid:4,pipelin:[0,5,6,9],prefix:6,process:[2,4,5],progress:0,proj_dir:6,project:[0,4,6],projectcodeerror:4,python:5,rang:0,re:0,read:4,redo_pcod:0,remov:6,repeat:0,repeat_id:4,requir:[0,4],result:0,run:[0,4],run_comput:4,run_init:4,run_scan:4,run_valid:4,save:0,save_sel:0,savetyp:0,sbatch:4,scan:[4,5],script:[0,4,6],search:5,select:0,serial:[2,5],serial_process:9,set:[5,6],setup:4,show_opt:0,singl:4,single_run:4,slurm:4,some:0,sourc:[5,6],specif:[0,6],stage:0,start:4,step:5,str:0,structur:6,subset:4,summaris:0,take:[0,4],text:6,text_file_to_csv:6,tfile:9,thi:[0,6],tiff:[5,9],tiff_to_zarr:9,tool:5,type:[0,4,9],unus:0,up:6,updat:6,valid:[4,5],valu:[0,6],variabl:[0,4],variou:0,verbos:[0,4,6,9],warn:0,when:9,which:0,whole:0,work:[0,5,6],workdir:[0,4]},titles:["Assess Module","Assessor Tool","Initialisation Module","Worked Examples","Pipeline Execution","Welcome to kerchunk-builder\u2019s documentation!","Initialisation Module","Pipeline Overview","Scanner Module","Serial Processor","Validation Module"],titleterms:{advanc:5,assess:0,assessor:1,builder:5,content:5,document:5,exampl:3,execut:4,indic:5,initialis:[2,6],kerchunk:5,modul:[0,2,6,8,10],overview:7,pipelin:[4,7],processor:[2,9],s:5,scanner:8,serial:9,tabl:5,tool:1,valid:10,welcom:5,work:3}}) \ No newline at end of file