Merge branch 'develop' into 'main'

release v0.3.3 See merge request tron/covigator!74
TRON-Bioinformatics · Jul 30, 2021 · 012ff05 · 012ff05
2 parents b940494 + c6c0cb5
commit 012ff05
Show file tree

Hide file tree

Showing 7 changed files with 65 additions and 48 deletions.
diff --git a/covigator/__init__.py b/covigator/__init__.py
@@ -1 +1 @@
-VERSION = "0.3.2"
+VERSION = "0.3.3"
diff --git a/covigator/dashboard/dashboard.py b/covigator/dashboard/dashboard.py
@@ -57,11 +57,11 @@ def serve_layout(self):
                     [
                         dbc.Col(html.A(html.Img(
                             src="/assets/tron_logo_without_text.png", height="22px"),
-                            href="https://tron-mainz.de"), className="ml-2"),
+                            href="https://tron-mainz.de",  target="_blank"), className="ml-2"),
                         dbc.Col(html.Br(), className="ml-2"),
                         dbc.Col(html.A(html.Img(
                             src="https://github.githubassets.com/images/modules/logos_page/Octocat.png", height="25px"),
-                            href="https://github.com/TRON-bioinformatics/covigator"), className="ml-2"),
+                            href="https://github.com/TRON-bioinformatics/covigator",  target="_blank"), className="ml-2"),
                         dbc.Col(html.Br(), className="ml-2"),
 
                     ],

diff --git a/covigator/dashboard/tabs/download.py b/covigator/dashboard/tabs/download.py
@@ -4,16 +4,15 @@
 from flask import send_from_directory
 import dash_bootstrap_components as dbc
 import dash_core_components as dcc
+from logzero import logger
 
 
 def get_tab_download(content_folder):
     return dbc.Card(
         dbc.CardBody(
             children=[
                 dcc.Markdown("""
-                    ** Download the raw CoVigator data**
-                     
-                    Every file has a suffix with the version of the form `_v12` and corresponds to a table in our database. 
+                    ** Download the raw CoVigator data** 
                     
                     * `variant_observation` contains the variant calls from both ENA and GISAID
                     * `subclonal_variant_observation` contains the variant calls from ENA with a VAF < 80 %
@@ -45,12 +44,14 @@ def get_downloadable_files(content_folder):
     else:
         return html.Div(
             children=dbc.ListGroup(
-                [dbc.ListGroupItem(filename, href="/download/{}".format(urlquote(filename))) for filename in files]))
+                [dbc.ListGroupItem(html.A(filename, href="/download/{}".format(urlquote(filename)),  target="_blank"))
+                 for filename in files]))
 
 
 def set_callbacks_download_tab(app, content_folder):
 
     @app.server.route("/download/<path:path>")
     def download(path):
         """Serve a file from the upload directory."""
+        logger.info("hey")
         return send_from_directory(content_folder, path, as_attachment=True)
diff --git a/covigator/dashboard/tabs/footer.py b/covigator/dashboard/tabs/footer.py
@@ -1,4 +1,3 @@
-import dash_core_components
 import dash_html_components as html
 import dash_core_components as dcc
 
@@ -13,7 +12,7 @@ def get_footer():
                 children=[
                     html.Br(),
                     html.P("CoVigator {} © 2021 TRON. All Rights Reserved".format(covigator.VERSION)),
-                    html.A(html.Img(src=tron_logo, id="tron-logo"), href="https://tron-mainz.de"),
+                    html.A(html.Img(src=tron_logo, id="tron-logo"), href="https://tron-mainz.de",  target="_blank"),
                     html.Br(),
                     dcc.Markdown("""
                     TRON is an independent biopharmaceutical non-profit translational research organization pursuing 
@@ -31,18 +30,20 @@ def get_footer():
                     solution was funded in part by """,
                         html.A(
                             "Intel’s Pandemic Response Technology Initiative",
-                            href="https://newsroom.intel.com/news/intel-commits-technology-response-combat-coronavirus/"),
+                            href="https://newsroom.intel.com/news/intel-commits-technology-response-combat-coronavirus/",
+                            target="_blank"),
                         """. For more 
                     information about healthcare solutions from Intel, visit intel.com/healthcare. For more 
                     information about Intel’s COVID-19 response, visit """,
                         html.A(
                             "intel.com/COVID-19",
-                            href="https://www.intel.com/content/www/us/en/corporate-responsibility/covid-19-response.html"),
+                            href="https://www.intel.com/content/www/us/en/corporate-responsibility/covid-19-response.html",
+                            target="_blank"),
                         "."
                         ]),
                     html.P([
-                        html.A("DATA PROTECTION", href="https://tron-mainz.de/data-protection/"), " | ",
-                        html.A("IMPRINT", href="https://tron-mainz.de/imprint/")])
+                        html.A("DATA PROTECTION", href="https://tron-mainz.de/data-protection/",  target="_blank"), " | ",
+                        html.A("IMPRINT", href="https://tron-mainz.de/imprint/",  target="_blank")])
                 ],
                 # this bit makes sure the footer sticks at the bottom
                 #style={"position": "relative", "bottom": "0", "width": "100%", "overflow": "hidden", "height": "120px"}

diff --git a/covigator/dashboard/tabs/overview.py b/covigator/dashboard/tabs/overview.py
@@ -62,7 +62,7 @@ def get_tab_overview(queries: Queries):
                        style={"font-style": "italic", "margin-left": "50px"}),
                    html.P(
                        html.A("https://doi.org/10.1101/2021.02.04.429765",
-                              href="https://doi.org/10.1101/2021.02.04.429765"),
+                              href="https://doi.org/10.1101/2021.02.04.429765",  target="_blank"),
                        style={"text-indent": "50px"}
                    ),
 
@@ -73,7 +73,7 @@ def get_tab_overview(queries: Queries):
                            "It is open sourced under the MIT license, see the repository for "
                            "more details ",
                            html.A("https://github.com/TRON-Bioinformatics/covigator-ngs-pipeline",
-                                  href="https://github.com/TRON-Bioinformatics/covigator-ngs-pipeline"),
+                                  href="https://github.com/TRON-Bioinformatics/covigator-ngs-pipeline",  target="_blank"),
                            "."]),
                    html.Br(),
                    html.P("If you would like to cite the pipeline:"),
@@ -85,7 +85,7 @@ def get_tab_overview(queries: Queries):
                    ], style={"font-style": "italic", "margin-left": "50px"}),
                    html.P(
                        html.A("http://doi.org/10.5281/zenodo.5082444",
-                              href="http://doi.org/10.5281/zenodo.5082444"),
+                              href="http://doi.org/10.5281/zenodo.5082444",  target="_blank"),
                        style={"text-indent": "50px"}
                    ),
                    html.Br(),

diff --git a/scripts/export_db.sh b/scripts/export_db.sh
@@ -8,21 +8,26 @@ pg_uri=postgresql://$COVIGATOR_DB_USER:$COVIGATOR_DB_PASSWORD@$COVIGATOR_DB_HOST
 mkdir -p $output
 
 get_export_command() {
-  echo "\\copy $1$version to program 'gzip > $output/$1$version.csv.gz' csv header;"
+  echo "\\copy $1$version to program 'gzip > $output/$1.csv.gz' csv header;"
 }
 
 conservation=`get_export_command "conservation"`
 gene=`get_export_command "gene"`
-job_ena="\\copy job_ena$version(run_accession,status,created_at,queued_at,downloaded_at,analysed_at,cleaned_at,loaded_at,cooccurrence_at,failed_at,error_message,fastq_path,vcf_path,qc_path,horizontal_coverage_path,vertical_coverage_path,num_reads,covered_bases,coverage,mean_depth,mean_base_quality,mean_mapping_quality) to program 'gzip > $output/job_ena$version.csv.gz' csv header;"
+job_ena="\\copy job_ena$version(run_accession,status,created_at,queued_at,downloaded_at,analysed_at,cleaned_at,loaded_at,cooccurrence_at,failed_at,error_message,fastq_path,vcf_path,qc_path,horizontal_coverage_path,vertical_coverage_path,num_reads,covered_bases,coverage,mean_depth,mean_base_quality,mean_mapping_quality) to program 'gzip > $output/job_ena.csv.gz' csv header;"
 job_gisaid=`get_export_command "job_gisaid"`
 log=`get_export_command "log"`
 precomputed_annotation=`get_export_command "precomputed_annotation"`
 precomputed_indel_length=`get_export_command "precomputed_indel_length"`
 precomputed_substitutions_counts=`get_export_command "precomputed_substitutions_counts"`
 precomputed_top_occurrence=`get_export_command "precomputed_top_occurrence"`
 precomputed_variants_per_sample=`get_export_command "precomputed_variants_per_sample"`
+precomputed_table_counts=`get_export_command "precomputed_table_counts"`
+precomputed_dn_ds=`get_export_command "precomputed_dn_ds"`
+precomputed_dn_ds_by_domain=`get_export_command "precomputed_dn_ds_by_domain"`
+precomputed_variants_per_sample=`get_export_command "precomputed_variants_per_sample"`
+precomputed_variant_abundance_histogram=`get_export_command "precomputed_variant_abundance_histogram"`
 sample_ena=`get_export_command "sample_ena"`
-sample_gisaid="\\copy sample_gisaid$version(run_accession,date,host_tax_id,host,country_raw,region,country,country_alpha_2,country_alpha_3,continent,continent_alpha_2,site,site2,sequence_length,count_n_bases,count_ambiguous_bases,count_snvs,count_insertions,count_deletions,finished) to program 'gzip > $output/sample_gisaid$version.csv.gz' csv header;"
+sample_gisaid="\\copy sample_gisaid$version(run_accession,date,host_tax_id,host,country_raw,region,country,country_alpha_2,country_alpha_3,continent,continent_alpha_2,site,site2,sequence_length,count_n_bases,count_ambiguous_bases,count_snvs,count_insertions,count_deletions,finished) to program 'gzip > $output/sample_gisaid.csv.gz' csv header;"
 sample=`get_export_command "sample"`
 subclonal_variant_observation=`get_export_command "subclonal_variant_observation"`
 variant_cooccurrence=`get_export_command "variant_cooccurrence"`
@@ -39,6 +44,11 @@ psql $pg_uri -c "$precomputed_indel_length"
 psql $pg_uri -c "$precomputed_substitutions_counts"
 psql $pg_uri -c "$precomputed_top_occurrence"
 psql $pg_uri -c "$precomputed_variants_per_sample"
+psql $pg_uri -c "$precomputed_table_counts"
+psql $pg_uri -c "$precomputed_dn_ds"
+psql $pg_uri -c "$precomputed_dn_ds_by_domain"
+psql $pg_uri -c "$precomputed_variants_per_sample"
+psql $pg_uri -c "$precomputed_variant_abundance_histogram"
 psql $pg_uri -c "$sample_ena"
 psql $pg_uri -c "$sample_gisaid"
 psql $pg_uri -c "$sample"

diff --git a/scripts/import_db.sh b/scripts/import_db.sh
@@ -3,20 +3,19 @@
 
 input_folder=$1
 version=$COVIGATOR_TABLE_VERSION
-pg_uri=postgresql://$COVIGATOR_DB_USER:$COVIGATOR_DB_PASSWORD@$COVIGATOR_DB_HOST:$COVIGATOR_DB_PORT/$COVIGATOR_DB_NAME
+export PGPASSWORD=$COVIGATOR_DB_PASSWORD
+pg_uri=postgresql://$COVIGATOR_DB_USER@$COVIGATOR_DB_HOST:$COVIGATOR_DB_PORT/$COVIGATOR_DB_NAME
 
 
 get_import_command() {
-  echo "\\copy $1$version from program 'gzip -dc $input_folder/$1_v14.csv.gz' csv header;"
+  echo "\\copy $1$version from program 'gzip -dc $input_folder/$1.csv.gz' csv header;"
 }
 
 get_delete_command() {
   echo "delete from $1$version;"
 }
 
-conservation=`get_import_command "conservation"`
-gene=`get_import_command "gene"`
-job_ena="\\copy job_ena$version(run_accession,status,created_at,queued_at,downloaded_at,analysed_at,cleaned_at,loaded_at,cooccurrence_at,failed_at,error_message,fastq_path,vcf_path,qc_path,horizontal_coverage_path,vertical_coverage_path,num_reads,covered_bases,coverage,mean_depth,mean_base_quality,mean_mapping_quality) from program 'gzip -dc $input_folder/job_ena_v14.csv.gz' csv header;"
+job_ena="\\copy job_ena$version(run_accession,status,created_at,queued_at,downloaded_at,analysed_at,cleaned_at,loaded_at,cooccurrence_at,failed_at,error_message,fastq_path,vcf_path,qc_path,horizontal_coverage_path,vertical_coverage_path,num_reads,covered_bases,coverage,mean_depth,mean_base_quality,mean_mapping_quality) from program 'gzip -dc $input_folder/job_ena.csv.gz' csv header;"
 job_gisaid=`get_import_command "job_gisaid"`
 log=`get_import_command "log"`
 
@@ -27,6 +26,8 @@ precomputed_top_occurrence_delete=`get_delete_command "precomputed_top_occurrenc
 precomputed_variants_per_sample_delete=`get_delete_command "precomputed_variants_per_sample"`
 precomputed_table_counts_delete=`get_delete_command "precomputed_table_counts"`
 precomputed_variant_abundance_histogram_delete=`get_delete_command "precomputed_variant_abundance_histogram"`
+precomputed_dn_ds_delete=`get_delete_command "precomputed_dn_ds"`
+precomputed_dn_ds_by_domain_delete=`get_delete_command "precomputed_dn_ds_by_domain"`
 
 precomputed_annotation=`get_import_command "precomputed_annotation"`
 precomputed_indel_length=`get_import_command "precomputed_indel_length"`
@@ -35,38 +36,42 @@ precomputed_top_occurrence=`get_import_command "precomputed_top_occurrence"`
 precomputed_variants_per_sample=`get_import_command "precomputed_variants_per_sample"`
 precomputed_table_counts=`get_import_command "precomputed_table_counts"`
 precomputed_variant_abundance_histogram=`get_import_command "precomputed_variant_abundance_histogram"`
+precomputed_dn_ds=`get_import_command "precomputed_dn_ds"`
+precomputed_dn_ds_by_domain=`get_import_command "precomputed_dn_ds_by_domain"`
 
 sample_ena=`get_import_command "sample_ena"`
-sample_gisaid="\\copy sample_gisaid$version(run_accession,date,host_tax_id,host,country_raw,region,country,country_alpha_2,country_alpha_3,continent,continent_alpha_2,site,site2,sequence_length,count_n_bases,count_ambiguous_bases,count_snvs,count_insertions,count_deletions,finished) from program 'gzip -dc $input_folder/sample_gisaid_v14.csv.gz' csv header;"
+sample_gisaid="\\copy sample_gisaid$version(run_accession,date,host_tax_id,host,country_raw,region,country,country_alpha_2,country_alpha_3,continent,continent_alpha_2,site,site2,sequence_length,count_n_bases,count_ambiguous_bases,count_snvs,count_insertions,count_deletions,finished) from program 'gzip -dc $input_folder/sample_gisaid.csv.gz' csv header;"
 sample=`get_import_command "sample"`
 subclonal_variant_observation=`get_import_command "subclonal_variant_observation"`
 variant_cooccurrence=`get_import_command "variant_cooccurrence"`
 variant_observation=`get_import_command "variant_observation"`
 variant=`get_import_command "variant"`
 
-#psql $pg_uri -c "$conservation"
-#psql $pg_uri -c "$gene"
-#psql $pg_uri -c "$sample_ena"
-#psql $pg_uri -c "$job_ena"
+psql $pg_uri -c "$sample_ena"
+psql $pg_uri -c "$job_ena"
 psql $pg_uri -c "$sample_gisaid"
 psql $pg_uri -c "$job_gisaid"
 psql $pg_uri -c "$sample"
-#psql $pg_uri -c "$variant"
-#psql $pg_uri -c "$subclonal_variant_observation"
-#psql $pg_uri -c "$variant_cooccurrence"
-#psql $pg_uri -c "$variant_observation"
-#psql $pg_uri -c "$log"
-#psql $pg_uri -c "$precomputed_annotation_delete"
-#psql $pg_uri -c "$precomputed_annotation"
-#psql $pg_uri -c "$precomputed_indel_length_delete"
-#psql $pg_uri -c "$precomputed_indel_length"
-#psql $pg_uri -c "$precomputed_substitutions_counts_delete"
-#psql $pg_uri -c "$precomputed_substitutions_counts"
-#psql $pg_uri -c "$precomputed_top_occurrence_delete"
-#psql $pg_uri -c "$precomputed_top_occurrence"
-#psql $pg_uri -c "$precomputed_variants_per_sample_delete"
-#psql $pg_uri -c "$precomputed_variants_per_sample"
-#psql $pg_uri -c "$precomputed_table_counts_delete"
-#psql $pg_uri -c "$precomputed_table_counts"
-#psql $pg_uri -c "$precomputed_variant_abundance_histogram_delete"
-#psql $pg_uri -c "$precomputed_variant_abundance_histogram"
+psql $pg_uri -c "$variant"
+psql $pg_uri -c "$subclonal_variant_observation"
+psql $pg_uri -c "$variant_cooccurrence"
+psql $pg_uri -c "$variant_observation"
+psql $pg_uri -c "$log"
+psql $pg_uri -c "$precomputed_annotation_delete"
+psql $pg_uri -c "$precomputed_annotation"
+psql $pg_uri -c "$precomputed_indel_length_delete"
+psql $pg_uri -c "$precomputed_indel_length"
+psql $pg_uri -c "$precomputed_substitutions_counts_delete"
+psql $pg_uri -c "$precomputed_substitutions_counts"
+psql $pg_uri -c "$precomputed_top_occurrence_delete"
+psql $pg_uri -c "$precomputed_top_occurrence"
+psql $pg_uri -c "$precomputed_variants_per_sample_delete"
+psql $pg_uri -c "$precomputed_variants_per_sample"
+psql $pg_uri -c "$precomputed_table_counts_delete"
+psql $pg_uri -c "$precomputed_table_counts"
+psql $pg_uri -c "$precomputed_variant_abundance_histogram_delete"
+psql $pg_uri -c "$precomputed_variant_abundance_histogram"
+psql $pg_uri -c "$precomputed_dn_ds_delete"
+psql $pg_uri -c "$precomputed_dn_ds"
+psql $pg_uri -c "$precomputed_dn_ds_by_domain_delete"
+psql $pg_uri -c "$precomputed_dn_ds_by_domain"