add ctd, mesh, medgen; preserve source ids in merged nodes; move conf…

…ig to yaml

00_fetch_data/ctd/download.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+wget https://ctdbase.org/reports/CTD_anatomy.tsv.gz
+wget https://ctdbase.org/reports/CTD_curated_genes_diseases.tsv.gz
+wget https://ctdbase.org/reports/CTD_Disease-GO_biological_process_associations.tsv.gz
+wget https://ctdbase.org/reports/CTD_Disease-GO_cellular_component_associations.tsv.gz
+wget https://ctdbase.org/reports/CTD_Disease-GO_molecular_function_associations.tsv.gz
+wget https://ctdbase.org/reports/CTD_exposure_events.tsv.gz
+wget https://ctdbase.org/reports/CTD_exposure_studies.tsv.gz
+wget https://ctdbase.org/reports/CTD_pheno_term_ixns.tsv.gz
+wget https://ctdbase.org/reports/CTD_Phenotype-Disease_biological_process_associations.tsv.gz
+wget https://ctdbase.org/reports/CTD_Phenotype-Disease_cellular_component_associations.tsv.gz
+wget https://ctdbase.org/reports/CTD_Phenotype-Disease_molecular_function_associations.tsv.gz
+wget https://ctdbase.org/reports/CTD_chemicals.tsv.gz
+wget https://ctdbase.org/reports/CTD_chem_go_enriched.tsv.gz
+wget https://ctdbase.org/reports/CTD_chemicals_diseases.tsv.gz
+wget https://ctdbase.org/reports/CTD_chem_gene_ixns.tsv.gz
+wget https://ctdbase.org/reports/CTD_chem_pathways_enriched.tsv.gz
+wget https://ctdbase.org/reports/CTD_curated_cas_nbrs.tsv.gz
+wget https://ctdbase.org/reports/CTD_diseases_pathways.tsv.gz
+wget https://ctdbase.org/reports/CTD_UniProtToCTDIdMapping.txt.gz
+wget https://ctdbase.org/reports/CTD_genes.tsv.gz
+wget https://ctdbase.org/reports/CTD_genes_diseases.tsv.gz
+wget https://ctdbase.org/reports/CTD_genes_pathways.tsv.gz
+wget https://ctdbase.org/reports/CTD_pathways.tsv.gz
+
+
+wget https://ctdbase.org/reports/CTD_chem_gene_ixn_types.obo
+curl https://ctdbase.org/reports/CTD_diseases.obo.gz | gzip -d > CTD_diseases.obo
+

00_fetch_data/medgen/fetch.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+rm -f medgen.obo medgen.owl medgen.obo.gz medgen.owl.gz
+
+wget https://github.com/monarch-initiative/medgen/releases/download/2024-10-06/medgen.obo
+robot convert --input medgen.obo --output medgen.owl
+
+gzip -9 medgen.owl
+rm -f medgen.obo
+
+

00_fetch_data/mesh/fetch.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+wget https://nlmpubs.nlm.nih.gov/projects/mesh/rdf/2024/mesh2024.nt.gz
+wget https://nlmpubs.nlm.nih.gov/projects/mesh/rdf/2024/vocabulary_1.0.0.ttl
+

01_ingest/grebi_ingest_kgx_edges/src/main.rs

@@ -9,12 +9,6 @@ use serde_json::json;
 #[command(author, version, about, long_about = None)]
 struct Args {
 
-    #[arg(long)]
-    datasource_name: String,
-
-    #[arg(long)]
-    filename: String,
-
     #[arg(long)]
     kgx_rename_field:Option<Vec<String>>,
 

01_ingest/grebi_ingest_ols/src/main.rs

@@ -17,9 +17,6 @@ struct Args {
     #[arg(long)]
     datasource_name: String,
 
-    #[arg(long)]
-    filename: String,
-
     #[arg(long)]
     ontologies:String,
 
@@ -236,12 +233,33 @@ fn read_entities(json: &mut JsonStreamReader<BufReader<StdinLock<'_>>>, output_n
                 continue;
             }
 
+            let v = obj.get(k).unwrap();
+
             output_nodes.write_all(r#","#.as_bytes()).unwrap();
             output_nodes.write_all(r#"""#.as_bytes()).unwrap();
-            output_nodes.write_all(k.as_bytes()).unwrap();// already reprefixed on load
-            output_nodes.write_all(r#"":"#.as_bytes()).unwrap();
 
-            let v = obj.get(k).unwrap();
+            if k.eq("ols:relatedFrom") || k.eq("ols:relatedTo") {
+                let vals = {
+                    if v.is_array() {
+                        v.as_array().unwrap().iter().map(|x| x).collect()
+                    } else {
+                        vec!(v)
+                    }
+                };
+                for related in vals {
+                    let v_as_obj = related.as_object().unwrap();
+                    let pred = v_as_obj.get("http://www.w3.org/2002/07/owl#onProperty");
+                    if pred.is_some() {
+                        output_nodes.write_all(pred.unwrap().as_str().unwrap().as_bytes()).unwrap();
+                    } else {
+                        output_nodes.write_all(k.as_bytes()).unwrap();
+                    }
+                }
+            } else {
+                output_nodes.write_all(k.as_bytes()).unwrap();
+            }
+
+            output_nodes.write_all(r#"":"#.as_bytes()).unwrap();
 
             output_nodes.write_all(r#"["#.as_bytes()).unwrap();
                 if v.is_array() {

01_ingest/grebi_ingest_reactome/src/main.rs

@@ -10,21 +10,8 @@ use grebi_shared::prefix_map::PrefixMapBuilder;
 use serde_json::json;
 use serde_json::Value;
 
-#[derive(clap::Parser, Debug)]
-#[command(author, version, about, long_about = None)]
-struct Args {
-
-    #[arg(long)]
-    datasource_name: String,
-
-    #[arg(long)]
-    filename: String,
-}
-
 fn main() {
 
-    let args = Args::parse();
-
     let stdin = io::stdin().lock();
     let mut reader = BufReader::new(stdin);
 

01_ingest/grebi_ingest_sssom/src/main.rs

@@ -9,26 +9,11 @@ use grebi_shared::prefix_map::PrefixMapBuilder;
 use serde_json::json;
 use serde_yaml;
 
-#[derive(clap::Parser, Debug)]
-#[command(author, version, about, long_about = None)]
-struct Args {
-
-    #[arg(long)]
-    datasource_name: String,
-
-    #[arg(long)]
-    filename: String,
-}
-
 fn main() {
 
-    let args = Args::parse();
-
     let stdin = io::stdin().lock();
     let mut reader = BufReader::new(stdin);
 
-    let datasource_name = args.datasource_name.as_str();
-
     let stdout = io::stdout().lock();
     let mut output_nodes = BufWriter::new(stdout);
 

01_ingest/grebi_ingest_json/Cargo.toml → 01_ingest/grebi_nodes2edges/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "grebi_ingest_json"
+name = "grebi_nodes2edges"
 version = "0.1.0"
 edition = "2021"
 
@@ -10,3 +10,4 @@ serde_json = { version = "1.0.108", features=["preserve_order"] }
 struson = "0.3.0"
 jemallocator = "0.5.4"
 
+