add: metrics queries sql #29

lib/service/diabetes-research-hub/README.md

@@ -145,6 +145,11 @@ into this directory, then ingest and query the data:
 $ surveilr ingest files -r study-files/ && surveilr orchestrate transform-csv
 ```
 
+```bash
+# generate the combined  cgm tracing
+$ deno run -A ./combined-cgm-tracing-generator.ts
+```
+
 ```bash
 # Apply de-identification
 $ cat de-identification/drh-deidentification.sql| surveilr orchestrate -n "deidentification"

lib/service/diabetes-research-hub/combined-cgm-tracing-generator.ts

@@ -1,10 +1,7 @@
 import { DB } from "https://deno.land/x/sqlite/mod.ts";
 
-// Path to your existing SQLite database
-const dbFilePath = "resource-surveillance.sqlite.db";
-
-// Function to create a combined view of CGM tracing
-
+// Function to create the combined CGM tracing view
+export function createCombinedCGMView(dbFilePath: string): void {
   // Open the existing database
   const db = new DB(dbFilePath);
   console.log(`Opened database: ${dbFilePath}`);
@@ -39,7 +36,7 @@ const dbFilePath = "resource-surveillance.sqlite.db";
     const params = JSON.stringify({ message: error.message });
     db.execute(sqlQuery, [params]);
     db.close();
-    return; // Exit if there's an error creating the view
+    return;
   }
 
   // Get the list of participant IDs from the view
@@ -49,26 +46,17 @@ const dbFilePath = "resource-surveillance.sqlite.db";
   const sqlParts: string[] = [];
 
   for (const [patient_id_raw] of participants) {
-    // Cast patient_id_raw to a string type
     const patient_id: string = patient_id_raw as string;
 
-    //console.log(`Processing participant: ${patient_id}`);
-
-    // Get the file names associated with the participant
     const [file_names_row] = db.query("SELECT file_names FROM drh_participant_file_names WHERE patient_id = ?", [patient_id]);
     if (!file_names_row) {
       console.log(`No file names found for participant ${patient_id}.`);
       continue;
     }
 
     const file_names = file_names_row[0];
-    //console.log(`File names for participant ${patient_id}: ${file_names}`);
-
     if (file_names) {
-      // Construct table names for the participant's data
       const participantTableNames = file_names.split(', ').map(fileName => `uniform_resource_${fileName}`);
-
-      // Add the SQL part for each participant's table
       participantTableNames.forEach(tableName => {
         sqlParts.push(`
           SELECT 
@@ -78,28 +66,25 @@ const dbFilePath = "resource-surveillance.sqlite.db";
           FROM ${tableName}
         `);
       });
-    } else {
-      console.log(`No file names found for participant ${patient_id}.`);
     }
   }
 
-  // Create the combined view for all participants
   if (sqlParts.length > 0) {
     const combinedUnionAllQuery = sqlParts.join(' UNION ALL ');
     const createCombinedViewSql = `CREATE VIEW IF NOT EXISTS combined_cgm_tracing AS ${combinedUnionAllQuery};`;
-    //console.log(`Creating combined view with SQL:\n${createCombinedViewSql}`);
-
-    //return createCombinedViewSql;
-    db.execute(createCombinedViewSql);
-    // db.close();
 
+    db.execute(createCombinedViewSql);
     console.log("Combined view 'combined_cgm_tracing' created successfully.");
-
   } else {
     console.log("No participant tables found, so the combined view will not be created.");
   }
 
-  // Close the database connection
- db.close();
+  db.close();
   console.log(`Closed database: ${dbFilePath}`);
+}
 
+// If the script is being run directly, execute the function
+if (import.meta.main) {
+  const dbFilePath = "resource-surveillance.sqlite.db"; // You can modify this path as needed
+  createCombinedCGMView(dbFilePath);
+}

lib/service/diabetes-research-hub/orchestration/deidentification-orchestration.sql

@@ -0,0 +1,171 @@
+-- BEGIN TRANSACTION
+--BEGIN;
+
+-- Perform De-identification
+-- Anonymize email addresses in the uniform_resource_investigator table
+UPDATE uniform_resource_investigator
+SET email = anonymize_email(email)
+WHERE email IS NOT NULL;
+
+-- Anonymize email addresses in the uniform_resource_author table
+UPDATE uniform_resource_author
+SET email = anonymize_email(email)
+WHERE email IS NOT NULL;
+
+-- Insert into orchestration_nature only if it doesn't exist
+INSERT OR IGNORE INTO orchestration_nature (
+    orchestration_nature_id,
+    nature,
+    elaboration,
+    created_at,
+    created_by,
+    updated_at,
+    updated_by,
+    deleted_at,
+    deleted_by,
+    activity_log
+) 
+SELECT
+    'deidentification',         -- Unique ID for the orchestration nature
+    'De-identification',        -- Human-readable name for the orchestration nature
+    NULL,                       -- No elaboration provided at insert time
+    CURRENT_TIMESTAMP,          -- Timestamp of creation
+    d.device_id,                  -- Creator's name
+    NULL,                       -- No updated timestamp yet
+    NULL,                       -- No updater yet
+    NULL,                       -- Not deleted
+    NULL,                       -- No deleter yet
+    NULL                        -- No activity log yet
+FROM drh_device d
+LIMIT 1;  -- Limiting to 1 device
+
+-- Insert into orchestration_session only if it doesn't exist
+INSERT OR IGNORE INTO orchestration_session (
+    orchestration_session_id,
+    device_id,
+    orchestration_nature_id,
+    version,
+    orch_started_at,
+    orch_finished_at,
+    elaboration,
+    args_json,
+    diagnostics_json,
+    diagnostics_md
+)
+SELECT
+    'ORCHSESSID-' || hex(randomblob(16)),  -- Generate a random hex blob for orchestration_session_id
+    d.device_id,                             -- Pull device_id from the drh_device view
+    'deidentification',                      -- Reference to the orchestration_nature_id we just inserted
+    '',                                      -- Version (placeholder)
+    CURRENT_TIMESTAMP,                       -- Start time
+    NULL,                                    -- Finished time (to be updated later)
+    NULL,                                    -- Elaboration (if any)
+    NULL,                                    -- Args JSON (if any)
+    NULL,                                    -- Diagnostics JSON (if any)
+    NULL                                     -- Diagnostics MD (if any)
+FROM drh_device d
+LIMIT 1;  -- Limiting to 1 device
+
+-- Create a temporary view to retrieve orchestration session information
+CREATE TEMP VIEW IF NOT EXISTS session_info AS
+SELECT
+    orchestration_session_id
+FROM 
+    orchestration_session
+WHERE 
+    orchestration_nature_id = 'deidentification'
+LIMIT 1;
+
+-- Insert into orchestration_session_entry only if it doesn't exist
+INSERT OR IGNORE INTO orchestration_session_entry (
+    orchestration_session_entry_id,
+    session_id,
+    ingest_src,
+    ingest_table_name,
+    elaboration
+) VALUES (
+    'ORCHSESSENID-' || hex(randomblob(16)),  -- Generate a random hex blob for orchestration_session_entry_id
+    (SELECT orchestration_session_id FROM session_info limit 1),  -- Session ID from previous insert
+    'deidentification-orchestration.sql',  -- Replace with actual ingest source
+    '',  -- Placeholder for actual table name
+    NULL  -- Elaboration (if any)
+);
+
+-- Create or replace a temporary view for session execution tracking
+DROP VIEW IF EXISTS temp_session_info;  -- Remove any existing view
+CREATE TEMP VIEW temp_session_info AS
+SELECT
+    orchestration_session_id,
+    (SELECT orchestration_session_entry_id FROM orchestration_session_entry WHERE session_id = orchestration_session_id LIMIT 1) AS orchestration_session_entry_id
+FROM orchestration_session 
+WHERE orchestration_nature_id = 'deidentification'
+LIMIT 1;
+
+-- Insert into orchestration_session_exec for uniform_resource_investigator
+INSERT OR IGNORE INTO orchestration_session_exec (
+    orchestration_session_exec_id,
+    exec_nature,
+    session_id,
+    session_entry_id,
+    exec_code,
+    exec_status,
+    input_text,
+    output_text,
+    exec_error_text,
+    narrative_md
+)
+SELECT
+    'ORCHSESSEXID-' || ((SELECT COUNT(*) FROM orchestration_session_exec) + 1),  -- Unique ID based on count
+    'De-identification',                                -- Nature of execution
+    s.orchestration_session_id,                         -- Session ID from the temp view
+    s.orchestration_session_entry_id,                   -- Session Entry ID from the temp view
+    'UPDATE uniform_resource_investigator SET email = anonymize_email(email) executed',  -- Description of the executed code
+    'SUCCESS',                                          -- Execution status
+    'email column in uniform_resource_investigator',   -- Input text reference
+    'De-identification completed',                      -- Output text summary
+    CASE 
+        WHEN (SELECT changes() = 0) THEN 'No rows updated'  -- Capture update status
+        ELSE NULL 
+    END,
+    'username in email is masked'                       -- Narrative for clarification
+FROM temp_session_info s;  -- From the temporary session info view
+
+-- Insert into orchestration_session_exec for uniform_resource_author
+INSERT OR IGNORE INTO orchestration_session_exec (
+    orchestration_session_exec_id,
+    exec_nature,
+    session_id,
+    session_entry_id,
+    exec_code,
+    exec_status,
+    input_text,
+    output_text,
+    exec_error_text,
+    narrative_md
+)
+SELECT
+    'ORCHSESSEXID-' || ((SELECT COUNT(*) FROM orchestration_session_exec) + 1),  -- Unique ID based on count
+    'De-identification',                                -- Nature of execution
+    s.orchestration_session_id,                         -- Session ID from the temp view
+    s.orchestration_session_entry_id,                   -- Session Entry ID from the temp view
+    'UPDATE uniform_resource_author SET email = anonymize_email(email) executed',  -- Description of the executed code
+    'SUCCESS',                                          -- Execution status
+    'email column in uniform_resource_author',          -- Input text reference
+    'De-identification completed',                      -- Output text summary
+    CASE 
+        WHEN (SELECT changes() = 0) THEN 'No rows updated'  -- Capture update status
+        ELSE NULL 
+    END,
+    'username in email is masked'                       -- Narrative for clarification
+FROM temp_session_info s;  -- From the temporary session info view
+
+-- Update orchestration_session to set finished timestamp and diagnostics
+UPDATE orchestration_session
+SET 
+    orch_finished_at = CURRENT_TIMESTAMP,             -- Set the finish time
+    diagnostics_json = '{"status": "completed"}',     -- Diagnostics status in JSON format
+    diagnostics_md = 'De-identification process completed'  -- Markdown summary
+WHERE orchestration_session_id = (SELECT orchestration_session_id FROM temp_session_info LIMIT 1);  -- Update the session identified in the temp view
+
+-- COMMIT TRANSACTION
+--COMMIT;

lib/service/diabetes-research-hub/orchestration/stateless-orchestration.sql

@@ -0,0 +1,87 @@
+
+-- Drop and recreate the orch_session_view view
+DROP VIEW IF EXISTS drh_orch_session_view;
+CREATE VIEW drh_orch_session_view AS
+SELECT
+    orchestration_session_id, device_id, orchestration_nature_id,
+    version, orch_started_at, orch_finished_at,
+    diagnostics_json, diagnostics_md
+FROM orchestration_session;
+
+-- Drop and recreate the orch_session_deidentifyview view
+DROP VIEW IF EXISTS drh_orch_session_deidentifyview;
+CREATE VIEW drh_orch_session_deidentifyview AS
+SELECT
+    orchestration_session_id, device_id, orchestration_nature_id,
+    version, orch_started_at, orch_finished_at,
+    diagnostics_json, diagnostics_md
+FROM orchestration_session
+WHERE orchestration_nature_id = 'deidentification';
+
+
+-- Drop and recreate the orchestration_session_entry_view view
+DROP VIEW IF EXISTS drh_orchestration_session_entry_view;
+CREATE VIEW drh_orchestration_session_entry_view AS
+SELECT
+    orchestration_session_entry_id, session_id, ingest_src, ingest_table_name
+FROM orchestration_session_entry;
+
+-- Drop and recreate the orchestration_session_exec_view view
+DROP VIEW IF EXISTS drh_orchestration_session_exec_view;
+CREATE VIEW drh_orchestration_session_exec_view AS
+SELECT
+    orchestration_session_exec_id, exec_nature, session_id, session_entry_id,
+    parent_exec_id, namespace, exec_identity, exec_code, exec_status,
+    input_text, exec_error_text, output_text, output_nature, narrative_md
+FROM orchestration_session_exec;
+
+-- Drop and recreate the vw_orchestration_deidentify view
+DROP VIEW IF EXISTS drh_vw_orchestration_deidentify;
+CREATE VIEW drh_vw_orchestration_deidentify AS
+SELECT
+    osex.orchestration_session_exec_id,
+    osex.exec_nature,
+    osex.session_id,
+    osex.session_entry_id,
+    osex.parent_exec_id,
+    osex.namespace,
+    osex.exec_identity,
+    osex.exec_code,
+    osex.exec_status,
+    osex.input_text,
+    osex.exec_error_text,
+    osex.output_text,
+    osex.output_nature,
+    osex.narrative_md,    
+    os.device_id,
+    os.orchestration_nature_id,
+    os.version,
+    os.orch_started_at,
+    os.orch_finished_at,    
+    os.args_json,
+    os.diagnostics_json,
+    os.diagnostics_md
+FROM
+    orchestration_session_exec osex
+    JOIN orchestration_session os ON osex.session_id = os.orchestration_session_id
+WHERE
+    os.orchestration_nature_id = 'deidentification';
+
+
+DROP VIEW IF EXISTS drh_vandv_orch_issues;
+CREATE VIEW drh_vandv_orch_issues AS
+SELECT    
+    osi.issue_type as 'Issue Type',
+    osi.issue_message as 'Issue Message',
+    osi.issue_column as 'Issue column',
+    osi.remediation,
+    osi.issue_row as 'Issue Row',    
+    osi.invalid_value  
+FROM
+    orchestration_session_issue osi
+JOIN
+    orchestration_session os
+ON
+    osi.session_id = os.orchestration_session_id
+WHERE
+    os.orchestration_nature_id = 'V&V';