Merge pull request #82 from surveilr/ani/feat/ui-parity

update: disable deidentification feature , switch to using Deno 2.0 with the latest sqlite3@12.0 for SQRT function. #37
surveilr · Oct 15, 2024 · 16b8fae · 16b8fae
2 parents 024b51b + e798de5
commit 16b8fae
Show file tree

Hide file tree

Showing 4 changed files with 137 additions and 98 deletions.
diff --git a/lib/service/diabetes-research-hub/combined-cgm-tracing-generator.ts b/lib/service/diabetes-research-hub/combined-cgm-tracing-generator.ts
@@ -1,70 +1,49 @@
-import { DB } from "https://deno.land/x/sqlite/mod.ts";
+import { Database } from "https://deno.land/x/sqlite3@0.12.0/mod.ts";
 
 // Function to create the combined CGM tracing view
 export function createCombinedCGMView(dbFilePath: string): void {
-  // Open the existing database
-  const db = new DB(dbFilePath);
-  //console.log(`Opened database: ${dbFilePath}`);
+  const db = new Database(dbFilePath);
 
-  // Create error log table if it doesn't exist
-  db.execute(`
-    CREATE TABLE IF NOT EXISTS error_log (
-      errorLogId INTEGER PRIMARY KEY AUTOINCREMENT,
-      datetime TEXT DEFAULT (datetime('now')),
-      error_message TEXT
-    );
-  `);
-  //console.log("Error log table created or already exists.");
+  db.exec(`CREATE TABLE IF NOT EXISTS error_log (
+    errorLogId INTEGER PRIMARY KEY AUTOINCREMENT,
+    datetime TEXT DEFAULT (datetime('now')),
+    error_message TEXT
+  );`);
 
-  // Drop the view if it exists and create it
   try {
-    db.execute(`DROP VIEW IF EXISTS drh_participant_file_names;`);
-    db.execute(`
-      CREATE VIEW  drh_participant_file_names AS
-      SELECT
-        patient_id,
-        GROUP_CONCAT(file_name, ', ') AS file_names
-      FROM
-        uniform_resource_cgm_file_metadata
-      GROUP BY
-        patient_id;
+    db.exec(`DROP VIEW IF EXISTS drh_participant_file_names;`);
+    db.exec(`
+      CREATE VIEW drh_participant_file_names AS
+      SELECT patient_id, GROUP_CONCAT(file_name, ', ') AS file_names
+      FROM uniform_resource_cgm_file_metadata
+      GROUP BY patient_id;
     `);
     console.log("View 'drh_participant_file_names' created successfully.");
   } catch (error) {
     console.error("Error creating view 'drh_participant_file_names':", error);
-    const sqlQuery = "INSERT INTO error_log (error_message) VALUES (?);";
     const params = JSON.stringify({ message: error.message });
-    db.execute(sqlQuery, [params]);
+    db.prepare("INSERT INTO error_log (error_message) VALUES (?);").run(params);
     db.close();
     return;
   }
 
-  // Get the list of participant IDs from the view
-  const participants = db.query(
-    "SELECT DISTINCT patient_id FROM drh_participant_file_names;",
-  );
+  const participantsStmt = db.prepare("SELECT DISTINCT patient_id FROM drh_participant_file_names;");
+  const participants = participantsStmt.all();
 
-  // Array to hold SQL parts for the combined view
   const sqlParts: string[] = [];
+  for (const { patient_id } of participants) {
+    const fileNamesStmt = db.prepare("SELECT file_names FROM drh_participant_file_names WHERE patient_id = ?;");
+    const file_names_row = fileNamesStmt.get(patient_id);
 
-  for (const [patient_id_raw] of participants) {
-    const patient_id: string = patient_id_raw as string;
-
-    const [file_names_row] = db.query(
-      "SELECT file_names FROM drh_participant_file_names WHERE patient_id = ?",
-      [patient_id],
-    );
     if (!file_names_row) {
       console.log(`No file names found for participant ${patient_id}.`);
       continue;
     }
 
-    const file_names = file_names_row[0];
+    const file_names = file_names_row.file_names; // Access property directly
     if (file_names) {
-      const participantTableNames = file_names.split(", ").map((fileName) =>
-        `uniform_resource_${fileName}`
-      );
-      participantTableNames.forEach((tableName) => {
+      const participantTableNames = file_names.split(', ').map(fileName => `uniform_resource_${fileName}`);
+      participantTableNames.forEach(tableName => {
         sqlParts.push(`
           SELECT 
             '${patient_id}' as participant_id, 
@@ -74,27 +53,31 @@ export function createCombinedCGMView(dbFilePath: string): void {
         `);
       });
     }
+    fileNamesStmt.finalize(); // Clean up
   }
 
   if (sqlParts.length > 0) {
-    const combinedUnionAllQuery = sqlParts.join(" UNION ALL ");
-    const createCombinedViewSql =
-      `CREATE VIEW IF NOT EXISTS combined_cgm_tracing AS ${combinedUnionAllQuery};`;
-
-    db.execute(createCombinedViewSql);
-    console.log("Combined view 'combined_cgm_tracing' created successfully.");
+    const combinedUnionAllQuery = sqlParts.join(' UNION ALL ');
+    const createCombinedViewSql = `CREATE VIEW IF NOT EXISTS combined_cgm_tracing AS ${combinedUnionAllQuery};`;
+
+    try {
+      db.exec(createCombinedViewSql);
+      console.log("Combined view 'combined_cgm_tracing' created successfully.");
+    } catch (error) {
+      console.error("Error creating combined view:", error);
+      const params = JSON.stringify({ message: error.message });
+      db.prepare("INSERT INTO error_log (error_message) VALUES (?);").run(params);
+    }
   } else {
-    console.log(
-      "No participant tables found, so the combined view will not be created.",
-    );
+    console.log("No participant tables found, so the combined view will not be created.");
   }
 
+  participantsStmt.finalize(); // Clean up
   db.close();
-  //console.log(`Closed database: ${dbFilePath}`);
 }
 
 // If the script is being run directly, execute the function
 if (import.meta.main) {
-  const dbFilePath = "resource-surveillance.sqlite.db";
+  const dbFilePath = "resource-surveillance.sqlite.db"; 
   createCombinedCGMView(dbFilePath);
 }
diff --git a/lib/service/diabetes-research-hub/drhctl.ts b/lib/service/diabetes-research-hub/drhctl.ts
@@ -1,23 +1,27 @@
 #!/usr/bin/env -S deno run --allow-read --allow-write --allow-env --allow-run --allow-net
 
 import * as colors from "https://deno.land/std@0.224.0/fmt/colors.ts";
-import { DB } from "https://deno.land/x/sqlite@v3.9.1/mod.ts";
+import { Database } from "https://deno.land/x/sqlite3@0.12.0/mod.ts";
 import * as drhux from "./package.sql.ts";
+import { createCombinedCGMView } from "./combined-cgm-tracing-generator.ts";
 import {
   FlexibleTextSupplierSync,
   spawnedResult,
   textFromSupplierSync,
 } from "../../universal/spawn.ts";
 
+
+
 // Detect platform-specific command format
 const isWindows = Deno.build.os === "windows";
 const toolCmd = isWindows ? ".\\surveilr" : "surveilr";
 const dbFilePath = "resource-surveillance.sqlite.db"; // Path to your SQLite DB
 
-const RSC_BASE_URL =
-  "https://raw.githubusercontent.com/surveilr/www.surveilr.com/main/lib/service/diabetes-research-hub";
+const RSC_BASE_URL =  "https://raw.githubusercontent.com/surveilr/www.surveilr.com/main/lib/service/diabetes-research-hub";
 const UX_URL = "https://www.surveilr.com/lib/service/diabetes-research-hub";
-//const UX_URL = "http://localhost:4321/lib/service/diabetes-research-hub"; // can be used if local server is accessible
+
+
+
 
 // Helper function to fetch SQL content
 async function fetchSqlContent(url: string): Promise<string> {
@@ -30,13 +34,15 @@ async function fetchSqlContent(url: string): Promise<string> {
   } catch (error) {
     console.error(
       colors.cyan(`Error fetching SQL content from ${url}:`),
-      error.message,
+      error.message,      
     );
     Deno.exit(1);
-    return "";
+    return '';
   }
 }
 
+
+
 // Helper function to execute a command
 async function executeCommand(
   cmd: string[],
@@ -99,23 +105,54 @@ async function fetchUxSqlContent(): Promise<string> {
       colors.red("Error fetching UX SQL content:"),
       error.message,
     );
-    Deno.exit(1);
+    return '';
+    //Deno.exit(1);
   }
 }
 
 // Function to execute SQL commands directly on SQLite database
-function executeSqlCommands(sqlCommands: string) {
+function executeSqlCommands(sqlCommands: string): void {
+  let db: Database | null = null; // Initialize db variable
+
   try {
-    const db = new DB(dbFilePath);
-    db.execute(sqlCommands); // Execute the SQL commands
-    db.close();
-    console.log(colors.green("UX SQL executed successfully."));
+    db = new Database(dbFilePath); // Open the database
+    db.exec(sqlCommands); // Execute the SQL commands
+    console.log(colors.green("SQL executed successfully."));
   } catch (error) {
     console.error(colors.red("Error executing SQL commands:"), error.message);
     Deno.exit(1);
+  } finally {
+    if (db) {
+      db.close(); // Close the database if it was opened
+    }
+  }
+}
+
+// Function to check for table existence and create combined view
+async function checkAndCreateCombinedView(dbFilePath: string) {
+  const db = new Database(dbFilePath);
+
+  try {
+    const tableName = 'uniform_resource_cgm_file_metadata';
+    // Check if the required table exists
+    const stmt = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`);
+    const rows = stmt.all(tableName);
+
+    if (rows.length > 0) {
+      console.log(colors.green("Required table exists. Proceeding to create the combined view."));
+      await createCombinedCGMView(dbFilePath); // Ensure this function is defined elsewhere
+    } else {
+      console.error(colors.red("The required table does not exist. Cannot create the combined view."));
+    }
+  } catch (error) {
+    console.error(colors.red("Error in checkAndCreateCombinedView:"), error.message);
+  } finally {
+    db.close();
   }
 }
 
+
+
 // Check if a folder name was provided
 if (Deno.args.length === 0) {
   console.error(
@@ -127,6 +164,7 @@ if (Deno.args.length === 0) {
 // Store the folder name in a variable
 const folderName = Deno.args[0];
 
+
 // Define synchronous suppliers
 const deidentificationSQLSupplier: FlexibleTextSupplierSync = () =>
   deidentificationSQL;
@@ -137,18 +175,20 @@ let deidentificationSQL: string;
 let vvSQL: string;
 let uxSQL: string;
 
+
+
 try {
   // Fetch SQL content for DeIdentification, Verification & Validation, and UX orchestration
   deidentificationSQL = await fetchSqlContent(
     `${RSC_BASE_URL}/de-identification/drh-deidentification.sql`,
   );
   vvSQL = await fetchSqlContent(
     `${RSC_BASE_URL}/verfication-validation/orchestrate-drh-vv.sql`,
-  );
-  uxSQL = await fetchSqlContent(
-    `${UX_URL}/package.sql`,
-  );
-  //uxSQL = await fetchUxSqlContent(); // Fetch UX SQL content
+  );  
+  // uxSQL = await fetchSqlContent(
+  //   `${UX_URL}/package.sql`,
+  // );
+  uxSQL = await fetchUxSqlContent(); // Fetch UX SQL content
 } catch (error) {
   console.error(
     colors.cyan(
@@ -173,6 +213,7 @@ try {
   Deno.exit(1);
 }
 
+
 try {
   await executeCommand([toolCmd, "orchestrate", "transform-csv"]);
   console.log(
@@ -183,6 +224,11 @@ try {
   Deno.exit(1);
 }
 
+
+// This function retrieves the SQL script for data de-identification.
+// Note: Deidentification functions are only available through the `surveilr shell` or `surveilr orchestrate` commands.
+// Issues prevail while executing these commands on Windows OS.
+// Therefore avoiding deidentification till the issue is resolved
 // try {
 //   console.log(colors.dim(`Performing DeIdentification: ${folderName}...`));
 //   await executeCommand(
@@ -192,31 +238,22 @@ try {
 //   console.log(colors.green("Deidentification successful."));
 // } catch (error) {
 //   console.error(colors.cyan("Error during DeIdentification:"), error.message);
-//   Deno.exit(1);
+//   //Deno.exit(1);
 // }
 
+// This function is for the dynamic combined view generation
 // try {
-//   console.log(
-//     colors.dim(`Performing Verification and Validation: ${folderName}...`),
-//   );
-//   await executeCommand([toolCmd, "orchestrate", "-n", "v&v"], vvSQLSupplier);
-//   console.log(
-//     colors.green(
-//       "Verification and validation orchestration completed successfully.",
-//     ),
-//   );
+//   await checkAndCreateCombinedView(dbFilePath);
+//   console.log(colors.green("View generation completed successfully."));
 // } catch (error) {
-//   console.error(
-//     colors.cyan("Error during Verification and Validation:"),
-//     error.message,
-//   );
-//   Deno.exit(1);
+//   console.error(colors.red("Error during view generation:"), error.message);
 // }
 
+
 try {
-  console.log(colors.dim(`Performing UX orchestration: ${folderName}...`));
-  await executeCommand([toolCmd, "shell"], uxSQLSupplier);
-  //executeSqlCommands(uxSQL); // Execute UX SQL commands
+  console.log(colors.dim(`Performing UX orchestration: ${folderName}...`));  
+  //await executeCommand([toolCmd, "shell"], uxSQLSupplier);
+  executeSqlCommands(uxSQL); // Execute UX SQL commands
   console.log(colors.green("UX orchestration completed successfully."));
 } catch (error) {
   console.error(colors.cyan("Error during UX orchestration:"), error.message);

diff --git a/lib/service/diabetes-research-hub/orchestration/vv-orchestration.sql b/lib/service/diabetes-research-hub/orchestration/vv-orchestration.sql
@@ -72,6 +72,10 @@ UNION ALL SELECT 'uniform_resource_cgm_file_metadata', 'data_start_date', 'TEXT'
 UNION ALL SELECT 'uniform_resource_cgm_file_metadata', 'data_end_date', 'TEXT', 0, 1
 UNION ALL SELECT 'uniform_resource_cgm_file_metadata', 'study_id', 'TEXT', 0, 1;
 
+CREATE TEMP VIEW IF NOT EXISTS device_info AS
+SELECT device_id, name, created_at
+FROM device d;
+
 
 INSERT OR IGNORE INTO orchestration_nature (
     orchestration_nature_id,