diff --git a/lib/service/diabetes-research-hub/combined-cgm-tracing-generator.ts b/lib/service/diabetes-research-hub/combined-cgm-tracing-generator.ts index 627c2c0e7..b0a72f68d 100644 --- a/lib/service/diabetes-research-hub/combined-cgm-tracing-generator.ts +++ b/lib/service/diabetes-research-hub/combined-cgm-tracing-generator.ts @@ -1,70 +1,49 @@ -import { DB } from "https://deno.land/x/sqlite/mod.ts"; +import { Database } from "https://deno.land/x/sqlite3@0.12.0/mod.ts"; // Function to create the combined CGM tracing view export function createCombinedCGMView(dbFilePath: string): void { - // Open the existing database - const db = new DB(dbFilePath); - //console.log(`Opened database: ${dbFilePath}`); + const db = new Database(dbFilePath); - // Create error log table if it doesn't exist - db.execute(` - CREATE TABLE IF NOT EXISTS error_log ( - errorLogId INTEGER PRIMARY KEY AUTOINCREMENT, - datetime TEXT DEFAULT (datetime('now')), - error_message TEXT - ); - `); - //console.log("Error log table created or already exists."); + db.exec(`CREATE TABLE IF NOT EXISTS error_log ( + errorLogId INTEGER PRIMARY KEY AUTOINCREMENT, + datetime TEXT DEFAULT (datetime('now')), + error_message TEXT + );`); - // Drop the view if it exists and create it try { - db.execute(`DROP VIEW IF EXISTS drh_participant_file_names;`); - db.execute(` - CREATE VIEW drh_participant_file_names AS - SELECT - patient_id, - GROUP_CONCAT(file_name, ', ') AS file_names - FROM - uniform_resource_cgm_file_metadata - GROUP BY - patient_id; + db.exec(`DROP VIEW IF EXISTS drh_participant_file_names;`); + db.exec(` + CREATE VIEW drh_participant_file_names AS + SELECT patient_id, GROUP_CONCAT(file_name, ', ') AS file_names + FROM uniform_resource_cgm_file_metadata + GROUP BY patient_id; `); console.log("View 'drh_participant_file_names' created successfully."); } catch (error) { console.error("Error creating view 'drh_participant_file_names':", error); - const sqlQuery = "INSERT INTO error_log (error_message) VALUES (?);"; const params = JSON.stringify({ message: error.message }); - db.execute(sqlQuery, [params]); + db.prepare("INSERT INTO error_log (error_message) VALUES (?);").run(params); db.close(); return; } - // Get the list of participant IDs from the view - const participants = db.query( - "SELECT DISTINCT patient_id FROM drh_participant_file_names;", - ); + const participantsStmt = db.prepare("SELECT DISTINCT patient_id FROM drh_participant_file_names;"); + const participants = participantsStmt.all(); - // Array to hold SQL parts for the combined view const sqlParts: string[] = []; + for (const { patient_id } of participants) { + const fileNamesStmt = db.prepare("SELECT file_names FROM drh_participant_file_names WHERE patient_id = ?;"); + const file_names_row = fileNamesStmt.get(patient_id); - for (const [patient_id_raw] of participants) { - const patient_id: string = patient_id_raw as string; - - const [file_names_row] = db.query( - "SELECT file_names FROM drh_participant_file_names WHERE patient_id = ?", - [patient_id], - ); if (!file_names_row) { console.log(`No file names found for participant ${patient_id}.`); continue; } - const file_names = file_names_row[0]; + const file_names = file_names_row.file_names; // Access property directly if (file_names) { - const participantTableNames = file_names.split(", ").map((fileName) => - `uniform_resource_${fileName}` - ); - participantTableNames.forEach((tableName) => { + const participantTableNames = file_names.split(', ').map(fileName => `uniform_resource_${fileName}`); + participantTableNames.forEach(tableName => { sqlParts.push(` SELECT '${patient_id}' as participant_id, @@ -74,27 +53,31 @@ export function createCombinedCGMView(dbFilePath: string): void { `); }); } + fileNamesStmt.finalize(); // Clean up } if (sqlParts.length > 0) { - const combinedUnionAllQuery = sqlParts.join(" UNION ALL "); - const createCombinedViewSql = - `CREATE VIEW IF NOT EXISTS combined_cgm_tracing AS ${combinedUnionAllQuery};`; - - db.execute(createCombinedViewSql); - console.log("Combined view 'combined_cgm_tracing' created successfully."); + const combinedUnionAllQuery = sqlParts.join(' UNION ALL '); + const createCombinedViewSql = `CREATE VIEW IF NOT EXISTS combined_cgm_tracing AS ${combinedUnionAllQuery};`; + + try { + db.exec(createCombinedViewSql); + console.log("Combined view 'combined_cgm_tracing' created successfully."); + } catch (error) { + console.error("Error creating combined view:", error); + const params = JSON.stringify({ message: error.message }); + db.prepare("INSERT INTO error_log (error_message) VALUES (?);").run(params); + } } else { - console.log( - "No participant tables found, so the combined view will not be created.", - ); + console.log("No participant tables found, so the combined view will not be created."); } + participantsStmt.finalize(); // Clean up db.close(); - //console.log(`Closed database: ${dbFilePath}`); } // If the script is being run directly, execute the function if (import.meta.main) { - const dbFilePath = "resource-surveillance.sqlite.db"; + const dbFilePath = "resource-surveillance.sqlite.db"; createCombinedCGMView(dbFilePath); } diff --git a/lib/service/diabetes-research-hub/drhctl.ts b/lib/service/diabetes-research-hub/drhctl.ts index 9c8eca32e..f26cfaebb 100644 --- a/lib/service/diabetes-research-hub/drhctl.ts +++ b/lib/service/diabetes-research-hub/drhctl.ts @@ -1,23 +1,27 @@ #!/usr/bin/env -S deno run --allow-read --allow-write --allow-env --allow-run --allow-net import * as colors from "https://deno.land/std@0.224.0/fmt/colors.ts"; -import { DB } from "https://deno.land/x/sqlite@v3.9.1/mod.ts"; +import { Database } from "https://deno.land/x/sqlite3@0.12.0/mod.ts"; import * as drhux from "./package.sql.ts"; +import { createCombinedCGMView } from "./combined-cgm-tracing-generator.ts"; import { FlexibleTextSupplierSync, spawnedResult, textFromSupplierSync, } from "../../universal/spawn.ts"; + + // Detect platform-specific command format const isWindows = Deno.build.os === "windows"; const toolCmd = isWindows ? ".\\surveilr" : "surveilr"; const dbFilePath = "resource-surveillance.sqlite.db"; // Path to your SQLite DB -const RSC_BASE_URL = - "https://raw.githubusercontent.com/surveilr/www.surveilr.com/main/lib/service/diabetes-research-hub"; +const RSC_BASE_URL = "https://raw.githubusercontent.com/surveilr/www.surveilr.com/main/lib/service/diabetes-research-hub"; const UX_URL = "https://www.surveilr.com/lib/service/diabetes-research-hub"; -//const UX_URL = "http://localhost:4321/lib/service/diabetes-research-hub"; // can be used if local server is accessible + + + // Helper function to fetch SQL content async function fetchSqlContent(url: string): Promise { @@ -30,13 +34,15 @@ async function fetchSqlContent(url: string): Promise { } catch (error) { console.error( colors.cyan(`Error fetching SQL content from ${url}:`), - error.message, + error.message, ); Deno.exit(1); - return ""; + return ''; } } + + // Helper function to execute a command async function executeCommand( cmd: string[], @@ -99,23 +105,54 @@ async function fetchUxSqlContent(): Promise { colors.red("Error fetching UX SQL content:"), error.message, ); - Deno.exit(1); + return ''; + //Deno.exit(1); } } // Function to execute SQL commands directly on SQLite database -function executeSqlCommands(sqlCommands: string) { +function executeSqlCommands(sqlCommands: string): void { + let db: Database | null = null; // Initialize db variable + try { - const db = new DB(dbFilePath); - db.execute(sqlCommands); // Execute the SQL commands - db.close(); - console.log(colors.green("UX SQL executed successfully.")); + db = new Database(dbFilePath); // Open the database + db.exec(sqlCommands); // Execute the SQL commands + console.log(colors.green("SQL executed successfully.")); } catch (error) { console.error(colors.red("Error executing SQL commands:"), error.message); Deno.exit(1); + } finally { + if (db) { + db.close(); // Close the database if it was opened + } + } +} + +// Function to check for table existence and create combined view +async function checkAndCreateCombinedView(dbFilePath: string) { + const db = new Database(dbFilePath); + + try { + const tableName = 'uniform_resource_cgm_file_metadata'; + // Check if the required table exists + const stmt = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`); + const rows = stmt.all(tableName); + + if (rows.length > 0) { + console.log(colors.green("Required table exists. Proceeding to create the combined view.")); + await createCombinedCGMView(dbFilePath); // Ensure this function is defined elsewhere + } else { + console.error(colors.red("The required table does not exist. Cannot create the combined view.")); + } + } catch (error) { + console.error(colors.red("Error in checkAndCreateCombinedView:"), error.message); + } finally { + db.close(); } } + + // Check if a folder name was provided if (Deno.args.length === 0) { console.error( @@ -127,6 +164,7 @@ if (Deno.args.length === 0) { // Store the folder name in a variable const folderName = Deno.args[0]; + // Define synchronous suppliers const deidentificationSQLSupplier: FlexibleTextSupplierSync = () => deidentificationSQL; @@ -137,6 +175,8 @@ let deidentificationSQL: string; let vvSQL: string; let uxSQL: string; + + try { // Fetch SQL content for DeIdentification, Verification & Validation, and UX orchestration deidentificationSQL = await fetchSqlContent( @@ -144,11 +184,11 @@ try { ); vvSQL = await fetchSqlContent( `${RSC_BASE_URL}/verfication-validation/orchestrate-drh-vv.sql`, - ); - uxSQL = await fetchSqlContent( - `${UX_URL}/package.sql`, - ); - //uxSQL = await fetchUxSqlContent(); // Fetch UX SQL content + ); + // uxSQL = await fetchSqlContent( + // `${UX_URL}/package.sql`, + // ); + uxSQL = await fetchUxSqlContent(); // Fetch UX SQL content } catch (error) { console.error( colors.cyan( @@ -173,6 +213,7 @@ try { Deno.exit(1); } + try { await executeCommand([toolCmd, "orchestrate", "transform-csv"]); console.log( @@ -183,6 +224,11 @@ try { Deno.exit(1); } + +// This function retrieves the SQL script for data de-identification. +// Note: Deidentification functions are only available through the `surveilr shell` or `surveilr orchestrate` commands. +// Issues prevail while executing these commands on Windows OS. +// Therefore avoiding deidentification till the issue is resolved // try { // console.log(colors.dim(`Performing DeIdentification: ${folderName}...`)); // await executeCommand( @@ -192,31 +238,22 @@ try { // console.log(colors.green("Deidentification successful.")); // } catch (error) { // console.error(colors.cyan("Error during DeIdentification:"), error.message); -// Deno.exit(1); +// //Deno.exit(1); // } +// This function is for the dynamic combined view generation // try { -// console.log( -// colors.dim(`Performing Verification and Validation: ${folderName}...`), -// ); -// await executeCommand([toolCmd, "orchestrate", "-n", "v&v"], vvSQLSupplier); -// console.log( -// colors.green( -// "Verification and validation orchestration completed successfully.", -// ), -// ); +// await checkAndCreateCombinedView(dbFilePath); +// console.log(colors.green("View generation completed successfully.")); // } catch (error) { -// console.error( -// colors.cyan("Error during Verification and Validation:"), -// error.message, -// ); -// Deno.exit(1); +// console.error(colors.red("Error during view generation:"), error.message); // } + try { - console.log(colors.dim(`Performing UX orchestration: ${folderName}...`)); - await executeCommand([toolCmd, "shell"], uxSQLSupplier); - //executeSqlCommands(uxSQL); // Execute UX SQL commands + console.log(colors.dim(`Performing UX orchestration: ${folderName}...`)); + //await executeCommand([toolCmd, "shell"], uxSQLSupplier); + executeSqlCommands(uxSQL); // Execute UX SQL commands console.log(colors.green("UX orchestration completed successfully.")); } catch (error) { console.error(colors.cyan("Error during UX orchestration:"), error.message); diff --git a/lib/service/diabetes-research-hub/orchestration/vv-orchestration.sql b/lib/service/diabetes-research-hub/orchestration/vv-orchestration.sql index 8eab14066..99daffe4c 100644 --- a/lib/service/diabetes-research-hub/orchestration/vv-orchestration.sql +++ b/lib/service/diabetes-research-hub/orchestration/vv-orchestration.sql @@ -72,6 +72,10 @@ UNION ALL SELECT 'uniform_resource_cgm_file_metadata', 'data_start_date', 'TEXT' UNION ALL SELECT 'uniform_resource_cgm_file_metadata', 'data_end_date', 'TEXT', 0, 1 UNION ALL SELECT 'uniform_resource_cgm_file_metadata', 'study_id', 'TEXT', 0, 1; +CREATE TEMP VIEW IF NOT EXISTS device_info AS +SELECT device_id, name, created_at +FROM device d; + INSERT OR IGNORE INTO orchestration_nature ( orchestration_nature_id, diff --git a/lib/service/diabetes-research-hub/package.sql.ts b/lib/service/diabetes-research-hub/package.sql.ts index b44503eda..a08a6a22c 100755 --- a/lib/service/diabetes-research-hub/package.sql.ts +++ b/lib/service/diabetes-research-hub/package.sql.ts @@ -1039,27 +1039,42 @@ Participants are individuals who volunteer to take part in CGM research studies. export async function drhSQL() { return await spn.TypicalSqlPageNotebook.SQL( new class extends spn.TypicalSqlPageNotebook { - async deidentifyDRHSQL() { - // read the file from either local or remote (depending on location of this file) - return await spn.TypicalSqlPageNotebook.fetchText( - import.meta.resolve( - "./orchestration/deidentification-orchestration.sql", - ), - ); - } + + + // async deidentifyDRHSQL() { + // // This function retrieves the SQL script for data de-identification. + // // Note: Deidentification functions are only available through the `surveilr shell` or `surveilr orchestrate` commands. + // // Issues prevail while executing these commands on Windows OS. + // return await spn.TypicalSqlPageNotebook.fetchText( + // import.meta.resolve( + // "./orchestration/deidentification-orchestration.sql", + // ), + // ); + // } async vandvDRHSQL() { - // read the file from either local or remote (depending on location of this file) + // This function retrieves the SQL script for verfication and validation return await spn.TypicalSqlPageNotebook.fetchText( import.meta.resolve("./orchestration/vv-orchestration.sql"), ); } + async statelessDRHSQL() { // read the file from either local or remote (depending on location of this file) return await spn.TypicalSqlPageNotebook.fetchText( import.meta.resolve("./stateless.sql"), ); } + + + // async metricsDRHSQL() { + // // This function fetches the SQL query for DRH metrics after the combined view has been generated. + // // Ensure that the combined view is created before calling this function. + // return await spn.TypicalSqlPageNotebook.fetchText( + // import.meta.resolve("./drh-metrics.sql"), + // ); + // } + }(), // new sh.ShellSqlPages(), new DrhShellSqlPages(),