Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

update: disable deidentification feature , switch to using Deno 2.0 with the latest sqlite3@12.0 for SQRT function. #37 #82

Merged
merged 3 commits into from
Oct 15, 2024
Merged

update: disable deidentification feature , switch to using Deno 2.0 with the latest sqlite3@12.0 for SQRT function. #37 #82

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c35d8f6
Update: Modify automation script to use `deno2.0` and integrate late…
MeetAnithaVarghese Oct 15, 2024
ab918b5
Update: Disable de-identification call until Windows OS issue with Su…
MeetAnithaVarghese Oct 15, 2024
e798de5
update: resolve typo issue #29
MeetAnithaVarghese Oct 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 37 additions & 54 deletions lib/service/diabetes-research-hub/combined-cgm-tracing-generator.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,70 +1,49 @@
import { DB } from "https://deno.land/x/sqlite/mod.ts";
import { Database } from "https://deno.land/x/sqlite3@0.12.0/mod.ts";

// Function to create the combined CGM tracing view
export function createCombinedCGMView(dbFilePath: string): void {
// Open the existing database
const db = new DB(dbFilePath);
//console.log(`Opened database: ${dbFilePath}`);
const db = new Database(dbFilePath);

// Create error log table if it doesn't exist
db.execute(`
CREATE TABLE IF NOT EXISTS error_log (
errorLogId INTEGER PRIMARY KEY AUTOINCREMENT,
datetime TEXT DEFAULT (datetime('now')),
error_message TEXT
);
`);
//console.log("Error log table created or already exists.");
db.exec(`CREATE TABLE IF NOT EXISTS error_log (
errorLogId INTEGER PRIMARY KEY AUTOINCREMENT,
datetime TEXT DEFAULT (datetime('now')),
error_message TEXT
);`);

// Drop the view if it exists and create it
try {
db.execute(`DROP VIEW IF EXISTS drh_participant_file_names;`);
db.execute(`
CREATE VIEW drh_participant_file_names AS
SELECT
patient_id,
GROUP_CONCAT(file_name, ', ') AS file_names
FROM
uniform_resource_cgm_file_metadata
GROUP BY
patient_id;
db.exec(`DROP VIEW IF EXISTS drh_participant_file_names;`);
db.exec(`
CREATE VIEW drh_participant_file_names AS
SELECT patient_id, GROUP_CONCAT(file_name, ', ') AS file_names
FROM uniform_resource_cgm_file_metadata
GROUP BY patient_id;
`);
console.log("View 'drh_participant_file_names' created successfully.");
} catch (error) {
console.error("Error creating view 'drh_participant_file_names':", error);
const sqlQuery = "INSERT INTO error_log (error_message) VALUES (?);";
const params = JSON.stringify({ message: error.message });
db.execute(sqlQuery, [params]);
db.prepare("INSERT INTO error_log (error_message) VALUES (?);").run(params);
db.close();
return;
}

// Get the list of participant IDs from the view
const participants = db.query(
"SELECT DISTINCT patient_id FROM drh_participant_file_names;",
);
const participantsStmt = db.prepare("SELECT DISTINCT patient_id FROM drh_participant_file_names;");
const participants = participantsStmt.all();

// Array to hold SQL parts for the combined view
const sqlParts: string[] = [];
for (const { patient_id } of participants) {
const fileNamesStmt = db.prepare("SELECT file_names FROM drh_participant_file_names WHERE patient_id = ?;");
const file_names_row = fileNamesStmt.get(patient_id);

for (const [patient_id_raw] of participants) {
const patient_id: string = patient_id_raw as string;

const [file_names_row] = db.query(
"SELECT file_names FROM drh_participant_file_names WHERE patient_id = ?",
[patient_id],
);
if (!file_names_row) {
console.log(`No file names found for participant ${patient_id}.`);
continue;
}

const file_names = file_names_row[0];
const file_names = file_names_row.file_names; // Access property directly
if (file_names) {
const participantTableNames = file_names.split(", ").map((fileName) =>
`uniform_resource_${fileName}`
);
participantTableNames.forEach((tableName) => {
const participantTableNames = file_names.split(', ').map(fileName => `uniform_resource_${fileName}`);
participantTableNames.forEach(tableName => {
sqlParts.push(`
SELECT
'${patient_id}' as participant_id,
Expand All @@ -74,27 +53,31 @@ export function createCombinedCGMView(dbFilePath: string): void {
`);
});
}
fileNamesStmt.finalize(); // Clean up
}

if (sqlParts.length > 0) {
const combinedUnionAllQuery = sqlParts.join(" UNION ALL ");
const createCombinedViewSql =
`CREATE VIEW IF NOT EXISTS combined_cgm_tracing AS ${combinedUnionAllQuery};`;

db.execute(createCombinedViewSql);
console.log("Combined view 'combined_cgm_tracing' created successfully.");
const combinedUnionAllQuery = sqlParts.join(' UNION ALL ');
const createCombinedViewSql = `CREATE VIEW IF NOT EXISTS combined_cgm_tracing AS ${combinedUnionAllQuery};`;

try {
db.exec(createCombinedViewSql);
console.log("Combined view 'combined_cgm_tracing' created successfully.");
} catch (error) {
console.error("Error creating combined view:", error);
const params = JSON.stringify({ message: error.message });
db.prepare("INSERT INTO error_log (error_message) VALUES (?);").run(params);
}
} else {
console.log(
"No participant tables found, so the combined view will not be created.",
);
console.log("No participant tables found, so the combined view will not be created.");
}

participantsStmt.finalize(); // Clean up
db.close();
//console.log(`Closed database: ${dbFilePath}`);
}

// If the script is being run directly, execute the function
if (import.meta.main) {
const dbFilePath = "resource-surveillance.sqlite.db";
const dbFilePath = "resource-surveillance.sqlite.db";
createCombinedCGMView(dbFilePath);
}
107 changes: 72 additions & 35 deletions lib/service/diabetes-research-hub/drhctl.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,23 +1,27 @@
#!/usr/bin/env -S deno run --allow-read --allow-write --allow-env --allow-run --allow-net

import * as colors from "https://deno.land/std@0.224.0/fmt/colors.ts";
import { DB } from "https://deno.land/x/sqlite@v3.9.1/mod.ts";
import { Database } from "https://deno.land/x/sqlite3@0.12.0/mod.ts";
import * as drhux from "./package.sql.ts";
import { createCombinedCGMView } from "./combined-cgm-tracing-generator.ts";
import {
FlexibleTextSupplierSync,
spawnedResult,
textFromSupplierSync,
} from "../../universal/spawn.ts";



// Detect platform-specific command format
const isWindows = Deno.build.os === "windows";
const toolCmd = isWindows ? ".\\surveilr" : "surveilr";
const dbFilePath = "resource-surveillance.sqlite.db"; // Path to your SQLite DB

const RSC_BASE_URL =
"https://raw.githubusercontent.com/surveilr/www.surveilr.com/main/lib/service/diabetes-research-hub";
const RSC_BASE_URL = "https://raw.githubusercontent.com/surveilr/www.surveilr.com/main/lib/service/diabetes-research-hub";
const UX_URL = "https://www.surveilr.com/lib/service/diabetes-research-hub";
//const UX_URL = "http://localhost:4321/lib/service/diabetes-research-hub"; // can be used if local server is accessible




// Helper function to fetch SQL content
async function fetchSqlContent(url: string): Promise<string> {
Expand All @@ -30,13 +34,15 @@ async function fetchSqlContent(url: string): Promise<string> {
} catch (error) {
console.error(
colors.cyan(`Error fetching SQL content from ${url}:`),
error.message,
error.message,
);
Deno.exit(1);
return "";
return '';
}
}



// Helper function to execute a command
async function executeCommand(
cmd: string[],
Expand Down Expand Up @@ -99,23 +105,54 @@ async function fetchUxSqlContent(): Promise<string> {
colors.red("Error fetching UX SQL content:"),
error.message,
);
Deno.exit(1);
return '';
//Deno.exit(1);
}
}

// Function to execute SQL commands directly on SQLite database
function executeSqlCommands(sqlCommands: string) {
function executeSqlCommands(sqlCommands: string): void {
let db: Database | null = null; // Initialize db variable

try {
const db = new DB(dbFilePath);
db.execute(sqlCommands); // Execute the SQL commands
db.close();
console.log(colors.green("UX SQL executed successfully."));
db = new Database(dbFilePath); // Open the database
db.exec(sqlCommands); // Execute the SQL commands
console.log(colors.green("SQL executed successfully."));
} catch (error) {
console.error(colors.red("Error executing SQL commands:"), error.message);
Deno.exit(1);
} finally {
if (db) {
db.close(); // Close the database if it was opened
}
}
}

// Function to check for table existence and create combined view
async function checkAndCreateCombinedView(dbFilePath: string) {
const db = new Database(dbFilePath);

try {
const tableName = 'uniform_resource_cgm_file_metadata';
// Check if the required table exists
const stmt = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`);
const rows = stmt.all(tableName);

if (rows.length > 0) {
console.log(colors.green("Required table exists. Proceeding to create the combined view."));
await createCombinedCGMView(dbFilePath); // Ensure this function is defined elsewhere
} else {
console.error(colors.red("The required table does not exist. Cannot create the combined view."));
}
} catch (error) {
console.error(colors.red("Error in checkAndCreateCombinedView:"), error.message);
} finally {
db.close();
}
}



// Check if a folder name was provided
if (Deno.args.length === 0) {
console.error(
Expand All @@ -127,6 +164,7 @@ if (Deno.args.length === 0) {
// Store the folder name in a variable
const folderName = Deno.args[0];


// Define synchronous suppliers
const deidentificationSQLSupplier: FlexibleTextSupplierSync = () =>
deidentificationSQL;
Expand All @@ -137,18 +175,20 @@ let deidentificationSQL: string;
let vvSQL: string;
let uxSQL: string;



try {
// Fetch SQL content for DeIdentification, Verification & Validation, and UX orchestration
deidentificationSQL = await fetchSqlContent(
`${RSC_BASE_URL}/de-identification/drh-deidentification.sql`,
);
vvSQL = await fetchSqlContent(
`${RSC_BASE_URL}/verfication-validation/orchestrate-drh-vv.sql`,
);
uxSQL = await fetchSqlContent(
`${UX_URL}/package.sql`,
);
//uxSQL = await fetchUxSqlContent(); // Fetch UX SQL content
);
// uxSQL = await fetchSqlContent(
// `${UX_URL}/package.sql`,
// );
uxSQL = await fetchUxSqlContent(); // Fetch UX SQL content
} catch (error) {
console.error(
colors.cyan(
Expand All @@ -173,6 +213,7 @@ try {
Deno.exit(1);
}


try {
await executeCommand([toolCmd, "orchestrate", "transform-csv"]);
console.log(
Expand All @@ -183,6 +224,11 @@ try {
Deno.exit(1);
}


// This function retrieves the SQL script for data de-identification.
// Note: Deidentification functions are only available through the `surveilr shell` or `surveilr orchestrate` commands.
// Issues prevail while executing these commands on Windows OS.
// Therefore avoiding deidentification till the issue is resolved
// try {
// console.log(colors.dim(`Performing DeIdentification: ${folderName}...`));
// await executeCommand(
Expand All @@ -192,31 +238,22 @@ try {
// console.log(colors.green("Deidentification successful."));
// } catch (error) {
// console.error(colors.cyan("Error during DeIdentification:"), error.message);
// Deno.exit(1);
// //Deno.exit(1);
// }

// This function is for the dynamic combined view generation
// try {
// console.log(
// colors.dim(`Performing Verification and Validation: ${folderName}...`),
// );
// await executeCommand([toolCmd, "orchestrate", "-n", "v&v"], vvSQLSupplier);
// console.log(
// colors.green(
// "Verification and validation orchestration completed successfully.",
// ),
// );
// await checkAndCreateCombinedView(dbFilePath);
// console.log(colors.green("View generation completed successfully."));
// } catch (error) {
// console.error(
// colors.cyan("Error during Verification and Validation:"),
// error.message,
// );
// Deno.exit(1);
// console.error(colors.red("Error during view generation:"), error.message);
// }


try {
console.log(colors.dim(`Performing UX orchestration: ${folderName}...`));
await executeCommand([toolCmd, "shell"], uxSQLSupplier);
//executeSqlCommands(uxSQL); // Execute UX SQL commands
console.log(colors.dim(`Performing UX orchestration: ${folderName}...`));
//await executeCommand([toolCmd, "shell"], uxSQLSupplier);
executeSqlCommands(uxSQL); // Execute UX SQL commands
console.log(colors.green("UX orchestration completed successfully."));
} catch (error) {
console.error(colors.cyan("Error during UX orchestration:"), error.message);
Expand Down
4 changes: 4 additions & 0 deletions lib/service/diabetes-research-hub/orchestration/vv-orchestration.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,10 @@ UNION ALL SELECT 'uniform_resource_cgm_file_metadata', 'data_start_date', 'TEXT'
UNION ALL SELECT 'uniform_resource_cgm_file_metadata', 'data_end_date', 'TEXT', 0, 1
UNION ALL SELECT 'uniform_resource_cgm_file_metadata', 'study_id', 'TEXT', 0, 1;

CREATE TEMP VIEW IF NOT EXISTS device_info AS
SELECT device_id, name, created_at
FROM device d;


INSERT OR IGNORE INTO orchestration_nature (
orchestration_nature_id,
Expand Down
Loading