-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
d33d2e7
commit c9f6b69
Showing
6 changed files
with
1,562 additions
and
30 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
171 changes: 171 additions & 0 deletions
171
lib/service/diabetes-research-hub/orchestration/deidentification-orchestration.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,171 @@ | ||
-- BEGIN TRANSACTION | ||
--BEGIN; | ||
|
||
-- Perform De-identification | ||
-- Anonymize email addresses in the uniform_resource_investigator table | ||
UPDATE uniform_resource_investigator | ||
SET email = anonymize_email(email) | ||
WHERE email IS NOT NULL; | ||
|
||
-- Anonymize email addresses in the uniform_resource_author table | ||
UPDATE uniform_resource_author | ||
SET email = anonymize_email(email) | ||
WHERE email IS NOT NULL; | ||
|
||
-- Insert into orchestration_nature only if it doesn't exist | ||
INSERT OR IGNORE INTO orchestration_nature ( | ||
orchestration_nature_id, | ||
nature, | ||
elaboration, | ||
created_at, | ||
created_by, | ||
updated_at, | ||
updated_by, | ||
deleted_at, | ||
deleted_by, | ||
activity_log | ||
) | ||
SELECT | ||
'deidentification', -- Unique ID for the orchestration nature | ||
'De-identification', -- Human-readable name for the orchestration nature | ||
NULL, -- No elaboration provided at insert time | ||
CURRENT_TIMESTAMP, -- Timestamp of creation | ||
d.device_id, -- Creator's name | ||
NULL, -- No updated timestamp yet | ||
NULL, -- No updater yet | ||
NULL, -- Not deleted | ||
NULL, -- No deleter yet | ||
NULL -- No activity log yet | ||
FROM drh_device d | ||
LIMIT 1; -- Limiting to 1 device | ||
|
||
-- Insert into orchestration_session only if it doesn't exist | ||
INSERT OR IGNORE INTO orchestration_session ( | ||
orchestration_session_id, | ||
device_id, | ||
orchestration_nature_id, | ||
version, | ||
orch_started_at, | ||
orch_finished_at, | ||
elaboration, | ||
args_json, | ||
diagnostics_json, | ||
diagnostics_md | ||
) | ||
SELECT | ||
'ORCHSESSID-' || hex(randomblob(16)), -- Generate a random hex blob for orchestration_session_id | ||
d.device_id, -- Pull device_id from the drh_device view | ||
'deidentification', -- Reference to the orchestration_nature_id we just inserted | ||
'', -- Version (placeholder) | ||
CURRENT_TIMESTAMP, -- Start time | ||
NULL, -- Finished time (to be updated later) | ||
NULL, -- Elaboration (if any) | ||
NULL, -- Args JSON (if any) | ||
NULL, -- Diagnostics JSON (if any) | ||
NULL -- Diagnostics MD (if any) | ||
FROM drh_device d | ||
LIMIT 1; -- Limiting to 1 device | ||
|
||
-- Create a temporary view to retrieve orchestration session information | ||
CREATE TEMP VIEW IF NOT EXISTS session_info AS | ||
SELECT | ||
orchestration_session_id | ||
FROM | ||
orchestration_session | ||
WHERE | ||
orchestration_nature_id = 'deidentification' | ||
LIMIT 1; | ||
|
||
-- Insert into orchestration_session_entry only if it doesn't exist | ||
INSERT OR IGNORE INTO orchestration_session_entry ( | ||
orchestration_session_entry_id, | ||
session_id, | ||
ingest_src, | ||
ingest_table_name, | ||
elaboration | ||
) VALUES ( | ||
'ORCHSESSENID-' || hex(randomblob(16)), -- Generate a random hex blob for orchestration_session_entry_id | ||
(SELECT orchestration_session_id FROM session_info limit 1), -- Session ID from previous insert | ||
'deidentification-orchestration.sql', -- Replace with actual ingest source | ||
'', -- Placeholder for actual table name | ||
NULL -- Elaboration (if any) | ||
); | ||
|
||
-- Create or replace a temporary view for session execution tracking | ||
DROP VIEW IF EXISTS temp_session_info; -- Remove any existing view | ||
CREATE TEMP VIEW temp_session_info AS | ||
SELECT | ||
orchestration_session_id, | ||
(SELECT orchestration_session_entry_id FROM orchestration_session_entry WHERE session_id = orchestration_session_id LIMIT 1) AS orchestration_session_entry_id | ||
FROM orchestration_session | ||
WHERE orchestration_nature_id = 'deidentification' | ||
LIMIT 1; | ||
|
||
-- Insert into orchestration_session_exec for uniform_resource_investigator | ||
INSERT OR IGNORE INTO orchestration_session_exec ( | ||
orchestration_session_exec_id, | ||
exec_nature, | ||
session_id, | ||
session_entry_id, | ||
exec_code, | ||
exec_status, | ||
input_text, | ||
output_text, | ||
exec_error_text, | ||
narrative_md | ||
) | ||
SELECT | ||
'ORCHSESSEXID-' || ((SELECT COUNT(*) FROM orchestration_session_exec) + 1), -- Unique ID based on count | ||
'De-identification', -- Nature of execution | ||
s.orchestration_session_id, -- Session ID from the temp view | ||
s.orchestration_session_entry_id, -- Session Entry ID from the temp view | ||
'UPDATE uniform_resource_investigator SET email = anonymize_email(email) executed', -- Description of the executed code | ||
'SUCCESS', -- Execution status | ||
'email column in uniform_resource_investigator', -- Input text reference | ||
'De-identification completed', -- Output text summary | ||
CASE | ||
WHEN (SELECT changes() = 0) THEN 'No rows updated' -- Capture update status | ||
ELSE NULL | ||
END, | ||
'username in email is masked' -- Narrative for clarification | ||
FROM temp_session_info s; -- From the temporary session info view | ||
|
||
-- Insert into orchestration_session_exec for uniform_resource_author | ||
INSERT OR IGNORE INTO orchestration_session_exec ( | ||
orchestration_session_exec_id, | ||
exec_nature, | ||
session_id, | ||
session_entry_id, | ||
exec_code, | ||
exec_status, | ||
input_text, | ||
output_text, | ||
exec_error_text, | ||
narrative_md | ||
) | ||
SELECT | ||
'ORCHSESSEXID-' || ((SELECT COUNT(*) FROM orchestration_session_exec) + 1), -- Unique ID based on count | ||
'De-identification', -- Nature of execution | ||
s.orchestration_session_id, -- Session ID from the temp view | ||
s.orchestration_session_entry_id, -- Session Entry ID from the temp view | ||
'UPDATE uniform_resource_author SET email = anonymize_email(email) executed', -- Description of the executed code | ||
'SUCCESS', -- Execution status | ||
'email column in uniform_resource_author', -- Input text reference | ||
'De-identification completed', -- Output text summary | ||
CASE | ||
WHEN (SELECT changes() = 0) THEN 'No rows updated' -- Capture update status | ||
ELSE NULL | ||
END, | ||
'username in email is masked' -- Narrative for clarification | ||
FROM temp_session_info s; -- From the temporary session info view | ||
|
||
-- Update orchestration_session to set finished timestamp and diagnostics | ||
UPDATE orchestration_session | ||
SET | ||
orch_finished_at = CURRENT_TIMESTAMP, -- Set the finish time | ||
diagnostics_json = '{"status": "completed"}', -- Diagnostics status in JSON format | ||
diagnostics_md = 'De-identification process completed' -- Markdown summary | ||
WHERE orchestration_session_id = (SELECT orchestration_session_id FROM temp_session_info LIMIT 1); -- Update the session identified in the temp view | ||
|
||
-- COMMIT TRANSACTION | ||
--COMMIT; |
87 changes: 87 additions & 0 deletions
87
lib/service/diabetes-research-hub/orchestration/stateless-orchestration.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,87 @@ | ||
|
||
-- Drop and recreate the orch_session_view view | ||
DROP VIEW IF EXISTS drh_orch_session_view; | ||
CREATE VIEW drh_orch_session_view AS | ||
SELECT | ||
orchestration_session_id, device_id, orchestration_nature_id, | ||
version, orch_started_at, orch_finished_at, | ||
diagnostics_json, diagnostics_md | ||
FROM orchestration_session; | ||
|
||
-- Drop and recreate the orch_session_deidentifyview view | ||
DROP VIEW IF EXISTS drh_orch_session_deidentifyview; | ||
CREATE VIEW drh_orch_session_deidentifyview AS | ||
SELECT | ||
orchestration_session_id, device_id, orchestration_nature_id, | ||
version, orch_started_at, orch_finished_at, | ||
diagnostics_json, diagnostics_md | ||
FROM orchestration_session | ||
WHERE orchestration_nature_id = 'deidentification'; | ||
|
||
|
||
-- Drop and recreate the orchestration_session_entry_view view | ||
DROP VIEW IF EXISTS drh_orchestration_session_entry_view; | ||
CREATE VIEW drh_orchestration_session_entry_view AS | ||
SELECT | ||
orchestration_session_entry_id, session_id, ingest_src, ingest_table_name | ||
FROM orchestration_session_entry; | ||
|
||
-- Drop and recreate the orchestration_session_exec_view view | ||
DROP VIEW IF EXISTS drh_orchestration_session_exec_view; | ||
CREATE VIEW drh_orchestration_session_exec_view AS | ||
SELECT | ||
orchestration_session_exec_id, exec_nature, session_id, session_entry_id, | ||
parent_exec_id, namespace, exec_identity, exec_code, exec_status, | ||
input_text, exec_error_text, output_text, output_nature, narrative_md | ||
FROM orchestration_session_exec; | ||
|
||
-- Drop and recreate the vw_orchestration_deidentify view | ||
DROP VIEW IF EXISTS drh_vw_orchestration_deidentify; | ||
CREATE VIEW drh_vw_orchestration_deidentify AS | ||
SELECT | ||
osex.orchestration_session_exec_id, | ||
osex.exec_nature, | ||
osex.session_id, | ||
osex.session_entry_id, | ||
osex.parent_exec_id, | ||
osex.namespace, | ||
osex.exec_identity, | ||
osex.exec_code, | ||
osex.exec_status, | ||
osex.input_text, | ||
osex.exec_error_text, | ||
osex.output_text, | ||
osex.output_nature, | ||
osex.narrative_md, | ||
os.device_id, | ||
os.orchestration_nature_id, | ||
os.version, | ||
os.orch_started_at, | ||
os.orch_finished_at, | ||
os.args_json, | ||
os.diagnostics_json, | ||
os.diagnostics_md | ||
FROM | ||
orchestration_session_exec osex | ||
JOIN orchestration_session os ON osex.session_id = os.orchestration_session_id | ||
WHERE | ||
os.orchestration_nature_id = 'deidentification'; | ||
|
||
|
||
DROP VIEW IF EXISTS drh_vandv_orch_issues; | ||
CREATE VIEW drh_vandv_orch_issues AS | ||
SELECT | ||
osi.issue_type as 'Issue Type', | ||
osi.issue_message as 'Issue Message', | ||
osi.issue_column as 'Issue column', | ||
osi.remediation, | ||
osi.issue_row as 'Issue Row', | ||
osi.invalid_value | ||
FROM | ||
orchestration_session_issue osi | ||
JOIN | ||
orchestration_session os | ||
ON | ||
osi.session_id = os.orchestration_session_id | ||
WHERE | ||
os.orchestration_nature_id = 'V&V'; |
Oops, something went wrong.