Skip to content

Commit

Permalink
chore: removed unwanted changes
Browse files Browse the repository at this point in the history
  • Loading branch information
@achettyiitr
achettyiitr committed May 17, 2023
1 parent 942f2d3 commit c8cb345
Show file tree
Hide file tree
Showing 14 changed files with 658 additions and 537 deletions.
2 changes: 1 addition & 1 deletion warehouse/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -401,7 +401,7 @@ func (uploadReq *UploadReq) TriggerWHUpload(ctx context.Context) (response *prot
uploadJobT := UploadJob{
upload: upload,
dbHandle: uploadReq.API.dbHandle,
Now: timeutil.Now,
now: timeutil.Now,
ctx: ctx,
}

Expand Down
165 changes: 95 additions & 70 deletions warehouse/archive/testdata/dump.sql
View file
Original file line number Diff line number Diff line change
@@ -1,72 +1,97 @@
BEGIN;
INSERT INTO wh_uploads (id, source_id, namespace, destination_id,
destination_type, start_staging_file_id,
end_staging_file_id, start_load_file_id,
end_load_file_id, status, schema,
error, metadata, first_event_at,
last_event_at, created_at, updated_at)
VALUES (1, 'test-sourceID', 'test-namespace',
'test-destinationID', 'POSTGRES',
1, 1, 1, 1, 'exported_data', '{}', '{}',
'{}', now(), now(), now(), now()),
(2, 'test-sourceID', 'test-namespace',
'test-destinationID', 'POSTGRES',
2, 2, 2, 2, 'exported_data', '{}', '{}',
'{}', now(), now(), now(), now()),
(3, 'test-sourceID', 'test-namespace',
'test-destinationID', 'POSTGRES',
3, 3, 3, 3, 'exported_data', '{}', '{}',
'{}', now(), now(), now(), now()),
(4, 'test-sourceID', 'test-namespace',
'test-destinationID', 'POSTGRES',
4, 4, 4, 4, 'exported_data', '{}', '{}',
'{}', now(), now(), now(), now());
INSERT INTO wh_staging_files (id, location, schema, source_id, destination_id,
status, total_events, total_bytes, first_event_at,
last_event_at, created_at, updated_at,
metadata, upload_id)
VALUES (1,
'rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz',
'{}', 'test-sourceID', 'test-destinationID',
'succeeded', 1, 231, NOW(), NOW(), NOW(),
NOW(), '{}', NULL),
(2,
'rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz',
'{}', 'test-sourceID', 'test-destinationID',
'succeeded', 1, 231, NOW(), NOW(), NOW(),
NOW(), '{}', NULL),
(3,
'rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz',
'{}', 'test-sourceID', 'test-destinationID',
'succeeded', 1, 231, NOW(), NOW(), NOW(),
NOW(), '{}', 3),
(4,
'rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz',
'{}', 'test-sourceID', 'test-destinationID',
'succeeded', 1, 231, NOW(), NOW(), NOW(),
NOW(), '{}', 4);
INSERT INTO wh_load_files (id, staging_file_id, location, source_id,
destination_id, destination_type,
table_name, total_events, created_at,
metadata)
VALUES (1, 1,
'rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz',
'test-sourceID', 'test-destinationID',
'POSTGRES', 'test-table', 1, NOW(),
'{}'),
(2, 2,
'rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz',
'test-sourceID', 'test-destinationID',
'POSTGRES', 'test-table', 1, NOW(),
'{}'),
(3, 3,
'rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz',
'test-sourceID', 'test-destinationID',
'POSTGRES', 'test-table', 1, NOW(),
'{}'),
(4, 4,
'rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz',
'test-sourceID', 'test-destinationID',
'POSTGRES', 'test-table', 1, NOW(),
'{}');
INSERT INTO wh_uploads (
id, source_id, namespace, destination_id,
destination_type, start_staging_file_id,
end_staging_file_id, start_load_file_id,
end_load_file_id, status, schema,
error, metadata, first_event_at,
last_event_at, created_at, updated_at
)
VALUES
(
1, 'test-sourceID', 'test-namespace',
'test-destinationID', 'POSTGRES',
1, 1, 1, 1, 'exported_data', '{}', '{}',
'{}', now(), now(), now(), now()
),
(
2, 'test-sourceID', 'test-namespace',
'test-destinationID', 'POSTGRES',
2, 2, 2, 2, 'exported_data', '{}', '{}',
'{}', now(), now(), now(), now()
),
(
3, 'test-sourceID', 'test-namespace',
'test-destinationID', 'POSTGRES',
3, 3, 3, 3, 'exported_data', '{}', '{}',
'{}', now(), now(), now(), now()
),
(
4, 'test-sourceID', 'test-namespace',
'test-destinationID', 'POSTGRES',
4, 4, 4, 4, 'exported_data', '{}', '{}',
'{}', now(), now(), now(), now()
);
INSERT INTO wh_staging_files (
id, location, schema, source_id, destination_id,
status, total_events, total_bytes, first_event_at,
last_event_at, created_at, updated_at,
metadata, upload_id
)
VALUES
(
1, 'rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz',
'{}', 'test-sourceID', 'test-destinationID',
'succeeded', 1, 231, NOW(), NOW(), NOW(),
NOW(), '{}', NULL
),
(
2, 'rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz',
'{}', 'test-sourceID', 'test-destinationID',
'succeeded', 1, 231, NOW(), NOW(), NOW(),
NOW(), '{}', NULL
),
(
3, 'rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz',
'{}', 'test-sourceID', 'test-destinationID',
'succeeded', 1, 231, NOW(), NOW(), NOW(),
NOW(), '{}', 3
),
(
4, 'rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz',
'{}', 'test-sourceID', 'test-destinationID',
'succeeded', 1, 231, NOW(), NOW(), NOW(),
NOW(), '{}', 4
);
INSERT INTO wh_load_files (
id, staging_file_id, location, source_id,
destination_id, destination_type,
table_name, total_events, created_at,
metadata
)
VALUES
(
1, 1, 'rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz',
'test-sourceID', 'test-destinationID',
'POSTGRES', 'test-table', 1, NOW(),
'{}'
),
(
2, 2, 'rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz',
'test-sourceID', 'test-destinationID',
'POSTGRES', 'test-table', 1, NOW(),
'{}'
),
(
3, 3, 'rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz',
'test-sourceID', 'test-destinationID',
'POSTGRES', 'test-table', 1, NOW(),
'{}'
),
(
4, 4, 'rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz',
'test-sourceID', 'test-destinationID',
'POSTGRES', 'test-table', 1, NOW(),
'{}'
);
COMMIT;
10 changes: 5 additions & 5 deletions warehouse/archive/testdata/storage.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"test-prefix/wh-staging-files.test-sourceID.test-destinationID.1.unix_time.json.gz": "{\"id\":1,\"location\":\"rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz\",\"source_id\":\"test-sourceID\",\"destination_id\":\"test-destinationID\",\"schema\":{},\"error\":null,\"status\":\"succeeded\",\"first_event_at\":\"{{.Now}}\",\"last_event_at\":\"{{.Now}}\",\"total_events\":1,\"created_at\":\"{{.Now}}\",\"updated_at\":\"{{.Now}}\",\"metadata\":{},\"workspace_id\":\"1\",\"total_bytes\":231,\"upload_id\":null}\n",
"test-prefix/wh-staging-files.test-sourceID.test-destinationID.2.unix_time.json.gz": "{\"id\":2,\"location\":\"rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz\",\"source_id\":\"test-sourceID\",\"destination_id\":\"test-destinationID\",\"schema\":{},\"error\":null,\"status\":\"succeeded\",\"first_event_at\":\"{{.Now}}\",\"last_event_at\":\"{{.Now}}\",\"total_events\":1,\"created_at\":\"{{.Now}}\",\"updated_at\":\"{{.Now}}\",\"metadata\":{},\"workspace_id\":\"1\",\"total_bytes\":231,\"upload_id\":null}\n",
"test-prefix/wh-staging-files.test-sourceID.test-destinationID.3.unix_time.json.gz": "{\"id\":3,\"location\":\"rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz\",\"source_id\":\"test-sourceID\",\"destination_id\":\"test-destinationID\",\"schema\":{},\"error\":null,\"status\":\"succeeded\",\"first_event_at\":\"{{.Now}}\",\"last_event_at\":\"{{.Now}}\",\"total_events\":1,\"created_at\":\"{{.Now}}\",\"updated_at\":\"{{.Now}}\",\"metadata\":{},\"workspace_id\":\"1\",\"total_bytes\":231,\"upload_id\":3}\n",
"test-prefix/wh-staging-files.test-sourceID.test-destinationID.4.unix_time.json.gz": "{\"id\":4,\"location\":\"rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz\",\"source_id\":\"test-sourceID\",\"destination_id\":\"test-destinationID\",\"schema\":{},\"error\":null,\"status\":\"succeeded\",\"first_event_at\":\"{{.Now}}\",\"last_event_at\":\"{{.Now}}\",\"total_events\":1,\"created_at\":\"{{.Now}}\",\"updated_at\":\"{{.Now}}\",\"metadata\":{},\"workspace_id\":\"1\",\"total_bytes\":231,\"upload_id\":4}\n"
}
"test-prefix/wh-staging-files.test-sourceID.test-destinationID.1.unix_time.json.gz": "{\"id\":1,\"location\":\"rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz\",\"source_id\":\"test-sourceID\",\"destination_id\":\"test-destinationID\",\"schema\":{},\"error\":null,\"status\":\"succeeded\",\"first_event_at\":\"{{.Now}}\",\"last_event_at\":\"{{.Now}}\",\"total_events\":1,\"created_at\":\"{{.Now}}\",\"updated_at\":\"{{.Now}}\",\"metadata\":{},\"workspace_id\":\"1\",\"total_bytes\":231,\"upload_id\":null}\n",
"test-prefix/wh-staging-files.test-sourceID.test-destinationID.2.unix_time.json.gz": "{\"id\":2,\"location\":\"rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz\",\"source_id\":\"test-sourceID\",\"destination_id\":\"test-destinationID\",\"schema\":{},\"error\":null,\"status\":\"succeeded\",\"first_event_at\":\"{{.Now}}\",\"last_event_at\":\"{{.Now}}\",\"total_events\":1,\"created_at\":\"{{.Now}}\",\"updated_at\":\"{{.Now}}\",\"metadata\":{},\"workspace_id\":\"1\",\"total_bytes\":231,\"upload_id\":null}\n",
"test-prefix/wh-staging-files.test-sourceID.test-destinationID.3.unix_time.json.gz": "{\"id\":3,\"location\":\"rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz\",\"source_id\":\"test-sourceID\",\"destination_id\":\"test-destinationID\",\"schema\":{},\"error\":null,\"status\":\"succeeded\",\"first_event_at\":\"{{.Now}}\",\"last_event_at\":\"{{.Now}}\",\"total_events\":1,\"created_at\":\"{{.Now}}\",\"updated_at\":\"{{.Now}}\",\"metadata\":{},\"workspace_id\":\"1\",\"total_bytes\":231,\"upload_id\":3}\n",
"test-prefix/wh-staging-files.test-sourceID.test-destinationID.4.unix_time.json.gz": "{\"id\":4,\"location\":\"rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz\",\"source_id\":\"test-sourceID\",\"destination_id\":\"test-destinationID\",\"schema\":{},\"error\":null,\"status\":\"succeeded\",\"first_event_at\":\"{{.Now}}\",\"last_event_at\":\"{{.Now}}\",\"total_events\":1,\"created_at\":\"{{.Now}}\",\"updated_at\":\"{{.Now}}\",\"metadata\":{},\"workspace_id\":\"1\",\"total_bytes\":231,\"upload_id\":4}\n"
}
126 changes: 63 additions & 63 deletions warehouse/internal/api/testdata/process_request.json
View file
Original file line number Diff line number Diff line change
@@ -1,67 +1,67 @@
{
"WorkspaceID": "279L3V7FSpx43LaNJ0nIs9KRaNC",
"Schema": {
"product_track": {
"context_destination_id": "string",
"context_destination_type": "string",
"context_ip": "string",
"context_library_name": "string",
"context_passed_ip": "string",
"context_request_ip": "string",
"context_source_id": "string",
"context_source_type": "string",
"event": "string",
"event_text": "string",
"id": "string",
"original_timestamp": "datetime",
"product_id": "string",
"rating": "int",
"received_at": "datetime",
"revenue": "float",
"review_body": "string",
"review_id": "string",
"sent_at": "datetime",
"timestamp": "datetime",
"user_id": "string",
"uuid_ts": "datetime"
"WorkspaceID": "279L3V7FSpx43LaNJ0nIs9KRaNC",
"Schema": {
"product_track": {
"context_destination_id": "string",
"context_destination_type": "string",
"context_ip": "string",
"context_library_name": "string",
"context_passed_ip": "string",
"context_request_ip": "string",
"context_source_id": "string",
"context_source_type": "string",
"event": "string",
"event_text": "string",
"id": "string",
"original_timestamp": "datetime",
"product_id": "string",
"rating": "int",
"received_at": "datetime",
"revenue": "float",
"review_body": "string",
"review_id": "string",
"sent_at": "datetime",
"timestamp": "datetime",
"user_id": "string",
"uuid_ts": "datetime"
},
"tracks": {
"context_destination_id": "string",
"context_destination_type": "string",
"context_ip": "string",
"context_library_name": "string",
"context_passed_ip": "string",
"context_request_ip": "string",
"context_source_id": "string",
"context_source_type": "string",
"event": "string",
"event_text": "string",
"id": "string",
"original_timestamp": "datetime",
"received_at": "datetime",
"sent_at": "datetime",
"timestamp": "datetime",
"user_id": "string",
"uuid_ts": "datetime"
}
},
"tracks": {
"context_destination_id": "string",
"context_destination_type": "string",
"context_ip": "string",
"context_library_name": "string",
"context_passed_ip": "string",
"context_request_ip": "string",
"context_source_id": "string",
"context_source_type": "string",
"event": "string",
"event_text": "string",
"id": "string",
"original_timestamp": "datetime",
"received_at": "datetime",
"sent_at": "datetime",
"timestamp": "datetime",
"user_id": "string",
"uuid_ts": "datetime"
}
},
"BatchDestination": {
"Source": {
"ID": "279L3gEKqwruBoKGsXZtSVX7vIy"
"BatchDestination": {
"Source": {
"ID": "279L3gEKqwruBoKGsXZtSVX7vIy"
},
"Destination": {
"ID": "27CHciD6leAhurSyFAeN4dp14qZ"
}
},
"Destination": {
"ID": "27CHciD6leAhurSyFAeN4dp14qZ"
}
},
"Location": "rudder-warehouse-staging-logs/279L3gEKqwruBoKGsXZtSVX7vIy/2022-11-08/1667913810.279L3gEKqwruBoKGsXZtSVX7vIy.7a6e7785-7a75-4345-8d3c-d7a1ce49a43f.json.gz",
"FirstEventAt": "2022-11-08T13:23:07Z",
"LastEventAt": "2022-11-08T13:23:07Z",
"TotalEvents": 2,
"TotalBytes": 2000,
"UseRudderStorage": false,
"DestinationRevisionID": "2H1cLBvL3v0prRBNzpe8D34XTzU",
"SourceTaskRunID": "<source-task-run-id>",
"SourceJobID": "<source-job-id>",
"SourceJobRunID": "<source-job-run-id>",
"TimeWindow": "0001-01-01T00:40:00Z"
"Location": "rudder-warehouse-staging-logs/279L3gEKqwruBoKGsXZtSVX7vIy/2022-11-08/1667913810.279L3gEKqwruBoKGsXZtSVX7vIy.7a6e7785-7a75-4345-8d3c-d7a1ce49a43f.json.gz",
"FirstEventAt": "2022-11-08T13:23:07Z",
"LastEventAt": "2022-11-08T13:23:07Z",
"TotalEvents": 2,
"TotalBytes": 2000,
"UseRudderStorage": false,
"DestinationRevisionID": "2H1cLBvL3v0prRBNzpe8D34XTzU",
"SourceTaskRunID": "<source-task-run-id>",
"SourceJobID": "<source-job-id>",
"SourceJobRunID": "<source-job-run-id>",
"TimeWindow": "0001-01-01T00:40:00Z"
}
Loading

0 comments on commit c8cb345

Please sign in to comment.