Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Initial version of cloud schema files #75

Merged
merged 4 commits into from
Mar 28, 2017
Merged

Initial version of cloud schema files #75

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3a53072
Move ETLv2 job files from xdmod-xsede to xdmod repo
smgallo Mar 8, 2017
c41a80a
Merge branch 'xdmod6.6' of github.com:ubccr/xdmod into etl/cloud
smgallo Mar 10, 2017
5d77961
Initial implementation of cloud schema and added job dimensions. Incl…
smgallo Mar 27, 2017
61e78c9
Fix merge conflicts
smgallo Mar 27, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 92 additions & 0 deletions configuration/etl/etl.d/jobs_cloud.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
{
"defaults": {

"global": {
"endpoints": {
"source": {
"type": "mysql",
"name": "Cloud DB",
"config": "datawarehouse",
"schema": "modw_cloud"
},
"destination": {
"type": "mysql",
"name": "Cloud DB",
"config": "datawarehouse",
"schema": "modw_cloud",
"create_schema_if_not_exists": true
}
}
},

"cloud-jobs": {
"namespace": "ETL\\Ingestor",
"options_class": "IngestorOptions"
}
},

"#": "Current Cloud job ingestion",

"cloud-jobs": [
{
"name": "CloudTableManagement",
"class": "ManageTables",
"description": "Manage job tables",
"namespace": "ETL\\Maintenance",
"options_class": "MaintenanceOptions",
"definition_file_list": [
"cloud/account.json",
"cloud/asset.json",
"cloud/asset_event.json",
"cloud/cloud_resource_metadata.json",
"cloud/event.json",
"cloud/host.json",
"cloud/image.json",
"cloud/instance_data.json",
"cloud/instance.json",
"cloud/job_record_event.json"
],
"enabled": true
},
{
"name": "CloudEventTypeIngestor",
"description": "Cloud event types",
"class": "StructuredFileIngestor",
"definition_file": "cloud/event_type.json",
"enabled": true,
"truncate_destination": true
},
{
"name": "CloudAssetTypeIngestor",
"description": "Cloud asset types",
"class": "StructuredFileIngestor",
"definition_file": "cloud/asset_type.json",
"enabled": true,
"truncate_destination": true
},
{
"name": "CloudInstanceTypeIngestor",
"description": "Cloud instance types",
"class": "StructuredFileIngestor",
"definition_file": "cloud/instance_type.json",
"enabled": true,
"truncate_destination": true
},
{
"name": "CloudRegionIngestor",
"description": "Cloud regions",
"class": "StructuredFileIngestor",
"definition_file": "cloud/region.json",
"enabled": true,
"truncate_destination": true
},
{
"name": "CloudAvailabilityZoneIngestor",
"description": "Cloud availability zones",
"class": "StructuredFileIngestor",
"definition_file": "cloud/avail_zone.json",
"enabled": true,
"truncate_destination": true
}
]
}
60 changes: 56 additions & 4 deletions configuration/etl/etl.d/jobs_hpc.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@
"endpoints": {
"source": {
"type": "mysql",
"name": "Cloud redesign test DB",
"name": "Cloud DB",
"config": "datawarehouse",
"schema": "modw_cloud"
},
"destination": {
"type": "mysql",
"name": "Cloud timing test DB",
"name": "Cloud DB",
"config": "datawarehouse",
"schema": "modw_cloud",
"create_schema_if_not_exists": true
Expand All @@ -23,11 +23,63 @@
"#": "Current HPC job ingestion from the XDCDB mirror",

"xdcdb-jobs": [
{
"name": "JobsTableManagement",
"class": "ManageTables",
"description": "Manage job tables",
"namespace": "ETL\\Maintenance",
"options_class": "MaintenanceOptions",
"definition_file_list": [
"jobs/job_request_info.json",
"jobs/job_task_countable.json",
"jobs/meta_job.json",
"jobs/meta_job_record.json"
],
"enabled": true
},
{
"name": "CountableTypeIngestor",
"description": "Countable types",
"class": "StructuredFileIngestor",
"definition_file": "jobs/countable_type.json",
"enabled": true,
"truncate_destination": true
},
{
"name": "JobRecordTypeIngestor",
"description": "Job record types",
"class": "StructuredFileIngestor",
"definition_file": "jobs/job_record_type.json",
"enabled": true,
"truncate_destination": true
},
{
"name": "JobTaskTypeIngestor",
"description": "Job task types",
"class": "StructuredFileIngestor",
"definition_file": "jobs/job_task_type.json",
"enabled": true,
"truncate_destination": true
},
{
"name": "UnitIngestor",
"description": "Units of measure",
"class": "StructuredFileIngestor",
"definition_file": "jobs/unit.json",
"enabled": true,
"truncate_destination": true
},
{
"name": "SubmissionVenueIngestor",
"description": "Submission venues",
"class": "StructuredFileIngestor",
"definition_file": "jobs/submission_venue.json",
"enabled": true,
"truncate_destination": true
},
{
"name": "XdcdbJobRecordIngestor",
"class": "DatabaseIngestor",
"namespace": "ETL\\Ingestor",
"options_class": "IngestorOptions",
"#": "Table definition relative to paths.table_config_dir if path not specified",
"definition_file": "jobs/job_record_hpc.json",
"description": "Ingest HPC job records from the XDCDB",
Expand Down
45 changes: 45 additions & 0 deletions configuration/etl/etl_tables.d/cloud/account.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
{
"table_definition": {
"name": "account",
"engine": "MyISAM",
"comment": "Cloud provider account",
"columns": [
{
"name": "account_id",
"type": "int(11) unsigned",
"nullable": false,
"extra": "auto_increment"
},
{
"name": "resource_id",
"type": "int(11) unsigned",
"nullable": false
},
{
"name": "provider_account",
"type": "varchar(64)",
"nullable": false,
"comment": "Account number from cloud provider"
}
],
"indexes": [
{
"name": "fk_resource",
"columns": [
"resource_id"
],
"type": "BTREE",
"is_unique": false
},
{
"name": "PRIMARY",
"columns": [
"account_id"
],
"type": "BTREE",
"is_unique": true
}
],
"triggers": []
}
}
50 changes: 50 additions & 0 deletions configuration/etl/etl_tables.d/cloud/asset.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
{
"table_definition": {
"name": "asset",
"engine": "MyISAM",
"comment": "Generic assets",
"columns": [
{
"name": "asset_id",
"type": "bigint(20) unsigned",
"nullable": false
},
{
"name": "asset_type_id",
"type": "int(11)",
"nullable": false,
"default": -1
},
{
"name": "provider_identifier",
"type": "varchar(64)",
"nullable": false
},
{
"name": "size",
"type": "int(11)",
"nullable": true,
"default": null
}
],
"indexes": [
{
"name": "fk_asset_type",
"columns": [
"asset_type_id"
],
"type": "BTREE",
"is_unique": false
},
{
"name": "PRIMARY",
"columns": [
"asset_id"
],
"type": "BTREE",
"is_unique": true
}
],
"triggers": []
}
}
38 changes: 38 additions & 0 deletions configuration/etl/etl_tables.d/cloud/asset_event.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{
"table_definition": {
"name": "asset_event",
"engine": "MyISAM",
"comment": "Events associated with an asset",
"columns": [
{
"name": "event_id",
"type": "bigint(20) unsigned",
"nullable": false
},
{
"name": "asset_id",
"type": "bigint(20) unsigned",
"nullable": false
}
],
"indexes": [
{
"name": "fk_event",
"columns": [
"event_id"
],
"type": "BTREE",
"is_unique": false
},
{
"name": "fk_asset",
"columns": [
"asset_id"
],
"type": "BTREE",
"is_unique": false
}
],
"triggers": []
}
}
70 changes: 70 additions & 0 deletions configuration/etl/etl_tables.d/cloud/asset_type.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
{
"table_definition": {
"name": "asset_type",
"engine": "MyISAM",
"comment": "Type of an asset",
"columns": [
{
"name": "asset_type_id",
"type": "int(11)",
"nullable": false
},
{
"name": "asset_type",
"type": "varchar(64)",
"nullable": false,
"comment": "Short version or abbrev"
},
{
"name": "display",
"type": "varchar(256)",
"nullable": false,
"comment": "What to show the user"
},
{
"name": "description",
"type": "varchar(1024)",
"nullable": true,
"default": null
},
{
"name": "unit_id",
"type": "int(11)",
"nullable": true,
"default": null
}
],
"indexes": [
{
"name": "fk_unit",
"columns": [
"unit_id"
],
"type": "BTREE",
"is_unique": false
},
{
"name": "PRIMARY",
"columns": [
"asset_type_id"
],
"type": "BTREE",
"is_unique": true
}
],
"triggers": []
},

"#": "Define the mapping between the source value fields and the table columns.",
"destination_columns": ["asset_type_id", "asset_type", "display", "description"],

"#": "Provide the raw source values.",
"source_values": [
[-1, "unknown", "Unknown", "Unknown" ],
[1, "vol-ebs", "EBS Volume", "EBS Volume" ],
[2, "vol-instance", "Instance Volume", "Instance Store Volume" ],
[3, "ip", "IP", "IP Address" ],
[4, "s3-bucket", "S3 Bucket", "S3 Bucket" ]
]

}
Loading