From 286770fbd27895658c76d9215c6a42aa25b54ab0 Mon Sep 17 00:00:00 2001 From: Pieter Date: Mon, 17 Jan 2022 16:18:37 +0200 Subject: [PATCH] feat: postgres arrays --- .github/workflows/synth-postgres.yml | 2 + core/src/graph/mod.rs | 96 ++- synth/src/datasource/mysql_datasource.rs | 12 +- synth/src/datasource/postgres_datasource.rs | 172 +++++- synth/src/datasource/relational_datasource.rs | 14 +- synth/testing_harness/postgres/.gitignore | 5 +- .../postgres/0_hospital_schema.sql | 3 +- .../postgres/1_hospital_data.sql | 30 +- .../postgres/arrays/0_arrays.sql | 49 ++ .../postgres/arrays/arrays.json | 471 +++++++++++++++ .../postgres/arrays/unofficial_arrays.json | 292 ++++++++++ .../postgres/arrays_master/arrays.json | 546 ++++++++++++++++++ .../arrays_master/unofficial_arrays.json | 302 ++++++++++ .../complete_import_master/types.json | 4 +- .../unofficial_types.json | 6 +- synth/testing_harness/postgres/e2e.sh | 23 + .../postgres/hospital_master/hospitals.json | 26 + 17 files changed, 2005 insertions(+), 48 deletions(-) create mode 100644 synth/testing_harness/postgres/arrays/0_arrays.sql create mode 100644 synth/testing_harness/postgres/arrays/arrays.json create mode 100644 synth/testing_harness/postgres/arrays/unofficial_arrays.json create mode 100644 synth/testing_harness/postgres/arrays_master/arrays.json create mode 100644 synth/testing_harness/postgres/arrays_master/unofficial_arrays.json diff --git a/.github/workflows/synth-postgres.yml b/.github/workflows/synth-postgres.yml index 21636d19..aa73ecfd 100644 --- a/.github/workflows/synth-postgres.yml +++ b/.github/workflows/synth-postgres.yml @@ -48,3 +48,5 @@ jobs: working-directory: synth/testing_harness/postgres - run: ./e2e.sh test-warning working-directory: synth/testing_harness/postgres + - run: ./e2e.sh test-arrays + working-directory: synth/testing_harness/postgres diff --git a/core/src/graph/mod.rs b/core/src/graph/mod.rs index 09b8e8b3..0dde0fb5 100644 --- a/core/src/graph/mod.rs +++ b/core/src/graph/mod.rs @@ -367,7 +367,98 @@ impl Type for Value { } } +impl Value { + fn to_postgres_string(&self) -> String { + match self { + Self::Array(arr) => { + let (typ, _) = self.get_postgres_type(); + let inner = arr + .iter() + .map(|v| v.to_postgres_string()) + .collect::>() + .join(", "); + + if typ == "jsonb" { + format!("[{}]", inner) + } else { + format!("{{{}}}", inner) + } + } + Self::Null(_) => "NULL".to_string(), + Self::Bool(b) => b.to_string(), + Self::Number(num) => match num { + Number::F32(f32) => (*f32).to_string(), + Number::F64(f64) => (*f64).to_string(), + _ => num.to_string(), + }, + Self::String(str) => format!("\"{}\"", str), + Self::DateTime(date) => date.format_to_string(), + Self::Object(_) => { + serde_json::to_string(&json::synth_val_to_json(self.clone())).unwrap() + } + } + } + + pub fn get_postgres_type(&self) -> (&'static str, usize) { + let mut depth = 0; + let mut typ = ""; + + let mut current = Some(self); + + // Based on https://docs.rs/sqlx-core/0.5.9/sqlx_core/postgres/types/index.html + while let Some(c) = current { + let pair = match c { + Value::Null(_) => (None, "unknown"), + Value::Bool(_) => (None, "bool"), + Value::Number(num) => match *num { + Number::I8(_) => (None, "char"), + Number::I16(_) => (None, "int2"), + Number::I32(_) => (None, "int4"), + Number::I64(_) => (None, "int8"), + Number::I128(_) => (None, "numeric"), + Number::U8(_) => (None, "char"), + Number::U16(_) => (None, "int2"), + Number::U32(_) => (None, "int4"), + Number::U64(_) => (None, "int8"), + Number::U128(_) => (None, "numeric"), + Number::F32(_) => (None, "float4"), + Number::F64(_) => (None, "float8"), + }, + Value::String(_) => (None, "text"), + Value::DateTime(ChronoValueAndFormat { value, .. }) => match value { + ChronoValue::NaiveDate(_) => (None, "date"), + ChronoValue::NaiveTime(_) => (None, "time"), + ChronoValue::NaiveDateTime(_) => (None, "timestamp"), + ChronoValue::DateTime(_) => (None, "timestamptz"), + }, + Value::Object(_) => (None, "jsonb"), + Value::Array(arr) => { + depth += 1; + if arr.is_empty() { + (None, "unknown") + } else { + (Some(&arr[0]), "") + } + } + }; + + current = pair.0; + typ = pair.1; + } + + (typ, depth) + } +} + impl Encode<'_, Postgres> for Value { + fn produces(&self) -> Option { + // Only arrays needs a special type + match self { + Value::Array(_) => Some(PgTypeInfo::with_name("text")), + _ => None, + } + } + fn encode_by_ref(&self, buf: &mut PgArgumentBuffer) -> IsNull { match self { Value::Null(_) => IsNull::Yes, @@ -409,7 +500,10 @@ impl Encode<'_, Postgres> for Value { json::synth_val_to_json(self.clone()), buf, ), - Value::Array(arr) => arr.encode_by_ref(buf), //TODO special-case for BYTEA + Value::Array(_) => { + let s = self.to_postgres_string(); + >::encode_by_ref(&s, buf) + } //TODO special-case for BYTEA } } } diff --git a/synth/src/datasource/mysql_datasource.rs b/synth/src/datasource/mysql_datasource.rs index 6b2b4a82..35118f4d 100644 --- a/synth/src/datasource/mysql_datasource.rs +++ b/synth/src/datasource/mysql_datasource.rs @@ -56,7 +56,7 @@ impl RelationalDataSource for MySqlDataSource { async fn execute_query( &self, query: String, - query_params: Vec<&Value>, + query_params: Vec, ) -> Result { let mut query = sqlx::query(query.as_str()); @@ -199,7 +199,13 @@ impl RelationalDataSource for MySqlDataSource { Ok(content) } - fn extend_parameterised_query(query: &mut String, _curr_index: usize, extend: usize) { + fn extend_parameterised_query( + query: &mut String, + _curr_index: usize, + query_params: Vec, + ) { + let extend = query_params.len(); + query.push('('); for i in 0..extend { query.push('?'); @@ -299,7 +305,7 @@ fn try_match_value(row: &MySqlRow, column: &MySqlColumn) -> Result { return Ok(Value::Number(Number::from(truncated))); } - bail!("Failed to convert Postgresql numeric data type to 64 bit float") + bail!("Failed to convert Mysql numeric data type to 64 bit float") } "timestamp" => Value::String(row.try_get::(column.name())?), "date" => Value::String(format!( diff --git a/synth/src/datasource/postgres_datasource.rs b/synth/src/datasource/postgres_datasource.rs index 6a61d9e8..f3c53100 100644 --- a/synth/src/datasource/postgres_datasource.rs +++ b/synth/src/datasource/postgres_datasource.rs @@ -14,8 +14,8 @@ use std::collections::BTreeMap; use std::convert::TryFrom; use synth_core::schema::number_content::{F32, F64, I32, I64}; use synth_core::schema::{ - BoolContent, Categorical, ChronoValueType, DateTimeContent, NumberContent, RangeStep, - RegexContent, StringContent, Uuid, + ArrayContent, BoolContent, Categorical, ChronoValue, ChronoValueAndFormat, ChronoValueType, + DateTimeContent, NumberContent, ObjectContent, RangeStep, RegexContent, StringContent, Uuid, }; use synth_core::{Content, Value}; @@ -120,7 +120,7 @@ impl RelationalDataSource for PostgresDataSource { async fn execute_query( &self, query: String, - query_params: Vec<&Value>, + query_params: Vec, ) -> Result { let mut query = sqlx::query(query.as_str()); @@ -255,20 +255,20 @@ impl RelationalDataSource for PostgresDataSource { RegexContent::pattern(pattern).context("pattern will always compile")?, )) } - "int2" => Content::Number(NumberContent::I64(I64::Range(RangeStep::default()))), + "int2" => Content::Number(NumberContent::I32(I32::Range(RangeStep::default()))), "int4" => Content::Number(NumberContent::I32(I32::Range(RangeStep::default()))), "int8" => Content::Number(NumberContent::I64(I64::Range(RangeStep::default()))), "float4" => Content::Number(NumberContent::F32(F32::Range(RangeStep::default()))), "float8" => Content::Number(NumberContent::F64(F64::Range(RangeStep::default()))), "numeric" => Content::Number(NumberContent::F64(F64::Range(RangeStep::default()))), "timestamptz" => Content::DateTime(DateTimeContent { - format: "".to_string(), // todo + format: "%Y-%m-%dT%H:%M:%S%z".to_string(), type_: ChronoValueType::DateTime, begin: None, end: None, }), "timestamp" => Content::DateTime(DateTimeContent { - format: "".to_string(), // todo + format: "%Y-%m-%dT%H:%M:%S".to_string(), type_: ChronoValueType::NaiveDateTime, begin: None, end: None, @@ -279,20 +279,56 @@ impl RelationalDataSource for PostgresDataSource { begin: None, end: None, }), + "time" => Content::DateTime(DateTimeContent { + format: "%H:%M:%S".to_string(), + type_: ChronoValueType::NaiveTime, + begin: None, + end: None, + }), + "json" | "jsonb" => Content::Object(ObjectContent { + skip_when_null: false, + fields: BTreeMap::new(), + }), "uuid" => Content::String(StringContent::Uuid(Uuid)), - _ => bail!( - "We haven't implemented a converter for {}", - column_info.data_type - ), + _ => { + if let Some(data_type) = column_info.data_type.strip_prefix('_') { + let mut column_info = column_info.clone(); + column_info.data_type = data_type.to_string(); + + Content::Array(ArrayContent::from_content_default_length( + self.decode_to_content(&column_info)?, + )) + } else { + bail!( + "We haven't implemented a converter for {}", + column_info.data_type + ) + } + } }; Ok(content) } - fn extend_parameterised_query(query: &mut String, curr_index: usize, extend: usize) { + fn extend_parameterised_query(query: &mut String, curr_index: usize, query_params: Vec) { + let extend = query_params.len(); + query.push('('); - for i in 0..extend { - query.push_str(&format!("${}", curr_index + i + 1)); + for (i, param) in query_params.iter().enumerate() { + let extra = if let Value::Array(_) = param { + let (typ, depth) = param.get_postgres_type(); + if typ == "unknown" { + "".to_string() // This is currently not supported + } else if typ == "jsonb" { + "::jsonb".to_string() // Cannot have an array of jsonb - ie jsonb[] + } else { + format!("::{}{}", typ, "[]".repeat(depth)) + } + } else { + "".to_string() + }; + + query.push_str(&format!("${}{}", curr_index + i + 1, extra)); if i != extend - 1 { query.push(','); } @@ -347,7 +383,10 @@ impl TryFrom for ValueWrapper { let mut kv = BTreeMap::new(); for column in row.columns() { - let value = try_match_value(&row, column).unwrap_or(Value::Null(())); + let value = try_match_value(&row, column).unwrap_or_else(|err| { + debug!("try_match_value failed: {}", err); + Value::Null(()) + }); kv.insert(column.name().to_string(), value); } @@ -389,6 +428,111 @@ fn try_match_value(row: &PgRow, column: &PgColumn) -> Result { "{}", row.try_get::(column.name())? )), + "time" => Value::String(format!( + "{}", + row.try_get::(column.name())? + )), + "json" | "jsonb" => { + let serde_value = row.try_get::(column.name())?; + serde_json::from_value(serde_value)? + } + "char[]" | "varchar[]" | "text[]" | "citext[]" | "bpchar[]" | "name[]" | "unknown[]" => { + Value::Array( + row.try_get::, &str>(column.name()) + .map(|vec| vec.iter().map(|s| Value::String(s.to_string())).collect())?, + ) + } + "bool[]" => Value::Array( + row.try_get::, &str>(column.name()) + .map(|vec| vec.into_iter().map(Value::Bool).collect())?, + ), + "int2[]" => Value::Array( + row.try_get::, &str>(column.name()) + .map(|vec| vec.into_iter().map(|i| Value::Number(i.into())).collect())?, + ), + "int4[]" => Value::Array( + row.try_get::, &str>(column.name()) + .map(|vec| vec.into_iter().map(|i| Value::Number(i.into())).collect())?, + ), + "int8[]" => Value::Array( + row.try_get::, &str>(column.name()) + .map(|vec| vec.into_iter().map(|i| Value::Number(i.into())).collect())?, + ), + "float4[]" => Value::Array( + row.try_get::, &str>(column.name()) + .map(|vec| vec.into_iter().map(|i| Value::Number(i.into())).collect())?, + ), + "float8[]" => Value::Array( + row.try_get::, &str>(column.name()) + .map(|vec| vec.into_iter().map(|i| Value::Number(i.into())).collect())?, + ), + "numeric[]" => { + let vec = row.try_get::, &str>(column.name())?; + let result: Result, _> = vec + .into_iter() + .map(|d| { + if let Some(truncated) = d.to_f64() { + return Ok(Value::Number(truncated.into())); + } + + bail!("Failed to convert Postgresql numeric data type to 64 bit float") + }) + .collect(); + + Value::Array(result?) + } + "timestamp[]" => Value::Array( + row.try_get::, &str>(column.name()) + .map(|vec| { + vec.into_iter() + .map(|d| { + Value::DateTime(ChronoValueAndFormat { + format: Arc::from("%Y-%m-%dT%H:%M:%S".to_owned()), + value: ChronoValue::NaiveDateTime(d), + }) + }) + .collect() + })?, + ), + "timestamptz[]" => Value::Array( + row.try_get::>, &str>(column.name()) + .map(|vec| { + vec.into_iter() + .map(|d| { + Value::DateTime(ChronoValueAndFormat { + format: Arc::from("%Y-%m-%dT%H:%M:%S%z".to_owned()), + value: ChronoValue::DateTime(d), + }) + }) + .collect() + })?, + ), + "date[]" => Value::Array( + row.try_get::, &str>(column.name()) + .map(|vec| { + vec.into_iter() + .map(|d| { + Value::DateTime(ChronoValueAndFormat { + format: Arc::from("%Y-%m-%d".to_owned()), + value: ChronoValue::NaiveDate(d), + }) + }) + .collect() + })?, + ), + "time[]" => Value::Array( + row.try_get::, &str>(column.name()) + .map(|vec| { + vec.into_iter() + .map(|t| { + Value::DateTime(ChronoValueAndFormat { + format: Arc::from("%H:%M:%S".to_owned()), + value: ChronoValue::NaiveTime(t), + }) + }) + .collect() + })?, + ), _ => { bail!( "Could not convert value. Converter not implemented for {}", diff --git a/synth/src/datasource/relational_datasource.rs b/synth/src/datasource/relational_datasource.rs index 300fda42..41f1e466 100644 --- a/synth/src/datasource/relational_datasource.rs +++ b/synth/src/datasource/relational_datasource.rs @@ -10,7 +10,7 @@ const DEFAULT_INSERT_BATCH_SIZE: usize = 1000; //TODO: Remove this once https://github.com/rust-lang/rust/issues/88900 gets fixed #[allow(dead_code)] -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ColumnInfo { pub(crate) column_name: String, pub(crate) ordinal_position: i32, @@ -111,10 +111,10 @@ pub trait RelationalDataSource: DataSource { .as_object() .expect("This is always an object (sampler contract)"); - let extend = row_obj.values().len(); - Self::extend_parameterised_query(&mut query, curr_index, extend); - curr_index += extend; - query_params.extend(row_obj.values()); + let mut curr_query_params: Vec = row_obj.values().cloned().collect(); + Self::extend_parameterised_query(&mut query, curr_index, curr_query_params.clone()); + curr_index += curr_query_params.len(); + query_params.append(&mut curr_query_params); if i == rows.len() - 1 { query.push_str(";\n"); @@ -139,7 +139,7 @@ pub trait RelationalDataSource: DataSource { async fn execute_query( &self, query: String, - query_params: Vec<&Value>, + query_params: Vec, ) -> Result; async fn get_table_names(&self) -> Result>; @@ -157,5 +157,5 @@ pub trait RelationalDataSource: DataSource { fn decode_to_content(&self, column_info: &ColumnInfo) -> Result; // Returns extended query string + current index - fn extend_parameterised_query(query: &mut String, curr_index: usize, extend: usize); + fn extend_parameterised_query(query: &mut String, curr_index: usize, query_params: Vec); } diff --git a/synth/testing_harness/postgres/.gitignore b/synth/testing_harness/postgres/.gitignore index 76888235..18a7d615 100644 --- a/synth/testing_harness/postgres/.gitignore +++ b/synth/testing_harness/postgres/.gitignore @@ -1,3 +1,4 @@ hospital_data_generated.json -hospital_import -complete_import +hospital_import/ +complete_import/ +arrays_import/ diff --git a/synth/testing_harness/postgres/0_hospital_schema.sql b/synth/testing_harness/postgres/0_hospital_schema.sql index 0a6b8f99..be5991d1 100644 --- a/synth/testing_harness/postgres/0_hospital_schema.sql +++ b/synth/testing_harness/postgres/0_hospital_schema.sql @@ -8,7 +8,8 @@ create table hospitals ( id int primary key, hospital_name varchar(255), - address varchar(255) + address varchar(255), + specialities varchar(255)[] ); create table doctors diff --git a/synth/testing_harness/postgres/1_hospital_data.sql b/synth/testing_harness/postgres/1_hospital_data.sql index a42d5204..1c6359b6 100644 --- a/synth/testing_harness/postgres/1_hospital_data.sql +++ b/synth/testing_harness/postgres/1_hospital_data.sql @@ -1,20 +1,20 @@ -- Hospitals -INSERT INTO public.hospitals (id,hospital_name,address) VALUES -(1,'Garcia-Washington','194 Davis Ferry Suite 232\nJenningsmouth, NV 83701'), -(2,'Cruz, Bowman and Martinez','1938 Key Wall\nMartinshire, OR 24041'), -(3,'Bishop, Hartman and Zuniga','574 Snyder Crossing\nPort Christineland, VT 37567'), -(4,'Maxwell-Garcia','328 Williams Coves\nSmithside, HI 71878'), -(5,'Potter-Lindsey','5737 Carmen Trace Suite 312\nSouth Evelyn, WY 40089'), -(6,'Nielsen-Sanchez','70964 Carrillo Burg\nSouth Karichester, ID 67549'), -(7,'Burch-Daniels','Unit 4839 Box 1083\nDPO AA 25986'), -(8,'Marshall, Anderson and Jarvis','51322 Joseph Park\nMelissaton, AZ 67575'), -(9,'Nelson-Jones','8068 David Turnpike\nDelgadoside, FL 82542'), -(10,'Hall, Wells and Salas','5280 Kelley Crossroad Apt. 574\nLake Davidfort, CT 94005'), -(11,'Hardy-Obrien','19920 Brian Curve Suite 711\nThompsonville, KY 89805'), -(12,'Ayala LLC','0079 Michelle Skyway Suite 179\nPort Tony, CA 48596'), -(13,'Hale-Padilla','19876 Carroll Flats\nClaytonbury, IA 94229'), -(14,'Jones Inc','82451 Anita Rue Suite 317\nJustintown, WI 30269'); +INSERT INTO public.hospitals (id,hospital_name,address,specialities) VALUES +(1,'Garcia-Washington','194 Davis Ferry Suite 232\nJenningsmouth, NV 83701', '{"Neurology"}'), +(2,'Cruz, Bowman and Martinez','1938 Key Wall\nMartinshire, OR 24041', '{"Cardiology"}'), +(3,'Bishop, Hartman and Zuniga','574 Snyder Crossing\nPort Christineland, VT 37567', '{"Neurology"}'), +(4,'Maxwell-Garcia','328 Williams Coves\nSmithside, HI 71878', NULL), +(5,'Potter-Lindsey','5737 Carmen Trace Suite 312\nSouth Evelyn, WY 40089', NULL), +(6,'Nielsen-Sanchez','70964 Carrillo Burg\nSouth Karichester, ID 67549', '{"Neurology"}'), +(7,'Burch-Daniels','Unit 4839 Box 1083\nDPO AA 25986', '{"Cardiology"}'), +(8,'Marshall, Anderson and Jarvis','51322 Joseph Park\nMelissaton, AZ 67575', '{"Cardiology"}'), +(9,'Nelson-Jones','8068 David Turnpike\nDelgadoside, FL 82542', NULL), +(10,'Hall, Wells and Salas','5280 Kelley Crossroad Apt. 574\nLake Davidfort, CT 94005', NULL), +(11,'Hardy-Obrien','19920 Brian Curve Suite 711\nThompsonville, KY 89805', '{"Neurology", "Cardiology"}'), +(12,'Ayala LLC','0079 Michelle Skyway Suite 179\nPort Tony, CA 48596', '{"Neurology"}'), +(13,'Hale-Padilla','19876 Carroll Flats\nClaytonbury, IA 94229', '{"Neurology", "Cardiology"}'), +(14,'Jones Inc','82451 Anita Rue Suite 317\nJustintown, WI 30269', '{"Cardiology", "Neurology"}'); -- Doctors diff --git a/synth/testing_harness/postgres/arrays/0_arrays.sql b/synth/testing_harness/postgres/arrays/0_arrays.sql new file mode 100644 index 00000000..66c41071 --- /dev/null +++ b/synth/testing_harness/postgres/arrays/0_arrays.sql @@ -0,0 +1,49 @@ +DROP TABLE IF EXISTS arrays; +DROP TABLE IF EXISTS unofficial_arrays; + +CREATE TABLE arrays +( + boolean_array boolean[] NOT NULL, + char_array char[] NOT NULL, + varchar_array varchar[] NOT NULL, + text_array text[] NOT NULL, + bpchar_array text[] NOT NULL, + name_array name[] NOT NULL, + /* uuid_array uuid[] NOT NULL, */ + int2_array int2[] NOT NULL, + int4_array int4[] NOT NULL, + int8_array int8[] NOT NULL, + numeric_array numeric[] NOT NULL, + uint2_array int2[] NOT NULL, + uint4_array int4[] NOT NULL, + uint8_array int8[] NOT NULL, + unumeric_array numeric[] NOT NULL, + float4_array float4[] NOT NULL, + float8_array float8[] NOT NULL, + int_array_2d int[][] NOT NULL, + timestamp_array timestamp[] NOT NULL, + timestamptz_array timestamptz[] NOT NULL, + date_array date[] NOT NULL, + time_array time[] NOT NULL, + json_array json, + jsonb_array jsonb +); + +CREATE TABLE unofficial_arrays +( + bool_array bool[] NOT NULL, + character_array character[] NOT NULL, + character_varying_array character varying[] NOT NULL, + smallint_array smallint[] NOT NULL, + int_array int[] NOT NULL, + integer_array integer[] NOT NULL, + bigint_array bigint[] NOT NULL, + usmallint_array smallint[] NOT NULL, + uint_array int[] NOT NULL, + uinteger_array integer[] NOT NULL, + ubigint_array bigint[] NOT NULL, + real_array real[] NOT NULL, + double_precision_array double precision[] NOT NULL, + decimal_array decimal[] NOT NULL, + timestamp_with_time_zone_array timestamp with time zone[] NOT NULL +); diff --git a/synth/testing_harness/postgres/arrays/arrays.json b/synth/testing_harness/postgres/arrays/arrays.json new file mode 100644 index 00000000..06a1796d --- /dev/null +++ b/synth/testing_harness/postgres/arrays/arrays.json @@ -0,0 +1,471 @@ +{ + "type": "array", + "content": { + "type": "object", + "boolean_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "bool", + "frequency": 0.5 + } + }, + "char_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "string", + "pattern": "[a-z]" + } + }, + "varchar_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "string", + "faker": { + "generator": "company_name" + } + } + }, + "text_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "string", + "faker": { + "generator": "name" + } + } + }, + "bpchar_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "string", + "faker": { + "generator": "title" + } + } + }, + "name_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "string", + "faker": { + "generator": "username" + } + } + }, + "int2_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "number", + "subtype": "i32", + "range": { + "low": -32766, + "high": 32767, + "step": 1 + } + } + }, + "int4_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "number", + "subtype": "i32", + "range": { + "low": -32766, + "high": 32767, + "step": 1 + } + } + }, + "int8_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "number", + "subtype": "i64", + "range": { + "low": -32766, + "high": 32767, + "step": 1 + } + } + }, + "numeric_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "number", + "subtype": "f64", + "range": { + "low": -32766, + "high": 32767, + "step": 0.01 + } + } + }, + "uint2_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "number", + "subtype": "u32", + "range": { + "low": 0, + "high": 32767, + "step": 1 + } + } + }, + "uint4_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "number", + "subtype": "u32", + "range": { + "low": 0, + "high": 32767, + "step": 1 + } + } + }, + "uint8_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "number", + "subtype": "u64", + "range": { + "low": 0, + "high": 32767, + "step": 1 + } + } + }, + "unumeric_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "number", + "subtype": "f64", + "range": { + "low": 0, + "high": 32767, + "step": 0.01 + } + } + }, + "float4_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "number", + "subtype": "f32", + "range": { + "low": -32766, + "high": 32767, + "step": 0.01 + } + } + }, + "float8_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "number", + "subtype": "f64", + "range": { + "low": -32766, + "high": 32767, + "step": 0.01 + } + } + }, + "int_array_2d": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 8, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "array", + "length": { + "type": "number", + "constant": 8 + }, + "content": { + "type": "number", + "subtype": "i32", + "range": { + "low": -32768, + "high": 32767, + "step": 1 + } + } + } + }, + "timestamp_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "date_time", + "format": "%Y-%m-%dT%H:%M:%S", + "subtype": "naive_date_time", + "begin": "2015-01-01T00:00:00", + "end": "2020-01-01T12:00:00" + } + }, + "timestamptz_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "date_time", + "format": "%Y-%m-%dT%H:%M:%S%z", + "subtype": "date_time", + "begin": "2015-01-01T00:00:00+0000", + "end": "2020-01-01T12:00:00+0000" + } + }, + "date_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "date_time", + "format": "%Y-%m-%d", + "subtype": "naive_date", + "begin": "2015-01-01", + "end": "2020-01-01" + } + }, + "time_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "date_time", + "format": "%H:%M:%S", + "subtype": "naive_time", + "begin": "00:00:00", + "end": "23:00:00" + } + }, + "json_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "object", + "username": { + "type": "string", + "faker": { + "generator": "username" + } + }, + "scores": { + "type": "array", + "length": 5, + "content": { + "type": "number", + "subtype": "u32", + "range": { + "low": 0, + "high": 100, + "step": 1 + } + } + } + } + }, + "jsonb_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "object", + "username": { + "type": "string", + "faker": { + "generator": "username" + } + }, + "scores": { + "type": "array", + "length": 5, + "content": { + "type": "number", + "subtype": "u32", + "range": { + "low": 0, + "high": 100, + "step": 1 + } + } + } + } + } + }, + "length": 10 +} diff --git a/synth/testing_harness/postgres/arrays/unofficial_arrays.json b/synth/testing_harness/postgres/arrays/unofficial_arrays.json new file mode 100644 index 00000000..bae9d385 --- /dev/null +++ b/synth/testing_harness/postgres/arrays/unofficial_arrays.json @@ -0,0 +1,292 @@ +{ + "type": "array", + "content": { + "type": "object", + "bool_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "bool", + "frequency": 0.5 + } + }, + "character_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "string", + "pattern": "[a-z]" + } + }, + "character_varying_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "string", + "faker": { + "generator": "company_name" + } + } + }, + "smallint_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "number", + "subtype": "i32", + "range": { + "low": -32766, + "high": 32767, + "step": 1 + } + } + }, + "int_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "number", + "subtype": "i32", + "range": { + "low": -32766, + "high": 32767, + "step": 1 + } + } + }, + "integer_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "number", + "subtype": "i32", + "range": { + "low": -32766, + "high": 32767, + "step": 1 + } + } + }, + "bigint_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "number", + "subtype": "i64", + "range": { + "low": -32766, + "high": 32767, + "step": 1 + } + } + }, + "usmallint_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "number", + "subtype": "u32", + "range": { + "low": 0, + "high": 32767, + "step": 1 + } + } + }, + "uint_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "number", + "subtype": "u32", + "range": { + "low": 0, + "high": 32767, + "step": 1 + } + } + }, + "uinteger_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "number", + "subtype": "u32", + "range": { + "low": 0, + "high": 32767, + "step": 1 + } + } + }, + "ubigint_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "number", + "subtype": "u64", + "range": { + "low": 0, + "high": 32767, + "step": 1 + } + } + }, + "real_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "number", + "subtype": "f32", + "range": { + "low": -32766, + "high": 32767, + "step": 0.01 + } + } + }, + "double_precision_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "number", + "subtype": "f64", + "range": { + "low": -32766, + "high": 32767, + "step": 0.01 + } + } + }, + "decimal_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "number", + "subtype": "f64", + "range": { + "low": -32766, + "high": 32767, + "step": 0.01 + } + } + }, + "timestamp_with_time_zone_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + } + }, + "content": { + "type": "date_time", + "format": "%Y-%m-%dT%H:%M:%S%z", + "subtype": "date_time", + "begin": "2015-01-01T00:00:00+0000", + "end": "2020-01-01T12:00:00+0000" + } + } + }, + "length": 10 +} diff --git a/synth/testing_harness/postgres/arrays_master/arrays.json b/synth/testing_harness/postgres/arrays_master/arrays.json new file mode 100644 index 00000000..d42cb1c9 --- /dev/null +++ b/synth/testing_harness/postgres/arrays_master/arrays.json @@ -0,0 +1,546 @@ +{ + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "object", + "boolean_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "bool", + "frequency": 0.5 + } + }, + "bpchar_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "string", + "pattern": "[a-zA-Z0-9]{0, 1}" + } + }, + "char_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "string", + "pattern": "[a-zA-Z0-9]{0, 1}" + } + }, + "date_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "date_time", + "format": "%Y-%m-%d", + "subtype": "naive_date", + "begin": "2015-01-09", + "end": "2019-12-27" + } + }, + "float4_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "number", + "range": { + "low": -31572.580078125, + "high": 32721.41015625 + }, + "subtype": "f32" + } + }, + "float8_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 7, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "number", + "range": { + "low": -31195.78, + "high": 31716.440000000002 + }, + "subtype": "f64" + } + }, + "int2_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "number", + "range": { + "low": -30856, + "high": 32726 + }, + "subtype": "i32" + } + }, + "int4_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 8, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "number", + "range": { + "low": -31911, + "high": 31842 + }, + "subtype": "i32" + } + }, + "int8_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 7, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "number", + "range": { + "low": -25394, + "high": 32038 + }, + "subtype": "i64" + } + }, + "int_array_2d": { + "type": "one_of", + "variants": [ + { + "weight": 1.0, + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 2, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "number", + "range": {}, + "subtype": "i32" + } + }, + { + "weight": 1.0, + "type": "null" + } + ] + }, + "json_array": { + "type": "one_of", + "variants": [ + { + "weight": 1.0, + "type": "object" + }, + { + "weight": 1.0, + "type": "null" + }, + { + "weight": 1.0, + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "one_of", + "variants": [ + { + "weight": 1.0, + "type": "object", + "scores": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 5, + "high": 6, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "one_of", + "variants": [ + { + "weight": 1.0, + "type": "number", + "range": { + "low": 0, + "high": 99, + "step": 1 + }, + "subtype": "u64" + } + ] + } + }, + "username": { + "type": "string", + "pattern": "[a-zA-Z0-9]*" + } + } + ] + } + } + ] + }, + "jsonb_array": { + "type": "one_of", + "variants": [ + { + "weight": 1.0, + "type": "object" + }, + { + "weight": 1.0, + "type": "null" + }, + { + "weight": 1.0, + "type": "array", + "length": { + "type": "number", + "range": { + "low": 2, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "one_of", + "variants": [ + { + "weight": 1.0, + "type": "object", + "scores": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 5, + "high": 6, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "one_of", + "variants": [ + { + "weight": 1.0, + "type": "number", + "range": { + "low": 0, + "high": 99, + "step": 1 + }, + "subtype": "u64" + } + ] + } + }, + "username": { + "type": "string", + "pattern": "[a-zA-Z0-9]*" + } + } + ] + } + } + ] + }, + "name_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "string", + "pattern": "[a-zA-Z0-9]{0, 1}" + } + }, + "numeric_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 8, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "number", + "range": { + "low": -31762.67, + "high": 31443.75 + }, + "subtype": "f64" + } + }, + "text_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 8, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "string", + "pattern": "[a-zA-Z0-9]{0, 1}" + } + }, + "time_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "date_time", + "format": "%H:%M:%S", + "subtype": "naive_time", + "begin": "00:37:03", + "end": "22:29:41" + } + }, + "timestamp_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 8, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "date_time", + "format": "%Y-%m-%dT%H:%M:%S", + "subtype": "naive_date_time", + "begin": "2015-02-14T16:41:01", + "end": "2019-10-06T23:46:18" + } + }, + "timestamptz_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 8, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "date_time", + "format": "%Y-%m-%dT%H:%M:%S%z", + "subtype": "date_time", + "begin": "2015-01-20T01:00:41+0000", + "end": "2019-10-28T09:37:36+0000" + } + }, + "uint2_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "number", + "range": { + "low": 341, + "high": 32588 + }, + "subtype": "i32" + } + }, + "uint4_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "number", + "range": { + "low": 280, + "high": 32750 + }, + "subtype": "i32" + } + }, + "uint8_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 8, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "number", + "range": { + "low": 258, + "high": 31382 + }, + "subtype": "i64" + } + }, + "unumeric_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "number", + "range": { + "low": 420.14, + "high": 31531.13 + }, + "subtype": "f64" + } + }, + "varchar_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "string", + "pattern": "[a-zA-Z0-9]{0, 1}" + } + } + } +} \ No newline at end of file diff --git a/synth/testing_harness/postgres/arrays_master/unofficial_arrays.json b/synth/testing_harness/postgres/arrays_master/unofficial_arrays.json new file mode 100644 index 00000000..bd990e15 --- /dev/null +++ b/synth/testing_harness/postgres/arrays_master/unofficial_arrays.json @@ -0,0 +1,302 @@ +{ + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 10, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "object", + "bigint_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "number", + "range": { + "low": -32362, + "high": 31882 + }, + "subtype": "i64" + } + }, + "bool_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "bool", + "frequency": 0.5 + } + }, + "character_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "string", + "pattern": "[a-zA-Z0-9]{0, 1}" + } + }, + "character_varying_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 8, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "string", + "pattern": "[a-zA-Z0-9]{0, 1}" + } + }, + "decimal_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "number", + "range": { + "low": -29508.77, + "high": 32595.16 + }, + "subtype": "f64" + } + }, + "double_precision_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "number", + "range": { + "low": -32728.93, + "high": 30830.96 + }, + "subtype": "f64" + } + }, + "int_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 8, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "number", + "range": { + "low": -31922, + "high": 32726 + }, + "subtype": "i32" + } + }, + "integer_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "number", + "range": { + "low": -31954, + "high": 31577 + }, + "subtype": "i32" + } + }, + "real_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "number", + "range": { + "low": -30618.009765625, + "high": 30062.16015625 + }, + "subtype": "f32" + } + }, + "smallint_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "number", + "range": { + "low": -31803, + "high": 31156 + }, + "subtype": "i32" + } + }, + "timestamp_with_time_zone_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "date_time", + "format": "%Y-%m-%dT%H:%M:%S%z", + "subtype": "date_time", + "begin": "2015-01-18T10:24:22+0000", + "end": "2019-11-10T12:31:48+0000" + } + }, + "ubigint_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "number", + "range": { + "low": 934, + "high": 32630 + }, + "subtype": "i64" + } + }, + "uint_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "number", + "range": { + "low": 2034, + "high": 32596 + }, + "subtype": "i32" + } + }, + "uinteger_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "number", + "range": { + "low": 965, + "high": 32446 + }, + "subtype": "i32" + } + }, + "usmallint_array": { + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 9, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "number", + "range": { + "low": 531, + "high": 32717 + }, + "subtype": "i32" + } + } + } +} \ No newline at end of file diff --git a/synth/testing_harness/postgres/complete_import_master/types.json b/synth/testing_harness/postgres/complete_import_master/types.json index b2904a6b..eaf22486 100644 --- a/synth/testing_harness/postgres/complete_import_master/types.json +++ b/synth/testing_harness/postgres/complete_import_master/types.json @@ -92,7 +92,7 @@ { "type": "date_time", "subtype": "date_time", - "format": "", + "format": "%Y-%m-%dT%H:%M:%S%z", "weight": 1 }, { @@ -107,7 +107,7 @@ { "type": "date_time", "subtype": "naive_date_time", - "format": "", + "format": "%Y-%m-%dT%H:%M:%S", "weight": 1 }, { diff --git a/synth/testing_harness/postgres/complete_import_master/unofficial_types.json b/synth/testing_harness/postgres/complete_import_master/unofficial_types.json index 1289925e..f62b04ba 100644 --- a/synth/testing_harness/postgres/complete_import_master/unofficial_types.json +++ b/synth/testing_harness/postgres/complete_import_master/unofficial_types.json @@ -69,7 +69,7 @@ { "type": "date_time", "subtype": "date_time", - "format": "", + "format": "%Y-%m-%dT%H:%M:%S%z", "weight": 1 }, { @@ -80,7 +80,7 @@ }, "id_serial2": { "type": "number", - "subtype": "i64", + "subtype": "i32", "range": { "low": 1, "high": 10 @@ -88,7 +88,7 @@ }, "id_smallserial": { "type": "number", - "subtype": "i64", + "subtype": "i32", "range": { "low": 1, "high": 10 diff --git a/synth/testing_harness/postgres/e2e.sh b/synth/testing_harness/postgres/e2e.sh index 629bbf30..a4ac43df 100755 --- a/synth/testing_harness/postgres/e2e.sh +++ b/synth/testing_harness/postgres/e2e.sh @@ -27,6 +27,7 @@ commands: test-import|Test importing from postgres data test-complete|Test generating and importing all types to/from postgres test-warning|Test integer warnings + test-arrays|Test encoding array values test-local|Run all test on a local machine using the container from 'up' (no need to call 'up' first) up|Starts a local Docker instance for testing down|Stops container started with 'up' @@ -91,6 +92,23 @@ function test-warning() { fi } +function test-arrays() { + echo -e "${INFO}Testing arrays to postgres${NC}" + psql -c "CREATE DATABASE arrays;" postgres://postgres:$PASSWORD@localhost:$PORT/postgres + psql -f arrays/0_arrays.sql postgres://postgres:$PASSWORD@localhost:$PORT/arrays + errors=$($SYNTH generate --to postgres://postgres:$PASSWORD@localhost:$PORT/arrays arrays 2>&1) + if [ ! -z "$errors" ] + then + echo -e "${ERROR}Did not expect errors:${NC}" + echo -e $errors + return 1 + fi + + echo -e "${INFO}Testing importing postgres arrays${NC}" + $SYNTH import --from postgres://postgres:${PASSWORD}@localhost:${PORT}/arrays arrays_import || { echo -e "${ERROR}Array import failed${NC}"; return 1; } + diff <(jq --sort-keys . arrays_import/*) <(jq --sort-keys . arrays_master/*) || { echo -e "${ERROR}Import arrays do not match${NC}"; return 1; } +} + function test-local() { up || return 1 @@ -99,6 +117,7 @@ function test-local() { test-import || result=$? test-complete || result=$? test-warning || result=$? + test-arrays || result=$? down cleanup @@ -134,6 +153,7 @@ function cleanup() { echo -e "${DEBUG}Cleaning up local files${NC}" rm -Rf hospital_import rm -Rf complete_import + rm -Rf arrays_import rm -Rf .synth } @@ -153,6 +173,9 @@ case "${1-*}" in test-warning) test-warning || exit 1 ;; + test-arrays) + test-arrays || exit 1 + ;; test-local) test-local || exit 1 ;; diff --git a/synth/testing_harness/postgres/hospital_master/hospitals.json b/synth/testing_harness/postgres/hospital_master/hospitals.json index 65e1b335..ad3e1f92 100644 --- a/synth/testing_harness/postgres/hospital_master/hospitals.json +++ b/synth/testing_harness/postgres/hospital_master/hospitals.json @@ -43,6 +43,32 @@ "type": "number", "id": {}, "subtype": "i32" + }, + "specialities": { + "type": "one_of", + "variants": [ + { + "weight": 1.0, + "type": "array", + "length": { + "type": "number", + "range": { + "low": 1, + "high": 2, + "step": 1 + }, + "subtype": "u64" + }, + "content": { + "type": "string", + "pattern": "[a-zA-Z0-9]{0, 1}" + } + }, + { + "weight": 1.0, + "type": "null" + } + ] } } }