Skip to content

Commit

Permalink
Add json output
Browse files Browse the repository at this point in the history
  • Loading branch information
bhagenbourger committed Apr 24, 2024
1 parent b9c5460 commit 6361fd0
Show file tree
Hide file tree
Showing 20 changed files with 457 additions and 14 deletions.
1 change: 1 addition & 0 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,4 +24,5 @@ jobs:
- name: Generate all formats all options
run: |
target/release/fakelake generate tests/csv_all_options.yaml
target/release/fakelake generate tests/json_all_options.yaml
target/release/fakelake generate tests/parquet_all_options.yaml
4 changes: 2 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@

# Generated
/target
*.parquet
*.csv
/site
output.csv
output.json

# Coverage
*.profraw
Expand Down
5 changes: 3 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ log = "0.4"
once_cell = "1.8"
parquet = "50.0.0"
rayon = "1.8.0"
serde_json = "1.0.114"
yaml-rust = "0.4"

[dev-dependencies]
Expand Down
10 changes: 10 additions & 0 deletions docs/output/parameters.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,16 @@ info:
```
Default delimiter is ',' but you can specify any character.
##### JSON
```yaml
info:
output_format: json
wrap_up: false
```
By default, wrap_up is set to false.
When wrap_up is set to false, each line into the result file is a json object but the whole file is not a valid json.
When wrap_up is set to true, the whole file is a valid json, rows are wrapped up into an array.
### Rows
To choose the number of rows in the generated file, use rows.
```yaml
Expand Down
56 changes: 56 additions & 0 deletions src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ impl Column {
pub enum OutputType {
Parquet(),
Csv(u8),
Json(bool),
}

#[derive(Debug)]
Expand Down Expand Up @@ -173,6 +174,17 @@ impl Info {
};
Some(OutputType::Csv(delimiter))
}
Some(value) if value == "json" => {
let wrap_up = match section_info["wrap_up"] {
Yaml::Boolean(value) => value,
Yaml::BadValue => false,
_ => {
warn!("Wrap up should be a bool. Default value 'false' is taken.");
false
}
};
Some(OutputType::Json(wrap_up))
}
_ => None,
};

Expand Down Expand Up @@ -507,6 +519,50 @@ mod tests {
assert_eq!(info.rows, None);
}

#[test]
fn given_json_format_should_use_default_wrap_up() {
let yaml = "
info:
output_format: json
";
let info = generate_info_from_yaml(yaml);
expecting_ok(&info);
let info = &info.unwrap();
assert_eq!(info.output_name, None);
assert_eq!(info.output_format, Some(OutputType::Json(false)));
assert_eq!(info.rows, None);
}

#[test]
fn given_json_format_with_custom_wrap_up_should_use_custom_wrap_up() {
let yaml = "
info:
output_format: json
wrap_up: true
";
let info = generate_info_from_yaml(yaml);
expecting_ok(&info);
let info = &info.unwrap();
assert_eq!(info.output_name, None);
assert_eq!(info.output_format, Some(OutputType::Json(true)));
assert_eq!(info.rows, None);
}

#[test]
fn given_json_format_with_invalid_wrap_up_should_default_wrap_up() {
let yaml = "
info:
output_format: json
wrap_up: invalid
";
let info = generate_info_from_yaml(yaml);
expecting_ok(&info);
let info = &info.unwrap();
assert_eq!(info.output_name, None);
assert_eq!(info.output_format, Some(OutputType::Json(false)));
assert_eq!(info.rows, None);
}

#[test]
fn given_rows_int_should_config_return_in_rows() {
let yaml = "
Expand Down
1 change: 1 addition & 0 deletions src/errors.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ pub enum FakeLakeError {
BadYAMLFormat(String),
IOError(io::Error),
CSVError(csv::Error),
JSONError(serde_json::Error),
}

#[cfg(not(tarpaulin_include))]
Expand Down
4 changes: 2 additions & 2 deletions src/generate/csv/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ mod tests {

#[test]
fn given_config_should_write_file() {
let config = get_config(1, Some("output_name".to_string()), Some(1000));
let config = get_config(1, Some("target/output_name".to_string()), Some(1000));
let output = OutputCsv { delimiter: 5 };
match output.generate_from_config(&config) {
Ok(_) => (),
Expand Down Expand Up @@ -212,7 +212,7 @@ mod tests {
let config = Config {
columns,
info: Some(Info {
output_name: Some("output_name".to_string()),
output_name: Some("target/output_name".to_string()),
output_format: Some(OutputType::Csv(5)),
rows: Some(1000),
}),
Expand Down
Loading

0 comments on commit 6361fd0

Please sign in to comment.