Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add glue CSV classifier #9824

Merged
merged 28 commits into from
Oct 11, 2019
Merged

Add glue CSV classifier #9824

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
bc2b69d
Start work on adding csv classifier per #9083
Jun 25, 2019
337743c
Update conflictsWith
Jun 25, 2019
cf51098
Add CSV Classifier specific code
Jun 25, 2019
4efc343
Update forceNew with csv classifier options
Jun 25, 2019
5e404dd
Fix the requireds with info from https://docs.aws.amazon.com/glue/lat…
Jun 25, 2019
534aa1d
It's Optional true, not Required false
Jun 25, 2019
2267add
Fix typing for the header field
Jun 25, 2019
2843fb3
Fix the typing for Glue classifier
Jun 25, 2019
d247480
Start adding validation
Jun 26, 2019
a9eff8c
Add expand and flatten methods (need to fix the types)
Jun 26, 2019
d4661bb
Validate the header options
Jun 26, 2019
10d15a2
Go fmt issues
Jun 26, 2019
0902257
IDE is now working :)
Jun 26, 2019
533f4f4
Run go fmt on code
Aug 19, 2019
ccd4936
Fix boolean types
Aug 19, 2019
b8e744d
Fix rest of typing
Aug 19, 2019
393fa05
Update existing tests with csvClassifier and start on csvClassifier t…
Aug 19, 2019
29f0f40
WIP on tests. Need to write the tests for the csv classifier
Aug 19, 2019
0b64a2f
Finish writing the test
Aug 20, 2019
1501cd1
go fmt the file
Aug 20, 2019
a82316a
Add documentation for the csv classifier
Aug 20, 2019
8e0784b
Fix the typing in the tests
Aug 20, 2019
8e1c4e5
Add missing parameter
Aug 20, 2019
c63e761
Merge remote-tracking branch 'tpa/master' into twunderlich/9083-glue-…
Aug 25, 2019
a3f1934
Fix warnings
Aug 25, 2019
6518e9e
Remove name as being part of the csv_classifier section
Aug 25, 2019
2c76b7e
Add quote_symbol back
Aug 25, 2019
acf4811
Fix the expected format of list type attributes
Aug 25, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 132 additions & 3 deletions aws/resource_aws_glue_classifier.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,18 @@ func resourceAwsGlueClassifier() *schema.Resource {
func(diff *schema.ResourceDiff, v interface{}) error {
// ForceNew when changing classifier type
// InvalidInputException: UpdateClassifierRequest can't change the type of the classifier
if diff.HasChange("csv_classifier") && diff.HasChange("grok_classifier") {
diff.ForceNew("csv_classifier")
diff.ForceNew("grok_classifier")
}
if diff.HasChange("csv_classifier") && diff.HasChange("json_classifier") {
diff.ForceNew("csv_classifier")
diff.ForceNew("json_classifier")
}
if diff.HasChange("csv_classifier") && diff.HasChange("xml_classifier") {
diff.ForceNew("csv_classifier")
diff.ForceNew("xml_classifier")
}
if diff.HasChange("grok_classifier") && diff.HasChange("json_classifier") {
diff.ForceNew("grok_classifier")
diff.ForceNew("json_classifier")
Expand All @@ -41,11 +53,47 @@ func resourceAwsGlueClassifier() *schema.Resource {
),

Schema: map[string]*schema.Schema{
"csv_classifier": {
Type: schema.TypeList,
Optional: true,
MaxItems: 1,
ConflictsWith: []string{"grok_classifier", "json_classifier", "xml_classifier"},
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"allow_single_column": {
Type: schema.TypeBool,
Optional: true,
},
"contains_header": {
Type: schema.TypeString,
Optional: true,
ValidateFunc: validateHeaderOptions(),
},
"delimiter": {
Type: schema.TypeString,
Optional: true,
},
"disable_value_trimming": {
Type: schema.TypeBool,
Optional: true,
},
"header": {
Type: schema.TypeList,
Optional: true,
Elem: &schema.Schema{Type: schema.TypeString},
},
"quote_symbol": {
Type: schema.TypeString,
Optional: true,
},
},
},
},
"grok_classifier": {
Type: schema.TypeList,
Optional: true,
MaxItems: 1,
ConflictsWith: []string{"json_classifier", "xml_classifier"},
ConflictsWith: []string{"csv_classifier", "json_classifier", "xml_classifier"},
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"classification": {
Expand All @@ -69,7 +117,7 @@ func resourceAwsGlueClassifier() *schema.Resource {
Type: schema.TypeList,
Optional: true,
MaxItems: 1,
ConflictsWith: []string{"grok_classifier", "xml_classifier"},
ConflictsWith: []string{"csv_classifier", "grok_classifier", "xml_classifier"},
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"json_path": {
Expand All @@ -89,7 +137,7 @@ func resourceAwsGlueClassifier() *schema.Resource {
Type: schema.TypeList,
Optional: true,
MaxItems: 1,
ConflictsWith: []string{"grok_classifier", "json_classifier"},
ConflictsWith: []string{"csv_classifier", "grok_classifier", "json_classifier"},
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"classification": {
Expand All @@ -113,6 +161,11 @@ func resourceAwsGlueClassifierCreate(d *schema.ResourceData, meta interface{}) e

input := &glue.CreateClassifierInput{}

if v, ok := d.GetOk("csv_classifier"); ok {
m := v.([]interface{})[0].(map[string]interface{})
input.CsvClassifier = expandGlueCsvClassifierCreate(name, m)
}

if v, ok := d.GetOk("grok_classifier"); ok {
m := v.([]interface{})[0].(map[string]interface{})
input.GrokClassifier = expandGlueGrokClassifierCreate(name, m)
Expand Down Expand Up @@ -164,6 +217,10 @@ func resourceAwsGlueClassifierRead(d *schema.ResourceData, meta interface{}) err
return nil
}

if err := d.Set("csv_classifier", flattenGlueCsvClassifier(classifier.CsvClassifier)); err != nil {
return fmt.Errorf("error setting match_criteria: %s", err)
}

if err := d.Set("grok_classifier", flattenGlueGrokClassifier(classifier.GrokClassifier)); err != nil {
return fmt.Errorf("error setting match_criteria: %s", err)
}
Expand All @@ -186,6 +243,11 @@ func resourceAwsGlueClassifierUpdate(d *schema.ResourceData, meta interface{}) e

input := &glue.UpdateClassifierInput{}

if v, ok := d.GetOk("csv_classifier"); ok {
m := v.([]interface{})[0].(map[string]interface{})
input.CsvClassifier = expandGlueCsvClassifierUpdate(d.Id(), m)
}

if v, ok := d.GetOk("grok_classifier"); ok {
m := v.([]interface{})[0].(map[string]interface{})
input.GrokClassifier = expandGlueGrokClassifierUpdate(d.Id(), m)
Expand Down Expand Up @@ -222,6 +284,14 @@ func resourceAwsGlueClassifierDelete(d *schema.ResourceData, meta interface{}) e
return nil
}

func validateHeaderOptions() schema.SchemaValidateFunc {
return validation.StringInSlice([]string{
"UNKNOWN",
"PRESENT",
"ABSENT",
}, true)
}

func deleteGlueClassifier(conn *glue.Glue, name string) error {
input := &glue.DeleteClassifierInput{
Name: aws.String(name),
Expand All @@ -238,6 +308,48 @@ func deleteGlueClassifier(conn *glue.Glue, name string) error {
return nil
}

func expandGlueCsvClassifierCreate(name string, m map[string]interface{}) *glue.CreateCsvClassifierRequest {
csvClassifier := &glue.CreateCsvClassifierRequest{
AllowSingleColumn: aws.Bool(m["allow_single_column"].(bool)),
ContainsHeader: aws.String(m["contains_header"].(string)),
Delimiter: aws.String(m["delimiter"].(string)),
DisableValueTrimming: aws.Bool(m["disable_value_trimming"].(bool)),
Name: aws.String(name),
QuoteSymbol: aws.String(m["quote_symbol"].(string)),
}

if v, ok := m["header"]; ok {
header := make([]string, len(v.([]interface{})))
for i, item := range v.([]interface{}) {
header[i] = fmt.Sprint(item)
}
csvClassifier.Header = aws.StringSlice(header)
}

return csvClassifier
}

func expandGlueCsvClassifierUpdate(name string, m map[string]interface{}) *glue.UpdateCsvClassifierRequest {
csvClassifier := &glue.UpdateCsvClassifierRequest{
AllowSingleColumn: aws.Bool(m["allow_single_column"].(bool)),
ContainsHeader: aws.String(m["contains_header"].(string)),
Delimiter: aws.String(m["delimiter"].(string)),
DisableValueTrimming: aws.Bool(m["disable_value_trimming"].(bool)),
Name: aws.String(name),
QuoteSymbol: aws.String(m["quote_symbol"].(string)),
}

if v, ok := m["header"]; ok {
header := make([]string, len(v.([]interface{})))
for i, item := range v.([]interface{}) {
header[i] = fmt.Sprint(item)
}
csvClassifier.Header = aws.StringSlice(header)
}

return csvClassifier
}

func expandGlueGrokClassifierCreate(name string, m map[string]interface{}) *glue.CreateGrokClassifierRequest {
grokClassifier := &glue.CreateGrokClassifierRequest{
Classification: aws.String(m["classification"].(string)),
Expand Down Expand Up @@ -308,6 +420,23 @@ func expandGlueXmlClassifierUpdate(name string, m map[string]interface{}) *glue.
return xmlClassifier
}

func flattenGlueCsvClassifier(csvClassifier *glue.CsvClassifier) []map[string]interface{} {
if csvClassifier == nil {
return []map[string]interface{}{}
}

m := map[string]interface{}{
"allow_single_column": aws.BoolValue(csvClassifier.AllowSingleColumn),
"contains_header": aws.StringValue(csvClassifier.ContainsHeader),
"delimiter": aws.StringValue(csvClassifier.Delimiter),
"disable_value_trimming": aws.BoolValue(csvClassifier.DisableValueTrimming),
"header": aws.StringValueSlice(csvClassifier.Header),
"quote_symbol": aws.StringValue(csvClassifier.QuoteSymbol),
}

return []map[string]interface{}{m}
}

func flattenGlueGrokClassifier(grokClassifier *glue.GrokClassifier) []map[string]interface{} {
if grokClassifier == nil {
return []map[string]interface{}{}
Expand Down
Loading