From 7b86f414b275396944e6f1e758d75330f25b2ce6 Mon Sep 17 00:00:00 2001 From: Becca Petrin Date: Wed, 25 Jul 2018 13:08:15 -0700 Subject: [PATCH 01/11] add a header type field --- logical/framework/backend.go | 3 ++ logical/framework/backend_test.go | 7 +++++ logical/framework/field_data.go | 41 ++++++++++++++++++++++++-- logical/framework/field_data_test.go | 44 ++++++++++++++++++++++++++++ logical/framework/field_type.go | 8 +++++ 5 files changed, 101 insertions(+), 2 deletions(-) diff --git a/logical/framework/backend.go b/logical/framework/backend.go index 3c674e365fbf..4c787cbed1fb 100644 --- a/logical/framework/backend.go +++ b/logical/framework/backend.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net/http" "regexp" "sort" "strings" @@ -548,6 +549,8 @@ func (t FieldType) Zero() interface{} { return []string{} case TypeCommaIntSlice: return []int{} + case TypeHeader: + return http.Header{} default: panic("unknown type: " + t.String()) } diff --git a/logical/framework/backend_test.go b/logical/framework/backend_test.go index fa050ac60c46..09c6f9045485 100644 --- a/logical/framework/backend_test.go +++ b/logical/framework/backend_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "net/http" + "github.com/hashicorp/vault/logical" ) @@ -532,6 +534,11 @@ func TestFieldSchemaDefaultOrZero(t *testing.T) { &FieldSchema{Type: TypeDurationSecond}, 0, }, + + "default header not set": { + &FieldSchema{Type: TypeHeader}, + http.Header{}, + }, } for name, tc := range cases { diff --git a/logical/framework/field_data.go b/logical/framework/field_data.go index 2bbb34805129..bc59e12d7bc9 100644 --- a/logical/framework/field_data.go +++ b/logical/framework/field_data.go @@ -4,6 +4,8 @@ import ( "encoding/json" "errors" "fmt" + "net/http" + "net/textproto" "regexp" "strings" @@ -37,7 +39,7 @@ func (d *FieldData) Validate() error { switch schema.Type { case TypeBool, TypeInt, TypeMap, TypeDurationSecond, TypeString, TypeLowerCaseString, TypeNameString, TypeSlice, TypeStringSlice, TypeCommaStringSlice, - TypeKVPairs, TypeCommaIntSlice: + TypeKVPairs, TypeCommaIntSlice, TypeHeader: _, _, err := d.getPrimitive(field, schema) if err != nil { return errwrap.Wrapf(fmt.Sprintf("error converting input %v for field %q: {{err}}", value, field), err) @@ -126,7 +128,7 @@ func (d *FieldData) GetOkErr(k string) (interface{}, bool, error) { switch schema.Type { case TypeBool, TypeInt, TypeMap, TypeDurationSecond, TypeString, TypeLowerCaseString, TypeNameString, TypeSlice, TypeStringSlice, TypeCommaStringSlice, - TypeKVPairs, TypeCommaIntSlice: + TypeKVPairs, TypeCommaIntSlice, TypeHeader: return d.getPrimitive(k, schema) default: return nil, false, @@ -291,6 +293,41 @@ func (d *FieldData) getPrimitive(k string, schema *FieldSchema) (interface{}, bo } return result, true, nil + case TypeHeader: + // First try to parse this as a map + var mapResult map[string][]string + if err := mapstructure.WeakDecode(raw, &mapResult); err == nil { + // The http.Header Get method is case-insensitive in the key it takes, + // but searches the map for a matching canonicalized value. Most headers + // will arrive in this format, but just in case they don't, let's + // canonicalize all the keys. + result := http.Header{} + for k, slice := range mapResult { + canonicalKey := textproto.CanonicalMIMEHeaderKey(k) + for _, v := range slice { + result.Add(canonicalKey, v) + } + } + return result, true, nil + } + + // If map parse fails, parse as a string list of = delimited pairs + var listResult []string + if err := mapstructure.WeakDecode(raw, &listResult); err != nil { + return nil, true, err + } + + result := http.Header{} + for _, keyPair := range listResult { + keyPairSlice := strings.SplitN(keyPair, "=", 2) + if len(keyPairSlice) != 2 || keyPairSlice[0] == "" { + return nil, false, fmt.Errorf("invalid key pair %q", keyPair) + } + // See note above about why we're canonicalizing. + result.Add(textproto.CanonicalMIMEHeaderKey(keyPairSlice[0]), keyPairSlice[1]) + } + return result, true, nil + default: panic(fmt.Sprintf("Unknown type: %s", schema.Type)) } diff --git a/logical/framework/field_data_test.go b/logical/framework/field_data_test.go index 86ebdd13d59e..8ec7df14dcf9 100644 --- a/logical/framework/field_data_test.go +++ b/logical/framework/field_data_test.go @@ -1,6 +1,7 @@ package framework import ( + "net/http" "reflect" "testing" ) @@ -467,6 +468,40 @@ func TestFieldDataGet(t *testing.T) { }, }, + "type header, keypair string array": { + map[string]*FieldSchema{ + "foo": {Type: TypeHeader}, + }, + map[string]interface{}{ + "foo": []interface{}{"key1=value1", "key2=value2", "key3=1"}, + }, + "foo", + http.Header{ + "Key1": []string{"value1"}, + "Key2": []string{"value2"}, + "Key3": []string{"1"}, + }, + }, + + "type header, map string slice": { + map[string]*FieldSchema{ + "foo": {Type: TypeHeader}, + }, + map[string]interface{}{ + "foo": map[string][]string{ + "key1": {"value1"}, + "key2": {"value2"}, + "key3": {"1"}, + }, + }, + "foo", + http.Header{ + "Key1": []string{"value1"}, + "Key2": []string{"value2"}, + "Key3": []string{"1"}, + }, + }, + "name string type, not supplied": { map[string]*FieldSchema{ "foo": {Type: TypeNameString}, @@ -556,6 +591,15 @@ func TestFieldDataGet(t *testing.T) { "foo", map[string]string{}, }, + + "type header, not supplied": { + map[string]*FieldSchema{ + "foo": {Type: TypeHeader}, + }, + map[string]interface{}{}, + "foo", + http.Header{}, + }, } for name, tc := range cases { diff --git a/logical/framework/field_type.go b/logical/framework/field_type.go index d447eabfbfb4..64a6a56dc08c 100644 --- a/logical/framework/field_type.go +++ b/logical/framework/field_type.go @@ -42,6 +42,12 @@ const ( // TypeCommaIntSlice is a helper for TypeSlice that returns a sanitized // slice of Ints TypeCommaIntSlice + + // TypeHeader is a helper for sending request headers through to Vault. + // For instance, the AWS and AliCloud credential plugins both act as a + // benevolent MITM for a request, and the headers are sent through and + // parsed. + TypeHeader ) func (t FieldType) String() string { @@ -64,6 +70,8 @@ func (t FieldType) String() string { return "duration (sec)" case TypeSlice, TypeStringSlice, TypeCommaStringSlice, TypeCommaIntSlice: return "slice" + case TypeHeader: + return "header" default: return "unknown type" } From fdbb32a60110475f9f71b42bb8e241ba2ea10977 Mon Sep 17 00:00:00 2001 From: Becca Petrin Date: Fri, 27 Jul 2018 10:16:55 -0700 Subject: [PATCH 02/11] chris feedback changes --- logical/framework/field_data.go | 4 ++-- logical/framework/field_data_test.go | 17 ++++++++++++++++- logical/framework/field_type.go | 8 ++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/logical/framework/field_data.go b/logical/framework/field_data.go index bc59e12d7bc9..d7ecc97fe506 100644 --- a/logical/framework/field_data.go +++ b/logical/framework/field_data.go @@ -311,7 +311,7 @@ func (d *FieldData) getPrimitive(k string, schema *FieldSchema) (interface{}, bo return result, true, nil } - // If map parse fails, parse as a string list of = delimited pairs + // If map parse fails, parse as a string list of : delimited pairs var listResult []string if err := mapstructure.WeakDecode(raw, &listResult); err != nil { return nil, true, err @@ -319,7 +319,7 @@ func (d *FieldData) getPrimitive(k string, schema *FieldSchema) (interface{}, bo result := http.Header{} for _, keyPair := range listResult { - keyPairSlice := strings.SplitN(keyPair, "=", 2) + keyPairSlice := strings.SplitN(keyPair, ":", 2) if len(keyPairSlice) != 2 || keyPairSlice[0] == "" { return nil, false, fmt.Errorf("invalid key pair %q", keyPair) } diff --git a/logical/framework/field_data_test.go b/logical/framework/field_data_test.go index 8ec7df14dcf9..ba9ad01be23b 100644 --- a/logical/framework/field_data_test.go +++ b/logical/framework/field_data_test.go @@ -473,7 +473,7 @@ func TestFieldDataGet(t *testing.T) { "foo": {Type: TypeHeader}, }, map[string]interface{}{ - "foo": []interface{}{"key1=value1", "key2=value2", "key3=1"}, + "foo": []interface{}{"key1:value1", "key2:value2", "key3:1"}, }, "foo", http.Header{ @@ -483,6 +483,21 @@ func TestFieldDataGet(t *testing.T) { }, }, + "type header, keypair string array with dupe key": { + map[string]*FieldSchema{ + "foo": {Type: TypeHeader}, + }, + map[string]interface{}{ + "foo": []interface{}{"key1:value1", "key2:value2", "key3:1", "key3:true"}, + }, + "foo", + http.Header{ + "Key1": []string{"value1"}, + "Key2": []string{"value2"}, + "Key3": []string{"1", "true"}, + }, + }, + "type header, map string slice": { map[string]*FieldSchema{ "foo": {Type: TypeHeader}, diff --git a/logical/framework/field_type.go b/logical/framework/field_type.go index 64a6a56dc08c..84a212dddc44 100644 --- a/logical/framework/field_type.go +++ b/logical/framework/field_type.go @@ -47,6 +47,14 @@ const ( // For instance, the AWS and AliCloud credential plugins both act as a // benevolent MITM for a request, and the headers are sent through and // parsed. + // IMPORTANT NOTES: + // - Under the hood, http.Header is a map[string][]string and its Get + // method will only return the first value. To retrieve all values, + // you must access them like you would in a map. + // - This implementation of http.Header has CASE-INSENSITIVE keys. All + // keys are converted to Title Case on their way in so that the Get + // method will match the keys no matter what case you use when + // looking for them. TypeHeader ) From 7ad2511e172ce029409a487b61375560442c7a72 Mon Sep 17 00:00:00 2001 From: Becca Petrin Date: Fri, 27 Jul 2018 14:59:10 -0700 Subject: [PATCH 03/11] strip MIME conversion as Add does it --- logical/framework/field_data.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/logical/framework/field_data.go b/logical/framework/field_data.go index d7ecc97fe506..e791b4841ed6 100644 --- a/logical/framework/field_data.go +++ b/logical/framework/field_data.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "net/http" - "net/textproto" "regexp" "strings" @@ -303,9 +302,8 @@ func (d *FieldData) getPrimitive(k string, schema *FieldSchema) (interface{}, bo // canonicalize all the keys. result := http.Header{} for k, slice := range mapResult { - canonicalKey := textproto.CanonicalMIMEHeaderKey(k) for _, v := range slice { - result.Add(canonicalKey, v) + result.Add(k, v) } } return result, true, nil @@ -324,7 +322,7 @@ func (d *FieldData) getPrimitive(k string, schema *FieldSchema) (interface{}, bo return nil, false, fmt.Errorf("invalid key pair %q", keyPair) } // See note above about why we're canonicalizing. - result.Add(textproto.CanonicalMIMEHeaderKey(keyPairSlice[0]), keyPairSlice[1]) + result.Add(keyPairSlice[0], keyPairSlice[1]) } return result, true, nil From a3a7cb34138014c6a5234c5920cc37646423937a Mon Sep 17 00:00:00 2001 From: Becca Petrin Date: Mon, 30 Jul 2018 13:33:47 -0700 Subject: [PATCH 04/11] add wrapper for http header --- logical/framework/backend.go | 3 +- logical/framework/backend_test.go | 4 +-- logical/framework/field_data.go | 5 ++- logical/framework/field_data_test.go | 13 ++++---- logical/framework/field_type.go | 46 ++++++++++++++++++++++++++++ logical/framework/field_type_test.go | 21 +++++++++++++ 6 files changed, 78 insertions(+), 14 deletions(-) create mode 100644 logical/framework/field_type_test.go diff --git a/logical/framework/backend.go b/logical/framework/backend.go index 4c787cbed1fb..386ce6816484 100644 --- a/logical/framework/backend.go +++ b/logical/framework/backend.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "io/ioutil" - "net/http" "regexp" "sort" "strings" @@ -550,7 +549,7 @@ func (t FieldType) Zero() interface{} { case TypeCommaIntSlice: return []int{} case TypeHeader: - return http.Header{} + return NewHeader() default: panic("unknown type: " + t.String()) } diff --git a/logical/framework/backend_test.go b/logical/framework/backend_test.go index 09c6f9045485..5b843e758a43 100644 --- a/logical/framework/backend_test.go +++ b/logical/framework/backend_test.go @@ -7,8 +7,6 @@ import ( "testing" "time" - "net/http" - "github.com/hashicorp/vault/logical" ) @@ -537,7 +535,7 @@ func TestFieldSchemaDefaultOrZero(t *testing.T) { "default header not set": { &FieldSchema{Type: TypeHeader}, - http.Header{}, + NewHeader(), }, } diff --git a/logical/framework/field_data.go b/logical/framework/field_data.go index e791b4841ed6..3d22cec74c3d 100644 --- a/logical/framework/field_data.go +++ b/logical/framework/field_data.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "net/http" "regexp" "strings" @@ -300,7 +299,7 @@ func (d *FieldData) getPrimitive(k string, schema *FieldSchema) (interface{}, bo // but searches the map for a matching canonicalized value. Most headers // will arrive in this format, but just in case they don't, let's // canonicalize all the keys. - result := http.Header{} + result := NewHeader() for k, slice := range mapResult { for _, v := range slice { result.Add(k, v) @@ -315,7 +314,7 @@ func (d *FieldData) getPrimitive(k string, schema *FieldSchema) (interface{}, bo return nil, true, err } - result := http.Header{} + result := NewHeader() for _, keyPair := range listResult { keyPairSlice := strings.SplitN(keyPair, ":", 2) if len(keyPairSlice) != 2 || keyPairSlice[0] == "" { diff --git a/logical/framework/field_data_test.go b/logical/framework/field_data_test.go index ba9ad01be23b..49272bd57c71 100644 --- a/logical/framework/field_data_test.go +++ b/logical/framework/field_data_test.go @@ -476,11 +476,12 @@ func TestFieldDataGet(t *testing.T) { "foo": []interface{}{"key1:value1", "key2:value2", "key3:1"}, }, "foo", - http.Header{ + Header{http.Header{ "Key1": []string{"value1"}, "Key2": []string{"value2"}, "Key3": []string{"1"}, }, + }, }, "type header, keypair string array with dupe key": { @@ -491,11 +492,11 @@ func TestFieldDataGet(t *testing.T) { "foo": []interface{}{"key1:value1", "key2:value2", "key3:1", "key3:true"}, }, "foo", - http.Header{ + Header{http.Header{ "Key1": []string{"value1"}, "Key2": []string{"value2"}, "Key3": []string{"1", "true"}, - }, + }}, }, "type header, map string slice": { @@ -510,11 +511,11 @@ func TestFieldDataGet(t *testing.T) { }, }, "foo", - http.Header{ + Header{http.Header{ "Key1": []string{"value1"}, "Key2": []string{"value2"}, "Key3": []string{"1"}, - }, + }}, }, "name string type, not supplied": { @@ -613,7 +614,7 @@ func TestFieldDataGet(t *testing.T) { }, map[string]interface{}{}, "foo", - http.Header{}, + Header{http.Header{}}, }, } diff --git a/logical/framework/field_type.go b/logical/framework/field_type.go index 84a212dddc44..906bb4bf058a 100644 --- a/logical/framework/field_type.go +++ b/logical/framework/field_type.go @@ -1,5 +1,10 @@ package framework +import ( + "net/http" + "net/textproto" +) + // FieldType is the enum of types that a field can be. type FieldType uint @@ -84,3 +89,44 @@ func (t FieldType) String() string { return "unknown type" } } + +/* + The reason we wrap an http.Header here is - under the hood, + an http.Header is a map[string][]string and it does some magic + with casing that can make it difficult to correctly iterate + all header values. + + For example: + h := http.Header{} + h.Add("hello", "world") + h.Add("hello", "monde") + + You'd expect that the header now have "hello" for a key, and + both "world" and "monde" for a value. But it doesn't. "Hello" + is now capitalized in the map, but "world" isn't. + + Later, when you do this: + h.Get("hello") + + You'll receive only "world". + + If you try to solve for this by doing this: + h["hello"] + + You'll receive nothing back, because remember, it's now in the + map as "Hello. + + To avoid bugs like this, we provide one more method. GetAll, + which returns all the values for a key. +*/ +func NewHeader() Header { + return Header{http.Header{}} +} + +type Header struct { + http.Header +} + +func (h Header) GetAll(key string) []string { + return h.Header[textproto.CanonicalMIMEHeaderKey(key)] +} diff --git a/logical/framework/field_type_test.go b/logical/framework/field_type_test.go new file mode 100644 index 000000000000..6265873520ee --- /dev/null +++ b/logical/framework/field_type_test.go @@ -0,0 +1,21 @@ +package framework + +import ( + "testing" +) + +func TestHeaderGetAll(t *testing.T) { + h := NewHeader() + h.Add("hello", "world") + h.Add("hello", "monde") + results := h.GetAll("hello") + if len(results) != 2 { + t.Fatal("expected 2 results") + } + if results[0] != "world" { + t.Fatal("expected the first result to be world") + } + if results[1] != "monde" { + t.Fatal("expected the second result to be monde") + } +} From a3286a96baab6d3ee70024331decd6c44dadf6fb Mon Sep 17 00:00:00 2001 From: Becca Petrin Date: Mon, 30 Jul 2018 13:40:20 -0700 Subject: [PATCH 05/11] update comment --- logical/framework/field_type.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/logical/framework/field_type.go b/logical/framework/field_type.go index 906bb4bf058a..aaa03efb78f1 100644 --- a/logical/framework/field_type.go +++ b/logical/framework/field_type.go @@ -103,18 +103,18 @@ func (t FieldType) String() string { You'd expect that the header now have "hello" for a key, and both "world" and "monde" for a value. But it doesn't. "Hello" - is now capitalized in the map, but "world" isn't. + is now capitalized in the map, but "world" and "monde" aren't. Later, when you do this: h.Get("hello") - You'll receive only "world". + You'll receive only "world", silently missing "monde". - If you try to solve for this by doing this: + If you try to solve that by doing this: h["hello"] You'll receive nothing back, because remember, it's now in the - map as "Hello. + map as "Hello". To avoid bugs like this, we provide one more method. GetAll, which returns all the values for a key. From ef8943976854c3e683a0806c61be3f0f18295cca Mon Sep 17 00:00:00 2001 From: Becca Petrin Date: Mon, 30 Jul 2018 13:53:24 -0700 Subject: [PATCH 06/11] strip outdated comments --- logical/framework/field_data.go | 5 ----- logical/framework/field_type.go | 8 -------- 2 files changed, 13 deletions(-) diff --git a/logical/framework/field_data.go b/logical/framework/field_data.go index 3d22cec74c3d..fc5021a21ad6 100644 --- a/logical/framework/field_data.go +++ b/logical/framework/field_data.go @@ -295,10 +295,6 @@ func (d *FieldData) getPrimitive(k string, schema *FieldSchema) (interface{}, bo // First try to parse this as a map var mapResult map[string][]string if err := mapstructure.WeakDecode(raw, &mapResult); err == nil { - // The http.Header Get method is case-insensitive in the key it takes, - // but searches the map for a matching canonicalized value. Most headers - // will arrive in this format, but just in case they don't, let's - // canonicalize all the keys. result := NewHeader() for k, slice := range mapResult { for _, v := range slice { @@ -320,7 +316,6 @@ func (d *FieldData) getPrimitive(k string, schema *FieldSchema) (interface{}, bo if len(keyPairSlice) != 2 || keyPairSlice[0] == "" { return nil, false, fmt.Errorf("invalid key pair %q", keyPair) } - // See note above about why we're canonicalizing. result.Add(keyPairSlice[0], keyPairSlice[1]) } return result, true, nil diff --git a/logical/framework/field_type.go b/logical/framework/field_type.go index aaa03efb78f1..8ca162fe1a22 100644 --- a/logical/framework/field_type.go +++ b/logical/framework/field_type.go @@ -52,14 +52,6 @@ const ( // For instance, the AWS and AliCloud credential plugins both act as a // benevolent MITM for a request, and the headers are sent through and // parsed. - // IMPORTANT NOTES: - // - Under the hood, http.Header is a map[string][]string and its Get - // method will only return the first value. To retrieve all values, - // you must access them like you would in a map. - // - This implementation of http.Header has CASE-INSENSITIVE keys. All - // keys are converted to Title Case on their way in so that the Get - // method will match the keys no matter what case you use when - // looking for them. TypeHeader ) From d4fb65fd2a430117796bc4422302ace725e7ac45 Mon Sep 17 00:00:00 2001 From: Becca Petrin Date: Mon, 30 Jul 2018 14:01:29 -0700 Subject: [PATCH 07/11] move header to parseutil --- helper/parseutil/header.go | 47 +++++++++++++++++++ .../parseutil/header_test.go | 2 +- logical/framework/backend.go | 2 +- logical/framework/backend_test.go | 3 +- logical/framework/field_data.go | 4 +- logical/framework/field_data_test.go | 10 ++-- logical/framework/field_type.go | 46 ------------------ 7 files changed, 59 insertions(+), 55 deletions(-) create mode 100644 helper/parseutil/header.go rename logical/framework/field_type_test.go => helper/parseutil/header_test.go (95%) diff --git a/helper/parseutil/header.go b/helper/parseutil/header.go new file mode 100644 index 000000000000..f0a6172d7d69 --- /dev/null +++ b/helper/parseutil/header.go @@ -0,0 +1,47 @@ +package parseutil + +import ( + "net/http" + "net/textproto" +) + +/* + The reason we wrap an http.Header here is - under the hood, + an http.Header is a map[string][]string and it does some magic + with casing that can make it difficult to correctly iterate + all header values. + + For example: + h := http.Header{} + h.Add("hello", "world") + h.Add("hello", "monde") + + You'd expect that the header now have "hello" for a key, and + both "world" and "monde" for a value. But it doesn't. "Hello" + is now capitalized in the map, but "world" and "monde" aren't. + + Later, when you do this: + h.Get("hello") + + You'll receive only "world", silently missing "monde". + + If you try to solve that by doing this: + h["hello"] + + You'll receive nothing back, because remember, it's now in the + map as "Hello". + + To avoid bugs like this, we provide one more method. GetAll, + which returns all the values for a key. +*/ +func NewHeader() Header { + return Header{http.Header{}} +} + +type Header struct { + http.Header +} + +func (h Header) GetAll(key string) []string { + return h.Header[textproto.CanonicalMIMEHeaderKey(key)] +} diff --git a/logical/framework/field_type_test.go b/helper/parseutil/header_test.go similarity index 95% rename from logical/framework/field_type_test.go rename to helper/parseutil/header_test.go index 6265873520ee..cb0b9d26b797 100644 --- a/logical/framework/field_type_test.go +++ b/helper/parseutil/header_test.go @@ -1,4 +1,4 @@ -package framework +package parseutil import ( "testing" diff --git a/logical/framework/backend.go b/logical/framework/backend.go index 386ce6816484..3865269d75a6 100644 --- a/logical/framework/backend.go +++ b/logical/framework/backend.go @@ -549,7 +549,7 @@ func (t FieldType) Zero() interface{} { case TypeCommaIntSlice: return []int{} case TypeHeader: - return NewHeader() + return parseutil.NewHeader() default: panic("unknown type: " + t.String()) } diff --git a/logical/framework/backend_test.go b/logical/framework/backend_test.go index 5b843e758a43..9fb31818b5a0 100644 --- a/logical/framework/backend_test.go +++ b/logical/framework/backend_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/hashicorp/vault/helper/parseutil" "github.com/hashicorp/vault/logical" ) @@ -535,7 +536,7 @@ func TestFieldSchemaDefaultOrZero(t *testing.T) { "default header not set": { &FieldSchema{Type: TypeHeader}, - NewHeader(), + parseutil.NewHeader(), }, } diff --git a/logical/framework/field_data.go b/logical/framework/field_data.go index fc5021a21ad6..2e552c32a6ae 100644 --- a/logical/framework/field_data.go +++ b/logical/framework/field_data.go @@ -295,7 +295,7 @@ func (d *FieldData) getPrimitive(k string, schema *FieldSchema) (interface{}, bo // First try to parse this as a map var mapResult map[string][]string if err := mapstructure.WeakDecode(raw, &mapResult); err == nil { - result := NewHeader() + result := parseutil.NewHeader() for k, slice := range mapResult { for _, v := range slice { result.Add(k, v) @@ -310,7 +310,7 @@ func (d *FieldData) getPrimitive(k string, schema *FieldSchema) (interface{}, bo return nil, true, err } - result := NewHeader() + result := parseutil.NewHeader() for _, keyPair := range listResult { keyPairSlice := strings.SplitN(keyPair, ":", 2) if len(keyPairSlice) != 2 || keyPairSlice[0] == "" { diff --git a/logical/framework/field_data_test.go b/logical/framework/field_data_test.go index 49272bd57c71..bbbca94c3657 100644 --- a/logical/framework/field_data_test.go +++ b/logical/framework/field_data_test.go @@ -4,6 +4,8 @@ import ( "net/http" "reflect" "testing" + + "github.com/hashicorp/vault/helper/parseutil" ) func TestFieldDataGet(t *testing.T) { @@ -476,7 +478,7 @@ func TestFieldDataGet(t *testing.T) { "foo": []interface{}{"key1:value1", "key2:value2", "key3:1"}, }, "foo", - Header{http.Header{ + parseutil.Header{http.Header{ "Key1": []string{"value1"}, "Key2": []string{"value2"}, "Key3": []string{"1"}, @@ -492,7 +494,7 @@ func TestFieldDataGet(t *testing.T) { "foo": []interface{}{"key1:value1", "key2:value2", "key3:1", "key3:true"}, }, "foo", - Header{http.Header{ + parseutil.Header{http.Header{ "Key1": []string{"value1"}, "Key2": []string{"value2"}, "Key3": []string{"1", "true"}, @@ -511,7 +513,7 @@ func TestFieldDataGet(t *testing.T) { }, }, "foo", - Header{http.Header{ + parseutil.Header{http.Header{ "Key1": []string{"value1"}, "Key2": []string{"value2"}, "Key3": []string{"1"}, @@ -614,7 +616,7 @@ func TestFieldDataGet(t *testing.T) { }, map[string]interface{}{}, "foo", - Header{http.Header{}}, + parseutil.Header{http.Header{}}, }, } diff --git a/logical/framework/field_type.go b/logical/framework/field_type.go index 8ca162fe1a22..64a6a56dc08c 100644 --- a/logical/framework/field_type.go +++ b/logical/framework/field_type.go @@ -1,10 +1,5 @@ package framework -import ( - "net/http" - "net/textproto" -) - // FieldType is the enum of types that a field can be. type FieldType uint @@ -81,44 +76,3 @@ func (t FieldType) String() string { return "unknown type" } } - -/* - The reason we wrap an http.Header here is - under the hood, - an http.Header is a map[string][]string and it does some magic - with casing that can make it difficult to correctly iterate - all header values. - - For example: - h := http.Header{} - h.Add("hello", "world") - h.Add("hello", "monde") - - You'd expect that the header now have "hello" for a key, and - both "world" and "monde" for a value. But it doesn't. "Hello" - is now capitalized in the map, but "world" and "monde" aren't. - - Later, when you do this: - h.Get("hello") - - You'll receive only "world", silently missing "monde". - - If you try to solve that by doing this: - h["hello"] - - You'll receive nothing back, because remember, it's now in the - map as "Hello". - - To avoid bugs like this, we provide one more method. GetAll, - which returns all the values for a key. -*/ -func NewHeader() Header { - return Header{http.Header{}} -} - -type Header struct { - http.Header -} - -func (h Header) GetAll(key string) []string { - return h.Header[textproto.CanonicalMIMEHeaderKey(key)] -} From f0443e0afeca8fede17ccfac41b9f389db53466e Mon Sep 17 00:00:00 2001 From: Becca Petrin Date: Tue, 7 Aug 2018 14:58:10 -0700 Subject: [PATCH 08/11] use http.Header type --- helper/parseutil/header.go | 47 ---------------------------- helper/parseutil/header_test.go | 21 ------------- logical/framework/backend.go | 3 +- logical/framework/backend_test.go | 5 +-- logical/framework/field_data.go | 5 +-- logical/framework/field_data_test.go | 15 ++++----- 6 files changed, 14 insertions(+), 82 deletions(-) delete mode 100644 helper/parseutil/header.go delete mode 100644 helper/parseutil/header_test.go diff --git a/helper/parseutil/header.go b/helper/parseutil/header.go deleted file mode 100644 index f0a6172d7d69..000000000000 --- a/helper/parseutil/header.go +++ /dev/null @@ -1,47 +0,0 @@ -package parseutil - -import ( - "net/http" - "net/textproto" -) - -/* - The reason we wrap an http.Header here is - under the hood, - an http.Header is a map[string][]string and it does some magic - with casing that can make it difficult to correctly iterate - all header values. - - For example: - h := http.Header{} - h.Add("hello", "world") - h.Add("hello", "monde") - - You'd expect that the header now have "hello" for a key, and - both "world" and "monde" for a value. But it doesn't. "Hello" - is now capitalized in the map, but "world" and "monde" aren't. - - Later, when you do this: - h.Get("hello") - - You'll receive only "world", silently missing "monde". - - If you try to solve that by doing this: - h["hello"] - - You'll receive nothing back, because remember, it's now in the - map as "Hello". - - To avoid bugs like this, we provide one more method. GetAll, - which returns all the values for a key. -*/ -func NewHeader() Header { - return Header{http.Header{}} -} - -type Header struct { - http.Header -} - -func (h Header) GetAll(key string) []string { - return h.Header[textproto.CanonicalMIMEHeaderKey(key)] -} diff --git a/helper/parseutil/header_test.go b/helper/parseutil/header_test.go deleted file mode 100644 index cb0b9d26b797..000000000000 --- a/helper/parseutil/header_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package parseutil - -import ( - "testing" -) - -func TestHeaderGetAll(t *testing.T) { - h := NewHeader() - h.Add("hello", "world") - h.Add("hello", "monde") - results := h.GetAll("hello") - if len(results) != 2 { - t.Fatal("expected 2 results") - } - if results[0] != "world" { - t.Fatal("expected the first result to be world") - } - if results[1] != "monde" { - t.Fatal("expected the second result to be monde") - } -} diff --git a/logical/framework/backend.go b/logical/framework/backend.go index 3865269d75a6..4c787cbed1fb 100644 --- a/logical/framework/backend.go +++ b/logical/framework/backend.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net/http" "regexp" "sort" "strings" @@ -549,7 +550,7 @@ func (t FieldType) Zero() interface{} { case TypeCommaIntSlice: return []int{} case TypeHeader: - return parseutil.NewHeader() + return http.Header{} default: panic("unknown type: " + t.String()) } diff --git a/logical/framework/backend_test.go b/logical/framework/backend_test.go index 9fb31818b5a0..09c6f9045485 100644 --- a/logical/framework/backend_test.go +++ b/logical/framework/backend_test.go @@ -7,7 +7,8 @@ import ( "testing" "time" - "github.com/hashicorp/vault/helper/parseutil" + "net/http" + "github.com/hashicorp/vault/logical" ) @@ -536,7 +537,7 @@ func TestFieldSchemaDefaultOrZero(t *testing.T) { "default header not set": { &FieldSchema{Type: TypeHeader}, - parseutil.NewHeader(), + http.Header{}, }, } diff --git a/logical/framework/field_data.go b/logical/framework/field_data.go index 2e552c32a6ae..d9cdfcb60e0f 100644 --- a/logical/framework/field_data.go +++ b/logical/framework/field_data.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "net/http" "regexp" "strings" @@ -295,7 +296,7 @@ func (d *FieldData) getPrimitive(k string, schema *FieldSchema) (interface{}, bo // First try to parse this as a map var mapResult map[string][]string if err := mapstructure.WeakDecode(raw, &mapResult); err == nil { - result := parseutil.NewHeader() + result := http.Header{} for k, slice := range mapResult { for _, v := range slice { result.Add(k, v) @@ -310,7 +311,7 @@ func (d *FieldData) getPrimitive(k string, schema *FieldSchema) (interface{}, bo return nil, true, err } - result := parseutil.NewHeader() + result := http.Header{} for _, keyPair := range listResult { keyPairSlice := strings.SplitN(keyPair, ":", 2) if len(keyPairSlice) != 2 || keyPairSlice[0] == "" { diff --git a/logical/framework/field_data_test.go b/logical/framework/field_data_test.go index bbbca94c3657..ba9ad01be23b 100644 --- a/logical/framework/field_data_test.go +++ b/logical/framework/field_data_test.go @@ -4,8 +4,6 @@ import ( "net/http" "reflect" "testing" - - "github.com/hashicorp/vault/helper/parseutil" ) func TestFieldDataGet(t *testing.T) { @@ -478,12 +476,11 @@ func TestFieldDataGet(t *testing.T) { "foo": []interface{}{"key1:value1", "key2:value2", "key3:1"}, }, "foo", - parseutil.Header{http.Header{ + http.Header{ "Key1": []string{"value1"}, "Key2": []string{"value2"}, "Key3": []string{"1"}, }, - }, }, "type header, keypair string array with dupe key": { @@ -494,11 +491,11 @@ func TestFieldDataGet(t *testing.T) { "foo": []interface{}{"key1:value1", "key2:value2", "key3:1", "key3:true"}, }, "foo", - parseutil.Header{http.Header{ + http.Header{ "Key1": []string{"value1"}, "Key2": []string{"value2"}, "Key3": []string{"1", "true"}, - }}, + }, }, "type header, map string slice": { @@ -513,11 +510,11 @@ func TestFieldDataGet(t *testing.T) { }, }, "foo", - parseutil.Header{http.Header{ + http.Header{ "Key1": []string{"value1"}, "Key2": []string{"value2"}, "Key3": []string{"1"}, - }}, + }, }, "name string type, not supplied": { @@ -616,7 +613,7 @@ func TestFieldDataGet(t *testing.T) { }, map[string]interface{}{}, "foo", - parseutil.Header{http.Header{}}, + http.Header{}, }, } From b1cd5204391dd6297a96ae3835acdce20b08e232 Mon Sep 17 00:00:00 2001 From: Becca Petrin Date: Thu, 9 Aug 2018 13:45:14 -0700 Subject: [PATCH 09/11] add support for b64-encoded json headers --- logical/framework/field_data.go | 45 +++++++++++++++++++--------- logical/framework/field_data_test.go | 18 +++++++++++ 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/logical/framework/field_data.go b/logical/framework/field_data.go index d9cdfcb60e0f..fc412dfebcea 100644 --- a/logical/framework/field_data.go +++ b/logical/framework/field_data.go @@ -1,6 +1,8 @@ package framework import ( + "bytes" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -293,10 +295,27 @@ func (d *FieldData) getPrimitive(k string, schema *FieldSchema) (interface{}, bo return result, true, nil case TypeHeader: - // First try to parse this as a map + // There are 3 ways a header could be provided: + // 1. Marshalled as JSON, and then converted to a b64 string. + // 2. As a map[string][]string. + // 3. As comma-delimited key-value pairs associated with a colon. + // We go through these sequentially below. + result := http.Header{} + + // 1. + var headerStr string + if err := mapstructure.WeakDecode(raw, &headerStr); err == nil { + if b, err := base64.StdEncoding.DecodeString(headerStr); err == nil { + if err := json.NewDecoder(bytes.NewReader(b)).Decode(&result); err != nil { + return nil, true, err + } + return result, true, nil + } + } + + // 2. var mapResult map[string][]string if err := mapstructure.WeakDecode(raw, &mapResult); err == nil { - result := http.Header{} for k, slice := range mapResult { for _, v := range slice { result.Add(k, v) @@ -305,21 +324,19 @@ func (d *FieldData) getPrimitive(k string, schema *FieldSchema) (interface{}, bo return result, true, nil } - // If map parse fails, parse as a string list of : delimited pairs + // 3. var listResult []string - if err := mapstructure.WeakDecode(raw, &listResult); err != nil { - return nil, true, err - } - - result := http.Header{} - for _, keyPair := range listResult { - keyPairSlice := strings.SplitN(keyPair, ":", 2) - if len(keyPairSlice) != 2 || keyPairSlice[0] == "" { - return nil, false, fmt.Errorf("invalid key pair %q", keyPair) + if err := mapstructure.WeakDecode(raw, &listResult); err == nil { + for _, keyPair := range listResult { + keyPairSlice := strings.SplitN(keyPair, ":", 2) + if len(keyPairSlice) != 2 || keyPairSlice[0] == "" { + return nil, true, fmt.Errorf("invalid key pair %q", keyPair) + } + result.Add(keyPairSlice[0], keyPairSlice[1]) } - result.Add(keyPairSlice[0], keyPairSlice[1]) + return result, true, nil } - return result, true, nil + return nil, true, fmt.Errorf("%s not provided an expected format", raw) default: panic(fmt.Sprintf("Unknown type: %s", schema.Type)) diff --git a/logical/framework/field_data_test.go b/logical/framework/field_data_test.go index ba9ad01be23b..e55e423ab417 100644 --- a/logical/framework/field_data_test.go +++ b/logical/framework/field_data_test.go @@ -483,6 +483,24 @@ func TestFieldDataGet(t *testing.T) { }, }, + "type header, b64 string": { + map[string]*FieldSchema{ + "foo": {Type: TypeHeader}, + }, + map[string]interface{}{ + "foo": "eyJDb250ZW50LUxlbmd0aCI6IFsiNDMiXSwgIlVzZXItQWdlbnQiOiBbImF3cy1zZGstZ28vMS40LjEyIChnbzEuNy4xOyBsaW51eDsgYW1kNjQpIl0sICJYLVZhdWx0LUFXU0lBTS1TZXJ2ZXItSWQiOiBbInZhdWx0LmV4YW1wbGUuY29tIl0sICJYLUFtei1EYXRlIjogWyIyMDE2MDkzMFQwNDMxMjFaIl0sICJDb250ZW50LVR5cGUiOiBbImFwcGxpY2F0aW9uL3gtd3d3LWZvcm0tdXJsZW5jb2RlZDsgY2hhcnNldD11dGYtOCJdLCAiQXV0aG9yaXphdGlvbiI6IFsiQVdTNC1ITUFDLVNIQTI1NiBDcmVkZW50aWFsPWZvby8yMDE2MDkzMC91cy1lYXN0LTEvc3RzL2F3czRfcmVxdWVzdCwgU2lnbmVkSGVhZGVycz1jb250ZW50LWxlbmd0aDtjb250ZW50LXR5cGU7aG9zdDt4LWFtei1kYXRlO3gtdmF1bHQtc2VydmVyLCBTaWduYXR1cmU9YTY5ZmQ3NTBhMzQ0NWM0ZTU1M2UxYjNlNzlkM2RhOTBlZWY1NDA0N2YxZWI0ZWZlOGZmYmM5YzQyOGMyNjU1YiJdfQ==", + }, + "foo", + http.Header{ + "Content-Length": []string{"43"}, + "User-Agent": []string{"aws-sdk-go/1.4.12 (go1.7.1; linux; amd64)"}, + "X-Vault-AWSIAM-Server-Id": []string{"vault.example.com"}, + "X-Amz-Date": []string{"20160930T043121Z"}, + "Content-Type": []string{"application/x-www-form-urlencoded; charset=utf-8"}, + "Authorization": []string{"AWS4-HMAC-SHA256 Credential=foo/20160930/us-east-1/sts/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date;x-vault-server, Signature=a69fd750a3445c4e553e1b3e79d3da90eef54047f1eb4efe8ffbc9c428c2655b"}, + }, + }, + "type header, keypair string array with dupe key": { map[string]*FieldSchema{ "foo": {Type: TypeHeader}, From ab5a9065a6267f082647b7839ad7ff5d475ba1a4 Mon Sep 17 00:00:00 2001 From: Becca Petrin Date: Fri, 10 Aug 2018 11:06:24 -0700 Subject: [PATCH 10/11] support headers as json strings or b64d json --- logical/framework/field_data.go | 99 +++++++++++++++++++++------- logical/framework/field_data_test.go | 18 ++++- 2 files changed, 92 insertions(+), 25 deletions(-) diff --git a/logical/framework/field_data.go b/logical/framework/field_data.go index fc412dfebcea..7c76bd9f008a 100644 --- a/logical/framework/field_data.go +++ b/logical/framework/field_data.go @@ -295,39 +295,92 @@ func (d *FieldData) getPrimitive(k string, schema *FieldSchema) (interface{}, bo return result, true, nil case TypeHeader: - // There are 3 ways a header could be provided: - // 1. Marshalled as JSON, and then converted to a b64 string. - // 2. As a map[string][]string. - // 3. As comma-delimited key-value pairs associated with a colon. - // We go through these sequentially below. + /* + + There are multiple ways a header could be provided: + + 1. As a map[string]interface{} that resolves to a map[string]string or map[string][]string, or a mix of both + because that's permitted for headers. + This mainly comes from the API. + + 2. As a string... + a. That contains JSON that originally was JSON, but then was base64 encoded. + b. That contains JSON, ex. `{"content-type":"text/json","accept":["encoding/json"]}`. + This mainly comes from the API and is used to save space while sending in the header. + + 3. As an array of strings that contains comma-delimited key-value pairs associated via a colon, + ex: `content-type:text`,`json,accept:encoding/json`. + This mainly comes from the CLI. + + We go through these sequentially below. + + */ result := http.Header{} - // 1. - var headerStr string - if err := mapstructure.WeakDecode(raw, &headerStr); err == nil { - if b, err := base64.StdEncoding.DecodeString(headerStr); err == nil { - if err := json.NewDecoder(bytes.NewReader(b)).Decode(&result); err != nil { - return nil, true, err + toHeader := func(resultMap map[string]interface{}) (http.Header, error) { + header := http.Header{} + for headerKey, headerValGroup := range resultMap { + switch typedHeader := headerValGroup.(type) { + case string: + header.Add(headerKey, typedHeader) + case []string: + for _, headerVal := range typedHeader { + header.Add(headerKey, headerVal) + } + case []interface{}: + for _, headerVal := range typedHeader { + strHeaderVal, ok := headerVal.(string) + if !ok { + // All header values should already be strings when they're being sent in. + // Even numbers and booleans will be treated as strings. + return nil, fmt.Errorf("received non-string value for header key:%s, val:%s", headerKey, headerValGroup) + } + header.Add(headerKey, strHeaderVal) + } + default: + return nil, fmt.Errorf("unrecognized type for %s", headerValGroup) } - return result, true, nil } + return header, nil } - // 2. - var mapResult map[string][]string - if err := mapstructure.WeakDecode(raw, &mapResult); err == nil { - for k, slice := range mapResult { - for _, v := range slice { - result.Add(k, v) - } + resultMap := make(map[string]interface{}) + + // 1. Are we getting a map from the API? + if err := mapstructure.WeakDecode(raw, &resultMap); err == nil { + result, err = toHeader(resultMap) + if err != nil { + return nil, true, err } return result, true, nil } - // 3. - var listResult []string - if err := mapstructure.WeakDecode(raw, &listResult); err == nil { - for _, keyPair := range listResult { + // 2. Are we getting a JSON string? + if headerStr, ok := raw.(string); ok { + // a. Is it base64 encoded? + headerBytes, err := base64.StdEncoding.DecodeString(headerStr) + if err != nil { + // b. It's not base64 encoded, it's a straight-out JSON string. + headerBytes = []byte(headerStr) + } + if err := json.NewDecoder(bytes.NewReader(headerBytes)).Decode(&resultMap); err != nil { + return nil, true, err + } + result, err = toHeader(resultMap) + if err != nil { + return nil, true, err + } + return result, true, nil + } + + // 3. Are we getting an array of fields like "content-type:encoding/json" from the CLI? + var keyPairs []interface{} + if err := mapstructure.WeakDecode(raw, &keyPairs); err == nil { + for _, keyPairIfc := range keyPairs { + keyPair, ok := keyPairIfc.(string) + if !ok { + return nil, true, fmt.Errorf("invalid key pair %q", keyPair) + } keyPairSlice := strings.SplitN(keyPair, ":", 2) if len(keyPairSlice) != 2 || keyPairSlice[0] == "" { return nil, true, fmt.Errorf("invalid key pair %q", keyPair) diff --git a/logical/framework/field_data_test.go b/logical/framework/field_data_test.go index e55e423ab417..f4763af2aed6 100644 --- a/logical/framework/field_data_test.go +++ b/logical/framework/field_data_test.go @@ -494,13 +494,27 @@ func TestFieldDataGet(t *testing.T) { http.Header{ "Content-Length": []string{"43"}, "User-Agent": []string{"aws-sdk-go/1.4.12 (go1.7.1; linux; amd64)"}, - "X-Vault-AWSIAM-Server-Id": []string{"vault.example.com"}, + "X-Vault-Awsiam-Server-Id": []string{"vault.example.com"}, "X-Amz-Date": []string{"20160930T043121Z"}, "Content-Type": []string{"application/x-www-form-urlencoded; charset=utf-8"}, "Authorization": []string{"AWS4-HMAC-SHA256 Credential=foo/20160930/us-east-1/sts/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date;x-vault-server, Signature=a69fd750a3445c4e553e1b3e79d3da90eef54047f1eb4efe8ffbc9c428c2655b"}, }, }, + "type header, json string": { + map[string]*FieldSchema{ + "foo": {Type: TypeHeader}, + }, + map[string]interface{}{ + "foo": `{"hello":"world","bonjour":["monde","dieu"]}`, + }, + "foo", + http.Header{ + "Hello": []string{"world"}, + "Bonjour": []string{"monde", "dieu"}, + }, + }, + "type header, keypair string array with dupe key": { map[string]*FieldSchema{ "foo": {Type: TypeHeader}, @@ -642,7 +656,7 @@ func TestFieldDataGet(t *testing.T) { } if err := data.Validate(); err != nil { - t.Fatalf("bad: %#v", err) + t.Fatalf("bad: %s", err) } actual := data.Get(tc.Key) From f486653dc889d358ed4fcec7ae05555113c4e215 Mon Sep 17 00:00:00 2001 From: Becca Petrin Date: Fri, 10 Aug 2018 11:10:56 -0700 Subject: [PATCH 11/11] fix comment typo --- logical/framework/field_data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logical/framework/field_data.go b/logical/framework/field_data.go index 7c76bd9f008a..2ee529116d20 100644 --- a/logical/framework/field_data.go +++ b/logical/framework/field_data.go @@ -309,7 +309,7 @@ func (d *FieldData) getPrimitive(k string, schema *FieldSchema) (interface{}, bo This mainly comes from the API and is used to save space while sending in the header. 3. As an array of strings that contains comma-delimited key-value pairs associated via a colon, - ex: `content-type:text`,`json,accept:encoding/json`. + ex: `content-type:text/json`,`accept:encoding/json`. This mainly comes from the CLI. We go through these sequentially below.