Merge pull request #64 from grount/issue-63

avoid type loss with JSON unmarshal/marshal

getters.go

@@ -36,6 +36,21 @@ func (ko *Koanf) Int64s(path string) []int64 {
 
 	var out []int64
 	switch v := o.(type) {
+	case []int64:
+		return v
+	case []int:
+		out = make([]int64, 0, len(v))
+		for _, vi := range v {
+			i, err := toInt64(vi)
+
+			// On error, return as it's not a valid
+			// int slice.
+			if err != nil {
+				return []int64{}
+			}
+			out = append(out, i)
+		}
+		return out
 	case []interface{}:
 		out = make([]int64, 0, len(v))
 		for _, vi := range v {
@@ -128,16 +143,37 @@ func (ko *Koanf) MustInt(path string) int {
 // empty []int slice if the path does not exist or if the value
 // is not a valid int slice.
 func (ko *Koanf) Ints(path string) []int {
-	ints := ko.Int64s(path)
-	if len(ints) == 0 {
+	o := ko.Get(path)
+	if o == nil {
 		return []int{}
 	}
 
-	out := make([]int, len(ints))
-	for i, v := range ints {
-		out[i] = int(v)
+	var out []int
+	switch v := o.(type) {
+	case []int:
+		return v
+	case []int64:
+		out = make([]int, 0, len(v))
+		for _, vi := range v {
+			out = append(out, int(vi))
+		}
+		return out
+	case []interface{}:
+		out = make([]int, 0, len(v))
+		for _, vi := range v {
+			i, err := toInt64(vi)
+
+			// On error, return as it's not a valid
+			// int slice.
+			if err != nil {
+				return []int{}
+			}
+			out = append(out, int(i))
+		}
+		return out
 	}
-	return out
+
+	return []int{}
 }
 
 // MustInts returns the []int slice value of a given key path or panics
@@ -205,6 +241,8 @@ func (ko *Koanf) Float64s(path string) []float64 {
 
 	var out []float64
 	switch v := o.(type) {
+	case []float64:
+		return v
 	case []interface{}:
 		out = make([]float64, 0, len(v))
 		for _, vi := range v {
@@ -215,7 +253,7 @@ func (ko *Koanf) Float64s(path string) []float64 {
 			if err != nil {
 				return []float64{}
 			}
-			out = append(out, float64(i))
+			out = append(out, i)
 		}
 		return out
 	}
@@ -379,6 +417,7 @@ func (ko *Koanf) Strings(path string) []string {
 		copy(out[:], v[:])
 		return out
 	}
+
 	return []string{}
 }
 

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/hashicorp/hcl v1.0.0
 	github.com/hashicorp/vault/api v1.0.4
 	github.com/joho/godotenv v1.3.0
+	github.com/mitchellh/copystructure v1.1.1
 	github.com/mitchellh/mapstructure v1.2.2
 	github.com/pelletier/go-toml v1.7.0
 	github.com/rhnvrm/simples3 v0.6.1

go.sum

@@ -58,6 +58,8 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
+github.com/mitchellh/copystructure v1.1.1 h1:Bp6x9R1Wn16SIz3OfeDr0b7RnCG2OB66Y7PQyC/cvq4=
+github.com/mitchellh/copystructure v1.1.1/go.mod h1:EBArHfARyrSWO/+Wyr9zwEkc6XMFB9XyNgFNmRkZZU4=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
@@ -67,6 +69,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
 github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc35r0TV4=
 github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE=
+github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI=

koanf.go

@@ -2,8 +2,8 @@ package koanf
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
+	"github.com/mitchellh/copystructure"
 	"sort"
 	"strconv"
 	"strings"
@@ -300,11 +300,10 @@ func (ko *Koanf) Get(path string) interface{} {
 		return maps.Copy(v)
 	}
 
-	// Inefficient, but marshal and unmarshal to create a copy
-	// of reference types to not expose  internal references to slices and maps.
-	var out interface{}
-	b, _ := json.Marshal(res)
-	json.Unmarshal(b, &out)
+	out, _ := copystructure.Copy(&res)
+	if ptrOut, ok := out.(*interface{}); ok {
+		return *ptrOut
+	}
 	return out
 }
 

koanf_test.go

@@ -323,6 +323,58 @@ func TestLoadFileAllKeys(t *testing.T) {
 	}
 }
 
+func TestLoadMergeYamlJson(t *testing.T) {
+	var (
+		assert = assert.New(t)
+		k      = koanf.New(delim)
+	)
+
+	assert.NoError(k.Load(file.Provider(mockYAML), yaml.Parser()),
+		"error loading file")
+	// loading json after yaml causes the intbools to be loaded as []float64
+	assert.NoError(k.Load(file.Provider(mockJSON), yaml.Parser()),
+		"error loading file")
+
+	// checking that there is no issues with expecting it to be an []int64
+	v := k.Int64s("intbools")
+	assert.Len(v, 3)
+
+	defer func() {
+		if err := recover(); err != nil {
+			assert.Failf("panic", "received panic: %v", err)
+		}
+	}()
+
+	v2 := k.MustInt64s("intbools")
+	assert.Len(v2, 3)
+}
+
+func TestLoadMergeJsonYaml(t *testing.T) {
+	var (
+		assert = assert.New(t)
+		k      = koanf.New(delim)
+	)
+
+	assert.NoError(k.Load(file.Provider(mockJSON), yaml.Parser()),
+		"error loading file")
+	// loading yaml after json causes the intbools to be loaded as []int after json loaded it with []float64
+	assert.NoError(k.Load(file.Provider(mockYAML), yaml.Parser()),
+		"error loading file")
+
+	// checking that there is no issues with expecting it to be an []float64
+	v := k.Float64s("intbools")
+	assert.Len(v, 3)
+
+	defer func() {
+		if err := recover(); err != nil {
+			assert.Failf("panic", "received panic: %v", err)
+		}
+	}()
+
+	v2 := k.MustFloat64s("intbools")
+	assert.Len(v2, 3)
+}
+
 func TestWatchFile(t *testing.T) {
 	var (
 		assert = assert.New(t)
@@ -614,13 +666,59 @@ func TestMerge(t *testing.T) {
 	assert.Equal(cut1.All(), k2.All(), "conf map mismatch")
 }
 
-func TestMergeAt(t *testing.T) {
+func TestRaw_YamlTypes(t *testing.T) {
 	var (
 		assert = assert.New(t)
 		k      = koanf.New(delim)
 	)
+
+	assert.Nil(k.Load(file.Provider(mockYAML), yaml.Parser()),
+		"error loading file")
+	raw := k.Raw()
+
+	i, ok := raw["intbools"]
+	assert.True(ok, "ints key does not exist in the map")
+
+	arr, ok := i.([]interface{})
+	assert.True(ok, "arr slice is not array of integers")
+
+	for _, integer := range arr {
+		if _, ok := integer.(int); !ok {
+			assert.Failf("failure", "%v not an integer but %T", integer, integer)
+		}
+	}
+}
+
+func TestRaw_JsonTypes(t *testing.T) {
+	var (
+		assert = assert.New(t)
+		k      = koanf.New(delim)
+	)
+
 	assert.Nil(k.Load(file.Provider(mockJSON), json.Parser()),
 		"error loading file")
+	raw := k.Raw()
+
+	i, ok := raw["intbools"]
+	assert.True(ok, "ints key does not exist in the map")
+
+	arr, ok := i.([]interface{})
+	assert.True(ok, "arr slice is not array of integers")
+
+	for _, integer := range arr {
+		if _, ok := integer.(float64); !ok {
+			assert.Failf("failure", "%v not an integer but %T", integer, integer)
+		}
+	}
+}
+
+func TestMergeAt(t *testing.T) {
+	var (
+		assert = assert.New(t)
+		k      = koanf.New(delim)
+	)
+	assert.Nil(k.Load(file.Provider(mockYAML), yaml.Parser()),
+		"error loading file")
 
 	// Get expected koanf, and root data
 	var (

maps/maps.go

@@ -4,8 +4,8 @@
 package maps
 
 import (
-	"encoding/json"
 	"fmt"
+	"github.com/mitchellh/copystructure"
 	"strings"
 )
 
@@ -182,18 +182,17 @@ func Search(mp map[string]interface{}, path []string) interface{} {
 	return nil
 }
 
-// Copy returns a copy of a conf map by doing a JSON marshal+unmarshal
-// pass. Inefficient, but creates a true deep copy. There is a side
-// effect, that is, all numeric types change to float64.
+// Copy returns a deep copy of a conf map.
 //
 // It's important to note that all nested maps should be
 // map[string]interface{} and not map[interface{}]interface{}.
 // Use IntfaceKeysToStrings() to convert if necessary.
 func Copy(mp map[string]interface{}) map[string]interface{} {
-	var out map[string]interface{}
-	b, _ := json.Marshal(mp)
-	json.Unmarshal(b, &out)
-	return out
+	out, _ := copystructure.Copy(&mp)
+	if res, ok := out.(*map[string]interface{}); ok {
+		return *res
+	}
+	return map[string]interface{}{}
 }
 
 // IntfaceKeysToStrings recursively converts map[interface{}]interface{} to