Support parsing nested JSON properties and schema check for nested properties

api/go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/eclipse-symphony/symphony/packages/mage v0.0.0-00010101000000-000000000000
 	github.com/eclipse/paho.mqtt.golang v1.4.2
 	github.com/fsnotify/fsnotify v1.7.0
+	github.com/itchyny/gojq v0.12.16
 	github.com/princjef/mageutil v1.0.0
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
 	helm.sh/helm/v3 v3.14.4
@@ -47,6 +48,7 @@ require (
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/itchyny/timefmt-go v0.1.6 // indirect
 	github.com/karrick/godirwalk v1.17.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/magefile/mage v1.15.0 // indirect
@@ -57,6 +59,7 @@ require (
 	github.com/onsi/gomega v1.31.0 // indirect
 	github.com/openzipkin/zipkin-go v0.4.1 // indirect
 	github.com/redis/go-redis/v9 v9.5.3 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	go.opentelemetry.io/contrib/bridges/otellogrus v0.3.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
@@ -122,8 +125,8 @@ require (
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.17 // indirect
-	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect

api/go.sum

@@ -254,6 +254,10 @@ github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
 github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/itchyny/gojq v0.12.16 h1:yLfgLxhIr/6sJNVmYfQjTIv0jGctu6/DgDoivmxTr7g=
+github.com/itchyny/gojq v0.12.16/go.mod h1:6abHbdC2uB9ogMS38XsErnfqJ94UlngIJGlRAIj4jTM=
+github.com/itchyny/timefmt-go v0.1.6 h1:ia3s54iciXDdzWzwaVKXZPbiXzxxnv1SPGFfM/myJ5Q=
+github.com/itchyny/timefmt-go v0.1.6/go.mod h1:RRDZYC5s9ErkjQvTvvU7keJjxUYzIISJGxm9/mAERQg=
 github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
 github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -313,11 +317,11 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
-github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
-github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI=
 github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
@@ -411,6 +415,9 @@ github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+Pymzi
 github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
 github.com/redis/go-redis/v9 v9.5.3 h1:fOAp1/uJG+ZtcITgZOfYFmTKPE7n4Vclj1wZFgRciUU=
 github.com/redis/go-redis/v9 v9.5.3/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/rubenv/sql-migrate v1.5.2 h1:bMDqOnrJVV/6JQgQ/MxOpU+AdO8uzYYA/TxFUBzFtS0=
@@ -587,6 +594,7 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
 golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

api/pkg/apis/v1alpha1/providers/config/catalog/catalogprovider.go

@@ -14,7 +14,6 @@ import (
 
 	"github.com/eclipse-symphony/symphony/api/pkg/apis/v1alpha1/model"
 	"github.com/eclipse-symphony/symphony/api/pkg/apis/v1alpha1/utils"
-	api_utils "github.com/eclipse-symphony/symphony/api/pkg/apis/v1alpha1/utils"
 	"github.com/eclipse-symphony/symphony/coa/pkg/apis/v1alpha2"
 	"github.com/eclipse-symphony/symphony/coa/pkg/apis/v1alpha2/contexts"
 	"github.com/eclipse-symphony/symphony/coa/pkg/apis/v1alpha2/providers"
@@ -90,12 +89,12 @@ func CatalogConfigProviderConfigFromMap(properties map[string]string) (CatalogCo
 	return ret, nil
 }
 func (m *CatalogConfigProvider) unwindOverrides(override string, field string, namespace string) (string, error) {
-	override = api_utils.ReplaceSeperator(override)
+	override = utils.ReplaceSeperator(override)
 	catalog, err := m.ApiClient.GetCatalog(context.TODO(), override, namespace, m.Config.User, m.Config.Password)
 	if err != nil {
 		return "", err
 	}
-	if v, ok := catalog.Spec.Properties[field]; ok {
+	if v, ok := utils.JsonParseProperty(catalog.Spec.Properties, field); ok {
 		if vstring, ok := v.(string); ok {
 			return vstring, nil
 		} else {
@@ -110,13 +109,13 @@ func (m *CatalogConfigProvider) unwindOverrides(override string, field string, n
 func (m *CatalogConfigProvider) Read(object string, field string, localcontext interface{}) (interface{}, error) {
 	clog.Debug(" M (Catalog): Read, object: %s, field: %s", object, field)
 	namespace := utils.GetNamespaceFromContext(localcontext)
-	object = api_utils.ReplaceSeperator(object)
+	object = utils.ReplaceSeperator(object)
 	catalog, err := m.ApiClient.GetCatalog(context.TODO(), object, namespace, m.Config.User, m.Config.Password)
 	if err != nil {
 		return "", err
 	}
 
-	if v, ok := catalog.Spec.Properties[field]; ok {
+	if v, ok := utils.JsonParseProperty(catalog.Spec.Properties, field); ok {
 		return m.traceValue(v, localcontext)
 	}
 
@@ -135,7 +134,7 @@ func (m *CatalogConfigProvider) Read(object string, field string, localcontext i
 func (m *CatalogConfigProvider) ReadObject(object string, localcontext interface{}) (map[string]interface{}, error) {
 	clog.Debug(" M (Catalog): ReadObject, object: %s", object)
 	namespace := utils.GetNamespaceFromContext(localcontext)
-	object = api_utils.ReplaceSeperator(object)
+	object = utils.ReplaceSeperator(object)
 
 	catalog, err := m.ApiClient.GetCatalog(context.TODO(), object, namespace, m.Config.User, m.Config.Password)
 	if err != nil {
@@ -256,11 +255,11 @@ func (m *CatalogConfigProvider) Remove(object string, field string) error {
 }
 func (m *CatalogConfigProvider) RemoveObject(object string) error {
 	clog.Debug(" M (Catalog): RemoveObject, object: %s", object)
-	object = api_utils.ReplaceSeperator(object)
+	object = utils.ReplaceSeperator(object)
 	return m.ApiClient.DeleteCatalog(context.TODO(), object, m.Config.User, m.Config.Password)
 }
 
 func (m *CatalogConfigProvider) getCatalogInDefaultNamespace(context context.Context, catalog string) (model.CatalogState, error) {
-	catalog = api_utils.ReplaceSeperator(catalog)
+	catalog = utils.ReplaceSeperator(catalog)
 	return m.ApiClient.GetCatalog(context, catalog, "", m.Config.User, m.Config.Password)
 }

api/pkg/apis/v1alpha1/providers/config/catalog/catalogprovider_test.go

@@ -58,6 +58,12 @@ func TestRead(t *testing.T) {
 								Type: "type",
 							},
 						},
+						"a": map[string]interface{}{
+							"b": map[string]interface{}{
+								"c": "nested",
+							},
+						},
+						"a.b.d": "dot",
 					},
 				},
 			}
@@ -95,7 +101,7 @@ func TestRead(t *testing.T) {
 	}
 	assert.Nil(t, err)
 
-	res, err := provider.Read("catalog1:v1", "components", nil)
+	res, err := provider.Read("catalog1:v1", ".components", nil)
 	assert.Nil(t, err)
 	data, err := json.Marshal(res)
 	assert.Nil(t, err)
@@ -104,7 +110,24 @@ func TestRead(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, "name", summary[0].Name)
 
-	res, err = provider.Read("catalog1:v1", "parentAttribute", nil)
+	res, err = provider.Read("catalog1:v1", ".a.b.c", nil)
+	assert.Nil(t, err)
+	data, err = json.Marshal(res)
+	assert.Nil(t, err)
+	var val string
+	err = json.Unmarshal(data, &val)
+	assert.Nil(t, err)
+	assert.Equal(t, "nested", val)
+
+	res, err = provider.Read("catalog1:v1", "`.\"a.b.d\"`", nil)
+	assert.Nil(t, err)
+	data, err = json.Marshal(res)
+	assert.Nil(t, err)
+	err = json.Unmarshal(data, &val)
+	assert.Nil(t, err)
+	assert.Equal(t, "dot", val)
+
+	res, err = provider.Read("catalog1:v1", ".parentAttribute", nil)
 	assert.Nil(t, err)
 	v, ok := res.(string)
 	assert.True(t, ok)
@@ -159,6 +182,7 @@ func TestReadObject(t *testing.T) {
 	}))
 	defer ts.Close()
 	os.Setenv(constants.SymphonyAPIUrlEnvName, ts.URL+"/")
+	os.Setenv(constants.UseServiceAccountTokenEnvName, "false")
 	provider := CatalogConfigProvider{}
 	err := provider.Init(CatalogConfigProviderConfig{})
 	provider.Context = &contexts.ManagerContext{
@@ -211,6 +235,7 @@ func TestSetandRemove(t *testing.T) {
 	}))
 	defer ts.Close()
 	os.Setenv(constants.SymphonyAPIUrlEnvName, ts.URL+"/")
+	os.Setenv(constants.UseServiceAccountTokenEnvName, "false")
 	provider := CatalogConfigProvider{}
 	err := provider.Init(CatalogConfigProviderConfig{})
 	provider.Context = &contexts.ManagerContext{
@@ -269,6 +294,7 @@ func TestSetandRemoveObject(t *testing.T) {
 	}))
 	defer ts.Close()
 	os.Setenv(constants.SymphonyAPIUrlEnvName, ts.URL+"/")
+	os.Setenv(constants.UseServiceAccountTokenEnvName, "false")
 	provider := CatalogConfigProvider{}
 	err := provider.Init(CatalogConfigProviderConfig{})
 	provider.Context = &contexts.ManagerContext{

api/pkg/apis/v1alpha1/utils/schema.go

@@ -42,7 +42,7 @@ func (s *Schema) CheckProperties(properties map[string]interface{}, evaluationCo
 	ret := SchemaResult{Valid: true, Errors: make(map[string]RuleResult)}
 	for k, v := range s.Rules {
 		if v.Type != "" {
-			if val, ok := properties[k]; ok {
+			if val, ok := JsonParseProperty(properties, k); ok {
 				if v.Type == "int" {
 					if _, err := strconv.Atoi(FormatAsString(val)); err != nil {
 						ret.Valid = false
@@ -72,13 +72,13 @@ func (s *Schema) CheckProperties(properties map[string]interface{}, evaluationCo
 			}
 		}
 		if v.Required {
-			if _, ok := properties[k]; !ok {
+			if _, ok := JsonParseProperty(properties, k); !ok {
 				ret.Valid = false
 				ret.Errors[k] = RuleResult{Valid: false, Error: "missing required property"}
 			}
 		}
 		if v.Pattern != "" {
-			if val, ok := properties[k]; ok {
+			if val, ok := JsonParseProperty(properties, k); ok {
 				match, err := s.matchPattern(FormatAsString(val), v.Pattern)
 				if err != nil {
 					ret.Valid = false
@@ -91,7 +91,7 @@ func (s *Schema) CheckProperties(properties map[string]interface{}, evaluationCo
 			}
 		}
 		if v.Expression != "" {
-			if val, ok := properties[k]; ok {
+			if val, ok := JsonParseProperty(properties, k); ok {
 				context.Value = val
 				parser := NewParser(v.Expression)
 				res, err := parser.Eval(*context)