From cdd83d7c1bb868d36d151bd591375270db7b657a Mon Sep 17 00:00:00 2001 From: amrutasali <51424374+amrutasali@users.noreply.github.com> Date: Wed, 27 Sep 2023 15:01:25 -0700 Subject: [PATCH] Query Parameter support in transformer infra for REST GET operation (#102) * Transformer infra enhacements to support singleton container -Changes in DbSpec creation, inherit db-name from sonic table, use cvl db-name annotation for sonic yang, utility function changes -Support exclusion of sonic yangs xfmr support from models list * Sonic yang singleton container support in GET and SET/CRU code flow * Added a container in openconfig-test-xfmr.yang that maps to a sonic yang table containing a singleton container and UT cases exercising this mapping using table-name and key-name annotations * Add sonic singleton container in sonic yang and UT cases for CRUD and GET * Update unit test README file with the build tag information * Initial Query Params infra support in translib and common app * Support for depth query parameter in transformer infra * Content query parameter support in xfmr infra * Add fields query parameters support * Query parameter pruning API integration and error handling * 1)Handle query-parameter content mismatch check for list node since translib fills ygot in request iteslf and when there is onctent mismatch one cannot return that ygot and has to return empty.2)Handle singleton container case for fields query-parameter processing. * added query-parameter support in transformer test infra * fixed build error due to rebase/merge-conflict resolution * Add UT cases for depth and content query parameters support in infra * Add UT cases for depth and content query parameters support in infra for sonic yang * Add OC yang fields query parameters unit tests * Added sonic yang fields query-parameter UT cases * Infra enhancement for handling OC Yang list/config/key-leaf & list/state/key-leaf allowing application annotation to be skipped and also avoiding translating into db key-attribute in value hash * Add composite key handling changes for 1:1 OC to sonic key mapping * Run format check on test yang UT file, Move composite Key handling inside keyCreate function * when filling the list-keys in GET flow, update the list-keys map used as an optimized way to fill list-key leaves inside the config and state containers instead of extracting from the URI string --------- Co-authored-by: ranjinidn Co-authored-by: ranjinidn <51423501+ranjinidn@users.noreply.github.com> --- translib/api_tests_app.go | 2 + translib/app_interface.go | 18 +- translib/common_app.go | 40 +- translib/transformer/sflow_openconfig_test.go | 6 +- translib/transformer/sflow_sonic_test.go | 12 +- .../test/openconfig-test-xfmr-annot.yang | 18 +- .../test/openconfig-test-xfmr.yang | 125 ++- .../transformer/test/sonic-test-xfmr.yang | 23 + translib/transformer/testxfmryang_test.go | 584 +++++++++++--- translib/transformer/utils_test.go | 20 +- translib/transformer/xconst.go | 18 + translib/transformer/xfmr_interface.go | 1 + .../transformer/xfmr_testxfmr_callbacks.go | 65 +- translib/transformer/xlate.go | 76 +- translib/transformer/xlate_datastructs.go | 32 +- translib/transformer/xlate_from_db.go | 387 +++++++++- translib/transformer/xlate_to_db.go | 58 +- translib/transformer/xlate_utils.go | 724 +++++++++++++++++- translib/transformer/xlate_xfmr_handler.go | 19 + translib/transformer/xspec.go | 120 ++- translib/translib.go | 13 +- 21 files changed, 2068 insertions(+), 293 deletions(-) diff --git a/translib/api_tests_app.go b/translib/api_tests_app.go index 6b6e84249b2c..8c7525f2a40b 100644 --- a/translib/api_tests_app.go +++ b/translib/api_tests_app.go @@ -137,6 +137,8 @@ func (app *apiTests) processGet(dbs [db.MaxDB]*db.DB, fmtType TranslibFmtType) ( resp["message"] = app.echoMsg resp["path"] = app.path resp["depth"] = app.depth + resp["content"] = app.content + resp["fields"] = app.fields gr.Payload, err = json.Marshal(&resp) return gr, err diff --git a/translib/app_interface.go b/translib/app_interface.go index e15cc3a4e09d..225332bc12e0 100644 --- a/translib/app_interface.go +++ b/translib/app_interface.go @@ -60,10 +60,20 @@ type appData struct { // These include RESTCONF query parameters like - depth, fields etc. type appOptions struct { - // depth limits subtree levels in the response data. - // 0 indicates unlimited depth. - // Valid for GET API only. - depth uint + // depth limits subtree levels in the response data. + // 0 indicates unlimited depth. + // Valid for GET API only. + depth uint + + // content query parameter value receved from the URI + // possible value is one of 'config', 'nonconfig','all','state' or 'operational' + // Valid for GET API only. + content string + + //fields query parameters + // paths of the fields that needs to be filtered in GET payload response + // Valid for GET API only. + fields []string // deleteEmptyEntry indicates if the db entry should be deleted upon // deletion of last field. This is a non standard option. diff --git a/translib/common_app.go b/translib/common_app.go index 9523a32427ad..770c373acef2 100644 --- a/translib/common_app.go +++ b/translib/common_app.go @@ -419,8 +419,24 @@ func (app *CommonApp) processGet(dbs [db.MaxDB]*db.DB, fmtType TranslibFmtType) origXfmrYgotRoot, _ := ygot.DeepCopy((*app.ygotRoot).(ygot.GoStruct)) isEmptyPayload := false appYgotStruct := (*app.ygotRoot).(ygot.GoStruct) - payload, isEmptyPayload, err = transformer.GetAndXlateFromDB(app.pathInfo.Path, &appYgotStruct, dbs, txCache) + var qParams transformer.QueryParams + qParams, err = transformer.NewQueryParams(app.depth, app.content, app.fields) if err != nil { + log.Warning("transformer.NewQueryParams() returned : ", err) + resPayload = []byte("{}") + break + } + payload, isEmptyPayload, err = transformer.GetAndXlateFromDB(app.pathInfo.Path, &appYgotStruct, dbs, txCache, qParams) + if err != nil { + // target URI for list GET request with QP content!=all and node's content-type mismatches the requested content-type, return empty payload + if isEmptyPayload && qParams.IsContentEnabled() && transformer.IsListNode(app.pathInfo.Path) { + if err.Error() == transformer.QUERY_CONTENT_MISMATCH_ERR { + err = nil + } + } + if err != nil { + log.Warning("transformer.GetAndXlateFromDB() returned : ", err) + } resPayload = payload break } @@ -429,6 +445,12 @@ func (app *CommonApp) processGet(dbs [db.MaxDB]*db.DB, fmtType TranslibFmtType) resPayload = payload break } + if isEmptyPayload && (app.depth == 1) && !transformer.IsLeafNode(app.pathInfo.Path) && !transformer.IsLeafListNode(app.pathInfo.Path) { + // target URI for Container or list GET request with depth = 1, returns empty payload + resPayload = payload + break + } + targetObj, tgtObjCastOk := (*app.ygotTarget).(ygot.GoStruct) if !tgtObjCastOk { /*For ygotTarget populated by tranlib, for query on leaf level and list(without instance) level, @@ -476,13 +498,17 @@ func (app *CommonApp) processGet(dbs [db.MaxDB]*db.DB, fmtType TranslibFmtType) err = tlerr.NotFound("Resource not found") break } - resPayload = payload - log.Info("No data available") - //TODO: Return not found error - //err = tlerr.NotFound("Resource not found") - break + if !qParams.IsEnabled() { + resPayload = payload + log.Info("No data available") + //TODO: Return not found error + //err = tlerr.NotFound("Resource not found") + break + } + } + if !qParams.IsEnabled() { + resYgot = appYgotStruct } - resYgot = appYgotStruct } } if resYgot != nil { diff --git a/translib/transformer/sflow_openconfig_test.go b/translib/transformer/sflow_openconfig_test.go index ae000b313ead..d112cfa8360c 100644 --- a/translib/transformer/sflow_openconfig_test.go +++ b/translib/transformer/sflow_openconfig_test.go @@ -68,7 +68,7 @@ func Test_node_on_openconfig_sflow(t *testing.T) { loadDB(db.ConfigDB, pre_req_map) expected_get_json := "{\"openconfig-sampling-sflow:state\":{\"agent\":\"Ethernet8\",\"enabled\":true,\"polling-interval\":300}}" url = "/openconfig-sampling-sflow:sampling/sflow/state" - t.Run("Test get on sflow node", processGetRequest(url, expected_get_json, false)) + t.Run("Test get on sflow node", processGetRequest(url, nil, expected_get_json, false)) time.Sleep(1 * time.Second) unloadDB(db.ConfigDB, cleanuptbl) t.Log("\n\n+++++++++++++ Done Performing Get on Sflow node ++++++++++++") @@ -154,7 +154,7 @@ func Test_node_openconfig_sflow_collector(t *testing.T) { loadDB(db.ConfigDB, pre_req_map) expected_get_json := "{ \"openconfig-sampling-sflow:collectors\":{\"collector\":[{\"address\":\"3.3.3.3\",\"config\":{\"address\":\"3.3.3.3\",\"network-instance\":\"default\",\"port\":6666},\"network-instance\":\"default\",\"port\":6666,\"state\":{\"address\":\"3.3.3.3\",\"network-instance\":\"default\",\"port\":6666}}]}}" url = "/openconfig-sampling-sflow:sampling/sflow/collectors" - t.Run("Test get on collector node for sflow", processGetRequest(url, expected_get_json, false)) + t.Run("Test get on collector node for sflow", processGetRequest(url, nil, expected_get_json, false)) time.Sleep(1 * time.Second) cleanuptbl = map[string]interface{}{"SFLOW_COLLECTOR": map[string]interface{}{"3.3.3.3_6666_default": ""}} unloadDB(db.ConfigDB, cleanuptbl) @@ -215,7 +215,7 @@ func Test_node_openconfig_sflow_interface(t *testing.T) { loadDB(db.ApplDB, non_pre_req_map) expected_get_json := "{\"openconfig-sampling-sflow:state\":{\"enabled\":true,\"name\":\"Ethernet8\",\"sampling-rate\":30000}}" url = "/openconfig-sampling-sflow:sampling/sflow/interfaces/interface[name=Ethernet8]/state" - t.Run("Test get on sflow interface", processGetRequest(url, expected_get_json, false)) + t.Run("Test get on sflow interface", processGetRequest(url, nil, expected_get_json, false)) time.Sleep(2 * time.Second) cleanuptbl = map[string]interface{}{"SFLOW_SESSION": map[string]interface{}{"Ethernet8": ""}} unloadDB(db.ConfigDB, cleanuptbl) diff --git a/translib/transformer/sflow_sonic_test.go b/translib/transformer/sflow_sonic_test.go index 79081daa3623..7c3a7655ed26 100644 --- a/translib/transformer/sflow_sonic_test.go +++ b/translib/transformer/sflow_sonic_test.go @@ -58,7 +58,7 @@ func Test_node_sonic_sflow(t *testing.T) { // Verify global configurations url = "/sonic-sflow:sonic-sflow/SFLOW/global" url_body_json = "{\"sonic-sflow:global\":{\"admin_state\":\"up\",\"agent_id\":\"Ethernet4\",\"polling_interval\":100}}" - t.Run("Verify sFlow global configurations", processGetRequest(url, url_body_json, false)) + t.Run("Verify sFlow global configurations", processGetRequest(url, nil, url_body_json, false)) //Delete sflow global configurations url = "/sonic-sflow:sonic-sflow/SFLOW" @@ -68,7 +68,7 @@ func Test_node_sonic_sflow(t *testing.T) { //Verify deleted sflow global configuration url = "/sonic-sflow:sonic-sflow/SFLOW" url_body_json = "{}" - t.Run("Verify delete on sflow node", processGetRequest(url, url_body_json, false)) + t.Run("Verify delete on sflow node", processGetRequest(url, nil, url_body_json, false)) } func Test_node_sonic_sflow_collector(t *testing.T) { @@ -82,7 +82,7 @@ func Test_node_sonic_sflow_collector(t *testing.T) { // Verify sFlow collector configurations url = "/sonic-sflow:sonic-sflow/SFLOW_COLLECTOR/SFLOW_COLLECTOR_LIST[name=1.1.1.1_6343_default]" - t.Run("Verify sFlow collector", processGetRequest(url, url_body_json, false)) + t.Run("Verify sFlow collector", processGetRequest(url, nil, url_body_json, false)) // Set collector ip url = "/sonic-sflow:sonic-sflow/SFLOW_COLLECTOR/SFLOW_COLLECTOR_LIST[name=1.1.1.1_6343_default]/collector_ip" @@ -104,7 +104,7 @@ func Test_node_sonic_sflow_collector(t *testing.T) { //Verify collector port url = "/sonic-sflow:sonic-sflow/SFLOW_COLLECTOR/SFLOW_COLLECTOR_LIST" url_body_json = "{}" - t.Run("Verify delete on sFlow collector", processGetRequest(url, url_body_json, false)) + t.Run("Verify delete on sFlow collector", processGetRequest(url, nil, url_body_json, false)) } func Test_node_sonic_sflow_interface(t *testing.T) { @@ -137,7 +137,7 @@ func Test_node_sonic_sflow_interface(t *testing.T) { url_body_json = "{}" err_str := "Resource not found" expected_err := tlerr.NotFoundError{Format: err_str} - t.Run("Verify delete on sFlow Interface", processGetRequest(url, url_body_json, true, expected_err)) + t.Run("Verify delete on sFlow Interface", processGetRequest(url, nil, url_body_json, true, expected_err)) //Delete sflow global configurations url = "/sonic-sflow:sonic-sflow/SFLOW" @@ -147,5 +147,5 @@ func Test_node_sonic_sflow_interface(t *testing.T) { //Verify deleted sflow global configuration url = "/sonic-sflow:sonic-sflow/SFLOW" url_body_json = "{}" - t.Run("Verify delete on sFlow collector", processGetRequest(url, url_body_json, false)) + t.Run("Verify delete on sFlow collector", processGetRequest(url, nil, url_body_json, false)) } diff --git a/translib/transformer/test/openconfig-test-xfmr-annot.yang b/translib/transformer/test/openconfig-test-xfmr-annot.yang index 3190ba6db11c..97e96ba08674 100644 --- a/translib/transformer/test/openconfig-test-xfmr-annot.yang +++ b/translib/transformer/test/openconfig-test-xfmr-annot.yang @@ -22,24 +22,12 @@ module openconfig-test-xfmr-annot { } } - deviation /oc-test-xfmr:test-xfmr/oc-test-xfmr:test-sensor-groups/oc-test-xfmr:test-sensor-group/oc-test-xfmr:config/oc-test-xfmr:id { - deviate add { - sonic-ext:field-transformer "test_sensor_group_id_field_xfmr"; - } - } - deviation /oc-test-xfmr:test-xfmr/oc-test-xfmr:test-sensor-groups/oc-test-xfmr:test-sensor-group/oc-test-xfmr:config/oc-test-xfmr:group-colors { deviate add { sonic-ext:field-name "colors"; } } - deviation /oc-test-xfmr:test-xfmr/oc-test-xfmr:test-sensor-groups/oc-test-xfmr:test-sensor-group/oc-test-xfmr:state/oc-test-xfmr:id { - deviate add { - sonic-ext:field-transformer "test_sensor_group_id_field_xfmr"; - } - } - deviation /oc-test-xfmr:test-xfmr/oc-test-xfmr:test-sensor-groups/oc-test-xfmr:test-sensor-group/oc-test-xfmr:state/oc-test-xfmr:group-colors { deviate add { sonic-ext:field-name "colors"; @@ -90,6 +78,12 @@ module openconfig-test-xfmr-annot { } } + deviation /oc-test-xfmr:test-xfmr/oc-test-xfmr:test-sensor-groups/oc-test-xfmr:test-sensor-group/oc-test-xfmr:test-sensor-components/oc-test-xfmr:test-sensor-component { + deviate add { + sonic-ext:table-name "TEST_SENSOR_COMPONENT_TABLE"; + } + } + deviation /oc-test-xfmr:test-xfmr/oc-test-xfmr:test-sets/oc-test-xfmr:test-set { deviate add { sonic-ext:table-name "TEST_SET_TABLE"; diff --git a/translib/transformer/test/openconfig-test-xfmr.yang b/translib/transformer/test/openconfig-test-xfmr.yang index f04fff4e9551..e0de7db7bd26 100644 --- a/translib/transformer/test/openconfig-test-xfmr.yang +++ b/translib/transformer/test/openconfig-test-xfmr.yang @@ -127,6 +127,7 @@ module openconfig-test-xfmr { } } } + uses test-sensor-components-top; } } } @@ -274,7 +275,9 @@ module openconfig-test-xfmr { } } } + //////////////////////////// + grouping test-sensor-group-config { description "Config parameters related to the test sensor groups"; @@ -370,6 +373,126 @@ module openconfig-test-xfmr { } + /////////////////// + grouping test-sensor-component-config { + description + "Configuration data for sensor-components"; + + leaf name { + type string; + description + "Device name for the sensor component. "; + } + leaf type { + type enumeration { + enum TYPE1 { + description + "Component Type 1."; + } + enum TYPE2 { + description + "Component Type 2."; + } + enum TYPE3 { + description + "Component Type 3."; + } + } + } + leaf version { + type string; + description + "Version of the Component. "; + } + leaf description { + type string; + description + "Description, or comment, for the test sensor component"; + } + } + + grouping test-sensor-component-state { + description + "Operational State data for sensor components"; + leaf mfg-name { + type string; + description + "System-supplied identifier for the manufacturer of the component."; + } + + leaf mfg-date { + type string; + description + "System-supplied representation of the component's + manufacturing date."; + } + + leaf hardware-version { + type string; + description + "For hardware components, this is the hardware revision of + the component."; + } + + leaf firmware-version { + type string; + description + "For hardware components, this is the version of associated + firmware that is running on the component, if applicable."; + } + } + + grouping test-sensor-components-top { + container test-sensor-components { + description + "Enclosing container for test sensor component references"; + + list test-sensor-component { + key "name type version"; + description + "List of sensor component references"; + + leaf name { + type leafref { + path "../config/name"; + } + description + "Reference to the name of component"; + } + + leaf type { + type leafref { + path "../config/type"; + } + description + "Reference to the type of component"; + } + + leaf version { + type leafref { + path "../config/version"; + } + description + "Reference to the version of component"; + } + + container config { + description + "Configuration parameters to configure a sensor component"; + uses test-sensor-component-config; + } + + container state { + config false; + description + "Operational state parameters of a sensor component"; + uses test-sensor-component-config; + uses test-sensor-component-state; + } + } + } + } + grouping interfaces-config { description "Configuration data for interface references"; @@ -424,10 +547,8 @@ module openconfig-test-xfmr { } } } - /////////////////// - // data definition statements container test-xfmr { diff --git a/translib/transformer/test/sonic-test-xfmr.yang b/translib/transformer/test/sonic-test-xfmr.yang index afe30ad0e396..6008baade449 100644 --- a/translib/transformer/test/sonic-test-xfmr.yang +++ b/translib/transformer/test/sonic-test-xfmr.yang @@ -117,6 +117,29 @@ module sonic-test-xfmr { } } + container TEST_SENSOR_COMPONENT_TABLE { + + list TEST_SENSOR_COMPONENT_TABLE_LIST { + key "name type version"; + + leaf name { + type string; + } + + leaf type { + type string; + } + + leaf version { + type string; + } + + leaf description { + type string; + } + } + } + container TEST_SET_TABLE { list TEST_SET_TABLE_LIST { diff --git a/translib/transformer/testxfmryang_test.go b/translib/transformer/testxfmryang_test.go index bf8dc4b2d054..b3951ad9fe58 100644 --- a/translib/transformer/testxfmryang_test.go +++ b/translib/transformer/testxfmryang_test.go @@ -73,7 +73,7 @@ func Test_node_exercising_subtree_xfmr_and_virtual_table(t *testing.T) { loadDB(db.ConfigDB, pre_req_map) expected_get_json := "{\"openconfig-test-xfmr:ingress-test-set\":[{\"config\":{\"set-name\":\"TestSet_03\",\"type\":\"openconfig-test-xfmr:TEST_SET_IPV6\"},\"set-name\":\"TestSet_03\",\"state\":{\"set-name\":\"TestSet_03\",\"type\":\"openconfig-test-xfmr:TEST_SET_IPV6\"},\"type\":\"openconfig-test-xfmr:TEST_SET_IPV6\"}]}" url = "/openconfig-test-xfmr:test-xfmr/interfaces/interface[id=Eth_1]/ingress-test-sets/ingress-test-set[set-name=TestSet_03][type=TEST_SET_IPV6]" - t.Run("Test get on node exercising subtree-xfmr and virtual table.", processGetRequest(url, expected_get_json, false)) + t.Run("Test get on node exercising subtree-xfmr and virtual table.", processGetRequest(url, nil, expected_get_json, false)) cleanuptbl = map[string]interface{}{"TEST_SET_TABLE": map[string]interface{}{"TestSet_03_TEST_SET_IPV6": ""}} unloadDB(db.ConfigDB, cleanuptbl) t.Log("\n\n+++++++++++++ Done Performing Get on Yang Node Exercising Subtree-Xfmr and Virtual Table ++++++++++++") @@ -124,7 +124,7 @@ func Test_node_exercising_tableName_key_and_field_xfmr(t *testing.T) { loadDB(db.ConfigDB, pre_req_map) expected_get_json := "{\"openconfig-test-xfmr:test-sets\":{\"test-set\":[{\"config\":{\"description\":\"TestSet_03Description\",\"name\":\"TestSet_03\",\"type\":\"openconfig-test-xfmr:TEST_SET_IPV6\"},\"name\":\"TestSet_03\",\"state\":{\"description\":\"TestSet_03Description\",\"name\":\"TestSet_03\",\"type\":\"openconfig-test-xfmr:TEST_SET_IPV6\"},\"type\":\"openconfig-test-xfmr:TEST_SET_IPV6\"}]}}" url = "/openconfig-test-xfmr:test-xfmr/test-sets" - t.Run("Test get on node exercising Table-Name, Key-Xfmr and Field-Xfmr.", processGetRequest(url, expected_get_json, false)) + t.Run("Test get on node exercising Table-Name, Key-Xfmr and Field-Xfmr.", processGetRequest(url, nil, expected_get_json, false)) time.Sleep(1 * time.Second) cleanuptbl = map[string]interface{}{"TEST_SET_TABLE": map[string]interface{}{"TestSet_03_TEST_SET_IPV6": ""}} unloadDB(db.ConfigDB, cleanuptbl) @@ -179,7 +179,7 @@ func Test_node_with_child_tableXfmr_keyXfmr_fieldNameXfmrs_nonConfigDB_data(t *t get_expected := "{\"openconfig-test-xfmr:test-sensor-group\":[{\"config\":{\"color-hold-time\":10,\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"},\"id\":\"test_group_1\",\"state\":{\"color-hold-time\":10,\"counters\":{\"frame-in\":12345,\"frame-out\":678910},\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"},\"test-sensor-types\":{\"test-sensor-type\":[{\"config\":{\"exclude-filter\":\"filterB\",\"type\":\"sensora_testA\"},\"state\":{\"exclude-filter\":\"filterB\",\"type\":\"sensora_testA\"},\"type\":\"sensora_testA\"}]}}]}" - t.Run("Verify_get_on_node_with_child_table_key_field_xfmrs", processGetRequest(url, get_expected, false)) + t.Run("Verify_get_on_node_with_child_table_key_field_xfmrs", processGetRequest(url, nil, get_expected, false)) // Teardown unloadDB(db.ConfigDB, cleanuptbl) @@ -308,7 +308,7 @@ func Test_sonic_yang_node_operations(t *testing.T) { loadDB(db.CountersDB, prereq) get_expected := "{\"sonic-test-xfmr:TEST_SENSOR_MODE_TABLE\":{\"TEST_SENSOR_MODE_TABLE_LIST\":[{\"description\":\"Test sensor mode\",\"id\":3543,\"mode\":\"mode:testsensor123\"}]}}" - t.Run("Get on Sonic table with key xfmr", processGetRequest(url, get_expected, false)) + t.Run("Get on Sonic table with key xfmr", processGetRequest(url, nil, get_expected, false)) // Teardown unloadDB(db.CountersDB, cleanuptbl) @@ -348,147 +348,519 @@ func Test_leaflist_node(t *testing.T) { unloadDB(db.ConfigDB, cleanuptbl) time.Sleep(1 * time.Second) t.Log("\n\n+++++++++++++ Done Performing Put/Replace on Yang leaf-list Node demonstrating leaf-list contents swap ++++++++++++") +} + +func Test_node_exercising_singleton_container_and_keyname_mapping(t *testing.T) { + var pre_req_map, expected_map, cleanuptbl map[string]interface{} + var url, url_body_json string + t.Log("\n\n+++++++++++++ Performing Set on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") + url = "/openconfig-test-xfmr:test-xfmr/global-sensor" + url_body_json = "{ \"openconfig-test-xfmr:mode\": \"testmode\", \"openconfig-test-xfmr:description\": \"testdescription\"}" + expected_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description": "testdescription"}}} + cleanuptbl = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": ""}} loadDB(db.ConfigDB, pre_req_map) time.Sleep(1 * time.Second) + t.Run("Test set on yang node exercising mapping to sonic singleton conatiner and key-name.", processSetRequest(url, url_body_json, "POST", false, nil)) + time.Sleep(1 * time.Second) + t.Run("Verify set on yang node exercising mapping to sonic-yang singleton conatiner and key-name.", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", expected_map, false)) + unloadDB(db.ConfigDB, cleanuptbl) + time.Sleep(1 * time.Second) + t.Log("\n\n+++++++++++++ Done Performing Set on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") + t.Log("\n\n+++++++++++++ Performing Delete on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") + pre_req_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{ + "mode": "testmode", + "description": "testdescription"}}} loadDB(db.ConfigDB, pre_req_map) time.Sleep(1 * time.Second) -} - -func Test_node_exercising_singleton_container_and_keyname_mapping(t *testing.T) { - var pre_req_map, expected_map, cleanuptbl map[string]interface{} - var url, url_body_json string - - t.Log("\n\n+++++++++++++ Performing Set on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") - url = "/openconfig-test-xfmr:test-xfmr/global-sensor" - url_body_json = "{ \"openconfig-test-xfmr:mode\": \"testmode\", \"openconfig-test-xfmr:description\": \"testdescription\"}" - expected_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description":"testdescription"}}} - cleanuptbl = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": ""}} - loadDB(db.ConfigDB, pre_req_map) - time.Sleep(1 * time.Second) - t.Run("Test set on yang node exercising mapping to sonic singleton conatiner and key-name.", processSetRequest(url, url_body_json, "POST", false, nil)) - time.Sleep(1 * time.Second) - t.Run("Verify set on yang node exercising mapping to sonic-yang singleton conatiner and key-name.", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", expected_map, false)) - unloadDB(db.ConfigDB, cleanuptbl) - time.Sleep(1 * time.Second) - t.Log("\n\n+++++++++++++ Done Performing Set on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") - - t.Log("\n\n+++++++++++++ Performing Delete on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") - pre_req_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{ - "mode": "testmode", - "description": "testdescription"}}} - loadDB(db.ConfigDB, pre_req_map) - time.Sleep(1 * time.Second) - url = "/openconfig-test-xfmr:test-xfmr/global-sensor/description" - t.Run("Test delete on node exercising mapping to sonic-yang singleton conatiner and key-name.", processDeleteRequest(url, false)) - time.Sleep(1 * time.Second) - expected_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{ - "mode": "testmode"}}} - t.Run("Verify delete on node exercising mapping to sonic-yang singleton conatiner and key-name.", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", expected_map, false)) - unloadDB(db.ConfigDB, cleanuptbl) - time.Sleep(1 * time.Second) - t.Log("\n\n+++++++++++++ Done Performing Delete on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") + url = "/openconfig-test-xfmr:test-xfmr/global-sensor/description" + t.Run("Test delete on node exercising mapping to sonic-yang singleton conatiner and key-name.", processDeleteRequest(url, false)) + time.Sleep(1 * time.Second) + expected_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{ + "mode": "testmode"}}} + t.Run("Verify delete on node exercising mapping to sonic-yang singleton conatiner and key-name.", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", expected_map, false)) + unloadDB(db.ConfigDB, cleanuptbl) + time.Sleep(1 * time.Second) + t.Log("\n\n+++++++++++++ Done Performing Delete on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") t.Log("\n\n+++++++++++++ Performing Get on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") - pre_req_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{ - "mode": "testmode", - "description": "testdescription"}}} - loadDB(db.ConfigDB, pre_req_map) - expected_get_json := "{\"openconfig-test-xfmr:global-sensor\": {\"description\": \"testdescription\",\"mode\": \"testmode\"}}" - url = "/openconfig-test-xfmr:test-xfmr/global-sensor" - t.Run("Test get on node exercising mapping to sonic-yang singleton conatiner and key-name.", processGetRequest(url, expected_get_json, false)) - time.Sleep(1 * time.Second) - unloadDB(db.ConfigDB, cleanuptbl) - t.Log("\n\n+++++++++++++ Done Performing Get on Yang Node Exercising mapping to sonic-yang singleton conatiner and key-name ++++++++++++") + pre_req_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{ + "mode": "testmode", + "description": "testdescription"}}} + loadDB(db.ConfigDB, pre_req_map) + expected_get_json := "{\"openconfig-test-xfmr:global-sensor\": {\"description\": \"testdescription\",\"mode\": \"testmode\"}}" + url = "/openconfig-test-xfmr:test-xfmr/global-sensor" + t.Run("Test get on node exercising mapping to sonic-yang singleton conatiner and key-name.", processGetRequest(url, nil, expected_get_json, false)) + time.Sleep(1 * time.Second) + unloadDB(db.ConfigDB, cleanuptbl) + t.Log("\n\n+++++++++++++ Done Performing Get on Yang Node Exercising mapping to sonic-yang singleton conatiner and key-name ++++++++++++") } func Test_singleton_sonic_yang_node_operations(t *testing.T) { - cleanuptbl := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": ""}} - url := "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL" + cleanuptbl := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": ""}} + url := "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL" + + t.Log("++++++++++++++ Test_create_on_sonic_singleton_container_yang_node +++++++++++++") + + // Setup - Prerequisite + unloadDB(db.ConfigDB, cleanuptbl) + + // Payload + post_payload := "{ \"sonic-test-xfmr:global_sensor\": { \"mode\": \"testmode\", \"description\": \"testdescp\" }}" + post_sensor_global_expected := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description": "testdescp"}}} + + t.Run("Create on singleton sonic table yang node", processSetRequest(url, post_payload, "POST", false)) + time.Sleep(1 * time.Second) + t.Run("Verify Create on singleton sonic table yang node", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", post_sensor_global_expected, false)) + + // Teardown + unloadDB(db.ConfigDB, cleanuptbl) + + t.Log("++++++++++++++ Test_patch_on_sonic_singleton_container_node +++++++++++++") + + prereq := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description": "testdescp"}}} + url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL/global_sensor" + + // Setup - Prerequisite + loadDB(db.ConfigDB, prereq) + + // Payload + patch_payload := "{ \"sonic-test-xfmr:global_sensor\": { \"mode\": \"testmode\", \"description\": \"test description\" }}" + patch_sensor_global_expected := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description": "test description"}}} + + t.Run("Patch on singleton sonic container yang node", processSetRequest(url, patch_payload, "PATCH", false)) + time.Sleep(1 * time.Second) + t.Run("Verify patch on singleton sonic container yang node", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", patch_sensor_global_expected, false)) + + // Teardown + unloadDB(db.ConfigDB, cleanuptbl) + + t.Log("++++++++++++++ Test_replace_on_sonic_singleton_container +++++++++++++") + + url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL/global_sensor/mode" + + // Setup - Prerequisite + loadDB(db.ConfigDB, prereq) + + // Payload + put_payload := "{ \"sonic-test-xfmr:mode\": \"test_mode_1\"}" + put_sensor_global_expected := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "test_mode_1", "description": "testdescp"}}} + + t.Run("Put on singleton sonic yang node", processSetRequest(url, put_payload, "PUT", false)) + time.Sleep(1 * time.Second) + t.Run("Verify put on singleton sonic yang node", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", put_sensor_global_expected, false)) + + // Teardown + unloadDB(db.ConfigDB, cleanuptbl) + + t.Log("++++++++++++++ Test_delete_on_singleton_sonic_container +++++++++++++") + + url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL/global_sensor" + + // Setup - Prerequisite + loadDB(db.ConfigDB, prereq) + + delete_expected := make(map[string]interface{}) + + t.Run("Delete on singleton sonic container", processDeleteRequest(url, false)) + time.Sleep(1 * time.Second) + t.Run("Verify delete on sonic singleton container", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", delete_expected, false)) + + // Teardown + unloadDB(db.ConfigDB, cleanuptbl) + + t.Log("++++++++++++++ Test_get_on_sonic_singleton_container +++++++++++++") + + prereq = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "mode_test", "description": "test description for single container"}}} + url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL" + + // Setup - Prerequisite + loadDB(db.ConfigDB, prereq) + + get_expected := "{\"sonic-test-xfmr:TEST_SENSOR_GLOBAL\":{ \"global_sensor\": { \"mode\": \"mode_test\", \"description\": \"test description for single container\" }}}" + t.Run("Get on Sonic singleton container", processGetRequest(url, nil, get_expected, false)) + + // Teardown + unloadDB(db.ConfigDB, cleanuptbl) +} + +// Query parameter UT cases + +func Test_Query_Params_OC_Yang_Get(t *testing.T) { + + var qp queryParamsUT + qp.depth = 3 + + cleanuptbl := map[string]interface{}{"TEST_SENSOR_GROUP": map[string]interface{}{"test_group_1": ""}, "TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": ""}, "TEST_SENSOR_A_TABLE": map[string]interface{}{"test_group_1|sensor_type_a_testA": ""}} + prereq := map[string]interface{}{"TEST_SENSOR_GROUP": map[string]interface{}{"test_group_1": map[string]interface{}{"colors@": "red,blue,green", "color-hold-time": "30"}}} + prereq_cntr := map[string]interface{}{"TEST_SENSOR_GROUP_COUNTERS": map[string]interface{}{"test_group_1": map[string]interface{}{"frame-in": "3435", "frame-out": "3452"}}} + + // Setup - Prerequisite - None + unloadDB(db.ConfigDB, cleanuptbl) + unloadDB(db.CountersDB, prereq_cntr) + loadDB(db.ConfigDB, prereq) + loadDB(db.CountersDB, prereq_cntr) + + t.Log("++++++++++++++ Test_Query_Depth3_Container_Get +++++++++++++") + url := "/openconfig-test-xfmr:test-xfmr" + get_expected := "{}" + t.Run("Test_Query_Depth3_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Depth4_Container_Get +++++++++++++") + qp.depth = 4 + get_expected = "{\"openconfig-test-xfmr:test-xfmr\":{\"test-sensor-groups\":{\"test-sensor-group\":[{\"id\":\"test_group_1\"}]}}}" + t.Run("Test_Query_Depth4_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Depth8_Container_Get +++++++++++++") + qp.depth = 8 + get_expected = "{\"openconfig-test-xfmr:test-xfmr\":{\"test-sensor-groups\":{\"test-sensor-group\":[{\"config\":{\"color-hold-time\":30,\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"},\"id\":\"test_group_1\",\"state\":{\"color-hold-time\":30,\"counters\":{\"frame-in\":3435,\"frame-out\":3452},\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"}}]}}}" + t.Run("Test_Query_Depth8_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Depth0_Container_Get +++++++++++++") + qp.depth = 0 + t.Run("Test_Query_Depth0_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Depth3_List_Get +++++++++++++") + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group" + qp.depth = 3 + get_expected = "{\"openconfig-test-xfmr:test-sensor-group\":[{\"config\":{\"color-hold-time\":30,\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"},\"id\":\"test_group_1\",\"state\":{\"color-hold-time\":30,\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"}}]}" + t.Run("Test_Query_Depth3_List_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Depth3_List_Instance_Get +++++++++++++") + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]" + t.Run("Test_Query_Depth3_List_Instance_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Depth2_Leaf_Get +++++++++++++") + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]/config/color-hold-time" + qp.depth = 2 + get_expected = "{\"openconfig-test-xfmr:color-hold-time\":30}" + t.Run("Test_Query_Depth2_Leaf_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Content_All_Get +++++++++++++") + // Reset Depth + qp.depth = 0 + qp.content = "all" + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]" + get_expected = "{\"openconfig-test-xfmr:test-sensor-group\":[{\"config\":{\"color-hold-time\":30,\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"},\"id\":\"test_group_1\",\"state\":{\"color-hold-time\":30,\"counters\":{\"frame-in\":3435,\"frame-out\":3452},\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"}}]}" + t.Run("Test_Query_Content_All_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Content_Config_Get +++++++++++++") + qp.content = "config" + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group" + get_expected = "{\"openconfig-test-xfmr:test-sensor-group\":[{\"config\":{\"color-hold-time\":30,\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"},\"id\":\"test_group_1\"}]}" + t.Run("Test_Query_Content_Config_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Content_NonConfig_Get +++++++++++++") + qp.content = "nonconfig" + url = "/openconfig-test-xfmr:test-xfmr" + get_expected = "{\"openconfig-test-xfmr:test-xfmr\":{\"test-sensor-groups\":{\"test-sensor-group\":[{\"id\":\"test_group_1\",\"state\":{\"color-hold-time\":30,\"counters\":{\"frame-in\":3435,\"frame-out\":3452},\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"}}]}}}" + t.Run("Test_Query_Content_NonConfig_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Content_Operational_Get +++++++++++++") + qp.content = "operational" + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups" + get_expected = "{\"openconfig-test-xfmr:test-sensor-groups\":{\"test-sensor-group\":[{\"id\":\"test_group_1\",\"state\":{\"counters\":{\"frame-in\":3435,\"frame-out\":3452}}}]}}" + t.Run("Test_Query_Content_Operational_Get", processGetRequest(url, &qp, get_expected, false)) + t.Log("++++++++++++++ Test_Query_Content_Mismatch_Leaf_Get +++++++++++++") + qp.content = "config" + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id==test_group_1]/state/counters/frame-in" + get_expected = "{}" + expected_err := tlerr.InvalidArgsError{Format: "Bad Request - requested content type doesn't match content type of terminal node uri."} + t.Run("Test_Query_Content_Mismatch_Leaf_Get", processGetRequest(url, &qp, get_expected, true, expected_err)) + + t.Log("++++++++++++++ Test_Query_Content_Mismatch_Container_Get +++++++++++++") + qp.content = "nonconfig" + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id==test_group_1]/config" + get_expected = "{}" + t.Run("Test_Query_Content_Mismatch_Container_Get1", processGetRequest(url, &qp, get_expected, false)) + qp.content = "config" + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]/state" + t.Run("Test_Query_Content_Mismatch_Container_Get2", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Depth3_Content_Config_List_Get +++++++++++++") + qp.depth = 3 + qp.content = "config" + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group" + get_expected = "{\"openconfig-test-xfmr:test-sensor-group\":[{\"config\":{\"color-hold-time\":30,\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"},\"id\":\"test_group_1\"}]}" + t.Run("Test_Query_Depth3_Content_Config_List_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Depth4_Content_All_Container_Get +++++++++++++") + qp.depth = 4 + qp.content = "all" + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups" + get_expected = "{\"openconfig-test-xfmr:test-sensor-groups\":{\"test-sensor-group\":[{\"config\":{\"color-hold-time\":30,\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"},\"id\":\"test_group_1\",\"state\":{\"color-hold-time\":30,\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"}}]}}" + t.Run("Test_Query_Depth4_Content_All_Container_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Depth3_Content_Nonconfig_ListInstance_Get +++++++++++++") + qp.depth = 3 + qp.content = "nonconfig" + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]" + get_expected = "{\"openconfig-test-xfmr:test-sensor-group\":[{\"id\":\"test_group_1\",\"state\":{\"color-hold-time\":30,\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"}}]}" + t.Run("Test_Query_Depth3_Content_NonConfig_ListInstance_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Fields_Leaf_Get +++++++++++++") + // Reset Depth and Content + qp.depth = 0 + qp.content = "" + qp.fields = []string{"config/color-hold-time"} + get_expected = "{\"openconfig-test-xfmr:test-sensor-group\":[{\"config\":{\"color-hold-time\":30},\"id\":\"test_group_1\"}]}" + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]" + t.Run("Test_Query_Fields_Leaf_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Fields_MultiLeaf_Get +++++++++++++") + qp.fields = []string{"state/color-hold-time", "state/counters/frame-in"} + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group" + get_expected = "{\"openconfig-test-xfmr:test-sensor-group\":[{\"id\":\"test_group_1\",\"state\":{\"color-hold-time\":30,\"counters\":{\"frame-in\":3435}}}]}" + t.Run("Test_Query_Fields_MultiLeaf_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Fields_Container_Get +++++++++++++") + qp.fields = []string{"counters"} + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]/state" + get_expected = "{\"openconfig-test-xfmr:state\":{\"counters\":{\"frame-in\":3435,\"frame-out\":3452}}}" + t.Run("Test_Query_Fields_Container_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Fields_Error_IncorrectField_Get +++++++++++++") + qp.fields = []string{"state/color-hold-times"} + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]" + get_expected = "{}" + expected_err = tlerr.InvalidArgsError{Format: "Invalid field name/path: state/color-hold-times"} + t.Run("Test_Query_Fields_Error_IncorrectField_Get", processGetRequest(url, &qp, get_expected, true, expected_err)) + + t.Log("++++++++++++++ Test_Query_Fields_Error_Leaf_Get +++++++++++++") + qp.fields = []string{"color-hold-time"} + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]/config/color-hold-time" + get_expected = "{}" + expected_err = tlerr.InvalidArgsError{Format: "Bad Request - fields query parameter specified on a terminal node uri."} + t.Run("Test_Query_Fields_Error_Leaf_Get", processGetRequest(url, &qp, get_expected, true, expected_err)) - t.Log("++++++++++++++ Test_create_on_sonic_singleton_container_yang_node +++++++++++++") + // Teardown + unloadDB(db.ConfigDB, prereq) + unloadDB(db.CountersDB, prereq_cntr) +} - // Setup - Prerequisite - unloadDB(db.ConfigDB, cleanuptbl) +/* sonic yang GET operation query-parameter tests */ +func Test_sonic_yang_content_query_parameter_operations(t *testing.T) { + var qp queryParamsUT + + t.Log("++++++++++++++ Test_content_all_query_parameter_on_sonic_yang +++++++++++++") + prereq_sensor_global := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"description": "testdescription"}}} + prereq_sensor_mode := map[string]interface{}{"TEST_SENSOR_MODE_TABLE": map[string]interface{}{"mode:testsensor123:3543": map[string]interface{}{"description": "Test sensor mode"}}} + url := "/sonic-test-xfmr:sonic-test-xfmr" + qp.content = "all" + //Setup + loadDB(db.ConfigDB, prereq_sensor_global) + loadDB(db.CountersDB, prereq_sensor_mode) + get_expected := "{\"sonic-test-xfmr:sonic-test-xfmr\":{\"TEST_SENSOR_GLOBAL\":{\"global_sensor\":{\"description\":\"testdescription\"}},\"TEST_SENSOR_MODE_TABLE\":{\"TEST_SENSOR_MODE_TABLE_LIST\":[{\"description\":\"Test sensor mode\",\"id\":3543,\"mode\":\"mode:testsensor123\"}]}}}" + t.Run("Sonic yang query parameter content=all", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_content_config_query_parameter_on_sonic_yang +++++++++++++") + url = "/sonic-test-xfmr:sonic-test-xfmr" + qp.content = "config" + get_expected = "{\"sonic-test-xfmr:sonic-test-xfmr\":{\"TEST_SENSOR_GLOBAL\":{\"global_sensor\":{\"description\":\"testdescription\"}}}}" + t.Run("Sonic yang query parameter content=config", processGetRequest(url, &qp, get_expected, false)) - // Payload - post_payload := "{ \"sonic-test-xfmr:global_sensor\": { \"mode\": \"testmode\", \"description\": \"testdescp\" }}" - post_sensor_global_expected := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description": "testdescp"}}} + t.Log("++++++++++++++ Test_content_nonconfig_query_parameter_on_sonic_yang +++++++++++++") + url = "/sonic-test-xfmr:sonic-test-xfmr" + qp.content = "nonconfig" + get_expected = "{\"sonic-test-xfmr:sonic-test-xfmr\":{\"TEST_SENSOR_MODE_TABLE\":{\"TEST_SENSOR_MODE_TABLE_LIST\":[{\"description\":\"Test sensor mode\",\"id\":3543,\"mode\":\"mode:testsensor123\"}]}}}" + t.Run("Sonic yang query parameter content=nonconfig", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_content_mismatch_error_query_parameter_on_sonic_yang +++++++++++++") + url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_MODE_TABLE/TEST_SENSOR_MODE_TABLE_LIST[id=3543][mode=testsensor123]/description" + qp.content = "config" + get_expected = "{\"sonic-test-xfmr:sonic-test-xfmr\":{\"TEST_SENSOR_GLOBAL\":{\"global_sensor\":{\"description\":\"testdescription\"}},\"TEST_SENSOR_MODE_TABLE\":{\"TEST_SENSOR_MODE_TABLE_LIST\":[{\"description\":\"Test sensor mode\",\"id\":3543,\"mode\":\"mode:testsensor123\"}]}}}" + get_expected = "{}" + exp_err := tlerr.InvalidArgsError{Format: "Bad Request - requested content type doesn't match content type of terminal node uri."} + t.Run("Sonic yang query parameter simple terminal node content mismatch error.", processGetRequest(url, &qp, get_expected, true, exp_err)) + // Teardown + unloadDB(db.ConfigDB, prereq_sensor_global) + unloadDB(db.CountersDB, prereq_sensor_mode) - t.Run("Create on singleton sonic table yang node", processSetRequest(url, post_payload, "POST", false)) - time.Sleep(1 * time.Second) - t.Run("Verify Create on singleton sonic table yang node", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", post_sensor_global_expected, false)) +} - // Teardown - unloadDB(db.ConfigDB, cleanuptbl) +func Test_sonic_yang_depth_query_parameter_operations(t *testing.T) { + var qp queryParamsUT + + t.Log("++++++++++++++ Test_depth_level_1_query_parameter_on_sonic_yang +++++++++++++") + prereq_sensor_global := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"description": "testdescription"}}} + url := "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL/global_sensor/description" + qp.depth = 1 + //Setup + loadDB(db.ConfigDB, prereq_sensor_global) + get_expected := "{\"sonic-test-xfmr:description\":\"testdescription\"}" + t.Run("Sonic yang query parameter depth=1", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_depth_level_2_query_parameter_on_sonic_yang +++++++++++++") + url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL/global_sensor" + qp.depth = 2 + get_expected = "{\"sonic-test-xfmr:global_sensor\":{\"description\":\"testdescription\"}}" + t.Run("Sonic yang query parameter depth=2", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_depth_level_4_query_parameter_on_sonic_yang +++++++++++++") + url = "/sonic-test-xfmr:sonic-test-xfmr" + qp.depth = 4 + get_expected = "{\"sonic-test-xfmr:sonic-test-xfmr\":{\"TEST_SENSOR_GLOBAL\":{\"global_sensor\":{\"description\":\"testdescription\"}}}}" + t.Run("Sonic yang query parameter depth=4", processGetRequest(url, &qp, get_expected, false)) + // Teardown + unloadDB(db.ConfigDB, prereq_sensor_global) +} - t.Log("++++++++++++++ Test_patch_on_sonic_singleton_container_node +++++++++++++") +func Test_sonic_yang_content_plus_depth_query_parameter_operations(t *testing.T) { + var qp queryParamsUT + + prereq_sensor_global := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"description": "testdescription"}}} + prereq_sensor_mode := map[string]interface{}{"TEST_SENSOR_MODE_TABLE": map[string]interface{}{"mode:testsensor123:3543": map[string]interface{}{"description": "Test sensor mode"}}} + t.Log("++++++++++++++ Test_content_all_depth_level_4_query_parameter_on_sonic_yang +++++++++++++") + url := "/sonic-test-xfmr:sonic-test-xfmr" + qp.depth = 4 + qp.content = "all" + //Setup + loadDB(db.ConfigDB, prereq_sensor_global) + loadDB(db.CountersDB, prereq_sensor_mode) + get_expected := "{\"sonic-test-xfmr:sonic-test-xfmr\":{\"TEST_SENSOR_GLOBAL\":{\"global_sensor\":{\"description\":\"testdescription\"}},\"TEST_SENSOR_MODE_TABLE\":{\"TEST_SENSOR_MODE_TABLE_LIST\":[{\"description\":\"Test sensor mode\",\"id\":3543,\"mode\":\"mode:testsensor123\"}]}}}" + t.Run("Sonic yang query parameter content=all depth=4", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_content_config_depth_level_4_query_parameter_on_sonic_yang +++++++++++++") + url = "/sonic-test-xfmr:sonic-test-xfmr" + qp.depth = 4 + qp.content = "config" + get_expected = "{\"sonic-test-xfmr:sonic-test-xfmr\":{\"TEST_SENSOR_GLOBAL\":{\"global_sensor\":{\"description\":\"testdescription\"}}}}" + t.Run("Sonic yang query parameter content=config depth=4", processGetRequest(url, &qp, get_expected, false)) - prereq := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description": "testdescp"}}} - url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL/global_sensor" + t.Log("++++++++++++++ Test_content_nonconfig_depth_level_4_query_parameter_on_sonic_yang +++++++++++++") + url = "/sonic-test-xfmr:sonic-test-xfmr" + qp.depth = 4 + qp.content = "nonconfig" + get_expected = "{\"sonic-test-xfmr:sonic-test-xfmr\":{\"TEST_SENSOR_MODE_TABLE\":{\"TEST_SENSOR_MODE_TABLE_LIST\":[{\"description\":\"Test sensor mode\",\"id\":3543,\"mode\":\"mode:testsensor123\"}]}}}" + t.Run("Sonic yang query parameter content=nonconfig depth=4", processGetRequest(url, &qp, get_expected, false)) + // Teardown + unloadDB(db.ConfigDB, prereq_sensor_global) + unloadDB(db.CountersDB, prereq_sensor_mode) +} - // Setup - Prerequisite - loadDB(db.ConfigDB, prereq) +func Test_sonic_yang_fields_query_parameter_operations(t *testing.T) { + var qp queryParamsUT + qp.fields = make([]string, 0) - // Payload - patch_payload := "{ \"sonic-test-xfmr:global_sensor\": { \"mode\": \"testmode\", \"description\": \"test description\" }}" - patch_sensor_global_expected := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description": "test description"}}} + t.Log("++++++++++++++ Test_fields(single_field)_query_parameter_on_sonic_yang +++++++++++++") + prereq := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"description": "testdescription"}}, "TEST_SENSOR_A_TABLE": map[string]interface{}{"test_group_1|sensor_type_a_testA": map[string]interface{}{"exclude_filter": "filter_filterB", "description_a": "test group1 sensor type a descriptionXYZ"}, "test_group_2|sensor_type_a_testB": map[string]interface{}{"exclude_filter": "filter_filterA", "description_a": "test group2 sensor type a descriptionB"}}} + url := "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_A_TABLE/TEST_SENSOR_A_TABLE_LIST" + qp.fields = append(qp.fields, "exclude_filter") + loadDB(db.ConfigDB, prereq) + get_expected := "{\"sonic-test-xfmr:TEST_SENSOR_A_TABLE_LIST\":[{\"exclude_filter\": \"filter_filterB\",\"id\": \"test_group_1\",\"type\": \"sensor_type_a_testA\" },{\"exclude_filter\": \"filter_filterA\",\"id\": \"test_group_2\",\"type\": \"sensor_type_a_testB\"}]}" + t.Run("Sonic yang query parameter fields(single field) ", processGetRequest(url, &qp, get_expected, false)) - t.Run("Patch on singleton sonic container yang node", processSetRequest(url, patch_payload, "PATCH", false)) - time.Sleep(1 * time.Second) - t.Run("Verify patch on singleton sonic container yang node", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", patch_sensor_global_expected, false)) + t.Log("++++++++++++++ Test_fields(multiple_fields)_query_parameter_on_sonic_yang +++++++++++++") + url = "/sonic-test-xfmr:sonic-test-xfmr" + qp.fields = make([]string, 0) + qp.fields = append(qp.fields, "TEST_SENSOR_GLOBAL/global_sensor/description", "TEST_SENSOR_A_TABLE") + get_expected = "{\"sonic-test-xfmr:sonic-test-xfmr\": {\"TEST_SENSOR_A_TABLE\": {\"TEST_SENSOR_A_TABLE_LIST\": [{\"description_a\": \"test group1 sensor type a descriptionXYZ\",\"exclude_filter\": \"filter_filterB\",\"id\": \"test_group_1\",\"type\": \"sensor_type_a_testA\"},{\"description_a\": \"test group2 sensor type a descriptionB\",\"exclude_filter\": \"filter_filterA\",\"id\": \"test_group_2\",\"type\": \"sensor_type_a_testB\"}]},\"TEST_SENSOR_GLOBAL\": {\"global_sensor\": {\"description\": \"testdescription\"}}}}" + t.Run("Sonic yang query parameter fields(multiple field) ", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_invalid_fields_query_parameter_on_sonic_yang +++++++++++++") + url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL" + qp.fields = make([]string, 0) + qp.fields = append(qp.fields, "global_sensor/desc") + get_expected = "{}" + exp_err := tlerr.InvalidArgsError{Format: "Invalid field name/path: global_sensor/desc"} + t.Run("Sonic yang query parameter invalid fields", processGetRequest(url, &qp, get_expected, true, exp_err)) + + t.Log("++++++++++++++ Test_invalid_fields_query_parameter_target_on_sonic_yang +++++++++++++") + url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_A_TABLE/TEST_SENSOR_A_TABLE_LIST[id=test_group_1][type=sensor_type_a_testA]/exclude_filter" + qp.fields = make([]string, 0) + qp.fields = append(qp.fields, "exclude_filter") + get_expected = "{}" + exp_err = tlerr.InvalidArgsError{Format: "Bad Request - fields query parameter specified on a terminal node uri."} + t.Run("Sonic yang invalid fields query parameter request target", processGetRequest(url, &qp, get_expected, true, exp_err)) + // Teardown + unloadDB(db.ConfigDB, prereq) +} - // Teardown - unloadDB(db.ConfigDB, cleanuptbl) +func Test_OC_Sonic_OneOnOne_Composite_KeyMapping(t *testing.T) { - t.Log("++++++++++++++ Test_replace_on_sonic_singleton_container +++++++++++++") + parent_prereq := map[string]interface{}{"TEST_SENSOR_GROUP": map[string]interface{}{"test_group_1": map[string]interface{}{"NULL": "NULL"}}} + prereq := map[string]interface{}{"TEST_SENSOR_COMPONENT_TABLE": map[string]interface{}{"FAN|TYPE1|14.31": map[string]interface{}{"description": "Test fan sensor type1 v14.31"}}} - url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL/global_sensor/mode" + // Setup - Prerequisite + unloadDB(db.ConfigDB, prereq) + loadDB(db.ConfigDB, parent_prereq) - // Setup - Prerequisite - loadDB(db.ConfigDB, prereq) + url := "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]/test-sensor-components" - // Payload - put_payload := "{ \"sonic-test-xfmr:mode\": \"test_mode_1\"}" - put_sensor_global_expected := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "test_mode_1", "description": "testdescp"}}} + t.Log("++++++++++++++ Test_Set_OC_Sonic_OneOnOne_Composite_KeyMapping +++++++++++++") - t.Run("Put on singleton sonic yang node", processSetRequest(url, put_payload, "PUT", false)) - time.Sleep(1 * time.Second) - t.Run("Verify put on singleton sonic yang node", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", put_sensor_global_expected, false)) + url_body_json := "{\"openconfig-test-xfmr:test-sensor-component\":[{\"config\":{\"name\":\"FAN\",\"type\":\"TYPE1\",\"version\":\"14.31\",\"description\":\"Test fan sensor type1 v14.31\"},\"name\":\"FAN\",\"type\":\"TYPE1\",\"version\":\"14.31\"}]}" - // Teardown - unloadDB(db.ConfigDB, cleanuptbl) + expected_map := map[string]interface{}{"TEST_SENSOR_COMPONENT_TABLE": map[string]interface{}{"FAN|TYPE1|14.31": map[string]interface{}{"description": "Test fan sensor type1 v14.31"}}} + t.Run("SET on OC_Sonic_OneOnOne_Composite_KeyMapping", processSetRequest(url, url_body_json, "POST", false, nil)) + time.Sleep(1 * time.Second) + t.Run("Test OC-Sonic one-one composite key mapping", verifyDbResult(rclient, "TEST_SENSOR_COMPONENT_TABLE|FAN|TYPE1|14.31", expected_map, false)) - t.Log("++++++++++++++ Test_delete_on_singleton_sonic_container +++++++++++++") + // Teardown + unloadDB(db.ConfigDB, prereq) + unloadDB(db.ConfigDB, parent_prereq) - url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL/global_sensor" + t.Log("++++++++++++++ Test_Get_OC_Sonic_OneOnOne_Composite_KeyMapping +++++++++++++") - // Setup - Prerequisite - loadDB(db.ConfigDB, prereq) + loadDB(db.ConfigDB, parent_prereq) + loadDB(db.ConfigDB, prereq) - delete_expected := make(map[string]interface{}) + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]/test-sensor-components" - t.Run("Delete on singleton sonic container", processDeleteRequest(url, false)) - time.Sleep(1 * time.Second) - t.Run("Verify delete on sonic singleton container", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", delete_expected, false)) + get_expected := "{\"openconfig-test-xfmr:test-sensor-components\":{\"test-sensor-component\":[{\"config\":{\"description\":\"Test fan sensor type1 v14.31\",\"name\":\"FAN\",\"type\":\"TYPE1\",\"version\":\"14.31\"},\"name\":\"FAN\",\"state\":{\"description\":\"Test fan sensor type1 v14.31\",\"name\":\"FAN\",\"type\":\"TYPE1\",\"version\":\"14.31\"},\"type\":\"TYPE1\",\"version\":\"14.31\"}]}}" + t.Run("GET on List_OC_Sonic_OneOnOne_Composite_KeyMapping", processGetRequest(url, nil, get_expected, false)) - // Teardown - unloadDB(db.ConfigDB, cleanuptbl) + // Teardown + unloadDB(db.ConfigDB, prereq) + unloadDB(db.ConfigDB, parent_prereq) +} - t.Log("++++++++++++++ Test_get_on_sonic_singleton_container +++++++++++++") +/*Test OC List having config container with leaves, that are referenced by list key-leafs and have no annotation. + Also covers the list's state container that have leaves same as list keys */ +func Test_NodeWithListHavingConfigLeafRefByKey_OC_Yang(t *testing.T) { - prereq = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "mode_test", "description": "test description for single container"}}} - url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL" + t.Log("++++++++++++++ Test_set_on_OC_yang_node_with_list_having_config_leaf_referenced_by_list_key +++++++++++++") + pre_req := map[string]interface{}{"TEST_SENSOR_GROUP": map[string]interface{}{"test_group_1": map[string]interface{}{"color-hold-time": "10"}}} + url := "/openconfig-test-xfmr:test-xfmr/test-sensor-groups" + // Payload + post_payload := "{\"openconfig-test-xfmr:test-sensor-group\":[ { \"id\" : \"test_group_1\", \"config\": { \"id\": \"test_group_1\"} } ]}" + post_sensor_group_expected := map[string]interface{}{"TEST_SENSOR_GROUP": map[string]interface{}{"test_group_1": map[string]interface{}{"NULL": "NULL", "color-hold-time": "10"}}} + t.Run("Set on OC-Yang node with list having config leaf referenced by list key.", processSetRequest(url, post_payload, "POST", false)) + time.Sleep(1 * time.Second) + t.Run("Verify set on OC-Yang node with list having config leaf referenced by list key.", verifyDbResult(rclient, "TEST_SENSOR_GROUP|test_group_1", post_sensor_group_expected, false)) + // Teardown + unloadDB(db.ConfigDB, pre_req) - // Setup - Prerequisite - loadDB(db.ConfigDB, prereq) + t.Log("++++++++++++++ Test get on OC yang node with list having config leaf referenced by list key and state leaf same as list key +++++++++++++") + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group" + // Setup - Prerequisite + loadDB(db.ConfigDB, pre_req) + // Payload + get_expected := "{\"openconfig-test-xfmr:test-sensor-group\":[{\"config\":{\"color-hold-time\":10,\"id\":\"test_group_1\"},\"id\":\"test_group_1\",\"state\":{\"color-hold-time\":10,\"id\":\"test_group_1\"}}]}" + t.Run("Verify get on OC yang node with list having config leaf referenced by list key and state leaf same list key", processGetRequest(url, nil, get_expected, false)) + // Teardown + unloadDB(db.ConfigDB, pre_req) - get_expected := "{\"sonic-test-xfmr:TEST_SENSOR_GLOBAL\":{ \"global_sensor\": { \"mode\": \"mode_test\", \"description\": \"test description for single container\" }}}" - t.Run("Get on Sonic singleton container", processGetRequest(url, get_expected, false)) + t.Log("++++++++++++++ GET on OC YANG config container leaf that is referenced by immediate parent list's key and has no app annotations +++++++++++++") + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]/state/id" + // Setup - Prerequisite + loadDB(db.ConfigDB, pre_req) + // Payload + get_expected = "{\"openconfig-test-xfmr:id\":\"test_group_1\"}" + t.Run("Get on leaf in OC config container, with no app annotation, and is referenced by immediate parent list's key leaf", processGetRequest(url, nil, get_expected, false)) + // Teardown + unloadDB(db.ConfigDB, pre_req) - // Teardown - unloadDB(db.ConfigDB, cleanuptbl) + t.Log("++++++++++++++ GET on OC YANG State container leaf that is same as immediate parent list's key and has no app annotations +++++++++++++") + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]/state/id" + // Setup - Prerequisite + loadDB(db.ConfigDB, pre_req) + // Payload + get_expected = "{\"openconfig-test-xfmr:id\":\"test_group_1\"}" + t.Run("Get on leaf in OC state container, with no app annotation, and is same as immediate parent list's key leaf", processGetRequest(url, nil, get_expected, false)) + // Teardown + unloadDB(db.ConfigDB, pre_req) } diff --git a/translib/transformer/utils_test.go b/translib/transformer/utils_test.go index 4dd4c20fc475..ef3e0db3b965 100644 --- a/translib/transformer/utils_test.go +++ b/translib/transformer/utils_test.go @@ -30,6 +30,12 @@ import ( "github.com/go-redis/redis/v7" ) +type queryParamsUT struct { + depth uint + content string + fields []string +} + func checkErr(t *testing.T, err error, expErr error) { if err.Error() != expErr.Error() { t.Fatalf("Error %v, Expect Err: %v", err, expErr) @@ -38,12 +44,22 @@ func checkErr(t *testing.T, err error, expErr error) { } } -func processGetRequest(url string, expectedRespJson string, errorCase bool, expErr ...error) func(*testing.T) { +func processGetRequest(url string, qparams *queryParamsUT, expectedRespJson string, errorCase bool, expErr ...error) func(*testing.T) { return func(t *testing.T) { var expectedMap map[string]interface{} var receivedMap map[string]interface{} + var qp QueryParameters - response, err := Get(GetRequest{Path: url, User: UserRoles{Name: "admin", Roles: []string{"admin"}} }) + qp.Depth = 0 + qp.Content = "" + qp.Fields = make([]string, 0) + + if qparams != nil { + qp.Depth = qparams.depth + qp.Content = qparams.content + qp.Fields = qparams.fields + } + response, err := Get(GetRequest{Path: url, User: UserRoles{Name: "admin", Roles: []string{"admin"}}, QueryParams: qp}) if err != nil { if !errorCase { t.Fatalf("Error %v received for Url: %s", err, url) diff --git a/translib/transformer/xconst.go b/translib/transformer/xconst.go index 3bbfded343c5..1aae4d9e5c0e 100644 --- a/translib/transformer/xconst.go +++ b/translib/transformer/xconst.go @@ -32,6 +32,12 @@ const ( IANA_MDL_PFX = "iana-" PATH_XFMR_RET_ARGS = 1 PATH_XFMR_RET_ERR_INDX = 0 + + YANG_CONTAINER_NM_CONFIG = "config" + CONFIG_CNT_SUFFIXED_XPATH = "/config" + STATE_CNT_SUFFIXED_XPATH = "/state" + CONFIG_CNT_WITHIN_XPATH = "/config/" + STATE_CNT_WITHIN_XPATH = "/state/" ) const ( @@ -52,6 +58,18 @@ const ( XFMR_DEFAULT_ENABLE ) +const ( + QUERY_CONTENT_ALL ContentType = iota + QUERY_CONTENT_CONFIG + QUERY_CONTENT_NONCONFIG + QUERY_CONTENT_OPERATIONAL +) + +const ( + QUERY_CONTENT_MISMATCH_ERR = "Query Parameter Content mismatch" + QUERY_PARAMETER_SBT_PRUNING_ERR = "Query Parameter processing unsuccessful" +) + const ( GET Operation = iota + 1 CREATE diff --git a/translib/transformer/xfmr_interface.go b/translib/transformer/xfmr_interface.go index f3e4cd2d200b..7c4a63c7f86d 100644 --- a/translib/transformer/xfmr_interface.go +++ b/translib/transformer/xfmr_interface.go @@ -51,6 +51,7 @@ type XfmrParams struct { isVirtualTbl *bool pCascadeDelTbl *[]string //used to populate list of tables needed cascade delete by subtree overloaded methods yangDefValMap map[string]map[string]db.Value + queryParams QueryParams invokeCRUSubtreeOnce *bool } diff --git a/translib/transformer/xfmr_testxfmr_callbacks.go b/translib/transformer/xfmr_testxfmr_callbacks.go index e310c6d2131b..c4acdc5476dd 100644 --- a/translib/transformer/xfmr_testxfmr_callbacks.go +++ b/translib/transformer/xfmr_testxfmr_callbacks.go @@ -19,7 +19,6 @@ //go:build xfmrtest // +build xfmrtest - package transformer import ( @@ -50,7 +49,6 @@ func init() { XlateFuncBind("DbToYang_test_set_key_xfmr", DbToYang_test_set_key_xfmr) // Key leafrefed Field transformer functions - XlateFuncBind("DbToYang_test_sensor_group_id_field_xfmr", DbToYang_test_sensor_group_id_field_xfmr) XlateFuncBind("DbToYang_test_sensor_type_field_xfmr", DbToYang_test_sensor_type_field_xfmr) XlateFuncBind("DbToYang_test_set_name_field_xfmr", DbToYang_test_set_name_field_xfmr) @@ -68,7 +66,7 @@ func init() { XlateFuncBind("Subscribe_test_port_bindings_xfmr", Subscribe_test_port_bindings_xfmr) // Sonic yang Key transformer functions - XlateFuncBind("DbToYang_test_sensor_mode_key_xfmr", DbToYang_test_sensor_mode_key_xfmr) + XlateFuncBind("DbToYang_test_sensor_mode_key_xfmr", DbToYang_test_sensor_mode_key_xfmr) } const ( @@ -109,24 +107,24 @@ var test_pre_xfmr PreXfmrFunc = func(inParams XfmrParams) error { var test_post_xfmr PostXfmrFunc = func(inParams XfmrParams) (map[string]map[string]db.Value, error) { - pathInfo := NewPathInfo(inParams.uri) - groupId := pathInfo.Var("id") + pathInfo := NewPathInfo(inParams.uri) + groupId := pathInfo.Var("id") retDbDataMap := (*inParams.dbDataMap)[inParams.curDb] log.Info("Entering test_post_xfmr Request URI path = ", inParams.requestUri) if inParams.oper == UPDATE { xpath, _, _ := XfmrRemoveXPATHPredicates(inParams.requestUri) if xpath == "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group/config/color-hold-time" { - holdTime := retDbDataMap["TEST_SENSOR_GROUP"][groupId].Field["color-hold-time"] - key := groupId + "|" + "sensor_type_a_post" + holdTime - subOpCreateMap := make(map[db.DBNum]map[string]map[string]db.Value) - subOpCreateMap[db.ConfigDB] = make(map[string]map[string]db.Value) - subOpCreateMap[db.ConfigDB]["TEST_SENSOR_A_TABLE"] = make(map[string]db.Value) - subOpCreateMap[db.ConfigDB]["TEST_SENSOR_A_TABLE"][key] = db.Value{Field: make(map[string]string)} - subOpCreateMap[db.ConfigDB]["TEST_SENSOR_A_TABLE"][key].Field["description_a"] = "Added instance in post xfmr" - inParams.subOpDataMap[CREATE] = &subOpCreateMap - } - } + holdTime := retDbDataMap["TEST_SENSOR_GROUP"][groupId].Field["color-hold-time"] + key := groupId + "|" + "sensor_type_a_post" + holdTime + subOpCreateMap := make(map[db.DBNum]map[string]map[string]db.Value) + subOpCreateMap[db.ConfigDB] = make(map[string]map[string]db.Value) + subOpCreateMap[db.ConfigDB]["TEST_SENSOR_A_TABLE"] = make(map[string]db.Value) + subOpCreateMap[db.ConfigDB]["TEST_SENSOR_A_TABLE"][key] = db.Value{Field: make(map[string]string)} + subOpCreateMap[db.ConfigDB]["TEST_SENSOR_A_TABLE"][key].Field["description_a"] = "Added instance in post xfmr" + inParams.subOpDataMap[CREATE] = &subOpCreateMap + } + } return retDbDataMap, nil } @@ -262,19 +260,6 @@ var DbToYang_test_set_key_xfmr KeyXfmrDbToYang = func(inParams XfmrParams) (map[ } -var DbToYang_test_sensor_group_id_field_xfmr FieldXfmrDbtoYang = func(inParams XfmrParams) (map[string]interface{}, error) { - var err error - result := make(map[string]interface{}) - log.Info("DbToYang_test_sensor_group_id_field_xfmr - inParams.uri ", inParams.uri) - - if len(inParams.key) > 0 { - result["id"] = inParams.key - } - log.Info("DbToYang_test_sensor_group_id_field_xfmr returns ", result) - - return result, err -} - var DbToYang_test_sensor_type_field_xfmr FieldXfmrDbtoYang = func(inParams XfmrParams) (map[string]interface{}, error) { var err error result := make(map[string]interface{}) @@ -482,7 +467,7 @@ var YangToDb_test_port_bindings_xfmr SubTreeXfmrYangToDb = func(inParams XfmrPar testSetName := getTestSetKeyStrFromOCKey(inTestSetKey.SetName, inTestSetKey.Type) testSetInterfacesMap[testSetName] = append(testSetInterfacesMap[testSetName], *intf.Id) _, ok := testSetTableMap[testSetName] - if !ok && inParams.oper == DELETE { + if !ok && inParams.oper == DELETE { return res_map, tlerr.NotFound("Binding not found for test set %v on %v", inTestSetKey.SetName, *intf.Id) } if inParams.oper == DELETE { @@ -687,20 +672,20 @@ func convertSonicTestSetTypeToOC(testSetType string) ocbinds.E_OpenconfigTestXfm //Sonic yang key transformer functions var DbToYang_test_sensor_mode_key_xfmr SonicKeyXfmrDbToYang = func(inParams SonicXfmrParams) (map[string]interface{}, error) { - res_map := make(map[string]interface{}) - /* from DB-key string(inParams.key) extract mode and id to fill into the res_map + res_map := make(map[string]interface{}) + /* from DB-key string(inParams.key) extract mode and id to fill into the res_map * db key contains the separator as well eg: "mode:test123:3545" - */ - log.Info("DbToYang_test_sensor_mode_key_xfmr: key", inParams.key) - if len(inParams.key) > 0 { - /*split id and mode */ + */ + log.Info("DbToYang_test_sensor_mode_key_xfmr: key", inParams.key) + if len(inParams.key) > 0 { + /*split id and mode */ temp := strings.SplitN(inParams.key, ":", 3) if len(temp) >= 3 { res_map["mode"] = temp[0] + ":" + temp[1] - id := temp[2] + id := temp[2] i64, _ := strconv.ParseUint(id, 10, 32) i32 := uint32(i64) - res_map["id"] = i32 + res_map["id"] = i32 } else if len(temp) == 2 { res_map["mode"] = temp[0] res_map["id"] = temp[1] @@ -708,7 +693,7 @@ var DbToYang_test_sensor_mode_key_xfmr SonicKeyXfmrDbToYang = func(inParams Soni errStr := "Invalid Key in uri." return res_map, tlerr.InvalidArgsError{Format: errStr} } - } - log.Info("DbToYang_test_sensor_mode_key_xfmr: res_map - ", res_map) - return res_map, nil + } + log.Info("DbToYang_test_sensor_mode_key_xfmr: res_map - ", res_map) + return res_map, nil } diff --git a/translib/transformer/xlate.go b/translib/transformer/xlate.go index e6aaf51a9e2e..fdc7c2beb5b5 100644 --- a/translib/transformer/xlate.go +++ b/translib/transformer/xlate.go @@ -196,7 +196,7 @@ func updateDbDataMapAndKeyCache(dbKeyStr string, data *db.Value, spec *KeySpec, dbTblKeyGetCache[spec.DbNum][spec.Ts.Name][dbKeyStr] = readOk } -func XlateUriToKeySpec(uri string, requestUri string, ygRoot *ygot.GoStruct, t *interface{}, txCache interface{}) (*[]KeySpec, error) { +func XlateUriToKeySpec(uri string, requestUri string, ygRoot *ygot.GoStruct, t *interface{}, txCache interface{}, qParams QueryParams) (*[]KeySpec, error) { var err error var retdbFormat = make([]KeySpec, 0) @@ -214,17 +214,23 @@ func XlateUriToKeySpec(uri string, requestUri string, ygRoot *ygot.GoStruct, t * } } - retdbFormat = fillSonicKeySpec(xpath, tableName, keyStr) + retdbFormat = fillSonicKeySpec(xpath, tableName, keyStr, qParams.content) } else { + var reqUriXpath string /* Extract the xpath and key from input xpath */ retData, _ := xpathKeyExtract(nil, ygRoot, GET, uri, requestUri, nil, nil, txCache, nil) - retdbFormat = fillKeySpecs(retData.xpath, retData.dbKey, &retdbFormat) + if requestUri == uri { + reqUriXpath = retData.xpath + } else { + reqUriXpath, _, _ = XfmrRemoveXPATHPredicates(requestUri) + } + retdbFormat = fillKeySpecs(reqUriXpath, &qParams, retData.xpath, retData.dbKey, &retdbFormat) } return &retdbFormat, err } -func fillKeySpecs(yangXpath string, keyStr string, retdbFormat *[]KeySpec) []KeySpec { +func fillKeySpecs(reqUriXpath string, qParams *QueryParams, yangXpath string, keyStr string, retdbFormat *[]KeySpec) []KeySpec { var err error if xYangSpecMap == nil { return *retdbFormat @@ -261,8 +267,10 @@ func fillKeySpecs(yangXpath string, keyStr string, retdbFormat *[]KeySpec) []Key if chlen > 0 { children := make([]KeySpec, 0) for _, childXpath := range xDbSpecMap[child].yangXpath { - children = fillKeySpecs(childXpath, "", &children) - dbFormat.Child = append(dbFormat.Child, children...) + if isChildTraversalRequired(reqUriXpath, qParams, childXpath) { + children = fillKeySpecs(reqUriXpath, qParams, childXpath, "", &children) + dbFormat.Child = append(dbFormat.Child, children...) + } } } } @@ -276,7 +284,9 @@ func fillKeySpecs(yangXpath string, keyStr string, retdbFormat *[]KeySpec) []Key chlen := len(xDbSpecMap[child].yangXpath) if chlen > 0 { for _, childXpath := range xDbSpecMap[child].yangXpath { - *retdbFormat = fillKeySpecs(childXpath, "", retdbFormat) + if isChildTraversalRequired(reqUriXpath, qParams, childXpath) { + *retdbFormat = fillKeySpecs(reqUriXpath, qParams, childXpath, "", retdbFormat) + } } } } @@ -287,7 +297,7 @@ func fillKeySpecs(yangXpath string, keyStr string, retdbFormat *[]KeySpec) []Key return *retdbFormat } -func fillSonicKeySpec(xpath string, tableName string, keyStr string) []KeySpec { +func fillSonicKeySpec(xpath string, tableName string, keyStr string, content ContentType) []KeySpec { var retdbFormat = make([]KeySpec, 0) @@ -296,6 +306,9 @@ func fillSonicKeySpec(xpath string, tableName string, keyStr string) []KeySpec { dbFormat.Ts.Name = tableName cdb := db.ConfigDB if _, ok := xDbSpecMap[tableName]; ok { + if (xDbSpecMap[tableName].dbEntry == nil) || ((content == QUERY_CONTENT_CONFIG) && (xDbSpecMap[tableName].dbEntry.ReadOnly())) || ((content == QUERY_CONTENT_NONCONFIG) && (!xDbSpecMap[tableName].dbEntry.ReadOnly())) { + return retdbFormat + } cdb = xDbSpecMap[tableName].dbIndex } dbFormat.DbNum = cdb @@ -313,6 +326,9 @@ func fillSonicKeySpec(xpath string, tableName string, keyStr string) []KeySpec { for dir := range dbInfo.dbEntry.Dir { _, ok := xDbSpecMap[dir] if ok && xDbSpecMap[dir].yangType == YANG_CONTAINER { + if (xDbSpecMap[dir].dbEntry == nil) || ((content == QUERY_CONTENT_CONFIG) && (xDbSpecMap[dir].dbEntry.ReadOnly())) || ((content == QUERY_CONTENT_NONCONFIG) && (!xDbSpecMap[dir].dbEntry.ReadOnly())) { + continue + } cdb := xDbSpecMap[dir].dbIndex dbFormat := KeySpec{} dbFormat.Ts.Name = dir @@ -387,14 +403,42 @@ func XlateToDb(path string, oper int, d *db.DB, yg *ygot.GoStruct, yt *interface return result, yangDefValMap, yangAuxValMap, err } -func GetAndXlateFromDB(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, txCache interface{}) ([]byte, bool, error) { +func GetAndXlateFromDB(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, txCache interface{}, qParams QueryParams) ([]byte, bool, error) { var err error var payload []byte var inParamsForGet xlateFromDbParams + var processReq bool + inParamsForGet.queryParams = qParams xfmrLogInfo("received xpath = ", uri) requestUri := uri - keySpec, _ := XlateUriToKeySpec(uri, requestUri, ygRoot, nil, txCache) + if len(qParams.fields) > 0 { + xfmrLogDebug("Process fields QP") //todo + yngNdType, nd_err := getYangNodeTypeFromUri(uri) + if nd_err != nil { + return []byte("{}"), false, nd_err + } else if (yngNdType == YANG_LEAF) || (yngNdType == YANG_LEAF_LIST) { + err = tlerr.InvalidArgsError{Format: "Bad Request - fields query parameter specified on a terminal node uri."} + return []byte("{}"), false, err + } + } else { + processReq, err = contentQParamTgtEval(uri, qParams) + if err != nil { + return []byte("{}"), false, err + } + if !processReq { + xfmrLogInfo("further processing of request not needed due to content query param.") + /* translib fills requested list-instance into ygot, but when there is content-mismatch + we have to send empty payload response.So distinguish this case in common_app we send this err + */ + if IsListNode(uri) { + return []byte("{}"), true, tlerr.InternalError{Format: QUERY_CONTENT_MISMATCH_ERR} + } + return []byte("{}"), true, err + } + } + + keySpec, _ := XlateUriToKeySpec(uri, requestUri, ygRoot, nil, txCache, qParams) var dbresult = make(RedisDbMap) for i := db.ApplDB; i < db.MaxDB; i++ { dbresult[i] = make(map[string]map[string]db.Value) @@ -510,7 +554,8 @@ func XlateFromDb(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, data R } } dbTblKeyGetCache := inParamsForGet.dbTblKeyGetCache - inParamsForGet = formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, xpath, GET, "", "", &dbData, txCache, nil, false) + qparams := inParamsForGet.queryParams + inParamsForGet = formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, xpath, GET, "", "", &dbData, txCache, nil, false, qparams, nil) inParamsForGet.xfmrDbTblKeyCache = make(map[string]tblKeyCache) inParamsForGet.dbTblKeyGetCache = dbTblKeyGetCache payload, isEmptyPayload, err := dbDataToYangJsonCreate(inParamsForGet) @@ -726,6 +771,15 @@ func IsLeafListNode(uri string) bool { return result } +func IsListNode(uri string) bool { + result := false + yngNdType, err := getYangNodeTypeFromUri(uri) + if (err == nil) && (yngNdType == YANG_LIST) { + result = true + } + return result +} + func tableKeysToBeSorted(tblNm string) bool { /* function to decide whether to sort table keys. Required when a sonic table has more than 1 lists diff --git a/translib/transformer/xlate_datastructs.go b/translib/transformer/xlate_datastructs.go index 1f8868a67db8..25a7686f3af8 100644 --- a/translib/transformer/xlate_datastructs.go +++ b/translib/transformer/xlate_datastructs.go @@ -93,7 +93,9 @@ type xlateFromDbParams struct { resultMap map[string]interface{} validate bool xfmrDbTblKeyCache map[string]tblKeyCache + queryParams QueryParams dbTblKeyGetCache map[db.DBNum]map[string]map[string]bool + listKeysMap map[string]interface{} } type xlateToParams struct { @@ -122,8 +124,35 @@ type xlateToParams struct { invokeCRUSubtreeOnceMap map[string]map[string]bool } +type contentQPSpecMapInfo struct { + yangType yangElementType + yangName string + isReadOnly bool + isOperationalNd bool + hasNonTerminalNd bool + hasChildOperationalNd bool + isOcMdl bool +} + +type qpSubtreePruningErr struct { + subtreePath string +} + type Operation int + +type ContentType uint8 + +type QueryParams struct { + depthEnabled bool + curDepth uint + content ContentType + fields []string + fieldsFillAll bool + allowFieldsXpath map[string]bool + tgtFieldsXpathMap map[string][]string +} + type ygotUnMarshalCtx struct { ygParentObj *ygot.GoStruct relUri string @@ -135,4 +164,5 @@ type ygotUnMarshalCtx struct { type ygotXlator struct { ygotCtx *ygotUnMarshalCtx -} \ No newline at end of file +} + diff --git a/translib/transformer/xlate_from_db.go b/translib/transformer/xlate_from_db.go index 18342fc6b6e2..76d204af1c6b 100644 --- a/translib/transformer/xlate_from_db.go +++ b/translib/transformer/xlate_from_db.go @@ -247,6 +247,15 @@ func sonicDbToYangTerminalNodeFill(field string, inParamsForGet xlateFromDbParam resField := field value := "" + if len(inParamsForGet.queryParams.fields) > 0 { + curFldXpath := inParamsForGet.tbl + "/" + field + if _, ok := inParamsForGet.queryParams.tgtFieldsXpathMap[curFldXpath]; !ok { + if !inParamsForGet.queryParams.fieldsFillAll { + return + } + } + } + if inParamsForGet.dbDataMap != nil { tblInstFields, dbDataExists := (*inParamsForGet.dbDataMap)[inParamsForGet.curDb][inParamsForGet.tbl][inParamsForGet.tblKey] if dbDataExists { @@ -311,7 +320,7 @@ func sonicDbToYangListFill(inParamsForGet xlateFromDbParams) []typeMapOfInterfac curMap := make(map[string]interface{}) sonicKeyDataAdd(dbIdx, yangKeys, table, xDbSpecMap[xpath].dbEntry.Name, keyStr, curMap) if len(curMap) > 0 { - linParamsForGet := formXlateFromDbParams(inParamsForGet.dbs[dbIdx], inParamsForGet.dbs, dbIdx, inParamsForGet.ygRoot, inParamsForGet.uri, inParamsForGet.requestUri, xpath, inParamsForGet.oper, table, keyStr, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate) + linParamsForGet := formXlateFromDbParams(inParamsForGet.dbs[dbIdx], inParamsForGet.dbs, dbIdx, inParamsForGet.ygRoot, inParamsForGet.uri, inParamsForGet.requestUri, xpath, inParamsForGet.oper, table, keyStr, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate, inParamsForGet.queryParams, nil) sonicDbToYangDataFill(linParamsForGet) curMap = linParamsForGet.resultMap dbDataMap = linParamsForGet.dbDataMap @@ -364,7 +373,7 @@ func sonicDbToYangDataFill(inParamsForGet xlateFromDbParams) { fldName = fldName + "@" } curUri := inParamsForGet.uri + "/" + yangChldName - linParamsForGet := formXlateFromDbParams(nil, inParamsForGet.dbs, dbIdx, inParamsForGet.ygRoot, curUri, inParamsForGet.requestUri, chldXpath, inParamsForGet.oper, table, key, dbDataMap, inParamsForGet.txCache, resultMap, inParamsForGet.validate) + linParamsForGet := formXlateFromDbParams(nil, inParamsForGet.dbs, dbIdx, inParamsForGet.ygRoot, curUri, inParamsForGet.requestUri, chldXpath, inParamsForGet.oper, table, key, dbDataMap, inParamsForGet.txCache, resultMap, inParamsForGet.validate, inParamsForGet.queryParams, nil) dbEntry := yangNode.dbEntry.Dir[yangChldName] sonicDbToYangTerminalNodeFill(fldName, linParamsForGet, dbEntry) resultMap = linParamsForGet.resultMap @@ -372,14 +381,37 @@ func sonicDbToYangDataFill(inParamsForGet xlateFromDbParams) { } else if chldYangType == YANG_CONTAINER { curMap := make(map[string]interface{}) curUri := uri + "/" + yangChldName + if len(inParamsForGet.queryParams.fields) > 0 { + if _, ok := inParamsForGet.queryParams.tgtFieldsXpathMap[chldXpath]; ok { + inParamsForGet.queryParams.fieldsFillAll = true + } else if _, ok := inParamsForGet.queryParams.allowFieldsXpath[chldXpath]; !ok { + if !inParamsForGet.queryParams.fieldsFillAll { + for path := range inParamsForGet.queryParams.tgtFieldsXpathMap { + if strings.HasPrefix(chldXpath, path) { + inParamsForGet.queryParams.fieldsFillAll = true + } + } + if !inParamsForGet.queryParams.fieldsFillAll { + continue + } + } + } + } // container can have a static key, so extract key for current container _, curKey, curTable := sonicXpathKeyExtract(curUri) if _, specmapOk := xDbSpecMap[curTable]; !specmapOk || xDbSpecMap[curTable].dbEntry == nil { xfmrLogDebug("Yang entry not found for %v", curTable) continue } + if inParamsForGet.queryParams.content != QUERY_CONTENT_ALL { + processReq, _ := sonicContentQParamYangNodeProcess(curUri, xDbSpecMap[curTable].yangType, xDbSpecMap[curTable].dbEntry.ReadOnly(), inParamsForGet.queryParams) + if !processReq { + xfmrLogDebug("Further traversal not needed due to content query param, for of URI - %v", curUri) + continue + } + } d := inParamsForGet.dbs[xDbSpecMap[curTable].dbIndex] - linParamsForGet := formXlateFromDbParams(d, inParamsForGet.dbs, xDbSpecMap[curTable].dbIndex, inParamsForGet.ygRoot, curUri, inParamsForGet.requestUri, chldXpath, inParamsForGet.oper, curTable, curKey, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate) + linParamsForGet := formXlateFromDbParams(d, inParamsForGet.dbs, xDbSpecMap[curTable].dbIndex, inParamsForGet.ygRoot, curUri, inParamsForGet.requestUri, chldXpath, inParamsForGet.oper, curTable, curKey, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate, inParamsForGet.queryParams, nil) sonicDbToYangDataFill(linParamsForGet) curMap = linParamsForGet.resultMap dbDataMap = linParamsForGet.dbDataMap @@ -391,6 +423,7 @@ func sonicDbToYangDataFill(inParamsForGet xlateFromDbParams) { } else { xfmrLogDebug("Empty container for xpath(%v)", curUri) } + inParamsForGet.queryParams.fieldsFillAll = false inParamsForGet.dbDataMap = linParamsForGet.dbDataMap inParamsForGet.resultMap = resultMap } else if chldYangType == YANG_LIST { @@ -398,6 +431,22 @@ func sonicDbToYangDataFill(inParamsForGet xlateFromDbParams) { curUri := uri + "/" + yangChldName inParamsForGet.uri = curUri inParamsForGet.xpath = chldXpath + if len(inParamsForGet.queryParams.fields) > 0 { + if _, ok := inParamsForGet.queryParams.tgtFieldsXpathMap[chldXpath]; ok { + inParamsForGet.queryParams.fieldsFillAll = true + } else if _, ok := inParamsForGet.queryParams.allowFieldsXpath[chldXpath]; !ok { + if !inParamsForGet.queryParams.fieldsFillAll { + for path := range inParamsForGet.queryParams.tgtFieldsXpathMap { + if strings.HasPrefix(chldXpath, path) { + inParamsForGet.queryParams.fieldsFillAll = true + } + } + if !inParamsForGet.queryParams.fieldsFillAll { + continue + } + } + } + } mapSlice = sonicDbToYangListFill(inParamsForGet) dbDataMap = inParamsForGet.dbDataMap if len(key) > 0 && len(mapSlice) == 1 { // Single instance query. Don't return array of maps @@ -410,6 +459,7 @@ func sonicDbToYangDataFill(inParamsForGet xlateFromDbParams) { } else { xfmrLogDebug("Empty list for xpath(%v)", curUri) } + inParamsForGet.queryParams.fieldsFillAll = false inParamsForGet.resultMap = resultMap } else if chldYangType == YANG_CHOICE || chldYangType == YANG_CASE { inParamsForGet.xpath = chldXpath @@ -430,6 +480,7 @@ func sonicDbToYangDataFill(inParamsForGet xlateFromDbParams) { func directDbToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, error) { var err error uri := inParamsForGet.uri + jsonData := "{}" dbDataMap := inParamsForGet.dbDataMap resultMap := inParamsForGet.resultMap xpath, key, table := sonicXpathKeyExtract(uri) @@ -438,7 +489,25 @@ func directDbToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, e inParamsForGet.tblKey = key fieldName := "" - if len(xpath) > 0 { + traverse := true + if inParamsForGet.queryParams.depthEnabled { + pathList := strings.Split(xpath, "/") + // Depth at requested URI starts at 1. Hence reduce the reqDepth calculated by 1 + reqDepth := (len(pathList) - 1) + int(inParamsForGet.queryParams.curDepth) - 1 + xfmrLogInfo("xpath: %v ,Sonic Yang len(pathlist) %v, reqDepth %v, sonic Field Index %v", xpath, len(pathList), reqDepth, SONIC_FIELD_INDEX) + if reqDepth < SONIC_FIELD_INDEX { + traverse = false + } + } + + if len(inParamsForGet.queryParams.fields) > 0 { + flderr := validateAndFillSonicQpFields(inParamsForGet) + if flderr != nil { + return jsonData, true, flderr + } + } + + if traverse && len(xpath) > 0 { tokens := strings.Split(xpath, "/") if len(tokens) > SONIC_FIELD_INDEX { // Request is at levelf/leaflist level @@ -453,7 +522,7 @@ func directDbToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, e // Request is at table level xpath = table xfmrLogDebug("Request is at table level container - %v.", uri) - } else { + } else { // Request is at top level container prefixed by module name xfmrLogDebug("Request is at top level container - %v.", uri) } @@ -476,7 +545,7 @@ func directDbToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, e if yangType == YANG_LEAF_LIST { fieldName = fieldName + "@" } - linParamsForGet := formXlateFromDbParams(nil, inParamsForGet.dbs, cdb, inParamsForGet.ygRoot, xpath, inParamsForGet.requestUri, uri, inParamsForGet.oper, table, key, dbDataMap, inParamsForGet.txCache, resultMap, inParamsForGet.validate) + linParamsForGet := formXlateFromDbParams(nil, inParamsForGet.dbs, cdb, inParamsForGet.ygRoot, xpath, inParamsForGet.requestUri, uri, inParamsForGet.oper, table, key, dbDataMap, inParamsForGet.txCache, resultMap, inParamsForGet.validate, inParamsForGet.queryParams, nil) sonicDbToYangTerminalNodeFill(fieldName, linParamsForGet, dbEntry) resultMap = linParamsForGet.resultMap } else if yangType == YANG_CONTAINER { @@ -500,7 +569,7 @@ func directDbToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, e jsonMapData, _ := json.Marshal(resultMap) isEmptyPayload := isJsonDataEmpty(string(jsonMapData)) - jsonData := fmt.Sprintf("%v", string(jsonMapData)) + jsonData = fmt.Sprintf("%v", string(jsonMapData)) if isEmptyPayload { log.Warning("No data available") } @@ -589,7 +658,7 @@ func dbDataFromTblXfmrGet(tbl string, inParams XfmrParams, dbDataMap *map[db.DBN return nil } -func yangListDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool) error { +func yangListDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool, isOcMdl bool) error { var tblList []string dbs := inParamsForGet.dbs ygRoot := inParamsForGet.ygRoot @@ -605,6 +674,11 @@ func yangListDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool) error _, ok := xYangSpecMap[xpath] if ok { + // Do not handle list if the curdepth is 1 + if inParamsForGet.queryParams.depthEnabled && (inParamsForGet.queryParams.curDepth <= 1) { + return nil + } + if xYangSpecMap[xpath].xfmrTbl != nil { xfmrTblFunc := *xYangSpecMap[xpath].xfmrTbl if len(xfmrTblFunc) > 0 { @@ -631,10 +705,15 @@ func yangListDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool) error // Handling for case: Parent list is not associated with a tableName but has children containers/lists having tableNames. if tblKey != "" { var mapSlice []typeMapOfInterface - instMap, err := yangListInstanceDataFill(inParamsForGet, isFirstCall) + instMap, err := yangListInstanceDataFill(inParamsForGet, isFirstCall, isOcMdl) dbDataMap = inParamsForGet.dbDataMap if err != nil { xfmrLogDebug("Error(%v) returned for %v", err, uri) + // abort GET request if QP Subtree Pruning API returns error + _, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return err + } } else if (instMap != nil) && (len(instMap) > 0) { mapSlice = append(mapSlice, instMap) } @@ -658,7 +737,19 @@ func yangListDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool) error } if len(tblList) == 0 { - xfmrLogInfo("Unable to traverse list as no table information available at URI %v. Please check if table mapping available", uri) + if (strings.HasSuffix(uri, "]") || strings.HasSuffix(uri, "]/")) && (len(xYangSpecMap[xpath].xfmrFunc) > 0 && xYangSpecMap[xpath].hasChildSubTree) { + err := yangDataFill(inParamsForGet, isOcMdl) + if err != nil { + xfmrLogInfo("yangListDataFill: error in its child subtree traversal for the xpath: %v", xpath) + _, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return err + } + } + return nil + } else { + xfmrLogInfo("Unable to traverse list as no table information available at URI %v. Please check if table mapping available", uri) + } } for _, tbl = range tblList { @@ -670,10 +761,15 @@ func yangListDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool) error var mapSlice []typeMapOfInterface for dbKey := range tblData { inParamsForGet.tblKey = dbKey - instMap, err := yangListInstanceDataFill(inParamsForGet, isFirstCall) + instMap, err := yangListInstanceDataFill(inParamsForGet, isFirstCall, isOcMdl) dbDataMap = inParamsForGet.dbDataMap if err != nil { xfmrLogDebug("Error(%v) returned for %v", err, uri) + // abort GET request if QP Subtree Pruning API returns error + _, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return err + } } else if (instMap != nil) && (len(instMap) > 0) { mapSlice = append(mapSlice, instMap) } @@ -709,7 +805,7 @@ func yangListDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool) error return nil } -func yangListInstanceDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool) (typeMapOfInterface, error) { +func yangListInstanceDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool, isOcMdl bool) (typeMapOfInterface, error) { var err error curMap := make(map[string]interface{}) @@ -737,18 +833,31 @@ func yangListInstanceDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool (len(xYangSpecMap[parentXpath].xfmrFunc) > 0 && (xYangSpecMap[parentXpath].xfmrFunc != xYangSpecMap[xpath].xfmrFunc)))) { xfmrLogDebug("Parent subtree already handled cur uri: %v", xpath) inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, curUri, requestUri, GET, dbKey, dbDataMap, nil, nil, txCache) + inParams.queryParams = inParamsForGet.queryParams err := xfmrHandlerFunc(inParams, xYangSpecMap[xpath].xfmrFunc) inParamsForGet.ygRoot = ygRoot inParamsForGet.dbDataMap = dbDataMap if err != nil { xfmrLogDebug("Error returned by %v: %v", xYangSpecMap[xpath].xfmrFunc, err) + // abort GET request if QP Subtree Pruning API returns error + _, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return curMap, err + } } } if xYangSpecMap[xpath].hasChildSubTree { - linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, curUri, requestUri, xpath, inParamsForGet.oper, tbl, dbKey, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate) + linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, curUri, requestUri, xpath, inParamsForGet.oper, tbl, dbKey, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate, inParamsForGet.queryParams, nil) linParamsForGet.xfmrDbTblKeyCache = inParamsForGet.xfmrDbTblKeyCache linParamsForGet.dbTblKeyGetCache = inParamsForGet.dbTblKeyGetCache - yangDataFill(linParamsForGet) + err := yangDataFill(linParamsForGet, isOcMdl) + if err != nil { + // abort GET request if QP Subtree Pruning API returns error + _, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return nil, err + } + } curMap = linParamsForGet.resultMap dbDataMap = linParamsForGet.dbDataMap ygRoot = linParamsForGet.ygRoot @@ -761,16 +870,27 @@ func yangListInstanceDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool xpathKeyExtRet, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, curUri, requestUri, dbDataMap, nil, txCache, inParamsForGet.xfmrDbTblKeyCache) keyFromCurUri := xpathKeyExtRet.dbKey inParamsForGet.ygRoot = ygRoot + var listKeyMap map[string]interface{} if dbKey == keyFromCurUri || keyFromCurUri == "" { if dbKey == keyFromCurUri { + listKeyMap = make(map[string]interface{}) for k, kv := range curKeyMap { curMap[k] = kv + listKeyMap[k] = kv } + } - linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, curUri, requestUri, xpathKeyExtRet.xpath, inParamsForGet.oper, tbl, dbKey, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate) + linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, curUri, requestUri, xpathKeyExtRet.xpath, inParamsForGet.oper, tbl, dbKey, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate, inParamsForGet.queryParams, listKeyMap) linParamsForGet.xfmrDbTblKeyCache = inParamsForGet.xfmrDbTblKeyCache linParamsForGet.dbTblKeyGetCache = inParamsForGet.dbTblKeyGetCache - yangDataFill(linParamsForGet) + err := yangDataFill(linParamsForGet, isOcMdl) + if err != nil { + // abort GET request if QP Subtree Pruning API returns error + _, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return curMap, err + } + } curMap = linParamsForGet.resultMap dbDataMap = linParamsForGet.dbDataMap ygRoot = linParamsForGet.ygRoot @@ -797,6 +917,13 @@ func terminalNodeProcess(inParamsForGet xlateFromDbParams, terminalNodeQuery boo dbDataMap := inParamsForGet.dbDataMap txCache := inParamsForGet.txCache + if uri != requestUri { + //Chk should be already done in yangDataFill + if inParamsForGet.queryParams.depthEnabled && inParamsForGet.queryParams.curDepth == 0 { + return resFldValMap, err + } + } + _, ok := xYangSpecMap[xpath] if !ok || yangEntry == nil { logStr := fmt.Sprintf("No YANG entry found for xpath %v.", xpath) @@ -807,6 +934,7 @@ func terminalNodeProcess(inParamsForGet xlateFromDbParams, terminalNodeQuery boo cdb := xYangSpecMap[xpath].dbIndex if len(xYangSpecMap[xpath].xfmrField) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, tblKey, dbDataMap, nil, nil, txCache) + inParams.queryParams = inParamsForGet.queryParams fldValMap, err := leafXfmrHandlerFunc(inParams, xYangSpecMap[xpath].xfmrField) inParamsForGet.ygRoot = ygRoot inParamsForGet.dbDataMap = dbDataMap @@ -837,10 +965,12 @@ func terminalNodeProcess(inParamsForGet xlateFromDbParams, terminalNodeQuery boo if dbFldName == XFMR_NONE_STRING { return resFldValMap, err } - /* if there is no transformer extension/annotation then it means leaf-list in YANG is also leaflist in db */ - if len(dbFldName) > 0 && !xYangSpecMap[xpath].isKey { + fillLeafFromUriKey := false + yangDataType := yangEntry.Type.Kind + if terminalNodeQuery && xYangSpecMap[xpath].isKey { //GET request for list key-leaf(direct child of list) + fillLeafFromUriKey = true + } else if len(dbFldName) > 0 && !xYangSpecMap[xpath].isKey { yangType := xYangSpecMap[xpath].yangType - yngTerminalNdDtType := yangEntry.Type.Kind if yangType == YANG_LEAF_LIST { dbFldName += "@" val, ok := (*dbDataMap)[cdb][tbl][tblKey].Field[dbFldName] @@ -880,7 +1010,7 @@ func terminalNodeProcess(inParamsForGet xlateFromDbParams, terminalNodeQuery boo } return resFldValMap, err } else { - resLst := processLfLstDbToYang(xpath, val, yngTerminalNdDtType) + resLst := processLfLstDbToYang(xpath, val, yangDataType) resFldValMap[yangEntry.Name] = resLst } } else { @@ -892,15 +1022,57 @@ func terminalNodeProcess(inParamsForGet xlateFromDbParams, terminalNodeQuery boo } else { val, ok := (*dbDataMap)[cdb][tbl][tblKey].Field[dbFldName] if ok { - resVal, _, err := DbToYangType(yngTerminalNdDtType, xpath, val) + resVal, _, err := DbToYangType(yangDataType, xpath, val) if err != nil { log.Warning("Conversion of DB value type to YANG type for field didn't happen. Field-xfmr recommended if data types differ. Field xpath", xpath) } else { resFldValMap[yangEntry.Name] = resVal } } else { - xfmrLogDebug("Field value does not exist in DB for - %v", uri) - err = tlerr.NotFoundError{Format: "Resource not found"} + resNotFound := true + if xYangSpecMap[xpath].isRefByKey { + /*config container leaf is referenced by list-key that exists already before reaching here + state container leaf is not referenced by list-key, some state containers are mapped to + non-config DB(different from list level DB mapping), so check instance existence before + filling from uri*/ + _, stateContainerInstanceOk := (*dbDataMap)[cdb][tbl][tblKey] + if !yangEntry.ReadOnly() || stateContainerInstanceOk { + fillLeafFromUriKey = true + resNotFound = false + } + } + if resNotFound { + xfmrLogDebug("Field value does not exist in DB for - %v", uri) + err = tlerr.NotFoundError{Format: "Resource not found"} + } + } + } + } else if (len(dbFldName) == 0) && (xYangSpecMap[xpath].isRefByKey) { + _, stateContainerInstanceOk := (*dbDataMap)[cdb][tbl][tblKey] + if !yangEntry.ReadOnly() || stateContainerInstanceOk { + fillLeafFromUriKey = true + } + } + if fillLeafFromUriKey { + extractKeyLeafFromUri := true + xfmrLogDebug("Inferring isRefByKey Leaf %v value from list instance/keys", xpath) + if len(inParamsForGet.listKeysMap) > 0 { + if resVal, keyLeafExists := inParamsForGet.listKeysMap[yangEntry.Name]; keyLeafExists { + resFldValMap = make(map[string]interface{}) + resFldValMap[yangEntry.Name] = resVal + xfmrLogDebug("Filled isRefByKey leaf value from list keys map") + extractKeyLeafFromUri = false + } + } + if extractKeyLeafFromUri { + xfmrLogDebug("Filling isRefByKey leaf valuea from uri string.") + val := extractLeafValFromUriKey(uri, yangEntry.Name) + resVal, _, err := DbToYangType(yangDataType, xpath, val) + if err != nil { + log.Warning("Conversion of DB value type to YANG type for field didn't happen. Field-xfmr recommended if data types differ. Field xpath", xpath) + } else { + resFldValMap = make(map[string]interface{}) + resFldValMap[yangEntry.Name] = resVal } } } @@ -908,7 +1080,7 @@ func terminalNodeProcess(inParamsForGet xlateFromDbParams, terminalNodeQuery boo return resFldValMap, err } -func yangDataFill(inParamsForGet xlateFromDbParams) error { +func yangDataFill(inParamsForGet xlateFromDbParams, isOcMdl bool) error { var err error validate := inParamsForGet.validate isValid := validate @@ -925,8 +1097,19 @@ func yangDataFill(inParamsForGet xlateFromDbParams) error { yangNode, ok := xYangSpecMap[xpath] if ok && yangNode.yangEntry != nil { + if inParamsForGet.queryParams.depthEnabled { + // If fields are available, we need to evaluate the depth after fields level is met + // The depth upto the fields level is considered at level 1 + inParamsForGet.queryParams.curDepth = inParamsForGet.queryParams.curDepth - 1 + log.Infof("yangDataFill curdepth: %v, Path : %v", inParamsForGet.queryParams.curDepth, xpath) + if inParamsForGet.queryParams.curDepth == 0 { + return err + } + } + for yangChldName := range yangNode.yangEntry.Dir { chldXpath := xpath + "/" + yangChldName + chFieldsFillAll := inParamsForGet.queryParams.fieldsFillAll if xYangSpecMap[chldXpath] != nil && xYangSpecMap[chldXpath].nameWithMod != nil { chldUri = uri + "/" + *(xYangSpecMap[chldXpath].nameWithMod) } else { @@ -935,6 +1118,30 @@ func yangDataFill(inParamsForGet xlateFromDbParams) error { inParamsForGet.xpath = chldXpath inParamsForGet.uri = chldUri if xYangSpecMap[chldXpath] != nil && yangNode.yangEntry.Dir[yangChldName] != nil { + chldYangType := xYangSpecMap[chldXpath].yangType + if inParamsForGet.queryParams.content != QUERY_CONTENT_ALL { + yangNdInfo := contentQPSpecMapInfo{ + yangType: chldYangType, + yangName: yangChldName, + isReadOnly: yangNode.yangEntry.Dir[yangChldName].ReadOnly(), + isOperationalNd: xYangSpecMap[chldXpath].operationalQP, + hasNonTerminalNd: xYangSpecMap[chldXpath].hasNonTerminalNode, + hasChildOperationalNd: xYangSpecMap[chldXpath].hasChildOpertnlNd, + isOcMdl: isOcMdl, + } + processReq, _ := contentQParamYangNodeProcess(chldUri, yangNdInfo, inParamsForGet.queryParams) + if !processReq { + xfmrLogDebug("Further traversal not needed due to content query param, for of URI - %v", chldUri) + continue + } + } + + if inParamsForGet.queryParams.depthEnabled && inParamsForGet.queryParams.curDepth == 1 { + if (chldYangType == YANG_CONTAINER) || (chldYangType == YANG_LIST) { + continue + } + } + cdb := xYangSpecMap[chldXpath].dbIndex inParamsForGet.curDb = cdb if len(xYangSpecMap[chldXpath].validateFunc) > 0 && !validate { @@ -942,6 +1149,7 @@ func yangDataFill(inParamsForGet xlateFromDbParams) error { inParamsForGet.ygRoot = ygRoot // TODO - handle non CONFIG-DB inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, GET, xpathKeyExtRet.dbKey, dbDataMap, nil, nil, txCache) + inParams.queryParams = inParamsForGet.queryParams res := validateHandlerFunc(inParams, xYangSpecMap[chldXpath].validateFunc) if !res { xfmrLogDebug("Further traversal not needed. Validate xfmr returns false for URI %v", chldUri) @@ -953,11 +1161,18 @@ func yangDataFill(inParamsForGet xlateFromDbParams) error { inParamsForGet.dbDataMap = dbDataMap inParamsForGet.ygRoot = ygRoot } - chldYangType := xYangSpecMap[chldXpath].yangType if chldYangType == YANG_LEAF || chldYangType == YANG_LEAF_LIST { if len(xYangSpecMap[xpath].xfmrFunc) > 0 { continue } + if !xYangSpecMap[chldXpath].isKey && len(inParamsForGet.queryParams.fields) > 0 { + if _, ok := inParamsForGet.queryParams.tgtFieldsXpathMap[chldXpath]; !ok { + if !inParamsForGet.queryParams.fieldsFillAll { + xfmrLogDebug("Skip processing URI due to fields QP procesing - %v", chldUri) + continue + } + } + } yangEntry := yangNode.yangEntry.Dir[yangChldName] fldValMap, err := terminalNodeProcess(inParamsForGet, false, yangEntry) dbDataMap = inParamsForGet.dbDataMap @@ -975,6 +1190,23 @@ func yangDataFill(inParamsForGet xlateFromDbParams) error { chtbl := xpathKeyExtRet.tableName inParamsForGet.ygRoot = ygRoot + if len(inParamsForGet.queryParams.fields) > 0 { + if _, ok := inParamsForGet.queryParams.tgtFieldsXpathMap[chldXpath]; ok { + chFieldsFillAll = true + } else if _, ok := inParamsForGet.queryParams.allowFieldsXpath[chldXpath]; !ok { + if !inParamsForGet.queryParams.fieldsFillAll { + for path := range inParamsForGet.queryParams.tgtFieldsXpathMap { + if strings.HasPrefix(chldXpath, path) { + chFieldsFillAll = true + } + } + if !chFieldsFillAll { + continue + } + } + } + } + if _, ok := (*dbDataMap)[cdb][chtbl][tblKey]; !ok && len(chtbl) > 0 { qdbMapHasTblData := false qdbMapHasTblKeyData := false @@ -1019,11 +1251,17 @@ func yangDataFill(inParamsForGet xlateFromDbParams) error { (len(xYangSpecMap[xpath].xfmrFunc) > 0 && (xYangSpecMap[xpath].xfmrFunc != xYangSpecMap[chldXpath].xfmrFunc)) { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, GET, "", dbDataMap, nil, nil, txCache) + inParams.queryParams = inParamsForGet.queryParams err := xfmrHandlerFunc(inParams, xYangSpecMap[chldXpath].xfmrFunc) inParamsForGet.dbDataMap = dbDataMap inParamsForGet.ygRoot = ygRoot if err != nil { xfmrLogDebug("Error returned by %v: %v", xYangSpecMap[xpath].xfmrFunc, err) + // abort GET request if QP Subtree Pruning API returns error + _, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return err + } } } if !xYangSpecMap[chldXpath].hasChildSubTree { @@ -1032,10 +1270,18 @@ func yangDataFill(inParamsForGet xlateFromDbParams) error { } } cmap2 := make(map[string]interface{}) - linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, chldXpath, inParamsForGet.oper, chtbl, tblKey, dbDataMap, inParamsForGet.txCache, cmap2, inParamsForGet.validate) + linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, chldXpath, inParamsForGet.oper, chtbl, tblKey, dbDataMap, inParamsForGet.txCache, cmap2, inParamsForGet.validate, inParamsForGet.queryParams, inParamsForGet.listKeysMap) linParamsForGet.xfmrDbTblKeyCache = inParamsForGet.xfmrDbTblKeyCache linParamsForGet.dbTblKeyGetCache = inParamsForGet.dbTblKeyGetCache - err = yangDataFill(linParamsForGet) + linParamsForGet.queryParams.fieldsFillAll = chFieldsFillAll + err = yangDataFill(linParamsForGet, isOcMdl) + if err != nil { + // abort GET request if QP Subtree Pruning API returns error + _, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return err + } + } cmap2 = linParamsForGet.resultMap dbDataMap = linParamsForGet.dbDataMap ygRoot = linParamsForGet.ygRoot @@ -1059,9 +1305,15 @@ func yangDataFill(inParamsForGet xlateFromDbParams) error { (len(xYangSpecMap[xpath].xfmrFunc) > 0 && (xYangSpecMap[xpath].xfmrFunc != xYangSpecMap[chldXpath].xfmrFunc)) { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, GET, "", dbDataMap, nil, nil, txCache) + inParams.queryParams = inParamsForGet.queryParams err := xfmrHandlerFunc(inParams, xYangSpecMap[chldXpath].xfmrFunc) if err != nil { xfmrLogDebug("Error returned by %v: %v", xYangSpecMap[chldXpath].xfmrFunc, err) + // abort GET request if QP Subtree Pruning API returns error + _, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return err + } } inParamsForGet.dbDataMap = dbDataMap inParamsForGet.ygRoot = ygRoot @@ -1083,10 +1335,17 @@ func yangDataFill(inParamsForGet xlateFromDbParams) error { inParamsForGet.dbDataMap = dbDataMap } } - linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, chldXpath, inParamsForGet.oper, lTblName, xpathKeyExtRet.dbKey, dbDataMap, inParamsForGet.txCache, resultMap, inParamsForGet.validate) + linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, chldXpath, inParamsForGet.oper, lTblName, xpathKeyExtRet.dbKey, dbDataMap, inParamsForGet.txCache, resultMap, inParamsForGet.validate, inParamsForGet.queryParams, nil) linParamsForGet.xfmrDbTblKeyCache = inParamsForGet.xfmrDbTblKeyCache linParamsForGet.dbTblKeyGetCache = inParamsForGet.dbTblKeyGetCache - yangListDataFill(linParamsForGet, false) + err := yangListDataFill(linParamsForGet, false, isOcMdl) + if err != nil { + // abort GET request if QP Subtree Pruning API returns error + _, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return err + } + } resultMap = linParamsForGet.resultMap dbDataMap = linParamsForGet.dbDataMap ygRoot = linParamsForGet.ygRoot @@ -1095,6 +1354,14 @@ func yangDataFill(inParamsForGet xlateFromDbParams) error { inParamsForGet.ygRoot = ygRoot } else if chldYangType == YANG_CHOICE || chldYangType == YANG_CASE { + err := yangDataFill(inParamsForGet, isOcMdl) + if err != nil { + // abort GET request if QP Subtree Pruning API returns error + _, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return err + } + } resultMap = inParamsForGet.resultMap dbDataMap = inParamsForGet.dbDataMap } else { @@ -1111,6 +1378,7 @@ func dbDataToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, err var err error var fldSbtErr error // used only when direct query on leaf/leaf-list having subtree var fldErr error //used only when direct query on leaf/leaf-list having field transformer + var isOcMdl bool jsonData := "{}" resultMap := make(map[string]interface{}) d := inParamsForGet.d @@ -1134,12 +1402,27 @@ func dbDataToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, err inParamsForGet.ygRoot = ygRoot yangNode, ok := xYangSpecMap[xpathKeyExtRet.xpath] if ok { + yangType := yangNode.yangType + //Check if fields are valid + if len(inParamsForGet.queryParams.fields) > 0 { + flderr := validateAndFillQpFields(inParamsForGet) + if flderr != nil { + return jsonData, true, flderr + } + } + + //Check if the request depth is 1 + if inParamsForGet.queryParams.depthEnabled && inParamsForGet.queryParams.curDepth == 1 && (yangType == YANG_CONTAINER || yangType == YANG_LIST || yangType == YANG_MODULE) { + return jsonData, true, err + } + /* Invoke pre-xfmr is present for the YANG module */ moduleName := "/" + strings.Split(uri, "/")[1] xfmrLogInfo("Module name for URI %s is %s", uri, moduleName) if xYangModSpecMap != nil { if modSpecInfo, specOk := xYangModSpecMap[moduleName]; specOk && (len(modSpecInfo.xfmrPre) > 0) { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, "", dbDataMap, nil, nil, txCache) + inParams.queryParams = inParamsForGet.queryParams err = preXfmrHandlerFunc(modSpecInfo.xfmrPre, inParams) xfmrLogInfo("Invoked pre transformer: %v, dbDataMap: %v ", modSpecInfo.xfmrPre, dbDataMap) if err != nil { @@ -1151,13 +1434,13 @@ func dbDataToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, err } } - yangType := yangNode.yangType validateHandlerFlag := false tableXfmrFlag := false IsValidate := false if len(xYangSpecMap[xpathKeyExtRet.xpath].validateFunc) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, xpathKeyExtRet.dbKey, dbDataMap, nil, nil, txCache) + inParams.queryParams = inParamsForGet.queryParams res := validateHandlerFunc(inParams, xYangSpecMap[xpathKeyExtRet.xpath].validateFunc) inParamsForGet.dbDataMap = dbDataMap inParamsForGet.ygRoot = ygRoot @@ -1170,6 +1453,9 @@ func dbDataToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, err } inParamsForGet.validate = IsValidate isList := false + if strings.HasPrefix(requestUri, "/"+OC_MDL_PFX) { + isOcMdl = true + } switch yangType { case YANG_LIST: isList = true @@ -1226,13 +1512,22 @@ func dbDataToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, err } if len(xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, "", dbDataMap, nil, nil, txCache) - fldSbtErr = xfmrHandlerFunc(inParams, xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) - if fldSbtErr != nil { + inParams.queryParams = inParamsForGet.queryParams + xfmrFuncErr := xfmrHandlerFunc(inParams, xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) + if xfmrFuncErr != nil { /*For request Uri pointing to leaf/leaf-list having subtree, error will be propagated - to handle check of leaf/leaf-list-instance existence in DB , which will be performed - by subtree + to handle check of leaf/leaf-list-instance existence in DB , which will be performed + by subtree + Error will also be propagated if QP Pruning API for subtree returns error */ - xfmrLogInfo("Error returned by %v: %v", xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc, err) + _, qpSbtPruneErrOk := xfmrFuncErr.(*qpSubtreePruningErr) + + if qpSbtPruneErrOk { + err = xfmrFuncErr + } else { + // propagate err from subtree callback + fldSbtErr = xfmrFuncErr + } inParamsForGet.ygRoot = ygRoot break } @@ -1268,9 +1563,13 @@ func dbDataToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, err } if len(xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, "", dbDataMap, nil, nil, txCache) + inParams.queryParams = inParamsForGet.queryParams err := xfmrHandlerFunc(inParams, xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) if err != nil { - xfmrLogInfo("Error returned by %v: %v", xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc, err) + qpSbtPruneErr, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + err = tlerr.InternalError{Format: QUERY_PARAMETER_SBT_PRUNING_ERR, Path: qpSbtPruneErr.subtreePath} + } return jsonData, true, err } inParamsForGet.dbDataMap = dbDataMap @@ -1281,7 +1580,7 @@ func dbDataToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, err } } inParamsForGet.resultMap = make(map[string]interface{}) - err = yangDataFill(inParamsForGet) + err = yangDataFill(inParamsForGet, isOcMdl) if err != nil { xfmrLogInfo("Empty container(\"%v\").\r\n", uri) } @@ -1291,8 +1590,14 @@ func dbDataToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, err isFirstCall := true if len(xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, "", dbDataMap, nil, nil, txCache) + inParams.queryParams = inParamsForGet.queryParams err := xfmrHandlerFunc(inParams, xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) if err != nil { + qpSbtPruneErr, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + err = tlerr.InternalError{Format: QUERY_PARAMETER_SBT_PRUNING_ERR, Path: qpSbtPruneErr.subtreePath} + return jsonData, true, err + } if ((strings.HasSuffix(uri, "]")) || (strings.HasSuffix(uri, "]/"))) && (uri == requestUri) { // The error handling here is for the deferred resource check error being handled by the subtree for virtual table cases. log.Warningf("Subtree at list instance level returns error %v for URI - %v", err, uri) @@ -1312,7 +1617,7 @@ func dbDataToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, err } } inParamsForGet.resultMap = make(map[string]interface{}) - err = yangListDataFill(inParamsForGet, isFirstCall) + err = yangListDataFill(inParamsForGet, isFirstCall, isOcMdl) if err != nil { xfmrLogInfo("yangListDataFill failed for list case(\"%v\").\r\n", uri) } @@ -1344,6 +1649,12 @@ func dbDataToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, err */ return jsonData, isEmptyPayload, fldErr } + if err != nil { + qpSbtPruneErr, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return jsonData, isEmptyPayload, tlerr.InternalError{Format: QUERY_PARAMETER_SBT_PRUNING_ERR, Path: qpSbtPruneErr.subtreePath} + } + } return jsonData, isEmptyPayload, nil } diff --git a/translib/transformer/xlate_to_db.go b/translib/transformer/xlate_to_db.go index 4c11fcb556de..6e04a8ead42b 100644 --- a/translib/transformer/xlate_to_db.go +++ b/translib/transformer/xlate_to_db.go @@ -184,21 +184,37 @@ func mapFillDataUtil(xlateParams xlateToParams) error { } if len(xpathInfo.fieldName) == 0 { - xfmrLogInfo("Field for yang-path(\"%v\") not found in DB.", xlateParams.xpath) + if xpathInfo.isRefByKey || (xpathInfo.isKey && strings.HasPrefix(xlateParams.uri, "/"+IETF_MDL_PFX)) { + dataToDBMapAdd(xlateParams.tableName, xlateParams.keyName, xlateParams.result, "NULL", "NULL") + xfmrLogDebug("%v - Either an OC YANG leaf path referenced by list key or IETF YANG list key, ", xlateParams.xpath, + "maps to no actual field in sonic yang so dummy row added to create instance in DB.") + } else { + xfmrLogInfo("Field for YANG path(\"%v\") not found in DB.", xlateParams.xpath) + } return nil } fieldName := xpathInfo.fieldName valueStr := "" fieldXpath := xlateParams.tableName + "/" + fieldName - _, ok = xDbSpecMap[fieldXpath] + xDbSpecInfo, ok := xDbSpecMap[fieldXpath] dbEntry := getYangEntryForXPath(fieldXpath) - if !ok || (dbEntry == nil) { + if !ok || (xDbSpecInfo == nil) || (dbEntry == nil) { logStr := fmt.Sprintf("Failed to find the xDbSpecMap: xpath(\"%v\").", fieldXpath) log.Warning(logStr) return nil } - + if xDbSpecInfo.isKey { + if xpathInfo.isRefByKey || (xpathInfo.isKey && strings.HasPrefix(xlateParams.uri, "/"+IETF_MDL_PFX)) { + // apps use this leaf in payload to create an instance, redis needs atleast one field:val to create an instance + dataToDBMapAdd(xlateParams.tableName, xlateParams.keyName, xlateParams.result, "NULL", "NULL") // redis needs atleast one field:val to create an instance + xfmrLogDebug("%v - Either an OC YANG leaf path referenced by list key or IETF YANG list key, ", xlateParams.xpath, + "maps to key in sonic YANG - %v so dummy row added to create instance in DB.", fieldXpath) + } else { + xfmrLogInfo("OC YANG leaf path(%v), maps to key in sonic YANG - %v which cannot be filled as a field in DB instance", xlateParams.xpath, fieldXpath) + } + return nil + } if xpathInfo.yangType == YANG_LEAF_LIST { /* Both YANG side and DB side('@' suffix field) the data type is leaf-list */ xfmrLogDebug("Yang type and DB type is Leaflist for field = %v", xlateParams.xpath) @@ -506,7 +522,7 @@ func dbMapDefaultFieldValFill(xlateParams xlateToParams, tblUriList []string) er } } else if len(childNode.fieldName) > 0 { var xfmrErr error - if _, ok := xDbSpecMap[tblName+"/"+childNode.fieldName]; ok { + if xDbSpecInfo, ok := xDbSpecMap[tblName+"/"+childNode.fieldName]; ok && (xDbSpecInfo != nil) && (!xDbSpecInfo.isKey) { if tblXfmrPresent { chldTblNm, ctErr := tblNameFromTblXfmrGet(*childNode.xfmrTbl, inParamsTblXfmr, xlateParams.xfmrDbTblKeyCache) xfmrLogDebug("Table transformer %v for xpath %v returned table %v", *childNode.xfmrTbl, childXpath, chldTblNm) @@ -532,6 +548,18 @@ func dbMapDefaultFieldValFill(xlateParams xlateToParams, tblUriList []string) er } } } + } else if childNode.isRefByKey { + /* this case occurs only for static table-name, table-xfmr always has field-name + assigned by infra when there no explicit annotation from user. + Also key-leaf in config container has no default value, so here we handle only + REPLACE yangAuxValMap filling */ + if xlateParams.oper != REPLACE { + continue + } + _, ok := xlateParams.result[tblName][dbKey].Field["NULL"] + if !ok { + dataToDBMapAdd(tblName, dbKey, xlateParams.yangAuxValMap, "NULL", "") + } } } } @@ -827,15 +855,15 @@ func yangReqToDbMapCreate(xlateParams xlateToParams) error { curUri, _ := uriWithKeyCreate(xlateParams.uri, xlateParams.xpath, data) _, ok := xYangSpecMap[xlateParams.xpath] if ok && len(xYangSpecMap[xlateParams.xpath].xfmrKey) > 0 { - /* key transformer present */ + // key transformer present curYgotNode, nodeErr := yangNodeForUriGet(curUri, xlateParams.ygRoot) if nodeErr != nil { curYgotNode = nil } - inParams := formXfmrInputRequest(xlateParams.d, dbs, db.MaxDB, xlateParams.ygRoot, curUri, xlateParams.requestUri, xlateParams.oper, "", nil, xlateParams.subOpDataMap, curYgotNode, xlateParams.txCache) + inParams := formXfmrInputRequest(xlateParams.d, dbs, db.ConfigDB, xlateParams.ygRoot, curUri, xlateParams.requestUri, xlateParams.oper, "", nil, xlateParams.subOpDataMap, curYgotNode, xlateParams.txCache) ktRetData, err := keyXfmrHandler(inParams, xYangSpecMap[xlateParams.xpath].xfmrKey) - //if key transformer is called without key values in curUri ignore the error + // if key transformer is called without key values in curUri ignore the error if err != nil && strings.HasSuffix(curUri, "]") { if xlateParams.xfmrErr != nil && *xlateParams.xfmrErr == nil { *xlateParams.xfmrErr = err @@ -846,7 +874,7 @@ func yangReqToDbMapCreate(xlateParams xlateToParams) error { } else if ok && xYangSpecMap[xlateParams.xpath].keyName != nil { curKey = *xYangSpecMap[xlateParams.xpath].keyName } else { - curKey = keyCreate(xlateParams.keyName, xlateParams.xpath, data, xlateParams.d.Opts.KeySeparator) + curKey = keyCreate(xlateParams, curUri, data) } curXlateParams := formXlateToDbParam(xlateParams.d, xlateParams.ygRoot, xlateParams.oper, curUri, xlateParams.requestUri, xlateParams.xpath, curKey, data, xlateParams.resultMap, xlateParams.result, xlateParams.txCache, xlateParams.tblXpathMap, xlateParams.subOpDataMap, xlateParams.pCascadeDelTbl, xlateParams.xfmrErr, "", "", "", xlateParams.invokeCRUSubtreeOnceMap) retErr = yangReqToDbMapCreate(curXlateParams) @@ -969,7 +997,7 @@ func yangReqToDbMapCreate(xlateParams xlateToParams) error { _, ok := xYangSpecMap[xpath] // Process the terminal node only if the targetUri is at terminal Node or if the leaf is not at parent level if ok && strings.HasPrefix(curXpath, reqXpath) { - if (!xYangSpecMap[xpath].isKey) || (len(xYangSpecMap[xpath].xfmrField) > 0) { + if (!xYangSpecMap[xpath].isKey) || (len(xYangSpecMap[xpath].xfmrField) > 0) || (xYangSpecMap[xpath].isKey && strings.HasPrefix(xlateParams.uri, "/"+IETF_MDL_PFX)) { if len(xYangSpecMap[xpath].xfmrFunc) == 0 { value := jData.MapIndex(key).Interface() xfmrLogDebug("data field: key(\"%v\"), value(\"%v\").", key, value) @@ -1047,11 +1075,11 @@ func verifyParentTableSonic(d *db.DB, dbs [db.MaxDB]*db.DB, oper Operation, uri tableExists, derr = dbTableExists(d, table, dbKey, oper) if hasSingletonContainer && oper == DELETE { // Special case when we delete at container that does'nt exist. Return true to skip translation. - if !tableExists { - return true, derr - } else { - return true, nil - } + if !tableExists { + return true, derr + } else { + return true, nil + } } if derr != nil { return false, derr diff --git a/translib/transformer/xlate_utils.go b/translib/transformer/xlate_utils.go index 8f490ebd047d..82911a872a36 100644 --- a/translib/transformer/xlate_utils.go +++ b/translib/transformer/xlate_utils.go @@ -46,15 +46,38 @@ func initRegex() { } /* Create db key from data xpath(request) */ -func keyCreate(keyPrefix string, xpath string, data interface{}, dbKeySep string) string { - _, ok := xYangSpecMap[xpath] +func keyCreate(xlateParams xlateToParams, curUri string, data interface{}) string { + keyPrefix := xlateParams.keyName + var dbs [db.MaxDB]*db.DB + dbKeySep := xlateParams.d.Opts.KeySeparator + xpathInfo, ok := xYangSpecMap[xlateParams.xpath] if ok { - if xYangSpecMap[xpath].yangEntry != nil { - yangEntry := xYangSpecMap[xpath].yangEntry + var tableName string + var err error + tblPtr := xpathInfo.tableName + if tblPtr != nil && *tblPtr != XFMR_NONE_STRING { + tableName = *tblPtr + } else if xpathInfo.xfmrTbl != nil { + inParams := formXfmrInputRequest(xlateParams.d, dbs, db.ConfigDB, xlateParams.ygRoot, curUri, xlateParams.requestUri, xlateParams.oper, "", nil, xlateParams.subOpDataMap, nil, xlateParams.txCache) + tableName, err = tblNameFromTblXfmrGet(*xpathInfo.xfmrTbl, inParams, nil) + if err != nil { + if xlateParams.xfmrErr != nil && *xlateParams.xfmrErr == nil { + *xlateParams.xfmrErr = err + } + } + } + if hasSameOcSonicKeys(curUri, xlateParams.xpath, tableName) { + // If the oc and sonic yangs have same keys then discard the parent key accumulated. Else the parent key is concatenated with the key generated by infra + keyPrefix = "" + xfmrLogDebug("1:1 mapping case. Concatenating YANG keys at current list", curUri) + } + + if xpathInfo.yangEntry != nil { + yangEntry := xpathInfo.yangEntry delim := dbKeySep - if len(xYangSpecMap[xpath].delim) > 0 { - delim = xYangSpecMap[xpath].delim - xfmrLogDebug("key concatenater(\"%v\") found for xpath %v ", delim, xpath) + if len(xpathInfo.delim) > 0 { + delim = xpathInfo.delim + xfmrLogDebug("key concatenater(\"%v\") found for xpath %v ", delim, xlateParams.xpath) } if len(keyPrefix) > 0 { @@ -65,7 +88,7 @@ func keyCreate(keyPrefix string, xpath string, data interface{}, dbKeySep string if i > 0 { keyVal = keyVal + delim } - fieldXpath := xpath + "/" + k + fieldXpath := xlateParams.xpath + "/" + k fVal, err := unmarshalJsonToDbData(yangEntry.Dir[k], fieldXpath, k, data.(map[string]interface{})[k]) if err != nil { log.Warningf("Couldn't unmarshal Json to DbData: path(\"%v\") error (\"%v\").", fieldXpath, err) @@ -181,8 +204,7 @@ func dbKeyToYangDataConvert(uri string, requestUri string, xpath string, tableNa } keyNameList := yangKeyFromEntryGet(xYangSpecMap[xpath].yangEntry) - id := xYangSpecMap[xpath].keyLevel - keyDataList := strings.SplitN(dbKey, dbKeySep, int(id)) + keyDataList := strings.SplitN(dbKey, dbKeySep, -1) uriWithKey := fmt.Sprintf("%v", xpath) uriWithKeyCreate := true if len(keyDataList) == 0 { @@ -224,34 +246,38 @@ func dbKeyToYangDataConvert(uri string, requestUri string, xpath string, tableNa } rmap := make(map[string]interface{}) - if len(keyNameList) > 1 { - log.Warningf("No key transformer found for multi element YANG key mapping to a single redis key string, for URI %v", uri) + keyNameValSame := len(keyNameList) == len(keyDataList) + if len(keyNameList) > 1 && !keyNameValSame && !hasSameOcSonicKeys(uri, xpath, tableName) { + log.Warningf("No key transformer found for multi element yang key mapping to a single redis key string, for uri %v", uri) errStr := fmt.Sprintf("Error processing key for list %v", uri) err = fmt.Errorf("%v", errStr) return rmap, uriWithKey, err } - keyXpath := xpath + "/" + keyNameList[0] - yangEntry, ok := xYangSpecMap[xpath].yangEntry.Dir[keyNameList[0]] - if !ok || yangEntry == nil { - errStr := fmt.Sprintf("Failed to find key xpath %v in xYangSpecMap or is nil, needed to fetch the yangEntry data-type", keyXpath) - err = fmt.Errorf("%v", errStr) - return rmap, uriWithKey, err - } - yngTerminalNdDtType := yangEntry.Type.Kind - resVal, _, err := DbToYangType(yngTerminalNdDtType, keyXpath, keyDataList[0]) - if err != nil { - err = fmt.Errorf("Failed in convert DB value type to YANG type for field %v. Key-xfmr recommended if data types differ", keyXpath) - return rmap, uriWithKey, err - } else { - rmap[keyNameList[0]] = resVal - } - if uriWithKeyCreate { - if reflect.TypeOf(resVal).Kind() == reflect.String { - resVal = escapeKeyValForSplitPathAndNewPathInfo(resVal.(string)) + + for i := range keyNameList { + keyXpath := xpath + "/" + keyNameList[i] + yangEntry, ok := xYangSpecMap[xpath].yangEntry.Dir[keyNameList[i]] + if !ok || yangEntry == nil { + errStr := fmt.Sprintf("Failed to find key xpath %v in xYangSpecMap or is nil, needed to fetch the yangEntry data-type", keyXpath) + err = fmt.Errorf("%v", errStr) + return rmap, uriWithKey, err + } + yngTerminalNdDtType := yangEntry.Type.Kind + resVal, _, err := DbToYangType(yngTerminalNdDtType, keyXpath, keyDataList[i]) + if err != nil { + err = fmt.Errorf("Failure in converting Db value type to yang type for field %v", keyXpath) + return rmap, uriWithKey, err + } else { + rmap[keyNameList[i]] = resVal + } + if uriWithKeyCreate { + if reflect.TypeOf(resVal).Kind() == reflect.String { + resVal = escapeKeyValForSplitPathAndNewPathInfo(resVal.(string)) + } + uriWithKey += fmt.Sprintf("[%v=%v]", keyNameList[i], resVal) } - uriWithKey += fmt.Sprintf("[%v=%v]", keyNameList[0], resVal) } - + xfmrLogDebug("dbKeyToYangDataConvert: uri %v dbkey: %v, uriWithkey %v, rmap :%v", uri, dbKey, uriWithKey, rmap) return rmap, uriWithKey, nil } @@ -749,6 +775,24 @@ func xpathKeyExtract(d *db.DB, ygRoot *ygot.GoStruct, oper Operation, path strin will be concatenated with respective default DB type key-delimiter */ if (yangType == YANG_LIST) && (xpathInfo.yangEntry != nil) { + var tableName string + tblPtr := xpathInfo.tableName + if tblPtr != nil && *tblPtr != XFMR_NONE_STRING { + tableName = *tblPtr + } else if xpathInfo.xfmrTbl != nil { + inParams := formXfmrInputRequest(d, dbs, cdb, ygRoot, curPathWithKey, requestUri, oper, "", nil, subOpDataMap, nil, txCache) + if oper == GET { + inParams.dbDataMap = dbDataMap + } + tableName, err = tblNameFromTblXfmrGet(*xpathInfo.xfmrTbl, inParams, xfmrTblKeyCache) + if err != nil && oper != GET { + return retData, err + } + } + if hasSameOcSonicKeys(curPathWithKey, yangXpath, tableName) { + // If the oc and sonic yangs have same keys then discard the parent key accumulated. Else the parent key is concatenated with the key generated by infra + keyStr = "" + } xfmrLogDebug("No key-xfmr at list %v, 1:1 mapping case. Concatenating YANG keys", curPathWithKey) keyNmList := strings.Split(xpathInfo.yangEntry.Key, " ") pathInfo := NewPathInfo("/" + k) @@ -1160,7 +1204,7 @@ func dbDataXfmrHandler(resultMap map[Operation]map[db.DBNum]map[string]map[strin return nil } -func formXlateFromDbParams(d *db.DB, dbs [db.MaxDB]*db.DB, cdb db.DBNum, ygRoot *ygot.GoStruct, uri string, requestUri string, xpath string, oper Operation, tbl string, tblKey string, dbDataMap *RedisDbMap, txCache interface{}, resultMap map[string]interface{}, validate bool) xlateFromDbParams { +func formXlateFromDbParams(d *db.DB, dbs [db.MaxDB]*db.DB, cdb db.DBNum, ygRoot *ygot.GoStruct, uri string, requestUri string, xpath string, oper Operation, tbl string, tblKey string, dbDataMap *RedisDbMap, txCache interface{}, resultMap map[string]interface{}, validate bool, qParams QueryParams, listKeysMap map[string]interface{}) xlateFromDbParams { var inParamsForGet xlateFromDbParams inParamsForGet.d = d inParamsForGet.dbs = dbs @@ -1176,6 +1220,8 @@ func formXlateFromDbParams(d *db.DB, dbs [db.MaxDB]*db.DB, cdb db.DBNum, ygRoot inParamsForGet.txCache = txCache inParamsForGet.resultMap = resultMap inParamsForGet.validate = validate + inParamsForGet.queryParams = qParams + inParamsForGet.listKeysMap = listKeysMap return inParamsForGet } @@ -1559,6 +1605,123 @@ func getYangNodeTypeFromUri(uri string) (yangElementType, error) { return yangNodeType, nil } +func NewQueryParams(depth uint, content string, fields []string) (QueryParams, error) { + var err error + var qparams QueryParams + + xfmrLogInfo("NewQueryParams: depth:%v, content: %v, fields: %v", depth, content, fields) + qpDepthContentPresent := false + if depth > 0 { + qparams.depthEnabled = true + qparams.curDepth = depth + } else { + qparams.curDepth = 0 + } + if len(content) > 0 { + content = strings.ToLower(content) + if content == "all" { + qparams.content = QUERY_CONTENT_ALL + } else if content == "config" { + qparams.content = QUERY_CONTENT_CONFIG + } else if content == "nonconfig" || content == "state" { + qparams.content = QUERY_CONTENT_NONCONFIG + } else if content == "operational" { + qparams.content = QUERY_CONTENT_OPERATIONAL + } + } else { + qparams.content = QUERY_CONTENT_ALL + } + if len(fields) > 0 { + if qpDepthContentPresent { + err = tlerr.NotSupported("Fields query parameter is not supported along with other query parameters.") + return qparams, err + } + + // Need to fill based on how the field values are filled + // the fields can be xpaths for the fields to be filtered out + qparams.fields = fields + qparams.tgtFieldsXpathMap = make(map[string][]string) + qparams.allowFieldsXpath = make(map[string]bool) + } + return qparams, nil +} + +func isChildTraversalRequired(xpath string, qParams *QueryParams, childXpath string) bool { + traverse := false + if len(xpath) == 0 || len(childXpath) == 0 { + return traverse + } + if !strings.HasPrefix(childXpath, xpath) { + return traverse + } + if qParams == nil || (!qParams.depthEnabled && qParams.content == QUERY_CONTENT_ALL) { + // traversal required for all levels + return true + } + if strings.HasPrefix(childXpath, xpath) { + if qParams.content != QUERY_CONTENT_ALL { + if xYangSpecInfo, specOk := xYangSpecMap[childXpath]; specOk { + var yangEntry *yang.Entry + yangType := xYangSpecInfo.yangType + if yangType == YANG_LEAF || yangType == YANG_LEAF_LIST { + yangEntry = getYangEntryForXPath(childXpath) + } else { + yangEntry = xYangSpecInfo.yangEntry + } + if yangEntry != nil { + var isOcMdl bool + if strings.HasPrefix(childXpath, "/"+OC_MDL_PFX) { + isOcMdl = true + } + yangNdInfo := contentQPSpecMapInfo{ + yangType: yangType, + yangName: yangEntry.Name, + isReadOnly: yangEntry.ReadOnly(), + isOperationalNd: xYangSpecInfo.operationalQP, + hasNonTerminalNd: xYangSpecInfo.hasNonTerminalNode, + hasChildOperationalNd: xYangSpecInfo.hasChildOpertnlNd, + isOcMdl: isOcMdl, + } + traverse, _ = contentQParamYangNodeProcess(childXpath, yangNdInfo, *qParams) + } else { + xfmrLogInfo("yang entry is nil for xpath ", childXpath) + } + } + if !traverse { + xfmrLogInfo("Child traversal not needed for xpath %v due to content QP(%v)", childXpath, qParams.content) + } + } + if !qParams.depthEnabled { + return traverse + } + + xpathList := strings.Split(xpath, "/") + xpathList = xpathList[1:] + childXpathList := strings.Split(childXpath, "/") + childXpathList = childXpathList[1:] + + // Evaluate traversal required for childXpath w.r t requested depth + depthDiff := uint(len(childXpathList) - len(xpathList)) + // The depth begins from 1 which is already included in the xpath considered + // Hence the difference in depth to be considered is 1 less than request depth + reqDiff := qParams.curDepth - 1 + if depthDiff < reqDiff { + // If the childXpath is non terminal container/list then table read is required + traverse = true + } else if depthDiff == reqDiff { + // When depth is met and the childXpath is container/list then table read is not required + // only for terminal nodes DB read would be required + if xspecInfo, ok := xYangSpecMap[childXpath]; ok { + yangType := xspecInfo.yangType + if yangType == YANG_LEAF_LIST || yangType == YANG_LEAF { + traverse = true + } + } + } + } // else not a child YANG path + return traverse +} + func getXfmrSpecInfoFromUri(uri string) (interface{}, error) { // function to extract xfmr spec info for a given sonic uri/path var err error @@ -1614,6 +1777,377 @@ func getXfmrSpecInfoFromUri(uri string) (interface{}, error) { return specInfo, err } +func contentQParamTgtEval(uri string, qParams QueryParams) (bool, error) { + /*function to evaluate target URI , based on content query parameter to decide : + 1.)Return bad-request if Target Uri points to leaf/leaf-list and content + query-param doesn't match to the read-only flag of that YANG node or + doesn't match to the operationalQP flag in spec map. + 2.)Don't further process URI if its of OC Yang and points to container named + "config" and content query-param is non-config or operational + 3.)Don't further process URI if its read-only container or list and content + query-param is config + Return values are true/false(whether to process request further) and error. + */ + var err error + var specInfo interface{} + var specInfoOk, isSonicUri bool + var yangEntry *yang.Entry + var yangType yangElementType + var dbSpecInfo *dbInfo + var xYangSpecInfo *yangXpathInfo + var processReq bool = true + + yangXpath, _, _ := XfmrRemoveXPATHPredicates(uri) + if qParams.content == QUERY_CONTENT_ALL { + err = nil + processReq = true + goto exitContentTgtEval + } + if (qParams.content == QUERY_CONTENT_OPERATIONAL) && (isSonicYang(uri) || !strings.HasPrefix(uri, "/"+OC_MDL_PFX)) { + processReq = false + err = tlerr.InvalidArgsError{Format: "Bad Request - operational content type is invalid for sonic and non-openconfig YANG request."} + goto exitContentTgtEval + } + specInfo, err = getXfmrSpecInfoFromUri(uri) + if err != nil { + processReq = false + goto exitContentTgtEval + } + + if isSonicYang(uri) { + isSonicUri = true + dbSpecInfo, specInfoOk = specInfo.(*dbInfo) + if specInfoOk { + yangType = dbSpecInfo.yangType + if yangType == YANG_LEAF || yangType == YANG_LEAF_LIST { + pathList := strings.Split(yangXpath, "/") + if len(pathList) > SONIC_FIELD_INDEX { + dbXpath := pathList[SONIC_TABLE_INDEX] + "/" + pathList[SONIC_FIELD_INDEX] + yangEntry = getYangEntryForXPath(dbXpath) + } + } else { + yangEntry = dbSpecInfo.dbEntry + } + } + } else { + xYangSpecInfo, specInfoOk = specInfo.(*yangXpathInfo) + if specInfoOk { + yangType = xYangSpecInfo.yangType + if yangType == YANG_LEAF || yangType == YANG_LEAF_LIST { + yangEntry = getYangEntryForXPath(yangXpath) + } else { + yangEntry = xYangSpecInfo.yangEntry + } + } + } + + if !specInfoOk || (yangEntry == nil) { + processReq = false + err = fmt.Errorf("no YANG meta-data for translation for URI - %v", uri) + xfmrLogInfo("%v", err) + goto exitContentTgtEval + } + if !isSonicUri { + var isOcMdl bool + if strings.HasPrefix(uri, "/"+OC_MDL_PFX) { + isOcMdl = true + } + yangNdInfo := contentQPSpecMapInfo{ + yangType: yangType, + yangName: yangEntry.Name, + isReadOnly: yangEntry.ReadOnly(), + isOperationalNd: xYangSpecInfo.operationalQP, + hasNonTerminalNd: xYangSpecInfo.hasNonTerminalNode, + hasChildOperationalNd: xYangSpecInfo.hasChildOpertnlNd, + isOcMdl: isOcMdl, + } + processReq, err = contentQParamYangNodeProcess(uri, yangNdInfo, qParams) + } else { + xfmrLogDebug("Sonic YANG content query param target URI eval") + //top container query + if strings.Count(uri, "/") == 1 { + processReq = true + goto exitContentTgtEval + } + processReq, err = sonicContentQParamYangNodeProcess(uri, yangType, yangEntry.ReadOnly(), qParams) + } + +exitContentTgtEval: + xfmrLogDebug("content target eval returning - process request further %v, error - %v", processReq, err) + return processReq, err + +} + +var rejectComplexYangNodeForQParamContent = func(content ContentType, isOcMdl bool, hasChildOpertnlNd bool) bool { + /* IETF model containeri/list having none RO child nodes */ + ietf_pure_rw_complex_node := (content == QUERY_CONTENT_NONCONFIG) && (!isOcMdl) && (!hasChildOpertnlNd) + + /* containeri/list having none opearational child nodes */ + no_operational_children_complex_node := (content == QUERY_CONTENT_OPERATIONAL) && (!hasChildOpertnlNd) + + if ietf_pure_rw_complex_node || no_operational_children_complex_node { + return true + } + return false +} + +func contentQParamYangNodeProcess(uri string, yangNdInfo contentQPSpecMapInfo, qParams QueryParams) (bool, error) { + /*function to decide whether to traverse a Non-Sonic(OC & Ietf) YANG node based + on content query params QUERY_CONTENT_CONFIG/QUERY_CONTENT_NONCONFIG/QUERY_CONTENT_OPERATIONAL. + */ + var processReq bool + var err error + var yangNdType yangElementType + content_mismatch_err := "Bad Request - requested content type doesn't match content type of terminal node uri." + + if uri == "" { + processReq = false + err = fmt.Errorf("Empty URI sent") + goto contentQParamYangNodeProcessExit + } + yangNdType = yangNdInfo.yangType + if (qParams.content == QUERY_CONTENT_CONFIG) && (yangNdInfo.isReadOnly) { //applies to leaf, leaf-list, container and list + processReq = false + if yangNdType == YANG_LEAF || yangNdType == YANG_LEAF_LIST { + err = tlerr.InvalidArgsError{Format: content_mismatch_err} + } + goto contentQParamYangNodeProcessExit + } + + switch yangNdType { + case YANG_LEAF, YANG_LEAF_LIST: + if (yangNdInfo.isReadOnly) && (qParams.content == QUERY_CONTENT_OPERATIONAL) { + if yangNdInfo.isOperationalNd { + processReq = true + } else { + processReq = false + err = tlerr.InvalidArgsError{Format: content_mismatch_err} + } + } else if ((qParams.content == QUERY_CONTENT_NONCONFIG) || (qParams.content == QUERY_CONTENT_OPERATIONAL)) && (!yangNdInfo.isReadOnly) { + processReq = false + err = tlerr.InvalidArgsError{Format: content_mismatch_err} + } else { + processReq = true + } + case YANG_CONTAINER: + containerNm := yangNdInfo.yangName + if strings.Contains(containerNm, ":") { + containerNm = strings.SplitN(containerNm, ":", 2)[1] + } + + var oc_config_or_terminal_rw_container bool + /* OC model container either having name "config" or is a terminal RW container for content=nonconfig|operational */ + if (qParams.content == QUERY_CONTENT_NONCONFIG) || (qParams.content == QUERY_CONTENT_OPERATIONAL) { + if yangNdInfo.isOcMdl { + if (containerNm == YANG_CONTAINER_NM_CONFIG) || ((!yangNdInfo.hasNonTerminalNd) && (!yangNdInfo.isReadOnly)) { + oc_config_or_terminal_rw_container = true + } + } + } + if oc_config_or_terminal_rw_container || rejectComplexYangNodeForQParamContent(qParams.content, yangNdInfo.isOcMdl, yangNdInfo.hasChildOperationalNd) { + processReq = false + } else { + processReq = true + } + case YANG_LIST: + var oc_terminal_rw_list bool + + /* OC Model terminal RW list */ + if (qParams.content == QUERY_CONTENT_NONCONFIG) || (qParams.content == QUERY_CONTENT_OPERATIONAL) { + if yangNdInfo.isOcMdl { + if (!yangNdInfo.hasNonTerminalNd) && (!yangNdInfo.isReadOnly) { + oc_terminal_rw_list = true + } + } + } + if oc_terminal_rw_list || rejectComplexYangNodeForQParamContent(qParams.content, yangNdInfo.isOcMdl, yangNdInfo.hasChildOperationalNd) { + processReq = false + } else { + processReq = true + } + default: + if ((qParams.content == QUERY_CONTENT_NONCONFIG) || (qParams.content == QUERY_CONTENT_OPERATIONAL)) && !yangNdInfo.isReadOnly { + processReq = false + } else { + processReq = true + } + xfmrLogDebug("default content QP processing for node type %v, for URI %v", yangNdType, uri) + } +contentQParamYangNodeProcessExit: + xfmrLogDebug("returning process request - %v, error -%v , for URI - %v", processReq, err, uri) + return processReq, err +} + +func sonicContentQParamYangNodeProcess(uri string, yangNdType yangElementType, ReadOnly bool, qParams QueryParams) (bool, error) { + var processReq bool + var err error + content_mismatch_err := "Bad Request - requested content type doesn't match content type of terminal node uri." + + if uri == "" { + processReq = false + err = fmt.Errorf("Empty URI sent") + goto sonicContentQParamYangNodeProcessExit + } + if (qParams.content == QUERY_CONTENT_CONFIG) && (ReadOnly) { + processReq = false + if yangNdType == YANG_LEAF || yangNdType == YANG_LEAF_LIST { + err = tlerr.InvalidArgsError{Format: content_mismatch_err} + } + goto sonicContentQParamYangNodeProcessExit + } + + switch yangNdType { + case YANG_LEAF, YANG_LEAF_LIST: + if (qParams.content == QUERY_CONTENT_NONCONFIG) && !ReadOnly { + processReq = false + err = tlerr.InvalidArgsError{Format: content_mismatch_err} + } else { + processReq = true + } + case YANG_CONTAINER, YANG_LIST: + /*as per sonic YANG guidelines when inner container(table) and list are ReadWrite + then they DON"T have ReadOnly elements + */ + if (qParams.content == QUERY_CONTENT_NONCONFIG) && !ReadOnly { + processReq = false + } else { + processReq = true + } + default: + if (qParams.content == QUERY_CONTENT_NONCONFIG) && !ReadOnly { + processReq = false + } else { + processReq = true + } + xfmrLogDebug("default content QP processing for node type %v, for URI %v", yangNdType, uri) + } +sonicContentQParamYangNodeProcessExit: + xfmrLogDebug("returning process request - %v, error -%v , for URI - %v", processReq, err, uri) + return processReq, err +} + +/* Validate "fields" query parameter for sonic YANG */ +func validateAndFillSonicQpFields(inParamsForGet xlateFromDbParams) error { + var err error + curAllowedXpath := "" + if len(strings.Split(inParamsForGet.xpath, "/")) > 2 { + curAllowedXpath = strings.Split(inParamsForGet.xpath, "/")[2] + } + if len(curAllowedXpath) > 0 { + inParamsForGet.queryParams.allowFieldsXpath[curAllowedXpath] = true + } + for _, field := range inParamsForGet.queryParams.fields { + curXpath := inParamsForGet.tbl + curAllowedXpath = strings.Join(strings.Split(inParamsForGet.xpath, "/")[2:], "/") + fpath := strings.Split(field, "/") + curTable := inParamsForGet.tbl + for _, p := range fpath { + if strings.Contains(p, "[") { + err = tlerr.NotSupported("Yang node type list not supported in fields query parameter(%v).", field) + return err + } + if len(p) == 0 { + continue + } + if len(curXpath) > 0 { + curXpath += "/" + p + } else { + curXpath = p + curAllowedXpath = p + curTable = p + } + yNode, ok := xDbSpecMap[curXpath] + if !ok || yNode == nil { + err = tlerr.InvalidArgs("Invalid field name/path: %v", field) + return err + } + /* check for list */ + if yNode.yangType == YANG_LIST { + err = tlerr.NotSupported("Yang node type list not supported in fields query parameter(%v).", field) + return err + } else if yNode.yangType == YANG_CONTAINER && (yNode.dbEntry != nil) && (yNode.dbEntry.Parent != nil) && (curTable == yNode.dbEntry.Parent.Name) { + // singleton container case + curAllowedXpath = curXpath + curXpath = curTable + } + inParamsForGet.queryParams.allowFieldsXpath[curAllowedXpath] = true + } + if _, ok := inParamsForGet.queryParams.tgtFieldsXpathMap[curXpath]; !ok { + if xDbSpecMap[curXpath].yangType == YANG_LEAF_LIST { + curXpath = curXpath + string('@') + } + inParamsForGet.queryParams.tgtFieldsXpathMap[curXpath] = []string{} + } + } + return err +} + +/* Validate "fields" query parameter */ +func validateAndFillQpFields(inParamsForGet xlateFromDbParams) error { + var err error + for _, field := range inParamsForGet.queryParams.fields { + curXpath := inParamsForGet.xpath + fpath := strings.Split(field, "/") + curSubtree := xYangSpecMap[curXpath].xfmrFunc + fpathPrefix := "" + subtreeXpath := "" + subTreeFld := "" + if len(curSubtree) > 0 { + subtreeXpath = curXpath + subTreeFld = field + } + for _, p := range fpath { + /* check for list instance */ + if strings.Contains(p, "[") { + err = tlerr.NotSupported("Yang node type list not in fields query parameter(%v).", field) + return err + } + if len(p) == 0 { + continue + } + if strings.Contains(p, ":") { + p = p[strings.LastIndex(p, ":")+1:] + } + curXpath += "/" + p + yNode, ok := xYangSpecMap[curXpath] + if !ok { + err = tlerr.InvalidArgs("Invalid field name/path: %v", field) + return err + } + /* check for list */ + if yNode.yangType == YANG_LIST { + err = tlerr.NotSupported("Yang node type list not in fields query parameter(%v).", field) + return err + } + + fpathPrefix += p + "/" + if len(yNode.xfmrFunc) > 0 && (curSubtree == "" || curSubtree != yNode.xfmrFunc) { + subtreeXpath = curXpath + if field == strings.TrimSuffix(fpathPrefix, "/") { + subTreeFld = "" + } else { + subTreeFld = strings.Replace(field, fpathPrefix, "", 1) + } + curSubtree = yNode.xfmrFunc + } + inParamsForGet.queryParams.allowFieldsXpath[curXpath] = true + } + + if len(subtreeXpath) > 0 { + /* xpaths handled by subtree-xfmr */ + if _, ok := inParamsForGet.queryParams.tgtFieldsXpathMap[subtreeXpath]; !ok { + inParamsForGet.queryParams.tgtFieldsXpathMap[subtreeXpath] = []string{} + } + inParamsForGet.queryParams.tgtFieldsXpathMap[subtreeXpath] = + append(inParamsForGet.queryParams.tgtFieldsXpathMap[subtreeXpath], subTreeFld) + } else { + /* xpaths handled by infra */ + inParamsForGet.queryParams.tgtFieldsXpathMap[curXpath] = []string{} + } + } + return err +} + func escapeKeyVal(val string) string { val = strings.Replace(val, "]", "\\]", -1) val = strings.Replace(val, "/", "\\/", -1) @@ -1770,6 +2304,43 @@ done: return entry } +func hasSameOcSonicKeys(listUri string, xpath string, table string) bool { + sameKeys := false + if len(listUri) == 0 || len(table) == 0 { + xfmrLogDebug("Empty uri / sonic table received") + return sameKeys + } + + xfmrLogDebug("hasSameOcSonicKeys:listUri: %v, xpath %v, table: %v", listUri, xpath, table) + if xspec, ok := xYangSpecMap[xpath]; ok { + if dbSpec, tableok := xDbSpecMap[table]; tableok { + if dbSpec.dbEntry != nil { + /* Traverse the lists to find if we have atleast one list with matching keys */ + /* For lists with not matching keys key-xfmr is always expected */ + ocKeyNameList := xspec.yangEntry.Key + if len(dbSpec.dbEntry.Dir) > 1 { + // Do not handle multi list cases. Key xfmr mandatory here + return false + } + for k := range dbSpec.dbEntry.Dir { + sncKeyNameList := dbSpec.dbEntry.Dir[k].Key + xfmrLogDebug("hasSameOcSonicKeys: ocKeyNameList : %v, sncKeyNameList: %v", ocKeyNameList, sncKeyNameList) + if len(ocKeyNameList) == len(sncKeyNameList) { + ocKey := strings.ToLower(ocKeyNameList) + ocKey = strings.ReplaceAll(ocKey, "-", "_") + sncKey := strings.ToLower(sncKeyNameList) + sncKey = strings.ReplaceAll(sncKey, "-", "_") + if ocKey == sncKey { + return true + } + } + } + } + } + } + return sameKeys +} + func (inPm XfmrParams) String() string { return fmt.Sprintf("{oper: %v, uri: %v, requestUri: %v, "+ "DB Name at current node: %v, table: %v, key: %v, dbDataMap: %v, subOpDataMap: %v, yangDefValMap: %v "+ @@ -1824,11 +2395,11 @@ func SonicUriHasSingletonContainer(uri string) bool { } xpath, _, err := XfmrRemoveXPATHPredicates(uri) - if err != nil || len(xpath) == 0 { - return hasSingletonContainer - } + if err != nil || len(xpath) == 0 { + return hasSingletonContainer + } - pathList := strings.Split(xpath, "/") + pathList := strings.Split(xpath, "/") if len(pathList) > SONIC_TBL_CHILD_INDEX { tblChldXpath := pathList[SONIC_TABLE_INDEX] + "/" + pathList[SONIC_TBL_CHILD_INDEX] @@ -1840,3 +2411,82 @@ func SonicUriHasSingletonContainer(uri string) bool { } return hasSingletonContainer } + +// IsEnabled : Exported version. translib.common_app needs this. +func (qp *QueryParams) IsEnabled() bool { + return qp.isEnabled() +} + +func (qp *QueryParams) isEnabled() bool { + return qp.isDepthEnabled() || + qp.isContentEnabled() || + qp.isFieldsEnabled() +} + +func (qp *QueryParams) isDepthEnabled() bool { + return qp.depthEnabled +} + +func (qp *QueryParams) IsContentEnabled() bool { + // IsEnabled : Exported version. translib.common_app needs this. + return qp.isContentEnabled() +} + +func (qp *QueryParams) isContentEnabled() bool { + return qp.content != QUERY_CONTENT_ALL +} + +func (qp *QueryParams) isFieldsEnabled() bool { + return len(qp.fields) != 0 +} + +func (ct ContentType) String() string { + ret := "Unknown" + switch ct { + case QUERY_CONTENT_ALL: + ret = "ALL" + case QUERY_CONTENT_CONFIG: + ret = "CONFIG" + case QUERY_CONTENT_NONCONFIG: + ret = "NONCONFIG" + case QUERY_CONTENT_OPERATIONAL: + ret = "OPERATIONAL" + } + return ret +} + +func (qp QueryParams) String() string { + qp_ptr := &qp + if qp_ptr.isEnabled() { + return fmt.Sprintf("QueryParams{depthEnabled: %v, curDepth: %v, content: %v, fields: %v, fieldsFillAll: %v, "+ + "allowFieldsXpath: %v, tgtFieldsXpathMap: %v}", qp.depthEnabled, qp.curDepth, qp.content, + qp.fields, qp.fieldsFillAll, qp.allowFieldsXpath, qp.tgtFieldsXpathMap) + } else { + return "" + } +} + +func (e *qpSubtreePruningErr) Error() string { + return fmt.Sprintf("Query Parameter pruning unsuccessful for subtree - %s", e.subtreePath) +} + +func extractLeafValFromUriKey(uri string, keyLeafNm string) string { + /* function to extract string value for key leaf from last list in uri + string value will have unescaped characters if they were escaped in + URI string + */ + var keyLeafVal string + xfmrLogDebug("Extract value for leaf %v from URI %v", keyLeafNm, uri) + uriElementList := splitUri(uri) + for idx := len(uriElementList) - 1; idx >= 0; idx-- { + pathInfo := NewPathInfo("/" + uriElementList[idx]) + if (pathInfo != nil) && (len(pathInfo.Vars) > 0) { + if pathInfo.HasVar(keyLeafNm) { + keyLeafVal = pathInfo.Var(keyLeafNm) + } + break + } + } + xfmrLogDebug("Returning value %v for leaf %v", keyLeafVal, keyLeafNm) + return keyLeafVal +} diff --git a/translib/transformer/xlate_xfmr_handler.go b/translib/transformer/xlate_xfmr_handler.go index 64ae3e9d5e30..0373ef41e71a 100644 --- a/translib/transformer/xlate_xfmr_handler.go +++ b/translib/transformer/xlate_xfmr_handler.go @@ -21,6 +21,7 @@ package transformer import ( "github.com/Azure/sonic-mgmt-common/translib/db" log "github.com/golang/glog" + "github.com/openconfig/ygot/ygot" ) func xfmrHandlerFunc(inParams XfmrParams, xfmrFuncNm string) error { @@ -48,9 +49,27 @@ func xfmrHandlerFunc(inParams XfmrParams, xfmrFuncNm string) error { } } } + if (err == nil) && inParams.queryParams.isEnabled() { + log.Infof("xfmrPruneQP: func %v URI %v, requestUri %v", + xfmrFuncNm, inParams.uri, inParams.requestUri) + err = xfmrPruneQP(inParams.ygRoot, inParams.queryParams, + inParams.uri, inParams.requestUri) + if err != nil { + xfmrLogInfo("xfmrPruneQP: returned error %v", err) + // following will allow xfmr to distinguish subtree vs pruning API err to abort GET request + err = &qpSubtreePruningErr{subtreePath: inParams.uri} + } + } return err } +/*Place holder for xfmrPruneQP API */ +func xfmrPruneQP(ygRoot *ygot.GoStruct, queryParams QueryParams, uri string, + requestUri string) error { + // TODO + return nil +} + func leafXfmrHandlerFunc(inParams XfmrParams, xfmrFieldFuncNm string) (map[string]interface{}, error) { const ( DBTY_FLD_XFMR_RET_ARGS = 2 diff --git a/translib/transformer/xspec.go b/translib/transformer/xspec.go index ceb873a38349..896b67cb0290 100644 --- a/translib/transformer/xspec.go +++ b/translib/transformer/xspec.go @@ -68,12 +68,15 @@ type yangXpathInfo struct { cascadeDel int8 virtualTbl *bool nameWithMod *string + operationalQP bool + hasChildOpertnlNd bool yangType yangElementType xfmrPath string compositeFields []string dbKeyCompCnt int subscriptionFlags utils.Bits isDataSrcDynamic *bool + isRefByKey bool } type dbInfo struct { @@ -91,6 +94,7 @@ type dbInfo struct { hasXfmrFn bool cascadeDel int8 yangType yangElementType + isKey bool } type moduleAnnotInfo struct { @@ -215,6 +219,29 @@ func childSubTreePresenceFlagSet(xpath string) { } } +func childOperationalNodeFlagSet(xpath string) { + if xpathData, ok := xYangSpecMap[xpath]; ok { + yangType := xpathData.yangType + if (yangType != YANG_LEAF) && (yangType != YANG_LEAF_LIST) { + xpathData.hasChildOpertnlNd = true + } + } + + parXpath := parentXpathGet(xpath) + for { + if parXpath == "" { + break + } + if parXpathData, ok := xYangSpecMap[parXpath]; ok { + if parXpathData.hasChildOpertnlNd { + break + } + parXpathData.hasChildOpertnlNd = true + } + parXpath = parentXpathGet(parXpath) + } +} + /* Recursive api to fill the map with yang details */ func yangToDbMapFill(keyLevel uint8, xYangSpecMap map[string]*yangXpathInfo, entry *yang.Entry, xpathPrefix string, xpathFull string) { xpath := "" @@ -389,8 +416,12 @@ func yangToDbMapFill(keyLevel uint8, xYangSpecMap map[string]*yangXpathInfo, ent if ((yangType == YANG_LEAF) || (yangType == YANG_LEAF_LIST)) && (len(xpathData.fieldName) > 0) && (xpathData.tableName != nil) { dbPath := *xpathData.tableName + "/" + xpathData.fieldName - if xDbSpecMap[dbPath] != nil { + _, ok := xDbSpecMap[dbPath] + if ok && xDbSpecMap[dbPath] != nil { xDbSpecMap[dbPath].yangXpath = append(xDbSpecMap[dbPath].yangXpath, xpath) + if xDbSpecMap[dbPath].isKey { + xpathData.fieldName = "" + } } } @@ -401,6 +432,7 @@ func yangToDbMapFill(keyLevel uint8, xYangSpecMap map[string]*yangXpathInfo, ent /* create list with current keys */ keyXpath := make([]string, len(strings.Split(entry.Key, " "))) + isOcMdl := strings.HasPrefix(xpath, "/"+OC_MDL_PFX) for id, keyName := range strings.Split(entry.Key, " ") { keyXpath[id] = xpath + "/" + keyName if _, ok := xYangSpecMap[xpath+"/"+keyName]; !ok { @@ -410,6 +442,34 @@ func yangToDbMapFill(keyLevel uint8, xYangSpecMap map[string]*yangXpathInfo, ent xYangSpecMap[xpath+"/"+keyName] = keyXpathData } xYangSpecMap[xpath+"/"+keyName].isKey = true + if isOcMdl { + var keyLfsInContainerXpaths []string + if configContEntry, ok := entry.Dir["config"]; ok && configContEntry != nil { //OC Mdl list has config container + if _, keyLfOk := configContEntry.Dir[keyName]; keyLfOk { + keyLfsInContainerXpaths = append(keyLfsInContainerXpaths, xpath+CONFIG_CNT_WITHIN_XPATH+keyName) + } + } + + /* Mark OC Model list state-container leaves that are also list key-leaves,as isRefByKey,even though there + is no yang leaf-reference from list-key leaves.This will enable xfmr infra to fill them and elliminate + app annotation + */ + if stateContEntry, ok := entry.Dir["state"]; ok && stateContEntry != nil { //OC Mdl list has state container + if _, keyLfOk := stateContEntry.Dir[keyName]; keyLfOk { + keyLfsInContainerXpaths = append(keyLfsInContainerXpaths, xpath+STATE_CNT_WITHIN_XPATH+keyName) + } + } + + for _, keyLfInContXpath := range keyLfsInContainerXpaths { + if _, ok := xYangSpecMap[keyLfInContXpath]; !ok { + xYangSpecMap[keyLfInContXpath] = new(yangXpathInfo) + xYangSpecMap[keyLfInContXpath].subscribeMinIntvl = XFMR_INVALID + xYangSpecMap[keyLfInContXpath].dbIndex = db.ConfigDB // default value + } + xYangSpecMap[keyLfInContXpath].isRefByKey = true + } + + } } xpathData.keyXpath = make(map[int]*[]string, (parentKeyLen + 1)) @@ -465,6 +525,44 @@ func yangToDbMapFill(keyLevel uint8, xYangSpecMap map[string]*yangXpathInfo, ent } } +/* find and set operation query parameter nodes */ +var xlateUpdateQueryParamInfo = func() { + + for xpath := range xYangSpecMap { + if xYangSpecMap[xpath].yangEntry != nil { + readOnly := xYangSpecMap[xpath].yangEntry.ReadOnly() + if xYangSpecMap[xpath].yangType == YANG_LEAF || xYangSpecMap[xpath].yangType == YANG_LEAF_LIST { + xYangSpecMap[xpath].yangEntry = nil //memory optimization - don't cache for leafy nodes + } + if !readOnly { + continue + } + } + + if strings.Contains(xpath, STATE_CNT_WITHIN_XPATH) { + cfgXpath := strings.Replace(xpath, STATE_CNT_WITHIN_XPATH, CONFIG_CNT_WITHIN_XPATH, -1) + if strings.HasSuffix(cfgXpath, STATE_CNT_SUFFIXED_XPATH) { + suffix_idx := strings.LastIndex(cfgXpath, STATE_CNT_SUFFIXED_XPATH) + cfgXpath = cfgXpath[:suffix_idx] + CONFIG_CNT_SUFFIXED_XPATH + } + if _, ok := xYangSpecMap[cfgXpath]; !ok { + xYangSpecMap[xpath].operationalQP = true + childOperationalNodeFlagSet(xpath) + } + } else if strings.HasSuffix(xpath, STATE_CNT_SUFFIXED_XPATH) { + suffix_idx := strings.LastIndex(xpath, STATE_CNT_SUFFIXED_XPATH) + cfgXpath := xpath[:suffix_idx] + CONFIG_CNT_SUFFIXED_XPATH + if _, ok := xYangSpecMap[cfgXpath]; !ok { + xYangSpecMap[xpath].operationalQP = true + childOperationalNodeFlagSet(xpath) + } + } else { + xYangSpecMap[xpath].operationalQP = true + childOperationalNodeFlagSet(xpath) + } + } +} + /* Build lookup table based of yang xpath */ func yangToDbMapBuild(entries map[string]*yang.Entry) { if entries == nil { @@ -489,6 +587,7 @@ func yangToDbMapBuild(entries map[string]*yang.Entry) { jsonfile := YangPath + TblInfoJsonFile xlateJsonTblInfoLoad(sonicOrdTblListMap, jsonfile) + xlateUpdateQueryParamInfo() sonicLeafRefMap = nil } @@ -559,11 +658,13 @@ func dbMapFill(tableName string, curPath string, moduleNm string, xDbSpecMap map dbXpath = tableName + "/" + entry.Name if tblSpecInfo, tblOk = xDbSpecMap[tableName]; tblOk { tblDbIndex = xDbSpecMap[tableName].dbIndex - } + } } xDbSpecPath = dbXpath - xDbSpecMap[dbXpath] = new(dbInfo) - xDbSpecMap[dbXpath].dbIndex = tblDbIndex + if _, ok := xDbSpecMap[dbXpath]; !ok { + xDbSpecMap[dbXpath] = new(dbInfo) + } + xDbSpecMap[dbXpath].dbIndex = tblDbIndex xDbSpecMap[dbXpath].yangType = entryType xDbSpecMap[dbXpath].dbEntry = entry xDbSpecMap[dbXpath].module = moduleNm @@ -592,6 +693,13 @@ func dbMapFill(tableName string, curPath string, moduleNm string, xDbSpecMap map } else if tblOk && (entryType == YANG_LIST && len(entry.Key) != 0) { tblSpecInfo.listName = append(tblSpecInfo.listName, entry.Name) xDbSpecMap[dbXpath].keyList = append(xDbSpecMap[dbXpath].keyList, strings.Split(entry.Key, " ")...) + for _, keyVal := range xDbSpecMap[dbXpath].keyList { + dbXpathForKeyLeaf := tableName + "/" + keyVal + if _, ok := xDbSpecMap[dbXpathForKeyLeaf]; !ok { + xDbSpecMap[dbXpathForKeyLeaf] = new(dbInfo) + } + xDbSpecMap[dbXpathForKeyLeaf].isKey = true + } } else if entryType == YANG_LEAF || entryType == YANG_LEAF_LIST { /* TODO - Uncomment this line once subscription changes for memory optimization ready xDbSpecMap[dbXpath].dbEntry = nil //memory optimization - don't cache for leafy nodes @@ -1114,6 +1222,9 @@ func mapPrint(fileName string) { fmt.Fprintf(fp, " %d. %#v\r\n", i, kd) } fmt.Fprintf(fp, "\r\n isKey : %v\r\n", d.isKey) + fmt.Fprintf(fp, "\r\n isRefByKey : %v\r\n", d.isRefByKey) + fmt.Fprintf(fp, "\r\n operQP : %v\r\n", d.operationalQP) + fmt.Fprintf(fp, "\r\n hasChildOperQP : %v\r\n", d.hasChildOpertnlNd) fmt.Fprintf(fp, "\r\n isDataSrcDynamic: ") if d.isDataSrcDynamic != nil { fmt.Fprintf(fp, "%v", *d.isDataSrcDynamic) @@ -1140,6 +1251,7 @@ func dbMapPrint(fname string) { for k, v := range xDbSpecMap { fmt.Fprintf(fp, " field:%v: \r\n", k) fmt.Fprintf(fp, " type :%v \r\n", getYangTypeStrId(v.yangType)) + fmt.Fprintf(fp, " isKey :%v \r\n", v.isKey) fmt.Fprintf(fp, " db-type :%v \r\n", v.dbIndex) fmt.Fprintf(fp, " hasXfmrFn:%v \r\n", v.hasXfmrFn) fmt.Fprintf(fp, " module :%v \r\n", v.module) diff --git a/translib/translib.go b/translib/translib.go index 191cf427afee..5105065504cd 100644 --- a/translib/translib.go +++ b/translib/translib.go @@ -79,16 +79,19 @@ type SetResponse struct { Err error } +type QueryParameters struct { + Depth uint // range 1 to 65535, default is 0 i.e. all + Content string // all, config, non-config(REST)/state(GNMI), operational(GNMI only) + Fields []string // list of fields from NBI +} + type GetRequest struct { Path string FmtType TranslibFmtType User UserRoles AuthEnabled bool ClientVersion Version - - // Depth limits the depth of data subtree in the response - // payload. Default value 0 indicates there is no limit. - Depth uint + QueryParams QueryParameters } type GetResponse struct { @@ -452,7 +455,7 @@ func Get(req GetRequest) (GetResponse, error) { return resp, err } - opts := appOptions{depth: req.Depth} + opts := appOptions{depth: req.QueryParams.Depth, content: req.QueryParams.Content, fields: req.QueryParams.Fields} err = appInitialize(app, appInfo, path, nil, &opts, GET) if err != nil {