diff --git a/cvl/Makefile b/cvl/Makefile index 28f0e240676e..278d97e6acfc 100644 --- a/cvl/Makefile +++ b/cvl/Makefile @@ -70,3 +70,5 @@ clean: $(RM) -r $(CVL_PKG) $(RM) -r $(CVL_TEST_DIR) +cleanall:clean + diff --git a/cvl/cvl.go b/cvl/cvl.go index 858aa08fafab..98922aa4b0ef 100644 --- a/cvl/cvl.go +++ b/cvl/cvl.go @@ -25,7 +25,6 @@ import ( "regexp" "time" log "github.com/golang/glog" - "encoding/json" "github.com/go-redis/redis" "github.com/antchfx/xmlquery" "github.com/antchfx/jsonquery" @@ -33,7 +32,6 @@ import ( . "github.com/Azure/sonic-mgmt-common/cvl/internal/util" "sync" "flag" - "runtime" ) //DB number @@ -557,14 +555,6 @@ func getRedisToYangKeys(tableName string, redisKey string)[]keyValuePairStruct{ return mkeys } - -//Add child node to a parent node -func(c *CVL) addChildNode(tableName string, parent *yparser.YParserNode, name string) *yparser.YParserNode { - - //return C.lyd_new(parent, modelInfo.tableInfo[tableName].module, C.CString(name)) - return c.yp.AddChildNode(modelInfo.tableInfo[tableName].module, parent, name) -} - //Check for path resolution func (c *CVL) checkPathForTableEntry(tableName string, currentValue string, cfgData *CVLEditConfigData, mustExpStk []string, token string) ([]string, string, CVLRetCode) { @@ -929,54 +919,6 @@ func (c *CVL) addTableEntryToCache(tableName string, redisKey string) { } } -//Check delete constraint for leafref if key/field is deleted -func (c *CVL) checkDeleteConstraint(cfgData []CVLEditConfigData, - tableName, keyVal, field string) CVLRetCode { - var leafRefs []tblFieldPair - if (field != "") { - //Leaf or field is getting deleted - leafRefs = c.findUsedAsLeafRef(tableName, field) - } else { - //Entire entry is getting deleted - leafRefs = c.findUsedAsLeafRef(tableName, modelInfo.tableInfo[tableName].keys[0]) - } - - //The entry getting deleted might have been referred from multiple tables - //Return failure if at-least one table is using this entry - for _, leafRef := range leafRefs { - TRACE_LOG(INFO_API, (TRACE_DELETE | TRACE_SEMANTIC), "Checking delete constraint for leafRef %s/%s", leafRef.tableName, leafRef.field) - //Check in dependent data first, if the referred entry is already deleted - leafRefDeleted := false - for _, cfgDataItem := range cfgData { - if (cfgDataItem.VType == VALIDATE_NONE) && - (cfgDataItem.VOp == OP_DELETE ) && - (strings.HasPrefix(cfgDataItem.Key, (leafRef.tableName + modelInfo.tableInfo[leafRef.tableName].redisKeyDelim + keyVal + modelInfo.tableInfo[leafRef.tableName].redisKeyDelim))) { - //Currently, checking for one entry is being deleted in same session - //We should check for all entries - leafRefDeleted = true - break - } - } - - if (leafRefDeleted == true) { - continue //check next leafref - } - - //Else, check if any referred enrty is present in DB - var nokey []string - refKeyVal, err := luaScripts["find_key"].Run(redisClient, nokey, leafRef.tableName, - modelInfo.tableInfo[leafRef.tableName].redisKeyDelim, leafRef.field, keyVal).Result() - if (err == nil && refKeyVal != "") { - CVL_LOG(ERROR, "Delete will violate the constraint as entry %s is referred in %s", tableName, refKeyVal) - - return CVL_SEMANTIC_ERROR - } - } - - - return CVL_SUCCESS -} - //Add the data which are referring this key func (c *CVL) updateDeleteDataToCache(tableName string, redisKey string) { if _, existing := c.tmpDbCache[tableName]; existing == false { @@ -1088,24 +1030,6 @@ func (c *CVL) addLeafRef(config bool, tableName string, name string, value strin } } - -func (c *CVL) addChildLeaf(config bool, tableName string, parent *yparser.YParserNode, name string, value string) { - - /* If there is no value then assign default space string. */ - if len(value) == 0 { - value = " " - } - - //Batch leaf creation - c.batchLeaf = c.batchLeaf + name + "#" + value + "#" - //Check if this leaf has leafref, - //If so add the add redis key to its table so that those - // details can be fetched for dependency validation - - c.addLeafRef(config, tableName, name, value) -} - - func (c *CVL) checkFieldMap(fieldMap *map[string]string) map[string]interface{} { fieldMapNew := map[string]interface{}{} @@ -1138,351 +1062,6 @@ func mergeMap(dest map[string]string, src map[string]string) { } } -// Fetch dependent data from validated data cache, -// Returns the data and flag to indicate that if requested data -// is found in update request, the data should be merged with Redis data -func (c *CVL) fetchDataFromRequestCache(tableName string, key string) (map[string]string, bool) { - cfgDataArr := c.requestCache[tableName][key] - if (cfgDataArr != nil) { - for _, cfgReqData := range cfgDataArr { - //Delete request doesn't have depedent data - if (cfgReqData.VOp == OP_CREATE) { - return cfgReqData.Data, false - } else if (cfgReqData.VOp == OP_UPDATE) { - return cfgReqData.Data, true - } - } - } - - return nil, false -} - -//Fetch given table entries using pipeline -func (c *CVL) fetchTableDataToTmpCache(tableName string, dbKeys map[string]interface{}) int { - - TRACE_LOG(INFO_API, TRACE_CACHE, "\n%v, Entered fetchTableDataToTmpCache", time.Now()) - - totalCount := len(dbKeys) - if (totalCount == 0) { - //No entry to be fetched - return 0 - } - - entryFetched := 0 - bulkCount := 0 - bulkKeys := []string{} - for dbKey, val := range dbKeys { //for all keys - - if (val != nil) { //skip entry already fetched - mapTable := c.tmpDbCache[tableName] - delete(mapTable.(map[string]interface{}), dbKey) //delete entry already fetched - totalCount = totalCount - 1 - if(bulkCount != totalCount) { - //If some entries are remaining go back to 'for' loop - continue - } - } else { - //Accumulate entries to be fetched - bulkKeys = append(bulkKeys, dbKey) - bulkCount = bulkCount + 1 - } - - if(bulkCount != totalCount) && ((bulkCount % MAX_BULK_ENTRIES_IN_PIPELINE) != 0) { - //If some entries are remaining and bulk bucket is not filled, - //go back to 'for' loop - continue - } - - mCmd := map[string]*redis.StringStringMapCmd{} - - pipe := redisClient.Pipeline() - - for _, dbKey := range bulkKeys { - - redisKey := tableName + modelInfo.tableInfo[tableName].redisKeyDelim + dbKey - //Check in validated cache first and add as dependent data - if entry, mergeNeeded := c.fetchDataFromRequestCache(tableName, dbKey); (entry != nil) { - c.tmpDbCache[tableName].(map[string]interface{})[dbKey] = entry - entryFetched = entryFetched + 1 - //Entry found in validated cache, so skip fetching from Redis - //if merging is not required with Redis DB - if (mergeNeeded == false) { - continue - } - } - - //Otherwise fetch it from Redis - mCmd[dbKey] = pipe.HGetAll(redisKey) //write into pipeline - if mCmd[dbKey] == nil { - CVL_LOG(ERROR, "Failed pipe.HGetAll('%s')", redisKey) - } - } - - _, err := pipe.Exec() - if err != nil { - CVL_LOG(ERROR, "Failed to fetch details for table %s", tableName) - return 0 - } - pipe.Close() - bulkKeys = nil - - mapTable := c.tmpDbCache[tableName] - - for key, val := range mCmd { - res, err := val.Result() - if (err != nil || len(res) == 0) { - //no data found, don't keep blank entry - delete(mapTable.(map[string]interface{}), key) - continue - } - //exclude table name and delim - keyOnly := key - - if (mapTable.(map[string]interface{})[keyOnly] != nil) { - tmpFieldMap := (mapTable.(map[string]interface{})[keyOnly]).(map[string]string) - //merge with validated cache data - mergeMap(res, tmpFieldMap) - fieldMap := c.checkFieldMap(&res) - mapTable.(map[string]interface{})[keyOnly] = fieldMap - } else { - fieldMap := c.checkFieldMap(&res) - mapTable.(map[string]interface{})[keyOnly] = fieldMap - } - - entryFetched = entryFetched + 1 - } - - runtime.Gosched() - } - - TRACE_LOG(INFO_API, TRACE_CACHE,"\n%v, Exiting fetchTableDataToTmpCache", time.Now()) - - return entryFetched -} - -//populate redis data to cache -func (c *CVL) fetchDataToTmpCache() *yparser.YParserNode { - TRACE_LOG(INFO_API, TRACE_CACHE, "\n%v, Entered fetchToTmpCache", time.Now()) - - entryToFetch := 0 - var root *yparser.YParserNode = nil - var errObj yparser.YParserError - - for entryToFetch = 1; entryToFetch > 0; { //Force to enter the loop for first time - //Repeat until all entries are fetched - entryToFetch = 0 - for tableName, dbKeys := range c.tmpDbCache { //for each table - entryToFetch = entryToFetch + c.fetchTableDataToTmpCache(tableName, dbKeys.(map[string]interface{})) - } //for each table - - //If no table entry delete the table itself - for tableName, dbKeys := range c.tmpDbCache { //for each table - if (len(dbKeys.(map[string]interface{})) == 0) { - delete(c.tmpDbCache, tableName) - continue - } - } - - if (entryToFetch == 0) { - //No more entry to fetch - break - } - - if (Tracing == true) { - jsonDataBytes, _ := json.Marshal(c.tmpDbCache) - jsonData := string(jsonDataBytes) - TRACE_LOG(INFO_API, TRACE_CACHE, "Top Node=%v\n", jsonData) - } - - data, err := jsonquery.ParseJsonMap(&c.tmpDbCache) - - if (err != nil) { - return nil - } - - //Build yang tree for each table and cache it - for jsonNode := data.FirstChild; jsonNode != nil; jsonNode=jsonNode.NextSibling { - TRACE_LOG(INFO_API, TRACE_CACHE, "Top Node=%v\n", jsonNode.Data) - //Visit each top level list in a loop for creating table data - topNode, _ := c.generateTableData(true, jsonNode) - if (root == nil) { - root = topNode - } else { - if root, errObj = c.yp.MergeSubtree(root, topNode); errObj.ErrCode != yparser.YP_SUCCESS { - return nil - } - } - } - } // until all dependent data is fetched - - if root != nil && Tracing == true { - dumpStr := c.yp.NodeDump(root) - TRACE_LOG(INFO_DETAIL, TRACE_CACHE, "Dependent Data = %v\n", dumpStr) - } - - TRACE_LOG(INFO_API, TRACE_CACHE, "\n%v, Exiting fetchToTmpCache", time.Now()) - return root -} - - -func (c *CVL) clearTmpDbCache() { - for key, _ := range c.tmpDbCache { - delete(c.tmpDbCache, key) - } -} - -func (c *CVL) generateTableFieldsData(config bool, tableName string, jsonNode *jsonquery.Node, -parent *yparser.YParserNode) CVLRetCode { - - //Traverse fields - for jsonFieldNode := jsonNode.FirstChild; jsonFieldNode!= nil; - jsonFieldNode = jsonFieldNode.NextSibling { - //Add fields as leaf to the list - if (jsonFieldNode.Type == jsonquery.ElementNode && - jsonFieldNode.FirstChild != nil && - jsonFieldNode.FirstChild.Type == jsonquery.TextNode) { - - if (len(modelInfo.tableInfo[tableName].mapLeaf) == 2) {//mapping should have two leaf always - //Values should be stored inside another list as map table - listNode := c.addChildNode(tableName, parent, tableName) //Add the list to the top node - c.addChildLeaf(config, tableName, - listNode, modelInfo.tableInfo[tableName].mapLeaf[0], - jsonFieldNode.Data) - - c.addChildLeaf(config, tableName, - listNode, modelInfo.tableInfo[tableName].mapLeaf[1], - jsonFieldNode.FirstChild.Data) - - } else { - //check if it is hash-ref, then need to add only key from "TABLE|k1" - hashRefMatch := reHashRef.FindStringSubmatch(jsonFieldNode.FirstChild.Data) - - if (hashRefMatch != nil && len(hashRefMatch) == 3) { - /*if (strings.HasPrefix(jsonFieldNode.FirstChild.Data, "[")) && - (strings.HasSuffix(jsonFieldNode.FirstChild.Data, "]")) && - (strings.Index(jsonFieldNode.FirstChild.Data, "|") > 0) {*/ - - c.addChildLeaf(config, tableName, - parent, jsonFieldNode.Data, - hashRefMatch[2]) //take hashref key value - } else { - c.addChildLeaf(config, tableName, - parent, jsonFieldNode.Data, - jsonFieldNode.FirstChild.Data) - } - } - - } else if (jsonFieldNode.Type == jsonquery.ElementNode && - jsonFieldNode.FirstChild != nil && - jsonFieldNode.FirstChild.Type == jsonquery.ElementNode) { - //Array data e.g. VLAN members - for arrayNode:=jsonFieldNode.FirstChild; arrayNode != nil; - - arrayNode = arrayNode.NextSibling { - c.addChildLeaf(config, tableName, - parent, jsonFieldNode.Data, - arrayNode.FirstChild.Data) - } - } - } - - return CVL_SUCCESS -} - -func (c *CVL) generateTableData(config bool, jsonNode *jsonquery.Node)(*yparser.YParserNode, CVLErrorInfo) { - var cvlErrObj CVLErrorInfo - - tableName := fmt.Sprintf("%s",jsonNode.Data) - c.batchLeaf = "" - - //Every Redis table is mapped as list within a container, - //E.g. ACL_RULE is mapped as - // container ACL_RULE { list ACL_RULE_LIST {} } - var topNode *yparser.YParserNode - - // Add top most conatiner e.g. 'container sonic-acl {...}' - if _, exists := modelInfo.tableInfo[tableName]; exists == false { - return nil, cvlErrObj - } - topNode = c.yp.AddChildNode(modelInfo.tableInfo[tableName].module, - nil, modelInfo.tableInfo[tableName].modelName) - - //Add the container node for each list - //e.g. 'container ACL_TABLE { list ACL_TABLE_LIST ...} - listConatinerNode := c.yp.AddChildNode(modelInfo.tableInfo[tableName].module, - topNode, tableName) - - //Traverse each key instance - for jsonNode = jsonNode.FirstChild; jsonNode != nil; jsonNode = jsonNode.NextSibling { - - //For each field check if is key - //If it is key, create list as child of top container - // Get all key name/value pairs - if yangListName := getRedisKeyToYangList(tableName, jsonNode.Data); yangListName!= "" { - tableName = yangListName - } - keyValuePair := getRedisToYangKeys(tableName, jsonNode.Data) - keyCompCount := len(keyValuePair) - totalKeyComb := 1 - var keyIndices []int - - //Find number of all key combinations - //Each key can have one or more key values, which results in nk1 * nk2 * nk2 combinations - idx := 0 - for i,_ := range keyValuePair { - totalKeyComb = totalKeyComb * len(keyValuePair[i].values) - keyIndices = append(keyIndices, 0) - } - - for ; totalKeyComb > 0 ; totalKeyComb-- { - //Get the YANG list name from Redis table name - //Ideally they are same except when one Redis table is split - //into multiple YANG lists - - //Add table i.e. create list element - listNode := c.addChildNode(tableName, listConatinerNode, tableName + "_LIST") //Add the list to the top node - - //For each key combination - //Add keys as leaf to the list - for idx = 0; idx < keyCompCount; idx++ { - c.addChildLeaf(config, tableName, - listNode, keyValuePair[idx].key, - keyValuePair[idx].values[keyIndices[idx]]) - } - - //Get all fields under the key field and add them as children of the list - c.generateTableFieldsData(config, tableName, jsonNode, listNode) - - //Check which key elements left after current key element - var next int = keyCompCount - 1 - for ((next > 0) && ((keyIndices[next] +1) >= len(keyValuePair[next].values))) { - next-- - } - //No more combination possible - if (next < 0) { - break - } - - keyIndices[next]++ - - //Reset indices for all other key elements - for idx = next+1; idx < keyCompCount; idx++ { - keyIndices[idx] = 0 - } - - TRACE_LOG(INFO_API, TRACE_CACHE, "Starting batch leaf creation - %s\n", c.batchLeaf) - //process batch leaf creation - if errObj := c.yp.AddMultiLeafNodes(modelInfo.tableInfo[tableName].module, listNode, c.batchLeaf); errObj.ErrCode != yparser.YP_SUCCESS { - cvlErrObj = CreateCVLErrObj(errObj) - return nil, cvlErrObj - } - c.batchLeaf = "" - } - } - - return topNode, cvlErrObj -} - func (c *CVL) translateToYang(jsonMap *map[string]interface{}) (*yparser.YParserNode, CVLErrorInfo) { var cvlErrObj CVLErrorInfo @@ -1587,45 +1166,6 @@ func (c *CVL) validateSyntax(data *yparser.YParserNode) (CVLErrorInfo, CVLRetCod return cvlErrObj, CVL_SUCCESS } -//Perform semantic checks -func (c *CVL) validateSemantics(data *yparser.YParserNode, appDepData *yparser.YParserNode) (CVLErrorInfo, CVLRetCode) { - var cvlErrObj CVLErrorInfo - - if (SkipSemanticValidation() == true) { - return cvlErrObj, CVL_SUCCESS - } - - //Get dependent data from - depData := c.fetchDataToTmpCache() //fetch data to temp cache for temporary validation - - if (Tracing == true) { - TRACE_LOG(INFO_API, TRACE_SEMANTIC, "Validating semantics data=%s\n depData =%s\n, appDepData=%s\n....", c.yp.NodeDump(data), c.yp.NodeDump(depData), c.yp.NodeDump(appDepData)) - } - - if errObj := c.yp.ValidateSemantics(data, depData, appDepData); errObj.ErrCode != yparser.YP_SUCCESS { - - retCode := CVLRetCode(errObj.ErrCode) - - cvlErrObj = CVLErrorInfo { - TableName : errObj.TableName, - ErrCode : CVLRetCode(errObj.ErrCode), - CVLErrDetails : cvlErrorMap[retCode], - Keys : errObj.Keys, - Value : errObj.Value, - Field : errObj.Field, - Msg : errObj.Msg, - ConstraintErrMsg : errObj.ErrTxt, - ErrAppTag : errObj.ErrAppTag, - } - - - - return cvlErrObj, retCode - } - - return cvlErrObj ,CVL_SUCCESS -} - //Add config data item to accumulate per table func (c *CVL) addCfgDataItem(configData *map[string]interface{}, cfgDataItem CVLEditConfigData) (string, string){ diff --git a/cvl/cvl_api.go b/cvl/cvl_api.go index 9e42d190a488..719467f31736 100644 --- a/cvl/cvl_api.go +++ b/cvl/cvl_api.go @@ -26,6 +26,7 @@ import ( "path/filepath" "github.com/Azure/sonic-mgmt-common/cvl/internal/yparser" . "github.com/Azure/sonic-mgmt-common/cvl/internal/util" + "time" ) type CVLValidateType uint @@ -108,6 +109,26 @@ type CVLEditConfigData struct { Data map[string]string //Value : {"alias": "40GE0/28", "mtu" : 9100, "admin_status": down} } +// ValidationTimeStats CVL validations stats +//Maintain time stats for call to ValidateEditConfig(). +//Hits : Total number of times ValidateEditConfig() called +//Time : Total time spent in ValidateEditConfig() +//Peak : Highest time spent in ValidateEditConfig() +type ValidationTimeStats struct { + Hits uint + Time time.Duration + Peak time.Duration +} + +//CVLDepDataForDelete Structure for dependent entry to be deleted +type CVLDepDataForDelete struct { + RefKey string //Ref Key which is getting deleted + Entry map[string]map[string]string //Entry or field which should be deleted as a result +} + +//Global data structure for maintaining validation stats +var cfgValidationStats ValidationTimeStats + func Initialize() CVLRetCode { if (cvlInitialized == true) { //CVL has already been initialized @@ -512,3 +533,34 @@ func (c *CVL) ValidateKeyData(key string, data string) CVLRetCode { func (c *CVL) ValidateFields(key string, field string, value string) CVLRetCode { return CVL_NOT_IMPLEMENTED } + +//SortDepTables Sort list of given tables as per their dependency +func (c *CVL) SortDepTables(inTableList []string) ([]string, CVLRetCode) { + return []string{}, CVL_NOT_IMPLEMENTED +} + +//GetOrderedTables Get the order list(parent then child) of tables in a given YANG module +//within a single model this is obtained using leafref relation +func (c *CVL) GetOrderedTables(yangModule string) ([]string, CVLRetCode) { + return []string{}, CVL_NOT_IMPLEMENTED +} + +//GetDepTables Get the list of dependent tables for a given table in a YANG module +func (c *CVL) GetDepTables(yangModule string, tableName string) ([]string, CVLRetCode) { + return []string{}, CVL_NOT_IMPLEMENTED +} + +//GetDepDataForDelete Get the dependent (Redis keys) to be deleted or modified +//for a given entry getting deleted +func (c *CVL) GetDepDataForDelete(redisKey string) ([]CVLDepDataForDelete) { + return []CVLDepDataForDelete{} +} + +//GetValidationTimeStats Retrieve global stats +func GetValidationTimeStats() ValidationTimeStats { + return cfgValidationStats +} + +//ClearValidationTimeStats Clear global stats +func ClearValidationTimeStats() { +} diff --git a/cvl/cvl_cache.go b/cvl/cvl_cache.go new file mode 100644 index 000000000000..66cc22bc7928 --- /dev/null +++ b/cvl/cvl_cache.go @@ -0,0 +1,234 @@ +//////////////////////////////////////////////////////////////////////////////// +// // +// Copyright 2020 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // +// its subsidiaries. // +// // +// Licensed under the Apache License, Version 2.0 (the "License"); // +// you may not use this file except in compliance with the License. // +// You may obtain a copy of the License at // +// // +// http://www.apache.org/licenses/LICENSE-2.0 // +// // +// Unless required by applicable law or agreed to in writing, software // +// distributed under the License is distributed on an "AS IS" BASIS, // +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // +// See the License for the specific language governing permissions and // +// limitations under the License. // +// // +//////////////////////////////////////////////////////////////////////////////// + +package cvl +import ( + "encoding/json" + "github.com/go-redis/redis" + //lint:ignore ST1001 This is safe to dot import for util package + . "github.com/Azure/sonic-mgmt-common/cvl/internal/util" + "github.com/Azure/sonic-mgmt-common/cvl/internal/yparser" + "time" + "runtime" + "github.com/antchfx/jsonquery" +) + +// Fetch dependent data from validated data cache, +// Returns the data and flag to indicate that if requested data +// is found in update request, the data should be merged with Redis data +func (c *CVL) fetchDataFromRequestCache(tableName string, key string) (d map[string]string, m bool) { + defer func() { + pd := &d + pm := &m + + TRACE_LOG(INFO_API, TRACE_CACHE, + "Returning data from request cache, data = %v, merge needed = %v", + *pd, *pm) + }() + + cfgDataArr := c.requestCache[tableName][key] + if (cfgDataArr != nil) { + for _, cfgReqData := range cfgDataArr { + //Delete request doesn't have depedent data + if (cfgReqData.VOp == OP_CREATE) { + return cfgReqData.Data, false + } else if (cfgReqData.VOp == OP_UPDATE) { + return cfgReqData.Data, true + } + } + } + + return nil, false +} + +//Fetch given table entries using pipeline +func (c *CVL) fetchTableDataToTmpCache(tableName string, dbKeys map[string]interface{}) int { + + TRACE_LOG(INFO_API, TRACE_CACHE, "\n%v, Entered fetchTableDataToTmpCache", time.Now()) + + totalCount := len(dbKeys) + if (totalCount == 0) { + //No entry to be fetched + return 0 + } + + entryFetched := 0 + bulkCount := 0 + bulkKeys := []string{} + for dbKey, val := range dbKeys { //for all keys + + if (val != nil) { //skip entry already fetched + mapTable := c.tmpDbCache[tableName] + delete(mapTable.(map[string]interface{}), dbKey) //delete entry already fetched + totalCount = totalCount - 1 + if(bulkCount != totalCount) { + //If some entries are remaining go back to 'for' loop + continue + } + } else { + //Accumulate entries to be fetched + bulkKeys = append(bulkKeys, dbKey) + bulkCount = bulkCount + 1 + } + + if(bulkCount != totalCount) && ((bulkCount % MAX_BULK_ENTRIES_IN_PIPELINE) != 0) { + //If some entries are remaining and bulk bucket is not filled, + //go back to 'for' loop + continue + } + + mCmd := map[string]*redis.StringStringMapCmd{} + + pipe := redisClient.Pipeline() + + for _, dbKey := range bulkKeys { + + redisKey := tableName + modelInfo.tableInfo[tableName].redisKeyDelim + dbKey + //Check in validated cache first and add as dependent data + if entry, mergeNeeded := c.fetchDataFromRequestCache(tableName, dbKey); (entry != nil) { + c.tmpDbCache[tableName].(map[string]interface{})[dbKey] = entry + entryFetched = entryFetched + 1 + //Entry found in validated cache, so skip fetching from Redis + //if merging is not required with Redis DB + if (mergeNeeded == false) { + continue + } + } + + //Otherwise fetch it from Redis + mCmd[dbKey] = pipe.HGetAll(redisKey) //write into pipeline + if mCmd[dbKey] == nil { + CVL_LOG(ERROR, "Failed pipe.HGetAll('%s')", redisKey) + } + } + + _, err := pipe.Exec() + if err != nil { + CVL_LOG(ERROR, "Failed to fetch details for table %s", tableName) + return 0 + } + pipe.Close() + bulkKeys = nil + + mapTable := c.tmpDbCache[tableName] + + for key, val := range mCmd { + res, err := val.Result() + if (err != nil || len(res) == 0) { + //no data found, don't keep blank entry + delete(mapTable.(map[string]interface{}), key) + continue + } + //exclude table name and delim + keyOnly := key + + if (len(mapTable.(map[string]interface{})) > 0) && (mapTable.(map[string]interface{})[keyOnly] != nil) { + tmpFieldMap := (mapTable.(map[string]interface{})[keyOnly]).(map[string]string) + //merge with validated cache data + mergeMap(res, tmpFieldMap) + fieldMap := c.checkFieldMap(&res) + mapTable.(map[string]interface{})[keyOnly] = fieldMap + } else { + fieldMap := c.checkFieldMap(&res) + mapTable.(map[string]interface{})[keyOnly] = fieldMap + } + + entryFetched = entryFetched + 1 + } + + runtime.Gosched() + } + + TRACE_LOG(INFO_API, TRACE_CACHE,"\n%v, Exiting fetchTableDataToTmpCache", time.Now()) + + return entryFetched +} + +//populate redis data to cache +func (c *CVL) fetchDataToTmpCache() *yparser.YParserNode { + TRACE_LOG(INFO_API, TRACE_CACHE, "\n%v, Entered fetchToTmpCache", time.Now()) + + entryToFetch := 0 + var root *yparser.YParserNode = nil + var errObj yparser.YParserError + + for entryToFetch = 1; entryToFetch > 0; { //Force to enter the loop for first time + //Repeat until all entries are fetched + entryToFetch = 0 + for tableName, dbKeys := range c.tmpDbCache { //for each table + entryToFetch = entryToFetch + c.fetchTableDataToTmpCache(tableName, dbKeys.(map[string]interface{})) + } //for each table + + //If no table entry delete the table itself + for tableName, dbKeys := range c.tmpDbCache { //for each table + if (len(dbKeys.(map[string]interface{})) == 0) { + delete(c.tmpDbCache, tableName) + continue + } + } + + if (entryToFetch == 0) { + //No more entry to fetch + break + } + + if Tracing { + jsonDataBytes, _ := json.Marshal(c.tmpDbCache) + jsonData := string(jsonDataBytes) + TRACE_LOG(INFO_API, TRACE_CACHE, "Top Node=%v\n", jsonData) + } + + data, err := jsonquery.ParseJsonMap(&c.tmpDbCache) + + if (err != nil) { + return nil + } + + //Build yang tree for each table and cache it + for jsonNode := data.FirstChild; jsonNode != nil; jsonNode=jsonNode.NextSibling { + TRACE_LOG(INFO_API, TRACE_CACHE, "Top Node=%v\n", jsonNode.Data) + //Visit each top level list in a loop for creating table data + topNode, _ := c.generateTableData(true, jsonNode) + if (root == nil) { + root = topNode + } else { + if root, errObj = c.yp.MergeSubtree(root, topNode); errObj.ErrCode != yparser.YP_SUCCESS { + return nil + } + } + } + } // until all dependent data is fetched + + if root != nil && Tracing { + dumpStr := c.yp.NodeDump(root) + TRACE_LOG(INFO_API, TRACE_CACHE, "Dependent Data = %v\n", dumpStr) + } + + TRACE_LOG(INFO_API, TRACE_CACHE, "\n%v, Exiting fetchToTmpCache", time.Now()) + return root +} + + +func (c *CVL) clearTmpDbCache() { + for key := range c.tmpDbCache { + delete(c.tmpDbCache, key) + } +} + + diff --git a/cvl/cvl_semantics.go b/cvl/cvl_semantics.go new file mode 100644 index 000000000000..b7eab1f539e5 --- /dev/null +++ b/cvl/cvl_semantics.go @@ -0,0 +1,123 @@ +//////////////////////////////////////////////////////////////////////////////// +// // +// Copyright 2020 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // +// its subsidiaries. // +// // +// Licensed under the Apache License, Version 2.0 (the "License"); // +// you may not use this file except in compliance with the License. // +// You may obtain a copy of the License at // +// // +// http://www.apache.org/licenses/LICENSE-2.0 // +// // +// Unless required by applicable law or agreed to in writing, software // +// distributed under the License is distributed on an "AS IS" BASIS, // +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // +// See the License for the specific language governing permissions and // +// limitations under the License. // +// // +//////////////////////////////////////////////////////////////////////////////// + +package cvl + +import ( + "strings" + "github.com/antchfx/xmlquery" + "github.com/Azure/sonic-mgmt-common/cvl/internal/yparser" + //lint:ignore ST1001 This is safe to dot import for util package + . "github.com/Azure/sonic-mgmt-common/cvl/internal/util" +) + +//YValidator YANG Validator used for external semantic +//validation including custom/platform validation +type YValidator struct { + root *xmlquery.Node //Top evel root for data + current *xmlquery.Node //Current position +} + +//Check delete constraint for leafref if key/field is deleted +func (c *CVL) checkDeleteConstraint(cfgData []CVLEditConfigData, + tableName, keyVal, field string) CVLRetCode { + var leafRefs []tblFieldPair + if (field != "") { + //Leaf or field is getting deleted + leafRefs = c.findUsedAsLeafRef(tableName, field) + } else { + //Entire entry is getting deleted + leafRefs = c.findUsedAsLeafRef(tableName, modelInfo.tableInfo[tableName].keys[0]) + } + + //The entry getting deleted might have been referred from multiple tables + //Return failure if at-least one table is using this entry + for _, leafRef := range leafRefs { + TRACE_LOG(INFO_API, (TRACE_DELETE | TRACE_SEMANTIC), "Checking delete constraint for leafRef %s/%s", leafRef.tableName, leafRef.field) + //Check in dependent data first, if the referred entry is already deleted + leafRefDeleted := false + for _, cfgDataItem := range cfgData { + if (cfgDataItem.VType == VALIDATE_NONE) && + (cfgDataItem.VOp == OP_DELETE ) && + (strings.HasPrefix(cfgDataItem.Key, (leafRef.tableName + modelInfo.tableInfo[leafRef.tableName].redisKeyDelim + keyVal + modelInfo.tableInfo[leafRef.tableName].redisKeyDelim))) { + //Currently, checking for one entry is being deleted in same session + //We should check for all entries + leafRefDeleted = true + break + } + } + + if (leafRefDeleted == true) { + continue //check next leafref + } + + //Else, check if any referred enrty is present in DB + var nokey []string + refKeyVal, err := luaScripts["find_key"].Run(redisClient, nokey, leafRef.tableName, + modelInfo.tableInfo[leafRef.tableName].redisKeyDelim, leafRef.field, keyVal).Result() + if (err == nil && refKeyVal != "") { + CVL_LOG(ERROR, "Delete will violate the constraint as entry %s is referred in %s", tableName, refKeyVal) + + return CVL_SEMANTIC_ERROR + } + } + + + return CVL_SUCCESS +} + +//Perform semantic checks +func (c *CVL) validateSemantics(data *yparser.YParserNode, appDepData *yparser.YParserNode) (CVLErrorInfo, CVLRetCode) { + var cvlErrObj CVLErrorInfo + + if (SkipSemanticValidation() == true) { + return cvlErrObj, CVL_SUCCESS + } + + //Get dependent data from + depData := c.fetchDataToTmpCache() //fetch data to temp cache for temporary validation + + if (Tracing == true) { + TRACE_LOG(INFO_API, TRACE_SEMANTIC, "Validating semantics data=%s\n depData =%s\n, appDepData=%s\n....", c.yp.NodeDump(data), c.yp.NodeDump(depData), c.yp.NodeDump(appDepData)) + } + + if errObj := c.yp.ValidateSemantics(data, depData, appDepData); errObj.ErrCode != yparser.YP_SUCCESS { + + retCode := CVLRetCode(errObj.ErrCode) + + cvlErrObj = CVLErrorInfo { + TableName : errObj.TableName, + ErrCode : CVLRetCode(errObj.ErrCode), + CVLErrDetails : cvlErrorMap[retCode], + Keys : errObj.Keys, + Value : errObj.Value, + Field : errObj.Field, + Msg : errObj.Msg, + ConstraintErrMsg : errObj.ErrTxt, + ErrAppTag : errObj.ErrAppTag, + } + + + + return cvlErrObj, retCode + } + + return cvlErrObj ,CVL_SUCCESS +} + diff --git a/cvl/cvl_syntax.go b/cvl/cvl_syntax.go new file mode 100644 index 000000000000..7cc812e0374a --- /dev/null +++ b/cvl/cvl_syntax.go @@ -0,0 +1,203 @@ +//////////////////////////////////////////////////////////////////////////////// +// // +// Copyright 2020 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // +// its subsidiaries. // +// // +// Licensed under the Apache License, Version 2.0 (the "License"); // +// you may not use this file except in compliance with the License. // +// You may obtain a copy of the License at // +// // +// http://www.apache.org/licenses/LICENSE-2.0 // +// // +// Unless required by applicable law or agreed to in writing, software // +// distributed under the License is distributed on an "AS IS" BASIS, // +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // +// See the License for the specific language governing permissions and // +// limitations under the License. // +// // +//////////////////////////////////////////////////////////////////////////////// + +package cvl +import ( + "fmt" + "github.com/antchfx/jsonquery" + "github.com/Azure/sonic-mgmt-common/cvl/internal/yparser" + //lint:ignore ST1001 This is safe to dot import for util package + . "github.com/Azure/sonic-mgmt-common/cvl/internal/util" +) + +//Add child node to a parent node +func(c *CVL) addChildNode(tableName string, parent *yparser.YParserNode, name string) *yparser.YParserNode { + + //return C.lyd_new(parent, modelInfo.tableInfo[tableName].module, C.CString(name)) + return c.yp.AddChildNode(modelInfo.tableInfo[tableName].module, parent, name) +} + +func (c *CVL) addChildLeaf(config bool, tableName string, parent *yparser.YParserNode, name string, value string) { + + /* If there is no value then assign default space string. */ + if len(value) == 0 { + value = " " + } + + //Batch leaf creation + c.batchLeaf = c.batchLeaf + name + "#" + value + "#" + //Check if this leaf has leafref, + //If so add the add redis key to its table so that those + // details can be fetched for dependency validation + + c.addLeafRef(config, tableName, name, value) +} + +func (c *CVL) generateTableFieldsData(config bool, tableName string, jsonNode *jsonquery.Node, +parent *yparser.YParserNode) CVLRetCode { + + //Traverse fields + for jsonFieldNode := jsonNode.FirstChild; jsonFieldNode!= nil; + jsonFieldNode = jsonFieldNode.NextSibling { + //Add fields as leaf to the list + if (jsonFieldNode.Type == jsonquery.ElementNode && + jsonFieldNode.FirstChild != nil && + jsonFieldNode.FirstChild.Type == jsonquery.TextNode) { + + if (len(modelInfo.tableInfo[tableName].mapLeaf) == 2) {//mapping should have two leaf always + //Values should be stored inside another list as map table + listNode := c.addChildNode(tableName, parent, tableName) //Add the list to the top node + c.addChildLeaf(config, tableName, + listNode, modelInfo.tableInfo[tableName].mapLeaf[0], + jsonFieldNode.Data) + + c.addChildLeaf(config, tableName, + listNode, modelInfo.tableInfo[tableName].mapLeaf[1], + jsonFieldNode.FirstChild.Data) + + } else { + //check if it is hash-ref, then need to add only key from "TABLE|k1" + hashRefMatch := reHashRef.FindStringSubmatch(jsonFieldNode.FirstChild.Data) + + if (hashRefMatch != nil && len(hashRefMatch) == 3) { + /*if (strings.HasPrefix(jsonFieldNode.FirstChild.Data, "[")) && + (strings.HasSuffix(jsonFieldNode.FirstChild.Data, "]")) && + (strings.Index(jsonFieldNode.FirstChild.Data, "|") > 0) {*/ + + c.addChildLeaf(config, tableName, + parent, jsonFieldNode.Data, + hashRefMatch[2]) //take hashref key value + } else { + c.addChildLeaf(config, tableName, + parent, jsonFieldNode.Data, + jsonFieldNode.FirstChild.Data) + } + } + + } else if (jsonFieldNode.Type == jsonquery.ElementNode && + jsonFieldNode.FirstChild != nil && + jsonFieldNode.FirstChild.Type == jsonquery.ElementNode) { + //Array data e.g. VLAN members + for arrayNode:=jsonFieldNode.FirstChild; arrayNode != nil; + + arrayNode = arrayNode.NextSibling { + c.addChildLeaf(config, tableName, + parent, jsonFieldNode.Data, + arrayNode.FirstChild.Data) + } + } + } + + return CVL_SUCCESS +} + +func (c *CVL) generateTableData(config bool, jsonNode *jsonquery.Node)(*yparser.YParserNode, CVLErrorInfo) { + var cvlErrObj CVLErrorInfo + + tableName := fmt.Sprintf("%s",jsonNode.Data) + c.batchLeaf = "" + + //Every Redis table is mapped as list within a container, + //E.g. ACL_RULE is mapped as + // container ACL_RULE { list ACL_RULE_LIST {} } + var topNode *yparser.YParserNode + + // Add top most conatiner e.g. 'container sonic-acl {...}' + if _, exists := modelInfo.tableInfo[tableName]; exists == false { + return nil, cvlErrObj + } + topNode = c.yp.AddChildNode(modelInfo.tableInfo[tableName].module, + nil, modelInfo.tableInfo[tableName].modelName) + + //Add the container node for each list + //e.g. 'container ACL_TABLE { list ACL_TABLE_LIST ...} + listConatinerNode := c.yp.AddChildNode(modelInfo.tableInfo[tableName].module, + topNode, tableName) + + //Traverse each key instance + for jsonNode = jsonNode.FirstChild; jsonNode != nil; jsonNode = jsonNode.NextSibling { + + //For each field check if is key + //If it is key, create list as child of top container + // Get all key name/value pairs + if yangListName := getRedisKeyToYangList(tableName, jsonNode.Data); yangListName!= "" { + tableName = yangListName + } + keyValuePair := getRedisToYangKeys(tableName, jsonNode.Data) + keyCompCount := len(keyValuePair) + totalKeyComb := 1 + var keyIndices []int + + //Find number of all key combinations + //Each key can have one or more key values, which results in nk1 * nk2 * nk2 combinations + idx := 0 + for i,_ := range keyValuePair { + totalKeyComb = totalKeyComb * len(keyValuePair[i].values) + keyIndices = append(keyIndices, 0) + } + + for ; totalKeyComb > 0 ; totalKeyComb-- { + //Get the YANG list name from Redis table name + //Ideally they are same except when one Redis table is split + //into multiple YANG lists + + //Add table i.e. create list element + listNode := c.addChildNode(tableName, listConatinerNode, tableName + "_LIST") //Add the list to the top node + + //For each key combination + //Add keys as leaf to the list + for idx = 0; idx < keyCompCount; idx++ { + c.addChildLeaf(config, tableName, + listNode, keyValuePair[idx].key, + keyValuePair[idx].values[keyIndices[idx]]) + } + + //Get all fields under the key field and add them as children of the list + c.generateTableFieldsData(config, tableName, jsonNode, listNode) + + //Check which key elements left after current key element + var next int = keyCompCount - 1 + for ((next > 0) && ((keyIndices[next] +1) >= len(keyValuePair[next].values))) { + next-- + } + //No more combination possible + if (next < 0) { + break + } + + keyIndices[next]++ + + //Reset indices for all other key elements + for idx = next+1; idx < keyCompCount; idx++ { + keyIndices[idx] = 0 + } + + TRACE_LOG(INFO_API, TRACE_CACHE, "Starting batch leaf creation - %s\n", c.batchLeaf) + //process batch leaf creation + if errObj := c.yp.AddMultiLeafNodes(modelInfo.tableInfo[tableName].module, listNode, c.batchLeaf); errObj.ErrCode != yparser.YP_SUCCESS { + cvlErrObj = CreateCVLErrObj(errObj) + return nil, cvlErrObj + } + c.batchLeaf = "" + } + } + + return topNode, cvlErrObj +} + diff --git a/cvl/cvl_test.go b/cvl/cvl_test.go index 3731311b63ed..655bf1eafc50 100644 --- a/cvl/cvl_test.go +++ b/cvl/cvl_test.go @@ -32,7 +32,6 @@ import ( "runtime" "github.com/Azure/sonic-mgmt-common/cvl" . "github.com/Azure/sonic-mgmt-common/cvl/internal/util" - "github.com/Azure/sonic-mgmt-common/cvl/internal/yparser" ) type testEditCfgData struct { @@ -2861,22 +2860,6 @@ func TestValidateEditConfig_Delete_Entry_Then_Dep_Leafref_Positive(t *testing.T) unloadConfigDB(rclient, depDataMap) } -func TestBadSchema(t *testing.T) { - badSchema, err := ioutil.TempFile("", "sonic-test-*.yin") - if err != nil { - t.Fatalf("could not create temp file") - } - - // write incomplete module data to temporary schema file - badSchema.WriteString("") - badSchema.Close() - - if module, _ := yparser.ParseSchemaFile(badSchema.Name()); module != nil { //should fail - t.Errorf("Bad schema parsing should fail.") - } -} - - func TestServicability_Debug_Trace(t *testing.T) { cvl.Debug(false) diff --git a/cvl/schema/Makefile b/cvl/schema/Makefile index e16280601b99..2642d1d96466 100644 --- a/cvl/schema/Makefile +++ b/cvl/schema/Makefile @@ -28,19 +28,27 @@ out_dir=$(TOPDIR)/build/cvl/schema src_files=$(wildcard $(sonic_yang)/*.yang) src_files += $(wildcard $(sonic_yang_common)/*.yang) -out=$(patsubst %.yang, $(out_dir)/%.yin, $(notdir $(src_files))) -search_path=$(std_yang_common):$(sonic_yang):$(sonic_yang_common) +out=$(patsubst %.yang, $(out_dir)/%.yin, $(shell ls -1 $(sonic_yang)/*.yang | cut -d'/' -f6)) +out_common=$(patsubst %.yang, $(out_dir)/%.yin, $(shell ls -1 $(sonic_yang_common)/*.yang | cut -d'/' -f7)) +out_platform=$(patsubst %.yang, $(out_dir)/%.yin, $(shell find $(sonic_yang_platform) -name '*.yang' | cut -d'/' -f6-8)) +out_platform_dep=$(shell find $(sonic_yang_platform) -name '*.yang') +out_tree=$(patsubst %.yang, $(out_dir)/%.tree, $(src_files)) -all: schema +search_path=$(sonic_yang):$(sonic_yang_common):$(sonic_yang_common)/ietf -schema: $(out) -.PRECIOUS: %/. -%/.: - mkdir -p $(@D) +all: precheck schema + +precheck: + mkdir -p $(out_dir) + +schema: $(out) $(out_common) $(out_platform) +# @$(call install_cvl_schema) + +schema-tree: $(out_tree) #Build YANG models -$(out_dir)/%.yin:$(sonic_yang)/%.yang | $(out_dir)/. +$(out_dir)/%.yin:$(sonic_yang)/%.yang @echo "Generating `basename $@` ..." @devFile="`echo $@ | cut -d . -f1`-deviation.yang"; \ if [ -f $$devFile ] ; then devOpt="--deviation-module $$devFile"; fi; \ @@ -49,12 +57,28 @@ $(out_dir)/%.yin:$(sonic_yang)/%.yang | $(out_dir)/. #Build common YANG models -$(out_dir)/%.yin:$(sonic_yang_common)/%.yang | $(out_dir)/. +$(out_dir)/%.yin:$(sonic_yang_common)/%.yang @echo "Generating `basename $@` ..." @devFile="`echo $@ | cut -d . -f1`-deviation.yang"; \ if [ -f $$devFile ] ; then devOpt="--deviation-module $$devFile"; fi; \ pyang -p $(search_path) --plugindir $(pyang_plugin_dir) \ -f yin-cvl $$devOpt $< -o $@ +#Build platform specific YANG models +$(out_platform):$(out_platform_dep) + @mkdir -p `dirname $@` + @echo "Generating $@ ..." + @pyang -p $(search_path) --plugindir $(pyang_plugin_dir) \ + -f yin-cvl "$(sonic_yang)/`echo $@ | grep -o 'platform/.*\.'`yang" -o $@ + +$(out_dir)/%.tree:%.yang + @echo "Generating `basename $@` ..." + @devFile="`echo $< | cut -d . -f1`-dev.yang"; \ + if [ -f $$devFile ] ; then devOpt="--deviation-module $$devFile"; fi; \ + pyang -p $(search_path) -f tree $$devOpt $< -o `basename $@` + clean: - $(RM) -r $(out_dir) + @echo "Removing files ..." + rm -rf $(out_dir) + rm -rf platform/ + diff --git a/cvl/testdata/schema/Makefile b/cvl/testdata/schema/Makefile index 93976f79f515..4bd09da99640 100644 --- a/cvl/testdata/schema/Makefile +++ b/cvl/testdata/schema/Makefile @@ -17,44 +17,38 @@ # # ################################################################################ -TOPDIR := ../../.. -YANGDIR=$(TOPDIR)/models/yang -sonic_yang=$(YANGDIR)/sonic -std_yang_common=$(YANGDIR)/common/ -sonic_yang_common=$(sonic_yang)/common -pyang_plugin_dir=$(TOPDIR)/tools/pyang/pyang_plugins +TOPDIR?=$(abspath ../../../) +sonic_yang=../../../models/yang/sonic +pyang_plugin_dir=../../../tools/pyang/pyang_plugins src_files=$(wildcard *.yang) +out_dir=$(TOPDIR)/build/tests/cvl/testdata/schema/ +out=$(patsubst %.yang, $(out_dir)/%.yin, $(src_files)) +out_ext=$(patsubst %.yang, $(out_dir)/%.tree, $(src_files)) -out_dir=$(TOPDIR)/build/tests/cvl/testdata/schema -out=$(patsubst %.yang, $(out_dir)/%.yin, $(notdir $(src_files)) ) -out_ext=$(patsubst %.yang, %.tree, $(src_files)) -search_path=$(std_yang_common):$(sonic_yang):$(sonic_yang_common) +all:precheck schema -all:schema +precheck: + mkdir -p $(out_dir) schema: $(out) schema-tree: $(out_ext) -.PRECIOUS: %/. -%/.: - mkdir -p $(@D) - -$(out_dir)/%.yin:%.yang | $(out_dir)/. +$(out_dir)/%.yin:%.yang @echo "Generating `basename $@` ..." - @devFile="`echo $< | cut -d . -f1`-dev.yang"; \ + @devFile="`echo $< | cut -d . -f1`-deviation.yang"; \ if [ -f $$devFile ] ; then devOpt="--deviation-module $$devFile"; fi; \ - pyang -p $(search_path) \ + pyang -p $(sonic_yang)/common:$(sonic_yang)/common/ietf \ --plugindir $(pyang_plugin_dir) -f yin-cvl $$devOpt $< -o $@ -%.tree:%.yang +$(out_dir)/%.tree:%.yang @echo "Generating `basename $@` ..." - @devFile="`echo $< | cut -d . -f1`-dev.yang"; \ + @devFile="`echo $< | cut -d . -f1`-deviation.yang"; \ if [ -f $$devFile ] ; then devOpt="--deviation-module $$devFile"; fi; \ - pyang -p $(search_path) \ - -f tree $$devOpt $< -o `basename $@` + pyang -p $(sonic_yang)/common:$(sonic_yang)/common/ietf \ + -f tree $$devOpt $< -o $@ clean: - $(RM) -r $(out_dir) - $(RM) *.tree + @echo "Removing files ..." + rm -rf platform diff --git a/cvl/testdata/schema/sonic-acl.yang b/cvl/testdata/schema/sonic-acl.yang new file mode 100644 index 000000000000..34b416940991 --- /dev/null +++ b/cvl/testdata/schema/sonic-acl.yang @@ -0,0 +1,241 @@ +module sonic-acl { + namespace "http://github.com/Azure/sonic-acl"; + prefix sacl; + yang-version 1.1; + + import ietf-inet-types { + prefix inet; + } + + import sonic-common { + prefix cmn; + } + + import sonic-extension { + prefix sonic-ext; + } + + import sonic-port { + prefix prt; + } + + import sonic-portchannel { + prefix po; + } + + import sonic-mirror-session { + prefix sms; + } + + organization + "SONiC"; + + contact + "SONiC"; + + description + "SONIC ACL"; + + revision 2019-05-15 { + description + "Initial revision."; + } + + container sonic-acl { + + container ACL_TABLE { + + list ACL_TABLE_LIST { + key "aclname"; + max-elements 1024; // Max 1K ACL tables for all platforms + + leaf aclname { + type string { + pattern '[a-zA-Z0-9]{1}([-a-zA-Z0-9_]{0,71})'; + length 1..72; + } + } + + leaf policy_desc { + type string { + length 1..255 { + error-app-tag policy-desc-invalid-length; + } + } + } + + leaf stage { + type enumeration { + enum INGRESS; + enum EGRESS; + } + } + + leaf type { + type enumeration { + enum MIRROR; + enum L2; + enum L3; + enum L3V6; + } + } + + leaf-list ports { + type union { + type leafref { + path "/prt:sonic-port/prt:PORT/prt:PORT_LIST/prt:ifname"; + } + type leafref { + path "/po:sonic-portchannel/po:PORTCHANNEL/po:PORTCHANNEL_LIST/po:name"; + } + } + } + } + } + + container ACL_RULE { + + list ACL_RULE_LIST { + key "aclname rulename"; + sonic-ext:custom-validation ValidateMaxAclRule; // Max 64K ACL rules for all platforms + + leaf aclname { + type leafref { + path "../../../ACL_TABLE/ACL_TABLE_LIST/aclname"; + } + must "(/cmn:operation/cmn:operation != 'DELETE') or " + + "count(current()/../../../ACL_TABLE/ACL_TABLE_LIST[aclname=current()]/ports) = 0" { + error-message "Ports are already bound to this rule."; + } + } + + leaf rulename { + type string; + } + + leaf PRIORITY { + type uint16 { + range "1..65535"{ + error-message "Invalid ACL rule priority."; + } + } + } + + leaf RULE_DESCRIPTION { + type string; + } + + leaf PACKET_ACTION { + type enumeration { + enum FORWARD; + enum DROP; + enum REDIRECT; + enum INT_INSERT; + enum INT_DELETE; + } + } + + leaf MIRROR_ACTION { + type leafref { + path "/sms:sonic-mirror-session/sms:MIRROR_SESSION/sms:MIRROR_SESSION_LIST/sms:name"; + } + } + + leaf IP_TYPE { + sonic-ext:custom-validation ValidateAclRuleIPAddress; + type enumeration { + enum ANY; + enum IP; + enum IPV4; + enum IPV4ANY; + enum NON_IPV4; + enum IPV6ANY; + enum NON_IPV6; + } + + default IPV4; + } + + leaf IP_PROTOCOL { + type uint8 { + range "1|2|6|17|46|47|51|103|115"; + } + } + + leaf ETHER_TYPE { + type string { + pattern "(0x88CC)|(0x8100)|(0x8915)|(0x0806)|(0x0800)|(0x86DD)|(0x8847)" { + error-message "Invalid ACL Rule Ether Type"; + error-app-tag ether-type-invalid; + } + } + } + + choice ip_src_dst { + case ipv4_src_dst { + when "boolean(IP_TYPE[.='ANY' or .='IP' or .='IPV4' or .='IPV4ANY'])"; + leaf SRC_IP { + mandatory true; + type inet:ipv4-prefix; + } + leaf DST_IP { + mandatory true; + type inet:ipv4-prefix; + } + } + case ipv6_src_dst { + when "boolean(IP_TYPE[.='ANY' or .='IP' or .='IPV6' or .='IPV6ANY'])"; + leaf SRC_IPV6 { + mandatory true; + type inet:ipv6-prefix; + } + leaf DST_IPV6 { + mandatory true; + type inet:ipv6-prefix; + } + } + } + + choice src_port { + case l4_src_port { + leaf L4_SRC_PORT { + type uint16; + } + } + case l4_src_port_range { + leaf L4_SRC_PORT_RANGE { + type string { + pattern "[0-9]{1,5}(-)[0-9]{1,5}"; + } + } + } + } + + choice dst_port { + case l4_dst_port { + leaf L4_DST_PORT { + type uint16; + } + } + case l4_dst_port_range { + leaf L4_DST_PORT_RANGE { + type string { + pattern "[0-9]{1,5}(-)[0-9]{1,5}"; + } + } + } + } + + leaf TCP_FLAGS { + type string { + pattern "0[xX][0-9a-fA-F]{2}[/]0[xX][0-9a-fA-F]{2}"; + } + } + + leaf DSCP { + type uint8; + } + } + } + } +} diff --git a/cvl/testdata/schema/sonic-port.yang b/cvl/testdata/schema/sonic-port.yang new file mode 100644 index 000000000000..44eb6eddd541 --- /dev/null +++ b/cvl/testdata/schema/sonic-port.yang @@ -0,0 +1,124 @@ +module sonic-port { + namespace "http://github.com/Azure/sonic-port"; + prefix prt; + + import sonic-common { + prefix scommon; + } + + organization + "SONiC"; + + contact + "SONiC"; + + description + "SONIC PORT"; + + revision 2019-05-15 { + description + "Initial revision."; + } + + + container sonic-port { + + container PORT { + + list PORT_LIST { + key "ifname"; + + leaf ifname { + type string { + pattern "Ethernet([1-3][0-9]{3}|[1-9][0-9]{2}|[1-9][0-9]|[0-9])" { + error-message "Invalid interface name"; + error-app-tag interface-name-invalid; + } + } + } + + leaf index { + type uint16; + } + + leaf speed { + type uint64; + } + + leaf valid_speeds { + type string; + } + + leaf alias { + type string; + } + + leaf description { + type string; + } + + leaf mtu{ + type uint32 { + range "1312..9216" { + error-message "Invalid MTU value"; + error-app-tag mtu-invalid; + } + } + } + + leaf lanes { + type string; + } + + leaf admin_status { + type scommon:admin-status; + } + } + } + container PORT_TABLE { + config false; + + list PORT_TABLE_LIST { + key "ifname"; + + leaf ifname { + type string { + pattern "Ethernet([1-3][0-9]{3}|[1-9][0-9]{2}|[1-9][0-9]|[0-9])"{ + error-message "Invalid interface name"; + error-app-tag interface-name-invalid; + } + } + } + + leaf index { + type uint16; + } + + leaf lanes { + type string; + } + + leaf mtu { + type uint32 { + range "1312..9216" { + error-message "Invalid MTU value"; + error-app-tag mtu-invalid; + } + } + } + + leaf valid_speeds { + type string; + } + + leaf alias { + type string; + } + + leaf oper_status { + type scommon:oper-status; + } + } + } + } +} diff --git a/models/yang/sonic/common/sonic-common.yang b/models/yang/sonic/common/sonic-common.yang index 45d608b684f8..42147e09757c 100644 --- a/models/yang/sonic/common/sonic-common.yang +++ b/models/yang/sonic/common/sonic-common.yang @@ -32,6 +32,14 @@ module sonic-common { } } + typedef oper-status { + type enumeration { + enum up; + enum down; + } + } + + container operation { description "This definition is used internally by CVL and is not exposed in NBI. Leaf 'operation' allows diff --git a/models/yang/sonic/common/sonic-extension.yang b/models/yang/sonic/common/sonic-extension.yang index f4888095863e..937c7adb3431 100644 --- a/models/yang/sonic/common/sonic-extension.yang +++ b/models/yang/sonic/common/sonic-extension.yang @@ -17,12 +17,6 @@ module sonic-extension { "Initial revision."; } - extension custom-handler { - description - "Node should be handled by custom handler"; - argument "name"; - } - extension db-name { description "DB name, e.g. APPL_DB, CONFIG_DB"; @@ -53,9 +47,10 @@ module sonic-extension { argument "value"; } - extension pf-check { + extension custom-validation { description - "Platform specific validation"; + "Extension for custom validation. + Platform specific validation can be implemented using custom validation."; argument "handler"; } }