WIP: Enhance Klish parser

CLI/Makefile

@@ -20,22 +20,50 @@ SHELL = /bin/bash
 .ONESHELL:
 .SHELLFLAGS += -e
 
+TOPDIR  ?= $(abspath ..)
 CLIDEPS ?= packages
 SUBDIRS := clitree renderer klish 
 export SONIC_CLI_ROOT=$(TOPDIR)/build
 TGT_DIR := $(SONIC_CLI_ROOT)/target
 
+CLI_DIR         := $(TOPDIR)/CLI
+PY_FILES        := $(shell find $(CLI_DIR)/actioner -name '*.py')
+PY_FILES        += $(shell find $(CLI_DIR)/renderer/scripts -name '*.py')
+PY_FILES        += $(shell find $(CLI_DIR)/clitree/scripts -name '*.py')
+PYLINT_DONE     := $(SONIC_CLI_ROOT)/pylint/.pylint_done
+TOOLS_DIR       := $(TOPDIR)/tools
+LINTER          := $(TOOLS_DIR)/test/pylint.sh
+
 .PHONY: clean
 
-all: $(CLIDEPS)
+all: $(CLIDEPS) $(PYLINT_DONE)
 	for dir in $(SUBDIRS); do \
     	$(MAKE) -C $$dir -f Makefile $@; \
 	done
 
-	rm -rf $(TOPDIR)/build/cli/*
+	rm -rf $(TOPDIR)/build/cli
 	mv -f $(TGT_DIR) $(TOPDIR)/build/cli
 
-clean:
+.SECONDEXPANSION:
+
+#======================================================================
+# Common rule for directories. Use "." suffix, like "xyz/."
+#======================================================================
+.PRECIOUS: %/. 
+%/.:
+	mkdir -p $@
+
+pylint: $(PYLINT_DONE)
+
+$(PYLINT_DONE): $(PY_FILES) | $$(@D)/.
+	$(LINTER) $?
+	touch $@
+
+pylint-clean:
+	$(RM) -r $(PYLINT_DONE)
+	$(RM) -r $(SONIC_CLI_ROOT)/cli/pylint.log*
+
+clean: pylint-clean
 	make --directory=klish clean
 	rm -rf $(TOPDIR)/build/cli
 	rm -rf $(TGT_DIR)
@@ -48,3 +76,4 @@ packages:
 	if ! dpkg -l | grep python-lxml -c >>/dev/null; then sudo apt-get install python-lxml; fi
 	if ! dpkg -l | grep libexpat1-dev -c >>/dev/null; then sudo apt-get install libexpat1-dev; fi
 
+

CLI/clitree/Makefile

@@ -18,23 +18,33 @@
 
 .PHONY: install
 
-TGT_DIR ?= $(SONIC_CLI_ROOT)/target
+TOPDIR               := ../..
+# TGT_DIR is the directory wher all prepreocssed cli model, actioner and
+# renderer scripts and temlates will be dumped. Can be overidden from caller.
+TGT_DIR              ?= $(SONIC_CLI_ROOT)/target
+CLI_XML_FILES        := $(shell find $(TOPDIR)/CLI/clitree/cli-xml -name '*.xml' | sort)
+CLI_XML_MACRO_FILES  := $(shell find $(TOPDIR)/CLI/clitree/macro -name '*.xml' | sort)
+cli_done             := ${TGT_DIR}/command-tree/.done
+cli_guide            := ${TGT_DIR}/command-tree/industry_standard_cli_reference_guide.md
 
-all:
+all: $(cli_done)
+
+$(cli_done): $(CLI_XML_FILES) $(CLI_XML_MACRO_FILES)
 	mkdir -p ${TGT_DIR}/command-tree ${TGT_DIR}/cli-macro ${TGT_DIR}/render-templates ${TGT_DIR}/scripts
 	cp -r cli-xml/include ${TGT_DIR}/command-tree
 	cp cli-xml/*.xml ${TGT_DIR}/command-tree
 	cp macro/*.xml ${TGT_DIR}/cli-macro
 	(cd scripts;./klish_platform_features_process.sh ../../clicfg ${TGT_DIR})
-	python scripts/klish_preproc_cmdtree.py ${TGT_DIR}/command-tree ${TGT_DIR}/cli-macro 3
+	python scripts/klish_preproc_cmdtree.py ${TGT_DIR}/command-tree ${TGT_DIR}/cli-macro 6 
 	cp ./../actioner/*.py ${TGT_DIR}/.
 	cp ../renderer/scripts/*.py ${TGT_DIR}/scripts
-	cp ../renderer/templates/* ${TGT_DIR}/render-templates
+	cp -r ../renderer/templates/* ${TGT_DIR}/render-templates
 	cp scripts/sonic-clish.xsd ${TGT_DIR}/command-tree
 	(cd ${TGT_DIR}/command-tree ; xmllint --noout --schema sonic-clish.xsd ${TGT_DIR}/command-tree/*.xml && \
             xmllint --noout --schema sonic-clish.xsd ${TGT_DIR}/command-tree/include/*.xml) || exit 1
-	rm -rf ${TGT_DIR}/cli-macro
+	rm -rf {TGT_DIR}/cli-macro
 	rm -rf scripts/*.pyc
+	touch $@
 
 clean:
 	rm -rf ${TGT_DIR}

CLI/clitree/cli-xml/include/pipe.xml

@@ -30,25 +30,6 @@ limitations under the License.
         ptype="STRING"
         mode="switch">
 
-        <PARAM name="display-xml"
-            help="Display in XML format"
-            ptype="SUBCOMMAND"
-            mode="subcommand"
-            test='"${__cmd}" != "show diff" -a "${__cmd}" != "do show diff"'>
-            <PARAM name="1_xpath"
-                value="xpath"
-                help="Show only text that does match a xpath expression"
-                ptype="SUBCOMMAND"
-                mode="subcommand"
-                optional="true"
-                >
-                <PARAM name="1_LINE"
-                    help="Xpath expression"
-                    ptype="STRING">
-                </PARAM>
-            </PARAM>
-        </PARAM>
-
         <PARAM name="except"
             help="Show only text that does not match a pattern"
             ptype="SUBCOMMAND"

CLI/clitree/scripts/klish_ins_def_cmd.py

@@ -79,7 +79,7 @@
 
 
 END_CMD = """<COMMAND name="end"
-                 help="Exit to the exec Mode"
+                 help="Exit to EXEC mode"
                  view="enable-view"/>"""
 
 VIEW_TAG_STR = """{http://www.dellemc.com/sonic/XMLSchema}VIEW"""

CLI/clitree/scripts/klish_platform_features_process.sh

@@ -57,8 +57,8 @@ function insert_in()
         printvalue="${pref}${value} ]>"
     fi
     #echo Insert_in $filename. Option $option
-    while read -r line; do
-        echo ${line} >> $outfile
+    while IFS= read -r line; do
+        echo "${line}" >> $outfile
         if [[ "${line}" =~ ${matchpattern} ]]; then
             #echo Match found for ${line}
             echo "${printvalue}" >> $outfile
@@ -81,7 +81,7 @@ function insert_entities()
     for i in ${list}; do
         echo Processing $i
         insert_in $i
-        xmllint $i >& /dev/null
+        xmllint $i > /dev/null
         if [ $? -ne 0 ]; then
             echo ENTITY insertion in $i failed
             exit 1

CLI/klish/Makefile

@@ -12,7 +12,7 @@ all : $(SRC_REPLACEMENTS)
 	tar xzvf klish-$(KLISH_VERSION).tgz -C $(SONIC_CLI_ROOT)
 	./patches/scripts/patchmake.sh -p VER=${KLISH_VERSION} TSP=${SONIC_CLI_ROOT} DSP=${CURDIR}/patches TWP=${SONIC_CLI_ROOT}
 
-	cd ${KLISH_SRC} && sh autogen.sh && ./configure --with-libxml2=/usr && make
+	cd ${KLISH_SRC} && sh autogen.sh && ./configure --with-libxml2=/usr --enable-debug=no LIBS='-l:libcurl-gnutls.so.4 -lpython2.7 -Wl,-rpath=/usr/lib/x86_64-linux-gnu -lcjson' CFLAGS='${CFLAGS} -g -I/usr/include/python2.7' && make
 
 	mkdir -p $(SONIC_CLI_ROOT)/target/.libs
 	cp $(CURDIR)/clish_start $(SONIC_CLI_ROOT)/target/.
@@ -24,4 +24,4 @@ all : $(SRC_REPLACEMENTS)
 
 .PHONY: clean
 clean:
-	rm -rf ${KLISH_SRC}/klish-$(KLISH_VERSION)
+	rm -rf ${KLISH_SRC}

CLI/klish/clish_start

@@ -2,10 +2,18 @@
 
 export SONIC_MGMT_ROOT=/usr/sbin
 export SONIC_CLI_ROOT=$SONIC_MGMT_ROOT/cli
-export SYSTEM_NAME="${HOSTNAME%%.*}"
-export PYTHONPATH=$SONIC_MGMT_ROOT:$SONIC_MGMT_ROOT/lib/swagger_client_py:$SONIC_CLI_ROOT:$SONIC_CLI_ROOT/scripts:$PYTHONPATH
+if [ -z $SYSTEM_NAME ]
+then
+ export SYSTEM_NAME="${HOSTNAME%%.*}"
+fi
+export PYTHONPATH=/usr/sbin/cli:/usr/sbin/cli/scripts:/usr/lib/python2.7:/usr/lib/python2.7/dist-packages:/usr/local/lib/python2.7/dist-packages:/usr/sbin/:/usr/sbin/lib/swagger_client_py:/usr/lib/python2.7:/usr/lib/python2.7/plat-x86_64-linux-gnu:/usr/lib/python2.7/lib-tk:/usr/lib/python2.7/lib-old:/usr/lib/python2.7/lib-dynload:/usr/local/lib/python2.7/dist-packages:/usr/local/lib/python2.7/dist-packages/pyang-2.0.1-py2.7.egg:/usr/local/lib/python2.7/dist-packages/lxml-4.3.4-py2.7-linux-x86_64.egg:/usr/local/lib/python2.7/dist-packages/pyang_json_schema_plugin-0.1-py2.7.egg:/usr/lib/python2.7/dist-packages:/usr/lib/python2.7/dist-packages/gtk-2.0
+export RENDERER_TEMPLATE_PATH=$SONIC_CLI_ROOT/render-templates
 export CLISH_PATH=$SONIC_CLI_ROOT/command-tree
 export LD_LIBRARY_PATH=/usr/local/lib:$SONIC_CLI_ROOT/.libs:$LD_LIBRARY_PATH
 export PATH=$PATH:/usr/local/sbin:/usr/sbin:/sbin:$SONIC_CLI_ROOT
 
-$SONIC_CLI_ROOT/clish "$@"
+MODE=$(sonic-cfggen -d -v "DEVICE_METADATA['localhost']['intf_naming_mode']")
+export SONIC_CLI_IFACE_MODE=$MODE
+export KLISH_CLI_USER=$CLI_USER
+
+$SONIC_CLI_ROOT/clish -o "$@"

CLI/klish/patches/klish-2.1.4/bin/clish.c.diff

@@ -0,0 +1,20 @@
+41a42,43
+> #include <stdbool.h>
+> 
+48a51,56
+> bool _nos_use_alt_name = false;
+> 
+> bool nos_use_alt_name() {
+>     return _nos_use_alt_name;
+> }
+> 
+88a97,101
+>     char *mode = getenv("SONIC_CLI_IFACE_MODE");
+>     if (mode) {
+>         _nos_use_alt_name = (strcmp(mode, "standard") == 0);
+>     }
+> 
+307c320
+< 		clish_shell__set_idle_timeout(shell, timeout);
+---
+> 		clish_shell__set_timeout(shell, timeout);

CLI/klish/patches/klish-2.1.4/clish/command.h.diff

@@ -0,0 +1,6 @@
+78a79,83
+> char *clish_command__get_test(const clish_command_t *instance);
+> void clish_command__set_hidden(clish_command_t * instance, bool_t hidden);
+> bool_t clish_command__get_hidden(const clish_command_t * instance);
+> void clish_command__set_enabled(clish_command_t * instance, bool_t enabled);
+> bool_t clish_command__get_enabled(const clish_command_t * instance);

CLI/klish/patches/klish-2.1.4/clish/command/command.c.diff

@@ -0,0 +1,37 @@
+43a44,46
+> 	this->test = NULL;
+>         this->hidden = BOOL_FALSE;
+>         this->enabled = BOOL_FALSE;
+305a309,340
+> 
+> /*--------------------------------------------------------- */
+> char *clish_command__get_test(const clish_command_t *this)
+> {
+>         return this->test;
+> }
+> 
+> /*--------------------------------------------------------- */
+> void clish_command__set_hidden(clish_command_t * this, bool_t hidden)
+> {
+>         this->hidden = hidden;
+> }
+> 
+> /*--------------------------------------------------------- */
+> bool_t clish_command__get_hidden(const clish_command_t * this)
+> {
+>         return this->hidden;
+> }
+> 
+> 
+> /*--------------------------------------------------------- */
+> void clish_command__set_enabled(clish_command_t * this, bool_t enabled)
+> {
+>         this->enabled = enabled;
+> }
+> 
+> /*--------------------------------------------------------- */
+> bool_t clish_command__get_enabled(const clish_command_t * this)
+> {
+>     return BOOL_TRUE;
+> }
+> 

CLI/klish/patches/klish-2.1.4/clish/command/private.h.diff

@@ -0,0 +1,4 @@
+26a27,29
+>       char *test;
+>       bool_t hidden;
+>       bool_t enabled;

CLI/klish/patches/klish-2.1.4/clish/nspace.h.diff

@@ -0,0 +1,2 @@
+17a18
+> typedef clish_nspace_visibility_e clish_nspace_visibility_t;

CLI/klish/patches/klish-2.1.4/clish/param.h.diff

@@ -0,0 +1,135 @@
+3,4d2
+<  * Parameter instances are assocated with a command line and used to validate the
+<  * the arguments which a user is inputing for a command.
+5a4,7
+> /**
+> \ingroup clish
+> \defgroup clish_param param
+> @{
+6a9,14
+> \brief This class represents an instance of a parameter type.
+> 
+> Parameter instances are assocated with a command line and used to validate the
+> the arguments which a user is inputing for a command.
+> 
+> */
+16a25
+> #include "mgmt_clish_extn_param.h"
+18c27,29
+< /* The means by which the param is interpreted */
+---
+> /**
+>  * The means by which the param is interpreted.
+>  */
+20c31,33
+< 	/* A common parameter */
+---
+>     /**
+>      * A common parameter.
+>     */
+22c35,38
+< 	/* A swich parameter. Choose the only one of nested parameters. */
+---
+>     /**
+>      * A swich parameter.
+>      * Choose the only one of nested parameters.
+>      */
+24c40,43
+< 	/* A subcomand. Identified by it's name. */
+---
+>     /**
+>      * A subcomand.
+>      * Identified by it's name.
+>      */
+28,29c47,52
+< /* Class param */
+< 
+---
+> /*=====================================
+>  * PARAM INTERFACE
+>  *===================================== */
+> /*-----------------
+>  * meta functions
+>  *----------------- */
+32c55,57
+< 
+---
+> /*-----------------
+>  * methods
+>  *----------------- */
+34c59
+< void clish_param_help(const clish_param_t * instance, clish_help_t *help);
+---
+> void clish_param_help(const clish_param_t * instance, clish_help_t *help, const char *pval);
+39,66c64,77
+< 
+< _CLISH_SET_STR(param, ptype_name);
+< _CLISH_GET_STR(param, ptype_name);
+< _CLISH_SET_STR(param, access);
+< _CLISH_GET_STR(param, access);
+< _CLISH_GET_STR(param, name);
+< _CLISH_GET_STR(param, text);
+< _CLISH_GET_STR(param, range);
+< _CLISH_SET_STR_ONCE(param, value);
+< _CLISH_GET_STR(param, value);
+< _CLISH_SET(param, clish_ptype_t *, ptype);
+< _CLISH_GET(param, clish_ptype_t *, ptype);
+< _CLISH_SET_STR_ONCE(param, defval);
+< _CLISH_GET_STR(param, defval);
+< _CLISH_SET_STR_ONCE(param, test);
+< _CLISH_GET_STR(param, test);
+< _CLISH_SET_STR_ONCE(param, completion);
+< _CLISH_GET_STR(param, completion);
+< _CLISH_SET(param, clish_param_mode_e, mode);
+< _CLISH_GET(param, clish_param_mode_e, mode);
+< _CLISH_GET(param, clish_paramv_t *, paramv);
+< _CLISH_SET(param, bool_t, optional);
+< _CLISH_GET(param, bool_t, optional);
+< _CLISH_SET(param, bool_t, order);
+< _CLISH_GET(param, bool_t, order);
+< _CLISH_SET(param, bool_t, hidden);
+< _CLISH_GET(param, bool_t, hidden);
+< _CLISH_GET(param, unsigned int, param_count);
+---
+> /*-----------------
+>  * attributes
+>  *----------------- */
+> void clish_param__set_ptype_name(clish_param_t *instance, const char *ptype_name);
+> const char * clish_param__get_ptype_name(const clish_param_t *instance);
+> const char *clish_param__get_name(const clish_param_t * instance);
+> const char *clish_param__get_text(const clish_param_t * instance);
+> const char *clish_param__get_range(const clish_param_t * instance);
+> const char *clish_param__get_default(const clish_param_t * instance);
+> clish_ptype_t *clish_param__get_ptype(const clish_param_t * instance);
+> void clish_param__set_ptype(clish_param_t *instance, clish_ptype_t *ptype);
+> void clish_param__set_default(clish_param_t * instance, const char *defval);
+> void clish_param__set_mode(clish_param_t * instance, clish_param_mode_e mode);
+> clish_param_mode_e clish_param__get_mode(const clish_param_t * instance);
+69,71c80,98
+< 	unsigned int index);
+< 
+< /* Class paramv */
+---
+> 				unsigned index);
+> unsigned int clish_param__get_param_count(const clish_param_t * instance);
+> clish_paramv_t *clish_param__get_paramv(clish_param_t * instance);
+> void clish_param__set_optional(clish_param_t * instance, bool_t optional);
+> bool_t clish_param__get_optional(const clish_param_t * instance);
+> void clish_param__set_order(clish_param_t * instance, bool_t order);
+> bool_t clish_param__get_order(const clish_param_t * instance);
+> void clish_param__set_value(clish_param_t * instance, const char * value);
+> char *clish_param__get_value(const clish_param_t * instance);
+> void clish_param__set_hidden(clish_param_t * instance, bool_t hidden);
+> bool_t clish_param__get_hidden(const clish_param_t * instance);
+> void clish_param__set_test(clish_param_t * instance, const char *test);
+> char *clish_param__get_test(const clish_param_t *instance);
+> void clish_param__set_completion(clish_param_t *instance, const char *completion);
+> char *clish_param__get_completion(const clish_param_t *instance);
+> void clish_param__set_access(clish_param_t *instance, const char *access);
+> char *clish_param__get_access(const clish_param_t *instance);
+> void clish_param__set_enabled(clish_param_t * instance, bool_t enabled);
+> bool_t clish_param__get_enabled(const clish_param_t * instance);
+72a100
+> /* paramv methods */
+85a114
+> /** @} clish_param */