zos-extension to nlc2cmd (#61)

* cloudbot refactor

Signed-off-by: Tathagata Chakraborti <tathagata.chakraborti1@ibm.com>

* duplicate requirements purge

Signed-off-by: Tathagata Chakraborti <tathagata.chakraborti1@ibm.com>

* wa skills refactor

Signed-off-by: Tathagata Chakraborti <tathagata.chakraborti1@ibm.com>

* add remote server

Signed-off-by: Tathagata Chakraborti <tathagata.chakraborti1@ibm.com>

* tested without z

Signed-off-by: Tathagata Chakraborti <tathagata.chakraborti1@ibm.com>

* dummy config

Signed-off-by: Tathagata Chakraborti <tathagata.chakraborti1@ibm.com>

* merge ready

Signed-off-by: Tathagata Chakraborti <tathagata.chakraborti1@ibm.com>

* added try prefix in suggestion

Signed-off-by: Tathagata Chakraborti <tathagata.chakraborti1@ibm.com>

* Update README.md

* Update README.md

* Create README.md

Co-authored-by: Tathagata Chakraborti <tathagata.chakraborti1@ibm.com>

.gitignore

@@ -112,6 +112,6 @@ data
 
 nohup.out
 
-# hide kube bot config
-*/kube_agent/config/*
-*/kube_agent/bb-survey-server/*
+# hide wa config
+*nlc2cmd/remote/config.json
+

clai/server/plugins/nlc2cmd/README.md

@@ -14,6 +14,8 @@ If there is a significant match with known patterns of `tar` and `grep`,
 the user command is translated into the corresponding command line syntax [here](./wa_skills/).
 The confidence of the skill is the same as the confidence of the underlying NLC layer.
 
+> An example of the remote router to the Watson Assistant instance is hosted [here](./remote/).
+
 This skill is merely illustrative and can be made as accurate as desired for these
 specific use cases. However, this approach would not scale to Bash commands
 in general and is quite brittle.
@@ -33,7 +35,8 @@ Right now this skills only supports `grep` and `tar` commands. Try these out!
 6. `>> grep for all files with "clai" in this directory, show me the line numbers`
 7. `>> grep for the number of files with "clai" in this directory`
 
-See [here](./wa_skills/) for some more examples on the [IBM Cloud CLI](https://www.ibm.com/cloud/cli).
+See [here](./wa_skills/) for some more examples on the [IBM Cloud CLI](https://www.ibm.com/cloud/cli)
+and in the z/OS operating system on mainframes.
 
 
 ## [xkcd](https://uni.xkcd.com/)

clai/server/plugins/nlc2cmd/manifest.properties

@@ -1,4 +1,4 @@
 [DEFAULT]
 name=nlc2cmd
-description=This skill helps you compress and uncompress archives using natural language commands.
+description=This skill helps you compress archives, find files, etc. using English commands.
 default=yes

clai/server/plugins/nlc2cmd/remote/Procfile

@@ -0,0 +1 @@
+web: python3 run.py > stdout.txt 2>stderr.txt

clai/server/plugins/nlc2cmd/remote/README.md

@@ -0,0 +1,7 @@
+## Remote WA router 
+
+This flask application is a router for requests to the watson assistant instance 
+that processes most of the `nlc2cmd` use cases. To bring up the server:
+
+1. Fill in the details of your assistant and skill in `config.json`
+2. Run `>> python3 run.py`

clai/server/plugins/nlc2cmd/remote/config.json

@@ -0,0 +1,9 @@
+{
+    "i_am_id" : "",
+    "skills"  : { 
+
+        "tarbot"  : "",
+        "grepbot" : "",
+        "zosbot"  : ""
+    }
+}

clai/server/plugins/nlc2cmd/remote/manifest.yml

@@ -0,0 +1,6 @@
+applications:
+- name: nlc2cmd
+  memory: 1G
+  path: .
+  buildpacks: 
+    - python_buildpack

clai/server/plugins/nlc2cmd/remote/requirements.txt

@@ -0,0 +1,2 @@
+Flask==0.12.2
+ibm-watson>=4.1.0

clai/server/plugins/nlc2cmd/remote/run.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+
+'''
+imports
+'''
+from flask import Flask, request, render_template
+from typing import Dict
+
+from ibm_watson import AssistantV2
+from ibm_cloud_sdk_core.authenticators import IAMAuthenticator
+
+import json
+import os
+import inspect
+
+'''
+globals
+'''
+
+config = json.loads( open('config.json').read() )
+
+authenticator = IAMAuthenticator(config["i_am_id"])
+assistant = AssistantV2(
+    version='2020-13-05',
+    authenticator=authenticator
+)
+
+assistant.set_service_url('https://gateway.watsonplatform.net/assistant/api')
+
+''''''
+app = Flask(__name__)
+
+@app.route("/")
+def hello():
+    return "Welcome to the router for the WA instances for the nlc2cmd skill in CLAI."
+
+'''
+endpoint for tarbot
+'''
+@app.route("/tarbot", methods=['GET', 'POST', 'PUT'])
+def tarbot():
+    return get_response( 
+        assistant_id = config["skills"][inspect.stack()[0][3]], 
+        data = json.loads(request.get_data().decode('utf-8')) )
+
+'''
+endpoint for grepbot
+'''
+@app.route("/grepbot", methods=['GET', 'POST', 'PUT'])
+def grepbot():
+    return get_response( 
+        assistant_id = config["skills"][inspect.stack()[0][3]], 
+        data = json.loads(request.get_data().decode('utf-8')) )
+
+'''
+endpoint for zosbot
+'''
+@app.route("/zosbot", methods=['GET', 'POST', 'PUT'])
+def zosbot():
+    return get_response( 
+        assistant_id = config["skills"][inspect.stack()[0][3]], 
+        data = json.loads(request.get_data().decode('utf-8')) )
+
+
+''' get response from WA workspace '''
+def get_response(assistant_id: str, data: Dict) -> Dict:
+
+    try:  
+
+        session_info = assistant.create_session(assistant_id=assistant_id).get_result()
+        response = assistant.message(
+            assistant_id=assistant_id,
+            session_id=session_info['session_id'],
+            input={
+                'message_type': 'text',
+                'text': data['text'],
+                'options' : {'alternate_intents' : True}
+            }
+        ).get_result()
+
+        return json.dumps( { 'result' : 'success', 'response' : response } )
+
+    except Exception as e:
+        return json.dumps( { 'result' : str(e) } )
+
+
+''' main '''
+if __name__ == "__main__":
+    app.run(host='0.0.0.0', port=int(os.getenv('PORT', 3456)))
+

clai/server/plugins/nlc2cmd/remote/runtime.txt

@@ -0,0 +1 @@
+python-3.6.9

clai/server/plugins/nlc2cmd/wa_skills/README.md

@@ -1,10 +1,11 @@
-# nlc2cmd for IBM Cloud CLI
+# nlc2cmd for IBM Cloud CLI and z/OS
 
 `NLP` `Support` `ibmcloud`
 
 This contains scripts that turn the recognized intents into actionable commands on the command line. 
 Previously in [../](../) we looked at some common examples using the `tar` and `grep` commands. 
-Here we want to highlight another instantiation of the nlc2cmd use case, this time for the [IBM Cloud CLI](https://www.ibm.com/cloud/cli) and the `ibmcloud` family of commands.
+Here we want to highlight another instantiation of the nlc2cmd use case, this time for the [IBM Cloud CLI](https://www.ibm.com/cloud/cli) and the `ibmcloud` family of commands on the shell, as well as more specialized
+commands in the z/OS USS terminal in mainframes.
 
 ## Example Usage
 
@@ -19,6 +20,15 @@ The script [cloudbot.py](cloudbot.py) supports some illustrative commands from t
 5. `>> available plugins`
 6. `>> I want to invite someone to my cloud`
 
+<img src="https://www.dropbox.com/s/jxct2mdlphhcua2/ss3.png?raw=1" width="400px"/>
+
+Similarly, the script [zosbot.py](zosbot.py) supports some illustrative commands from the 
+USS (UNIX System Services) terminal in the z/OS operating system.
+
+1. `>> copy file to PDS member`
+2. `>> lookup reason codes`
+3. `>> view mvs file`
+
 ## [xkcd](https://uni.xkcd.com/)
 
 There's planned downtime every night when we turn on the Roomba and it runs over the cord.

clai/server/plugins/nlc2cmd/wa_skills/__init__.py

@@ -8,3 +8,4 @@
 from clai.server.plugins.nlc2cmd.wa_skills.tarbot import wa_skill_processor_tarbot
 from clai.server.plugins.nlc2cmd.wa_skills.grepbot import wa_skill_processor_grepbot
 from clai.server.plugins.nlc2cmd.wa_skills.cloudbot import wa_skill_processor_cloudbot
+from clai.server.plugins.nlc2cmd.wa_skills.zosbot import wa_skill_processor_zosbot

clai/server/plugins/nlc2cmd/wa_skills/cloudbot.py

@@ -8,30 +8,40 @@
 ''' cloud starter-sheet command handler '''
 
 ''' imports '''
-import requests
+from clai.server.plugins.nlc2cmd.wa_skills.utils import call_wa_skill, get_own_name
 
 ''' globals '''
-wa_endpoint = 'https://ibmcloud-helper.mybluemix.net/message'
+__self = get_own_name(__file__) 
+__intents = { 'add_tag_resource'                 : "resource tag-attach --tag-names TAG --resource-name NAME",
+              'assign_access_create_policy'      : "iam user-policy-create USER_NAME [OPTIONS]",
+              'create_access_group'              : "iam user-policy-create USER_NAME [OPTIONS]",
+              'create_api_key'                   : "iam api-key-create NAME -d DESCRIPTION",
+              'create_org_cloudfoundry_services' : "account org-create ORG_NAME",
+              'create_resource'                  : "resource service-instance-create NAME SERVICE_NAME PLAN_NAME LOCATION",
+              'create_resource_group'            : "resource group create GROUP_NAME",
+              'create_space_services_org'        : "account space-create SPACE_NAME",
+              'install_plugin'                   : "plugin install NAME",
+              'invite_user_to_account'           : "account user-invite USER_EMAIL",
+              'list_plugins'                     : "plugin repo-plugins",
+              'list_services_resource_group'     : "resource service-instances -g GROUP_NAME",
+              'list_tags_account'                : "resource tags",
+              'list_users_in_account'            : "account users (works only for account owners)",
+              'search_ibm_cloud_catalog'         : "catalog search QUERY",
+              'target_cloudfoundry_org'          : "target --cf",
+              'view_service_details'             : "catalog service NAME",
+              'view_usage_costs_month'           : "billing account-usage [-d YYYY-MM]",
+              'login'                            : "login",
+              'help'                             : "help COMMAND",
+              'generic'                          : "help" }
 
 def wa_skill_processor_cloudbot(msg):
 
-    # Confidence remains at 0 unless an intent has been detected
-    confidence = 0.0 
-
     # Make sure we are not intercepting real IBM Cloud CLI commands
     if msg.startswith('ibmcloud'):
         return None, 0.0
 
-    try:
-        response = requests.put(wa_endpoint, json={'text': msg}).json()
-
-        if response['result'] == 'success':
-            response = response['response']['output']
-        else:
-            return [ { "text" : response['result'] }, confidence ]
-
-    except Exception as ex:
-        return [ { "text" : "Method failed with status " + str(ex) }, confidence ]
+    response, success = call_wa_skill(msg, __self)
+    if not success: return {"text" : response}, 0.0
 
     # Identify the intent in the user message
     try:
@@ -40,33 +50,10 @@ def wa_skill_processor_cloudbot(msg):
         confidence = response["intents"][0]["confidence"]
 
     except IndexError or KeyError:
-        intent = "ibmcloud help"
 
-    intents = { 'add_tag_resource': {"text": "Try >> ibmcloud resource tag-attach --tag-names TAG --resource-name NAME"},
-                'assign_access_create_policy': {"text": "Try >> ibmcloud iam user-policy-create USER_NAME [OPTIONS]"},
-                'create_access_group': {"text": "Try >> ibmcloud iam user-policy-create USER_NAME [OPTIONS]"},
-                'create_api_key': {"text": "Try >> ibmcloud iam api-key-create NAME -d DESCRIPTION"},
-                'create_org_cloudfoundry_services': {"text": "Try >> ibmcloud account org-create ORG_NAME"},
-                'create_resource': {"text": "Try >> ibmcloud resource service-instance-create NAME SERVICE_NAME PLAN_NAME LOCATION"},
-                'create_resource_group': {"text": "Try >> ibmcloud resource group create GROUP_NAME"},
-                'create_space_services_org': {"text": "Try >> ibmcloud account space-create SPACE_NAME"},
-                'help': {"text": "Try >> ibmcloud help COMMAND"},
-                'install_plugin': {"text": "Try >> ibmcloud plugin install NAME"},
-                'invite_user_to_account': {"text": "Try >> ibmcloud account user-invite USER_EMAIL"},
-                'list_plugins': {"text": "Try >> ibmcloud plugin repo-plugins"},
-                'list_services_resource_group': {"text": "Try >> ibmcloud resource service-instances -g GROUP_NAME"},
-                'list_tags_account': {"text": "Try >> ibmcloud resource tags"},
-                'list_users_in_account': {"text": "Try >> ibmcloud account users (works only for account owners)"},
-                'login': {"text": "Try >> ibmcloud login"},
-                'search_ibm_cloud_catalog': {"text": "Try >> ibmcloud catalog search QUERY"},
-                'target_cloudfoundry_org': {"text": "Try >> ibmcloud target --cf"},
-                'view_service_details': {"text": "Try >> ibmcloud catalog service NAME"},
-                'view_usage_costs_month': {"text": "Try >> ibmcloud billing account-usage [-d YYYY-MM]"}}
-
-    if intent in intents: data = intents[intent]
-    else: pass
+        intent = "generic"
+        confidence = 0.0
 
+    data = { "text" : "Try >> ibmcloud " + __intents[intent] } 
     return data, confidence
 
-
-

clai/server/plugins/nlc2cmd/wa_skills/config.json

@@ -0,0 +1,6 @@
+{
+	"cloudbot" : "https://ibmcloud-helper.mybluemix.net/message",
+	"tarbot"   : "http://nlc2cmd.mybluemix.net/tarbot",
+	"grepbot"  : "http://nlc2cmd.mybluemix.net/grepbot",
+	"zosbot"   : "http://nlc2cmd.mybluemix.net/zosbot"
+}

clai/server/plugins/nlc2cmd/wa_skills/grepbot.py

@@ -8,11 +8,11 @@
 ''' grep command handler '''
 
 ''' imports '''
-import requests, re
+from clai.server.plugins.nlc2cmd.wa_skills.utils import call_wa_skill, get_own_name
+import re
 
 ''' globals '''
-wa_endpoint = 'http://nlc2cmd-chipper-impala.us-east.mybluemix.net/grepbot'
-
+__self = get_own_name(__file__) 
 __flags = { "line-number" : 'n',
             "match-case"  : 'i',
             "count"       : 'c',
@@ -32,16 +32,8 @@ def wa_skill_processor_grepbot(msg):
     except:
         match_string = "<string>"
 
-    try:
-        response = requests.put(wa_endpoint, json={'text': msg}).json()
-
-        if response['result'] == 'success':
-            response = response['response']['output']
-        else:
-            return [ { "text" : response['result'] }, confidence ]
-
-    except Exception as ex:
-        return [ { "text" : "Method failed with status " + str(ex) }, confidence ]
+    response, success = call_wa_skill(msg, __self)
+    if not success: return {"text" : response}, 0.0
 
     if msg.startswith('grep for'): confidence = 1.0
     else: confidence = max([item['confidence'] for item in response['intents']])

clai/server/plugins/nlc2cmd/wa_skills/tarbot.py

@@ -8,30 +8,23 @@
 ''' tar command handler '''
 
 ''' imports '''
-import requests
+from clai.server.plugins.nlc2cmd.wa_skills.utils import call_wa_skill, get_own_name
 
 ''' globals '''
-wa_endpoint = 'http://nlc2cmd-chipper-impala.us-east.mybluemix.net/tarbot'
+__self = get_own_name(__file__) 
 
 def wa_skill_processor_tarbot(msg):
 
     # Confidence remains at 0 and data is None unless an intent has been detected
     confidence = 0.0
     data = None
 
+    # Make sure we are not intercepting real tar commands
     if msg.startswith('tar'):
         return None, 0.0
 
-    try:
-        response = requests.put(wa_endpoint, json={'text': msg}).json()
-
-        if response['result'] == 'success':
-            response = response['response']['output']
-        else:
-            return [ { "text" : response['result'] }, confidence ]
-
-    except Exception as ex:
-        return [ { "text" : "Method failed with status " + str(ex) }, confidence ]
+    response, success = call_wa_skill(msg, __self)
+    if not success: return {"text" : response}, 0.0
 
     # Identify the intent in the user message
     try:
@@ -40,16 +33,13 @@ def wa_skill_processor_tarbot(msg):
         confidence = response["intents"][0]["confidence"]
 
     except IndexError or KeyError:
-        intent = "xkcd"
+        pass
 
     # Identify entities in the user message
     entities = {}
     for item in response["entities"]:
-
-        if item['entity'] in entities:
-            entities[item['entity']].append(item['value'])
-        else:
-            entities[item['entity']] = [item['value']]
+        if item['entity'] in entities: entities[item['entity']].append(item['value'])
+        else: entities[item['entity']] = [item['value']]
 
     # Identify file and directory name if present
     filename = "<archive-file>"
@@ -157,6 +147,3 @@ def wa_skill_processor_tarbot(msg):
 
     return data, confidence
 
-
-
-