Watson Openscale Components to manage models

components/ibm-components/watson/manage/monitor_fairness/Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.6.8-stretch
+
+RUN pip install --upgrade pip
+RUN pip install --upgrade watson-machine-learning-client ibm-ai-openscale --no-cache | tail -n 1
+RUN pip install psycopg2-binary | tail -n 1
+
+ENV APP_HOME /app
+COPY src $APP_HOME
+WORKDIR $APP_HOME
+
+ENTRYPOINT ["python"]
+CMD ["enable_fairness_monitoring.py"]

components/ibm-components/watson/manage/monitor_fairness/src/enable_fairness_monitoring.py

@@ -0,0 +1,88 @@
+import json
+import argparse
+import ibm_boto3
+from ibm_botocore.client import Config
+from ibm_ai_openscale import APIClient
+from ibm_ai_openscale.engines import *
+from ibm_ai_openscale.utils import *
+from ibm_ai_openscale.supporting_classes import PayloadRecord, Feature
+from ibm_ai_openscale.supporting_classes.enums import *
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--model_name', type=str, help='Deployed model name', default='AIOS Spark German Risk Model - Final')
+    parser.add_argument('--fairness_threshold', type=float, help='Amount of threshold for fairness monitoring', default=0.95)
+    parser.add_argument('--fairness_min_records', type=int, help='Minimum amount of records for performing a fairness monitor', default=5)
+    parser.add_argument('--aios_manifest_path', type=str, help='Object storage file path for the aios manifest file', default='aios.json')
+    parser.add_argument('--cos_bucket_name', type=str, help='Object storage bucket name', default='bucket-name')
+    args = parser.parse_args()
+
+    model_name = args.model_name
+    fairness_threshold = args.fairness_threshold
+    fairness_min_records = args.fairness_min_records
+    cos_bucket_name = args.cos_bucket_name
+    aios_manifest_path = args.aios_manifest_path
+
+    with open("/app/secrets/aios_guid", 'r') as f:
+        aios_guid = f.readline().strip('\'')
+    f.close()
+    with open("/app/secrets/cloud_api_key", 'r') as f:
+        cloud_api_key = f.readline().strip('\'')
+    f.close()
+    with open("/app/secrets/cos_url", 'r') as f:
+        cos_url = f.readline().strip('\'')
+    f.close()
+    with open("/app/secrets/cos_apikey", 'r') as f:
+        cos_apikey = f.readline().strip('\'')
+    f.close()
+    with open("/app/secrets/cos_resource_id", 'r') as f:
+        cos_resource_instance_id = f.readline().strip('\'')
+    f.close()
+
+    ''' Upload data to IBM Cloud object storage '''
+    cos = ibm_boto3.resource('s3',
+                             ibm_api_key_id=cos_apikey,
+                             ibm_service_instance_id=cos_resource_instance_id,
+                             ibm_auth_endpoint='https://iam.bluemix.net/oidc/token',
+                             config=Config(signature_version='oauth'),
+                             endpoint_url=cos_url)
+
+    cos.Bucket(cos_bucket_name).download_file(aios_manifest_path, 'aios.json')
+
+    print('Fairness definition file ' + aios_manifest_path + ' is downloaded')
+
+    """ Load manifest JSON file """
+    with open('aios.json') as f:
+        aios_manifest = json.load(f)
+
+    """ Initiate AIOS client """
+
+    AIOS_CREDENTIALS = {
+        "instance_guid": aios_guid,
+        "apikey": cloud_api_key,
+        "url": "https://api.aiopenscale.cloud.ibm.com"
+    }
+
+    ai_client = APIClient(aios_credentials=AIOS_CREDENTIALS)
+    print('AIOS client version:' + ai_client.version)
+
+    ''' Setup fairness monitoring '''
+    subscriptions_uids = ai_client.data_mart.subscriptions.get_uids()
+    for sub in subscriptions_uids:
+        if ai_client.data_mart.subscriptions.get_details(sub)['entity']['asset']['name'] == model_name:
+            subscription = ai_client.data_mart.subscriptions.get(sub)
+
+    feature_list = []
+    for feature in aios_manifest['fairness_features']:
+        feature_list.append(Feature(feature['feature_name'], majority=feature['majority'], minority=feature['minority'], threshold=feature['threshold']))
+
+    subscription.fairness_monitoring.enable(
+            features=feature_list,
+            prediction_column='predictedLabel',
+            favourable_classes=aios_manifest['fairness_favourable_classes'],
+            unfavourable_classes=aios_manifest['fairness_unfavourable_classes'],
+            min_records=fairness_min_records
+        )
+
+    run_details = subscription.fairness_monitoring.run()
+    print('Fairness monitoring is enabled.')

components/ibm-components/watson/manage/monitor_quality/Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.6.8-stretch
+
+RUN pip install --upgrade pip
+RUN pip install --upgrade watson-machine-learning-client ibm-ai-openscale --no-cache | tail -n 1
+RUN pip install psycopg2-binary | tail -n 1
+
+ENV APP_HOME /app
+COPY src $APP_HOME
+WORKDIR $APP_HOME
+
+ENTRYPOINT ["python"]
+CMD ["enable_quality_monitoring.py"]

components/ibm-components/watson/manage/monitor_quality/src/enable_quality_monitoring.py

@@ -0,0 +1,56 @@
+import json
+import argparse
+from ibm_ai_openscale import APIClient
+from ibm_ai_openscale.engines import *
+from ibm_ai_openscale.utils import *
+from ibm_ai_openscale.supporting_classes import PayloadRecord, Feature
+from ibm_ai_openscale.supporting_classes.enums import *
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--model_name', type=str, help='Deployed model name', default="AIOS Spark German Risk Model - Final")
+    parser.add_argument('--problem_type', type=str, help='Model problem type', default="BINARY_CLASSIFICATION")
+    parser.add_argument('--quality_threshold', type=float, help='Amount of threshold for quality monitoring', default=0.7)
+    parser.add_argument('--quality_min_records', type=int, help='Minimum amount of records for performing a quality monitor', default=5)
+    args = parser.parse_args()
+
+    model_name = args.model_name
+    problem_type = args.problem_type
+    quality_threshold = args.quality_threshold
+    quality_min_records = args.quality_min_records
+
+    with open("/app/secrets/aios_guid", 'r') as f:
+        aios_guid = f.readline().strip('\'')
+    f.close()
+    with open("/app/secrets/cloud_api_key", 'r') as f:
+        cloud_api_key = f.readline().strip('\'')
+    f.close()
+
+    AIOS_CREDENTIALS = {
+        "instance_guid": aios_guid,
+        "apikey": cloud_api_key,
+        "url": "https://api.aiopenscale.cloud.ibm.com"
+    }
+
+    ai_client = APIClient(aios_credentials=AIOS_CREDENTIALS)
+    print('AIOS client version:' + ai_client.version)
+
+    ''' Setup quality monitoring '''
+    subscriptions_uids = ai_client.data_mart.subscriptions.get_uids()
+    for sub in subscriptions_uids:
+        if ai_client.data_mart.subscriptions.get_details(sub)['entity']['asset']['name'] == model_name:
+            subscription = ai_client.data_mart.subscriptions.get(sub)
+
+    PROBLEMTYPE = ProblemType.BINARY_CLASSIFICATION
+    if problem_type == 'BINARY_CLASSIFICATION':
+        PROBLEMTYPE = ProblemType.BINARY_CLASSIFICATION
+    elif problem_type == 'MULTICLASS_CLASSIFICATION':
+        PROBLEMTYPE = ProblemType.MULTICLASS_CLASSIFICATION
+    elif problem_type == 'REGRESSION':
+        PROBLEMTYPE = ProblemType.REGRESSION
+
+    subscription.quality_monitoring.enable(problem_type=PROBLEMTYPE, threshold=quality_threshold, min_records=quality_min_records)
+    # Runs need to post the minial payload records in order to trigger the monitoring run.
+    # run_details = subscription.quality_monitoring.run()
+
+    print('Quality monitoring is enabled.')

components/ibm-components/watson/manage/subscribe/Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.6.8-stretch
+
+RUN pip install --upgrade pip
+RUN pip install --upgrade watson-machine-learning-client ibm-ai-openscale --no-cache | tail -n 1
+RUN pip install psycopg2-binary | tail -n 1
+
+ENV APP_HOME /app
+COPY src $APP_HOME
+WORKDIR $APP_HOME
+
+ENTRYPOINT ["python"]
+CMD ["openscale_subscriptions.py"]

components/ibm-components/watson/manage/subscribe/src/openscale_subscriptions.py

@@ -0,0 +1,123 @@
+import json
+import argparse
+from ibm_ai_openscale import APIClient
+from ibm_ai_openscale.engines import *
+from ibm_ai_openscale.utils import *
+from ibm_ai_openscale.supporting_classes import PayloadRecord, Feature
+from ibm_ai_openscale.supporting_classes.enums import *
+from watson_machine_learning_client import WatsonMachineLearningAPIClient
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--aios_schema', type=str, help='AI OpenScale Schema Name', default="data_mart_credit_risk")
+    parser.add_argument('--model_name', type=str, help='Deployed model name', default="AIOS Spark German Risk Model - Final")
+    parser.add_argument('--model_uid', type=str, help='Deployed model uid', default="dummy uid")
+    parser.add_argument('--label_column', type=str, help='Model label column name', default="Risk")
+    args = parser.parse_args()
+
+    aios_schema = args.aios_schema
+    model_name = args.model_name
+    model_uid = args.model_uid
+    label_column = args.label_column
+
+    with open("/app/secrets/wml_credentials", 'r') as f:
+        wml_creds = f.readline().strip('\'')
+    f.close()
+    with open("/app/secrets/aios_guid", 'r') as f:
+        aios_guid = f.readline().strip('\'')
+    f.close()
+    with open("/app/secrets/cloud_api_key", 'r') as f:
+        cloud_api_key = f.readline().strip('\'')
+    f.close()
+    with open("/app/secrets/postgres_uri", 'r') as f:
+        postgres_uri = f.readline().strip('\'')
+    f.close()
+
+    WML_CREDENTIALS = json.loads(wml_creds)
+
+    AIOS_CREDENTIALS = {
+        "instance_guid": aios_guid,
+        "apikey": cloud_api_key,
+        "url": "https://api.aiopenscale.cloud.ibm.com"
+    }
+
+    if postgres_uri == '':
+        POSTGRES_CREDENTIALS = None
+    else:
+        POSTGRES_CREDENTIALS = {
+            "uri": postgres_uri
+        }
+
+    wml_client = WatsonMachineLearningAPIClient(WML_CREDENTIALS)
+    ai_client = APIClient(aios_credentials=AIOS_CREDENTIALS)
+    print('AIOS client version:' + ai_client.version)
+
+    ''' Setup Postgres SQL and AIOS binding '''
+    SCHEMA_NAME = aios_schema
+    try:
+        data_mart_details = ai_client.data_mart.get_details()
+        if 'internal_database' in data_mart_details['database_configuration'] and data_mart_details['database_configuration']['internal_database']:
+            if POSTGRES_CREDENTIALS is None:
+                print('Using existing internal datamart')
+            else:
+                print('Switching to external datamart')
+                ai_client.data_mart.delete(force=True)
+                create_postgres_schema(postgres_credentials=POSTGRES_CREDENTIALS, schema_name=SCHEMA_NAME)
+                ai_client.data_mart.setup(db_credentials=POSTGRES_CREDENTIALS, schema=SCHEMA_NAME)
+        else:
+            print('Using existing external datamart')
+    except:
+        if POSTGRES_CREDENTIALS is None:
+            print('Setting up internal datamart')
+            ai_client.data_mart.setup(internal_db=True)
+        else:
+            print('Setting up external datamart')
+            create_postgres_schema(postgres_credentials=POSTGRES_CREDENTIALS, schema_name=SCHEMA_NAME)
+            ai_client.data_mart.setup(db_credentials=POSTGRES_CREDENTIALS, schema=SCHEMA_NAME)
+
+    data_mart_details = ai_client.data_mart.get_details()
+    print(data_mart_details)
+
+    binding_uid = ai_client.data_mart.bindings.add('WML instance', WatsonMachineLearningInstance(WML_CREDENTIALS))
+    if binding_uid is None:
+        binding_uid = ai_client.data_mart.bindings.get_details()['service_bindings'][0]['metadata']['guid']
+    bindings_details = ai_client.data_mart.bindings.get_details()
+
+    print('\nWML binding ID is ' + binding_uid + '\n')
+
+    ''' Create subscriptions '''
+    subscriptions_uids = ai_client.data_mart.subscriptions.get_uids()
+    for subscription in subscriptions_uids:
+        sub_name = ai_client.data_mart.subscriptions.get_details(subscription)['entity']['asset']['name']
+        if sub_name == model_name:
+            ai_client.data_mart.subscriptions.delete(subscription)
+            print('Deleted existing subscription for', model_name)
+
+    subscription = ai_client.data_mart.subscriptions.add(WatsonMachineLearningAsset(
+        model_uid,
+        label_column=label_column,
+        prediction_column='predictedLabel',
+        probability_column='probability'
+    ))
+    if subscription is None:
+        print('Exists already')
+        # subscription already exists; get the existing one
+        subscriptions_uids = ai_client.data_mart.subscriptions.get_uids()
+        for sub in subscriptions_uids:
+            if ai_client.data_mart.subscriptions.get_details(sub)['entity']['asset']['name'] == model_name:
+                subscription = ai_client.data_mart.subscriptions.get(sub)
+
+    subscriptions_uids = ai_client.data_mart.subscriptions.get_uids()
+    print(subscription.get_details())
+
+    ''' Scoring the model and make sure the subscriptions are setup properly '''
+    credit_risk_scoring_endpoint = None
+    deployment_uid = subscription.get_deployment_uids()[0]
+
+    print('\n' + deployment_uid + '\n')
+
+    for deployment in wml_client.deployments.get_details()['resources']:
+        if deployment_uid in deployment['metadata']['guid']:
+            credit_risk_scoring_endpoint = deployment['entity']['scoring_url']
+
+    print('Scoring endpoint is: ' + credit_risk_scoring_endpoint + '\n')