Instructions/Annex/KnowledgeMiningSolution.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "trusted": true
      },
      "outputs": [],
      "source": [
        "import json\n",
        "import requests\n",
        "from pprint import pprint"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "trusted": true
      },
      "outputs": [],
      "source": [
        "#Define the names for the data source, skillset, index and indexer\n",
        "datasource_name=\"ds-scans\"\n",
        "skillset_name=\"scan-ocr\"\n",
        "index_name=\"scan-index\"\n",
        "indexer_name=\"scan-indexer\""
      ]
    },
    {
      "cell_type": "markdown",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "Add the name and key of your search service."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "trusted": true
      },
      "outputs": [],
      "source": [
        "#Setup the endpoint\n",
        "endpoint = 'https://<yourSearchInstance>.search.windows.net/'\n",
        "headers = {'Content-Type': 'application/json', 'api-key': '<yourAPIkey>' }\n",
        "params = {'api-version': '2019-05-06'}"
      ]
    },
    {
      "cell_type": "markdown",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "Add the full connection string to your storage account. This step assumes \"basic-demo-data-pr\" as the container name. Replace that string as well if your container name is different."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "trusted": true
      },
      "outputs": [],
      "source": [
        "#Create a data source\n",
        "datasourceConnectionString = \"\"\n",
        "datasource_payload = {\n",
        "    \"name\": datasource_name,\n",
        "    \"description\": \"Scanned pages from a book.\",\n",
        "    \"type\": \"azureblob\",\n",
        "    \"credentials\": {\n",
        "    \"connectionString\": datasourceConnectionString\n",
        "   },\n",
        "    \"container\": {\n",
        "     \"name\": \"cognitivesearch\",\n",
        "     \"query\": \"scans\"\n",
        "   }\n",
        "}\n",
        "r = requests.put( endpoint + \"/datasources/\" + datasource_name, data=json.dumps(datasource_payload), headers=headers, params=params )\n",
        "print(r.status_code)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "trusted": true
      },
      "outputs": [],
      "source": [
        "#Create a skillset\n",
        "skillset_payload = {\n",
        "  \"name\": skillset_name,\n",
        "  \"description\":\n",
        "  \"This skillset applies OCR on scanned book pages, extracts key phrases and the language from the text and eventually translates the text to a different language \",\n",
        "  \"skills\":\n",
        "  [\n",
        "          {\n",
        "          \"@odata.type\": \"#Microsoft.Skills.Vision.OcrSkill\",\n",
        "          \"textExtractionAlgorithm\": \"\",\n",
        "          \"lineEnding\": \"Space\",\n",
        "          \"defaultLanguageCode\": \"en\",\n",
        "          \"detectOrientation\": \"true\",\n",
        "          \"description\": \"scan-skill\",\n",
        "          \"context\": \"/document/normalized_images/*\",\n",
        "          \"inputs\": [\n",
        "            {\n",
        "              \"name\": \"image\",\n",
        "              \"source\": \"/document/normalized_images/*\"\n",
        "            }\n",
        "          ],\n",
        "          \"outputs\": [\n",
        "            {\n",
        "              \"name\": \"text\",\n",
        "              \"targetName\": \"mitext\"\n",
        "            }\n",
        "          ]\n",
        "        },\n",
        "        {\n",
        "        \"@odata.type\": \"#Microsoft.Skills.Text.MergeSkill\",\n",
        "        \"description\": \"Create merged_text, which includes all the textual representation of each image inserted at the right location in the content field.\",\n",
        "        \"context\": \"/document\",\n",
        "        \"insertPreTag\": \" \",\n",
        "        \"insertPostTag\": \" \",\n",
        "        \"inputs\": [\n",
        "            {\n",
        "              \"name\":\"text\", \"source\": \"/document/content\"\n",
        "            },\n",
        "            {\n",
        "              \"name\": \"itemsToInsert\", \"source\": \"/document/normalized_images/*/mitext\"\n",
        "            },\n",
        "            {\n",
        "              \"name\":\"offsets\", \"source\": \"/document/normalized_images/*/contentOffset\" \n",
        "            }\n",
        "        ],\n",
        "        \"outputs\": [\n",
        "            {\n",
        "              \"name\": \"mergedText\", \"targetName\" : \"mergedText\"\n",
        "            }\n",
        "          ]\n",
        "        },\n",
        "        {\n",
        "        \"@odata.type\": \"#Microsoft.Skills.Text.LanguageDetectionSkill\",\n",
        "        \"context\": \"/document\",\n",
        "        \"inputs\": [\n",
        "          {\n",
        "            \"name\": \"text\",\n",
        "            \"source\": \"/document/mergedText\"\n",
        "          }\n",
        "        ],\n",
        "        \"outputs\": [\n",
        "          {\n",
        "            \"name\": \"languageCode\",\n",
        "            \"targetName\": \"languageCode\"\n",
        "          }       \n",
        "        ]\n",
        "       },\n",
        "       {\n",
        "        \"@odata.type\": \"#Microsoft.Skills.Text.KeyPhraseExtractionSkill\",\n",
        "        \"context\": \"/document\",\n",
        "        \"inputs\": [\n",
        "          {\n",
        "            \"name\": \"text\",\n",
        "            \"source\": \"/document/mergedText\"\n",
        "          },\n",
        "          {\n",
        "            \"name\": \"languageCode\",\n",
        "            \"source\": \"/document/languageCode\" \n",
        "          }\n",
        "        ],\n",
        "        \"outputs\": [\n",
        "          {\n",
        "            \"name\": \"keyPhrases\",\n",
        "            \"targetName\": \"keyPhrases\"\n",
        "          }\n",
        "        ]\n",
        "       },\n",
        "       {\n",
        "       \"@odata.type\": \"#Microsoft.Skills.Text.TranslationSkill\",\n",
        "       \"defaultToLanguageCode\": \"de\",\n",
        "       \"context\": \"/document\",\n",
        "       \"inputs\": [\n",
        "          {\n",
        "            \"name\": \"text\",\n",
        "            \"source\": \"/document/mergedText\"\n",
        "          },\n",
        "          {\n",
        "            \"name\": \"fromLanguageCode\",\n",
        "            \"source\": \"/document/languageCode\"\n",
        "          },\n",
        "            \n",
        "        ],\n",
        "        \"outputs\": [\n",
        "          {\n",
        "            \"name\": \"translatedText\",\n",
        "            \"targetName\": \"translatedText\"\n",
        "          }\n",
        "        ]\n",
        "      }\n",
        "    ],\n",
        "    \n",
        "    \"cognitiveServices\": {\n",
        "    \"@odata.type\": \"#Microsoft.Azure.Search.CognitiveServicesByKey\",\n",
        "    \"description\": \"/subscriptions/2653c6f1-db32-41c9-8444-69430d77ef0e/resourceGroups/sapsmbinnovationsummit/providers/Microsoft.CognitiveServices/accounts/mismbsummitcognitiveservices\",\n",
        "    \"key\": \"059695e6bf6a42d48e70ddb1ab2cf963\"\n",
        "  },\n",
        "}\n",
        "\n",
        "r = requests.put(endpoint + \"/skillsets/\" + skillset_name, data=json.dumps(skillset_payload), headers=headers, params=params)\n",
        "print(r.status_code)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "trusted": true
      },
      "outputs": [],
      "source": [
        "#Create an index\n",
        "index_payload = {\n",
        "    \"name\": index_name,\n",
        "    \"fields\": [\n",
        "      {\n",
        "        \"name\": \"id\",\n",
        "        \"type\": \"Edm.String\",\n",
        "        \"key\": \"true\",\n",
        "        \"searchable\": \"true\",\n",
        "        \"filterable\": \"false\",\n",
        "        \"facetable\": \"false\",\n",
        "        \"sortable\": \"true\"\n",
        "      },\n",
        "      {\n",
        "        \"name\": \"content\",\n",
        "        \"type\": \"Edm.String\",\n",
        "        \"sortable\": \"false\",\n",
        "        \"searchable\": \"true\",\n",
        "        \"filterable\": \"false\",\n",
        "        \"facetable\": \"false\"\n",
        "      },\n",
        "        \n",
        "      {\n",
        "        \"name\": \"documentPath\",\n",
        "        \"type\": \"Edm.String\",\n",
        "        \"searchable\": \"true\",\n",
        "        \"filterable\": \"false\",\n",
        "        \"facetable\": \"false\"\n",
        "      },\n",
        "            {\n",
        "        \"name\": \"extractedText\",\n",
        "        \"type\": \"Edm.String\",\n",
        "        \"searchable\": \"true\",\n",
        "        \"filterable\": \"false\",\n",
        "        \"facetable\": \"false\"\n",
        "      },\n",
        "      {\n",
        "        \"name\": \"languageCode\",\n",
        "        \"type\": \"Edm.String\",\n",
        "        \"searchable\": \"true\",\n",
        "        \"filterable\": \"false\",\n",
        "        \"facetable\": \"false\"\n",
        "      },\n",
        "      {\n",
        "        \"name\": \"keyPhrases\",\n",
        "        \"type\": \"Collection(Edm.String)\",\n",
        "        \"searchable\": \"true\",\n",
        "        \"filterable\": \"false\",\n",
        "        \"facetable\": \"false\"\n",
        "      },\n",
        "      {\n",
        "        \"name\": \"translatedText\",\n",
        "        \"type\": \"Edm.String\",\n",
        "        \"searchable\": \"true\",\n",
        "        \"filterable\": \"false\",\n",
        "        \"facetable\": \"false\"\n",
        "      } \n",
        "   ]\n",
        "}\n",
        "\n",
        "r = requests.put(endpoint + \"/indexes/\" + index_name, data=json.dumps(index_payload), headers=headers, params=params)\n",
        "print(r.status_code)"
      ]
    },
    {
      "cell_type": "markdown",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "The next step, Create an indexer, is where all the deep processing occurs. This step takes several minutes to complete. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "trusted": true
      },
      "outputs": [],
      "source": [
        "# Create an indexer\n",
        "indexer_payload = {\n",
        "    \"name\": indexer_name,\n",
        "    \"dataSourceName\": datasource_name,\n",
        "    \"targetIndexName\": index_name,\n",
        "    \"skillsetName\": skillset_name,\n",
        "    \"fieldMappings\" : [\n",
        "    {\n",
        "      \"sourceFieldName\" : \"metadata_storage_path\",\n",
        "      \"targetFieldName\" : \"id\",\n",
        "      \"mappingFunction\" :\n",
        "        { \"name\" : \"base64Encode\" }\n",
        "    },\n",
        "    {\n",
        "      \"sourceFieldName\" : \"content\",\n",
        "      \"targetFieldName\" : \"content\"\n",
        "    }\n",
        "  ],\n",
        "   \"outputFieldMappings\" :\n",
        "  [ \n",
        "       {\n",
        "      \"sourceFieldName\" : \"document/metadata_storage_path\", \n",
        "      \"targetFieldName\" : \"documentPath\"\n",
        "      },\n",
        "      \n",
        "      {\n",
        "      \"sourceFieldName\" : \"/document/mergedText\", \n",
        "      \"targetFieldName\" : \"extractedText\"\n",
        "      },\n",
        "       {\n",
        "      \"sourceFieldName\" : \"/document/languageCode\", \n",
        "      \"targetFieldName\" : \"languageCode\"\n",
        "      },\n",
        "       {\n",
        "      \"sourceFieldName\" : \"/document/keyPhrases\", \n",
        "      \"targetFieldName\" : \"keyPhrases\"\n",
        "      },\n",
        "      {\n",
        "      \"sourceFieldName\" : \"/document/translatedText\", \n",
        "      \"targetFieldName\" : \"translatedText\"\n",
        "      }\n",
        "  ],\n",
        "   \"parameters\":\n",
        "  {\n",
        "    \"maxFailedItems\":-1,\n",
        "    \"maxFailedItemsPerBatch\":-1,\n",
        "    \"configuration\":\n",
        "    {\n",
        "      \"dataToExtract\": \"contentAndMetadata\",\n",
        "      \"imageAction\": \"generateNormalizedImages\"\n",
        "    }\n",
        "  }\n",
        "}\n",
        "\n",
        "r = requests.put(endpoint + \"/indexers/\" + indexer_name, data=json.dumps(indexer_payload), headers=headers, params=params)\n",
        "print(r.content)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "trusted": true
      },
      "outputs": [],
      "source": [
        "#Get indexer status\n",
        "r = requests.get(endpoint + \"/indexers/\" + indexer_name + \"/status\", headers=headers,params=params)\n",
        "pprint(json.dumps(r.json(), indent=1))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "trusted": true
      },
      "outputs": [],
      "source": [
        "#Query the index for all fields\n",
        "r = requests.get(endpoint + \"/indexes/\" + index_name, headers=headers,params=params)\n",
        "print(json.dumps(r.json(), indent=1))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "trusted": true
      },
      "outputs": [],
      "source": [
        "#Query the index to return the contents of organizations\n",
        "#Note: Index creation may take time. If this step returns no data, wait a few minutes\n",
        "#      and then try again\n",
        "r = requests.get(endpoint + \"/indexes/\" + index_name + \"/docs?&search=*&$select=organizations\", headers=headers, params=params)\n",
        "pprint(r.json())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "trusted": true
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3",
      "language": "python"
    },
    "language_info": {
      "mimetype": "text/x-python",
      "nbconvert_exporter": "python",
      "name": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.4",
      "file_extension": ".py",
      "codemirror_mode": {
        "version": 3,
        "name": "ipython"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}