Skip to content

Commit

Permalink
#85 Savepoint
Browse files Browse the repository at this point in the history
  • Loading branch information
docktermj committed Sep 27, 2024
1 parent 9a76ab0 commit b82dd83
Show file tree
Hide file tree
Showing 6 changed files with 53 additions and 36 deletions.
7 changes: 5 additions & 2 deletions rootfs/notebooks/hello_world.ipynb
100755 → 100644
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,14 @@
"metadata": {},
"outputs": [],
"source": [
"import grpc, json\n",
"import json\n",
"\n",
"import grpc\n",
"from senzing_grpc import SzAbstractFactory\n",
"\n",
"sz_abstract_factory = SzAbstractFactory(grpc_channel=grpc.insecure_channel(\"localhost:8261\"))\n",
"sz_abstract_factory = SzAbstractFactory(\n",
" grpc_channel=grpc.insecure_channel(\"localhost:8261\")\n",
")\n",
"sz_product = sz_abstract_factory.create_sz_product()\n",
"print(json.dumps(json.loads(sz_product.get_version()), indent=2))"
]
Expand Down
13 changes: 9 additions & 4 deletions rootfs/notebooks/load_truthsets.ipynb
100755 → 100644
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,11 @@
"metadata": {},
"outputs": [],
"source": [
"import grpc, json, requests, shutil\n",
"import json\n",
"import shutil\n",
"\n",
"import grpc\n",
"import requests\n",
"from senzing_grpc import SzAbstractFactory, SzEngineFlags, SzError"
]
},
Expand Down Expand Up @@ -80,9 +84,10 @@
"source": [
"for filename in truth_set_filenames:\n",
" url = truth_set_url_prefix + filename\n",
" response = requests.get(url, stream=True)\n",
" filepath = home_path + filename\n",
" response = requests.get(url, stream=True, timeout=10)\n",
" response.raw.decode_content = True\n",
" with open(filename, 'wb') as file:\n",
" with open(filepath, \"wb\") as file:\n",
" shutil.copyfileobj(response.raw, file)"
]
},
Expand Down Expand Up @@ -113,7 +118,7 @@
"\n",
"for filename in truth_set_filenames:\n",
" filepath = home_path + filename\n",
" with open(filepath, \"r\") as file:\n",
" with open(filepath, \"r\", encoding=\"utf-8\") as file:\n",
" for line in file:\n",
" line_as_dict = json.loads(line)\n",
" datasource = line_as_dict.get(\"DATA_SOURCE\")\n",
Expand Down
Empty file modified rootfs/notebooks/method_help.ipynb
100755 → 100644
Empty file.
5 changes: 3 additions & 2 deletions src/senzing_quickstart/hello_world.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
# coding: utf-8

# # Senzing Hello World
#
#
# The following is a quick test of connectivity to Senzing using Python over gRPC.
#
#

# In[ ]:

Expand All @@ -19,3 +19,4 @@
)
sz_product = sz_abstract_factory.create_sz_product()
print(json.dumps(json.loads(sz_product.get_version()), indent=2))

62 changes: 34 additions & 28 deletions src/senzing_quickstart/load_truthsets.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
# coding: utf-8

# # Load Senzing truth-sets
#
#
# These instructions load the [Senzing truth-sets] into the Senzing engine.
#
#
# [Senzing truth-sets]: https://github.com/Senzing/truth-sets

# ## Prepare Python enviroment
Expand All @@ -21,14 +21,15 @@
import requests
from senzing_grpc import SzAbstractFactory, SzEngineFlags, SzError


# Set environment specific variables.

# In[ ]:


HOME_PATH = "/notebooks/"
TRUTH_SET_URL_PREFIX = "https://raw.githubusercontent.com/Senzing/truth-sets/refs/heads/main/truthsets/demo/"
TRUTH_SET_FILENAMES = [
home_path = "/notebooks/"
truth_set_url_prefix = "https://raw.githubusercontent.com/Senzing/truth-sets/refs/heads/main/truthsets/demo/"
truth_set_filenames = [
"customers.json",
"reference.json",
"watchlist.json",
Expand All @@ -40,11 +41,12 @@
# In[ ]:


for filename in TRUTH_SET_FILENAMES:
URL = TRUTH_SET_URL_PREFIX + filename
response = requests.get(URL, stream=True, timeout=10)
for filename in truth_set_filenames:
url = truth_set_url_prefix + filename
filepath = home_path + filename
response = requests.get(url, stream=True, timeout=10)
response.raw.decode_content = True
with open(filename, "wb") as file:
with open(filepath, "wb") as file:
shutil.copyfileobj(response.raw, file)


Expand All @@ -57,16 +59,17 @@

datasources = []

for filename in TRUTH_SET_FILENAMES:
FILEPATH = HOME_PATH + filename
with open(FILEPATH, "r", encoding="utf-8") as file:
for filename in truth_set_filenames:
filepath = home_path + filename
with open(filepath, "r", encoding="utf-8") as file:
for line in file:
line_as_dict = json.loads(line)
datasource = line_as_dict.get("DATA_SOURCE")
if datasource not in datasources:
datasources.append(datasource)

print(f"Found the following DATA_SOURCE values in the data: {datasources}")



# ## Update Senzing configuration
Expand All @@ -91,8 +94,8 @@


old_config_id = sz_configmanager.get_default_config_id()
OLD_JSON_CONFIG = sz_configmanager.get_config(old_config_id)
config_handle = sz_config.import_config(OLD_JSON_CONFIG)
old_json_config = sz_configmanager.get_config(old_config_id)
config_handle = sz_config.import_config(old_json_config)


# Add DataSources to Senzing configuration.
Expand All @@ -112,9 +115,11 @@
# In[ ]:


NEW_JSON_CONFIG = sz_config.export_config(config_handle)
new_config_id = sz_configmanager.add_config(NEW_JSON_CONFIG, "Add TruthSet datasources")
sz_configmanager.replace_default_config_id(old_config_id, new_config_id)
new_json_config = sz_config.export_config(config_handle)
new_config_id = sz_configmanager.add_config(
new_json_config, "Add TruthSet datasources"
)
sz_configmanager.replace_default_config_id(old_config_id, new_config_id)


# With the change in Senzing configuration, Senzing objects need to be updated.
Expand All @@ -133,21 +138,21 @@
# In[ ]:


for filename in TRUTH_SET_FILENAMES:
FILEPATH = HOME_PATH + filename
with open(FILEPATH, "r", encoding="utf-8") as file:
for filename in truth_set_filenames:
filepath = home_path + filename
with open(filepath, "r") as file:
for line in file:
try:
try:
line_as_dict = json.loads(line)
INFO = sz_engine.add_record(
info = sz_engine.add_record(
line_as_dict.get("DATA_SOURCE"),
line_as_dict.get("RECORD_ID"),
line,
SzEngineFlags.SZ_WITH_INFO,
)
print(INFO)
print(info)
except SzError as err:
print(err)
print(err)


# ## View results
Expand All @@ -157,8 +162,8 @@
# In[ ]:


CUSTOMER_1070_ENTITY = sz_engine.get_entity_by_record_id("CUSTOMERS", "1070")
print(json.dumps(json.loads(CUSTOMER_1070_ENTITY), indent=2))
customer_1070_entity = sz_engine.get_entity_by_record_id("CUSTOMERS", "1070")
print(json.dumps(json.loads(customer_1070_entity), indent=2))


# Search for entities by attributes.
Expand All @@ -170,5 +175,6 @@
"name_full": "robert smith",
"date_of_birth": "11/12/1978",
}
SEARCH_RESULTS = sz_engine.search_by_attributes(json.dumps(search_query))
print(json.dumps(json.loads(SEARCH_RESULTS), indent=2))
search_result = sz_engine.search_by_attributes(json.dumps(search_query))
print(json.dumps(json.loads(search_result), indent=2))

2 changes: 2 additions & 0 deletions src/senzing_quickstart/method_help.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import grpc
from senzing_grpc import SzAbstractFactory


# Create Senzing object.

# In[ ]:
Expand All @@ -36,3 +37,4 @@


print(sz_engine.help("get_entity_by_record_id"))

0 comments on commit b82dd83

Please sign in to comment.