diff --git a/smsdk/Auth/auth.py b/smsdk/Auth/auth.py index 56de5ed..f8565e3 100644 --- a/smsdk/Auth/auth.py +++ b/smsdk/Auth/auth.py @@ -166,8 +166,8 @@ def check_auth(self): Determine if SDK has access to the client by checking the Cycle API. """ try: - url = "{}{}".format(self.host, ENDPOINTS["Cycle"]["alt_url"]) - resp = self._get_records(url, _limit=1, _only=["_id"]) + url = "{}{}".format(self.host, ENDPOINTS["Cycle"]["url_v1"]) + resp = self._get_records_v1(url, _limit=1, _only=["_id"]) return isinstance(resp, list) and "error" not in resp except Exception: # pylint:disable=broad-except return False diff --git a/smsdk/client.py b/smsdk/client.py index 64c3f80..4e65af3 100644 --- a/smsdk/client.py +++ b/smsdk/client.py @@ -11,7 +11,7 @@ except ImportError: from pandas.io.json import json_normalize -from smsdk.utils import get_url +from smsdk.utils import get_url, escape_mongo_field_name, dict_to_df from smsdk.Auth.auth import Authenticator, X_SM_DB_SCHEMA, X_SM_WORKSPACE_ID from smsdk.tool_register import smsdkentities from smsdk.client_v0 import ClientV0 @@ -37,35 +37,9 @@ def time_string_to_epoch(time_string): return time_epoch -def dict_to_df(data, normalize=True): - if normalize: - # special case to handle the 'stats' block - if data and "stats" in data[0]: - if isinstance(data[0]["stats"], dict): - # part stats are dict - df = json_normalize(data) - else: - # machine type stats are list - cols = [*data[0]] - cols.remove("stats") - df = json_normalize(data, "stats", cols, record_prefix="stats.") - else: - try: - df = json_normalize(data) - except: - # From cases like _distinct which don't have a "normal" return format - return pd.DataFrame({"values": data}) - else: - df = pd.DataFrame(data) - - if len(df) > 0: - if "_id" in df.columns: - df.set_index("_id", inplace=True) - - if "id" in df.columns: - df.set_index("id", inplace=True) - - return df +def generator_to_df(generator) -> pd.DataFrame: + data = pd.concat([page for page in generator]) + return data # We don't have a downtime schema, so hard code one @@ -170,15 +144,26 @@ def get_data_v1(self, ename, util_name, normalize=True, *args, **kwargs): # dict params strictly follow {'key':'value'} format # sub_kwargs = kwargs - if util_name in ["get_cycles", "get_downtime", "get_parts"]: + if util_name in [ + "get_cycles", + "get_downtime", + "get_parts", + "get_factories", + "get_machines", + "get_machine_types", + ]: sub_kwargs = [kwargs] else: sub_kwargs = self.fix_only(kwargs) if len(sub_kwargs) == 1: - data = dict_to_df( - getattr(cls, util_name)(*args, **sub_kwargs[0]), normalize - ) + if util_name in ["get_factories", "get_machines", "get_machine_types"]: + # data = dict_to_df(getattr(cls, util_name)(*args, **sub_kwargs[0]), normalize) + return getattr(cls, util_name)(normalize, *args, **sub_kwargs[0]) + else: + data = dict_to_df( + getattr(cls, util_name)(*args, **sub_kwargs[0]), normalize + ) else: data = dict_to_df( getattr(cls, util_name)(*args, **sub_kwargs[0]), normalize @@ -192,7 +177,7 @@ def get_data_v1(self, ename, util_name, normalize=True, *args, **kwargs): data.drop(joined_cols, axis=1) # To keep consistent, rename columns back from '.' to '__' - data.columns = [name.replace(".", "__") for name in data.columns] + data.columns = [escape_mongo_field_name(name) for name in data.columns] else: # raise error if requested for unregistered utility @@ -352,6 +337,123 @@ def get_fields_of_machine_type( return fields + def _get_factories(self, *args, normalize=True, **kwargs): + """ + Get list of factories and associated metadata. Note this includes extensive internal metadata. + + :param normalize: Flatten nested data structures + :type normalize: bool + :return: pandas dataframe + """ + return self.get_data_v1( + "factory_v1", "get_factories", normalize, *args, **kwargs + ) + + def _get_machines(self, *args, normalize=True, **kwargs) -> pd.DataFrame: + """ + Get list of machines and associated metadata. Note this includes extensive internal metadata. If you only want to get a list of machine names + then see also get_machine_names(). + + :param normalize: Flatten nested data structures + :type normalize: bool + :return: pandas dataframe + """ + return self.get_data_v1( + "machine_v1", "get_machines", normalize, *args, **kwargs + ) + + def _get_machine_types(self, *args, normalize=True, **kwargs): + """ + Get list of machine types and associated metadata. Note this includes extensive internal metadata. If you only want to get a list of machine type names + then see also get_machine_type_names(). + + :param normalize: Flatten nested data structures + :type normalize: bool + :return: pandas dataframe + """ + + return self.get_data_v1( + "machine_type_v1", "get_machine_types", normalize, *args, **kwargs + ) + + def get_factories(self, *args, normalize=True, **kwargs): + generator = self._get_factories(normalize=normalize, *args, **kwargs) + data = generator_to_df(generator) + return data + + def get_machines(self, *args, normalize=True, **kwargs): + generator = self._get_machines(normalize=normalize, *args, **kwargs) + data = generator_to_df(generator) + return data + + def get_machine_types(self, *args, normalize=True, **kwargs): + generator = self._get_machine_types(normalize=normalize, *args, **kwargs) + data = generator_to_df(generator) + return data + + def get_machine_names(self, source_type=None, clean_strings_out=True): + """ + Get a list of machine names. This is a simplified version of get_machines(). + + :param source_type: filter the list to only the specified source_type + :type source_type: str + :param clean_strings_out: If true, return the list using the UI-based display names. If false, the list contains the Sight Machine internal machine names. + :return: list + """ + + query_params = { + "select": ["source", "source_clean", "source_type"], + "order_by": [{"name": "source_clean"}], + } + + if source_type: + # Double check the type + mt = self.get_machine_types(source_type=source_type) + # If it was found, then no action to take, otherwise try looking up from clean string + mt = ( + self.get_machine_types(source_type_clean=source_type) + if not len(mt) + else [] + ) + if len(mt): + source_type = mt["source_type"].iloc[0] + else: + log.error("Machine Type not found") + return [] + + query_params["source_type"] = source_type + + machines = self.get_data_v1( + "machine_v1", "get_machines", normalize=True, **query_params + ) + machines = generator_to_df(machines) + + if clean_strings_out: + return machines["source_clean"].to_list() + else: + return machines["source"].to_list() + + def get_machine_type_names(self, clean_strings_out=True): + """ + Get a list of machine type names. This is a simplified version of get_machine_types(). + + :param clean_strings_out: If true, return the list using the UI-based display names. If false, the list contains the Sight Machine internal machine types. + :return: list + """ + query_params = { + "select": ["source_type", "source_type_clean"], + "order_by": [{"name": "source_type_clean"}], + } + machine_types = self.get_data_v1( + "machine_type_v1", "get_machine_types", normalize=True, **query_params + ) + machine_types = generator_to_df(machine_types) + + if clean_strings_out: + return machine_types["source_type_clean"].to_list() + else: + return machine_types["source_type"].to_list() + def get_cookbooks(self, **kwargs): """ Gets all of the cookbooks accessable to the logged in user. diff --git a/smsdk/client_v0.py b/smsdk/client_v0.py index 8d74333..7887e89 100644 --- a/smsdk/client_v0.py +++ b/smsdk/client_v0.py @@ -15,7 +15,7 @@ except ImportError: from pandas.io.json import json_normalize -from smsdk.utils import get_url +from smsdk.utils import get_url, escape_mongo_field_name, dict_to_df from smsdk.Auth.auth import Authenticator from smsdk.tool_register import smsdkentities @@ -42,37 +42,6 @@ def time_string_to_epoch(time_string): return time_epoch -def dict_to_df(data, normalize=True): - if normalize: - # special case to handle the 'stats' block - if data and "stats" in data[0]: - if isinstance(data[0]["stats"], dict): - # part stats are dict - df = json_normalize(data) - else: - # machine type stats are list - cols = [*data[0]] - cols.remove("stats") - df = json_normalize(data, "stats", cols, record_prefix="stats.") - else: - try: - df = json_normalize(data) - except: - # From cases like _distinct which don't have a "normal" return format - return pd.DataFrame({"values": data}) - else: - df = pd.DataFrame(data) - - if len(df) > 0: - if "_id" in df.columns: - df.set_index("_id", inplace=True) - - if "id" in df.columns: - df.set_index("id", inplace=True) - - return df - - # We don't have a downtime schema, so hard code one downmap = { "machine__source": "Machine", @@ -271,7 +240,7 @@ def get_data(self, ename, util_name, normalize=True, *args, **kwargs): data.drop(joined_cols, axis=1) # To keep consistent, rename columns back from '.' to '__' - data.columns = [name.replace(".", "__") for name in data.columns] + data.columns = [escape_mongo_field_name(name) for name in data.columns] else: # raise error if requested for unregistered utility @@ -526,13 +495,13 @@ def inner( try: stats = self.get_machine_types( normalize=False, _limit=1, source_type=machine_type - )["stats"][0] + ) except KeyError: # explicitly embed string to machine type names esp JCP machine_type = f"'{machine_type}'" stats = self.get_machine_types( normalize=False, _limit=1, source_type=machine_type - )["stats"][0] + ) except Exception as ex: print(f"Exception in getting machine type stats {ex}") kwargs["stats"] = stats @@ -715,7 +684,7 @@ def get_downtimes_with_cycles( return merged - def get_factories(self, normalize=True, *args, **kwargs): + def get_factories(self, *args, normalize=True, **kwargs): """ Get list of factories and associated metadata. Note this includes extensive internal metadata. @@ -725,7 +694,7 @@ def get_factories(self, normalize=True, *args, **kwargs): """ return self.get_data("factory", "get_factories", normalize, *args, **kwargs) - def get_machines(self, normalize=True, *args, **kwargs): + def get_machines(self, *args, normalize=True, **kwargs) -> pd.DataFrame: """ Get list of machines and associated metadata. Note this includes extensive internal metadata. If you only want to get a list of machine names then see also get_machine_names(). @@ -755,13 +724,16 @@ def get_machine_names(self, source_type=None, clean_strings_out=True): # Double check the type mt = self.get_machine_types(source_type=source_type) # If it was found, then no action to take, otherwise try looking up from clean string - if not len(mt): - mt = self.get_machine_types(source_type_clean=source_type) - if len(mt): - source_type = mt["source_type"].iloc[0] - else: - log.error("Machine Type not found") - return [] + mt = ( + self.get_machine_types(source_type_clean=source_type) + if not len(mt) + else [] + ) + if len(mt): + source_type = mt["source_type"].iloc[0] + else: + log.error("Machine Type not found") + return [] query_params["source_type"] = source_type @@ -822,7 +794,7 @@ def get_machine_timezone(self, machine_source): return timezone - def get_machine_types(self, normalize=True, *args, **kwargs): + def get_machine_types(self, *args, normalize=True, **kwargs): """ Get list of machine types and associated metadata. Note this includes extensive internal metadata. If you only want to get a list of machine type names then see also get_machine_type_names(). diff --git a/smsdk/config/api_endpoints.json b/smsdk/config/api_endpoints.json index ae15269..9976aff 100644 --- a/smsdk/config/api_endpoints.json +++ b/smsdk/config/api_endpoints.json @@ -3,27 +3,23 @@ "url": "/auth/password/login" }, "Cycle": { - "url_v1": "/v1/datatab/cycle", - "url": "/api/cycle", - "alt_url": "/api/cycle" + "url_v1": "/v1/datatab/cycle" }, "Factory": { - "url" : "/api/factory" + "url_v1" : "/v1/obj/factory" }, "MachineType": { - "url" : "/api/machinetype", + "url_v1" : "/v1/obj/machine_type", "fields": "/v1/selector/datatab/cycle/{}/field" }, "Machine": { - "url" : "/api/machine" + "url_v1" : "/v1/obj/machine" }, "Parts": { - "url" : "/api/part", "url_v1": "/v1/datatab/part", "part_schema": "/v1/obj/part_type" }, "Downtime": { - "url" : "/api/downtime", "url_v1" : "/v1/datatab/downtime" }, diff --git a/smsdk/ma_session.py b/smsdk/ma_session.py index f29a7e6..c66faf9 100644 --- a/smsdk/ma_session.py +++ b/smsdk/ma_session.py @@ -5,6 +5,7 @@ import requests import numpy as np +import pandas as pd from requests.structures import CaseInsensitiveDict from requests.sessions import Session @@ -16,6 +17,7 @@ import importlib_resources as pkg_resources from smsdk import config +from smsdk.utils import escape_mongo_field_name, dict_to_df RESOURCE_CONFIG = json.loads(pkg_resources.read_text(config, "message_config.json")) @@ -25,6 +27,7 @@ X_SM_DB_SCHEMA = RESOURCE_CONFIG["x_sm_db_schema"] X_SM_WORKSPACE_ID = RESOURCE_CONFIG["x_sm_workspace_id"] + import logging log = logging.getLogger(__name__) @@ -94,10 +97,8 @@ def _get_records( return records _offset += this_loop_limit - except: - import traceback - - print(traceback.print_exc()) + except Exception as e: + log.exception(str(e), exc_info=1) return records def _get_schema(self, endpoint, method="get", **url_params): @@ -199,10 +200,8 @@ def _get_records_v1( return records offset += this_loop_limit - except: - import traceback - - print(traceback.print_exc()) + except Exception as e: + log.exception(str(e), exc_info=1) return records def _complete_async_task( @@ -278,3 +277,87 @@ def get_starttime_endtime_keys(self, **kwargs): continue return starttime_key, endtime_key + + def _get_records_mongo_v1( + self, + endpoint, + normalize=True, + method="get", + limit=np.Inf, + offset=1, + **url_params, + ): + """ + Function to get api call and fetch data from MA APIs + :param endpoint: complete url endpoint + :param method: Reqested method. Default = get + :param enable_pagination: if pagination is enabled then + the records are fetched with limit offset pagination + :param limit: Limit the number of records for pagination + :param offset: pagination offset + :param url_params: dict of params for API ex filtering, columns etc + :return: List of records + """ + next_page = "" + offset = int(offset) + try: + limit = int(limit) + except: + limit = float(limit) + + if "machine_type" in url_params: + url_params.pop("machine_type") + max_page_size = 2000 + limit = min(max_page_size, limit) + if not url_params.get("per_page"): + url_params["per_page"] = 5 + + if limit < url_params["per_page"]: + url_params["per_page"] = limit + + def _fetch_data(endpoint, url_params): + response = getattr(self.session, method.lower())( + endpoint, params=url_params + ) + if response.text: + if response.status_code not in [200, 201]: + raise ValueError("Error - {}".format(response.text)) + try: + data = response.json() + try: + next_page = data["next_page"] + except: + next_page = "" + if data["success"]: + data = data["objects"] + except JSONDecodeError as e: + print(f"No valid JSON returned {e}") + data = [] + else: + data = [] + return data, next_page + + while limit > 0: + if next_page: + data, next_page = _fetch_data(endpoint=next_page, url_params={}) + if not next_page: + limit = 0 + else: + limit -= len(data) + else: + data, next_page = _fetch_data(endpoint=endpoint, url_params=url_params) + if not next_page: + limit = 0 + else: + limit -= len(data) + data = dict_to_df(data, normalize=normalize) + + # To keep consistent, rename columns back from '.' to '__' + data.columns = [escape_mongo_field_name(name) for name in data.columns] + + if "endtime" in data.columns: + data["endtime"] = pd.to_datetime(data["endtime"]) + if "starttime" in data.columns: + data["starttime"] = pd.to_datetime(data["starttime"]) + + yield data diff --git a/smsdk/smsdk_entities/cycle/cycleV1.py b/smsdk/smsdk_entities/cycle/cycleV1.py index cfefc58..787a5d5 100644 --- a/smsdk/smsdk_entities/cycle/cycleV1.py +++ b/smsdk/smsdk_entities/cycle/cycleV1.py @@ -8,7 +8,7 @@ import importlib_resources as pkg_resources from smsdk.tool_register import SmsdkEntities, smsdkentities -from smsdk.utils import module_utility +from smsdk.utils import module_utility, check_kw from smsdk import config from smsdk.ma_session import MaSession from datetime import datetime, timedelta @@ -52,11 +52,8 @@ def get_cycles(self, *args, **kwargs): # log.warn('Machine source not specified.') # return [] - if "/api/cycle" in url: - records = self._get_records(url, **kwargs) - else: - kwargs = self.modify_input_params(**kwargs) - records = self._get_records_v1(url, **kwargs) + kwargs = self.modify_input_params(**kwargs) + records = self._get_records_v1(url, **kwargs) if not isinstance(records, List): raise ValueError("Error - {}".format(records)) @@ -106,16 +103,8 @@ def modify_input_params(self, **kwargs): ) for kw in kwargs: - if ( - kw[0] != "_" - and "machine_type" not in kw - and "Machine" not in kw - and "machine__source" not in kw - and "End Time" not in kw - and "endtime" not in kw - and "Start Time" not in kw - and "starttime" not in kw - ): + if check_kw(kw): + # if kw[0] != '_' and 'machine_type' not in kw and 'Machine' not in kw and 'machine__source' not in kw and 'End Time' not in kw and 'endtime' not in kw and 'Start Time' not in kw and 'starttime' not in kw: if "__" not in kw: where.append({"name": kw, "op": "eq", "value": kwargs[kw]}) else: diff --git a/smsdk/smsdk_entities/downtime/downtimeV1.py b/smsdk/smsdk_entities/downtime/downtimeV1.py index a99c27e..4dd7903 100644 --- a/smsdk/smsdk_entities/downtime/downtimeV1.py +++ b/smsdk/smsdk_entities/downtime/downtimeV1.py @@ -11,7 +11,7 @@ import importlib_resources as pkg_resources from smsdk.tool_register import SmsdkEntities, smsdkentities -from smsdk.utils import module_utility +from smsdk.utils import module_utility, check_kw from smsdk import config from smsdk.ma_session import MaSession @@ -49,11 +49,8 @@ def get_downtime(self, *args, **kwargs): """ url = "{}{}".format(self.base_url, ENDPOINTS["Downtime"]["url_v1"]) - if "/api/downtime" in url: - records = self._get_records(url, **kwargs) - else: - kwargs = self.modify_input_params(**kwargs) - records = self._get_records_v1(url, **kwargs) + kwargs = self.modify_input_params(**kwargs) + records = self._get_records_v1(url, **kwargs) if not isinstance(records, List): raise ValueError("Error - {}".format(records)) @@ -92,16 +89,8 @@ def modify_input_params(self, **kwargs): ) for kw in kwargs: - if ( - kw[0] != "_" - and "machine_type" not in kw - and "Machine" not in kw - and "machine__source" not in kw - and "End Time" not in kw - and "endtime" not in kw - and "Start Time" not in kw - and "starttime" not in kw - ): + if check_kw(kw): + # if kw[0] != '_' and 'machine_type' not in kw and 'Machine' not in kw and 'machine__source' not in kw and 'End Time' not in kw and 'endtime' not in kw and 'Start Time' not in kw and 'starttime' not in kw: if "__" not in kw: where.append({"name": kw, "op": "eq", "value": kwargs[kw]}) else: diff --git a/smsdk/smsdk_entities/factory/factoryV1.py b/smsdk/smsdk_entities/factory/factoryV1.py new file mode 100644 index 0000000..e92c692 --- /dev/null +++ b/smsdk/smsdk_entities/factory/factoryV1.py @@ -0,0 +1,179 @@ +from typing import List +import json +from datetime import datetime, timedelta + +try: + import importlib.resources as pkg_resources +except ImportError: + # Try backported to PY<37 `importlib_resources`. + import importlib_resources as pkg_resources + + +import numpy as np + +from smsdk.tool_register import SmsdkEntities, smsdkentities +from smsdk.utils import module_utility, check_kw +from smsdk import config +from smsdk.ma_session import MaSession + +ENDPOINTS = json.loads(pkg_resources.read_text(config, "api_endpoints.json")) + + +@smsdkentities.register("factory_v1") +class Factory(SmsdkEntities, MaSession): + # Decorator to register a function as utility + # Only the registered utilites would be accessible + # to outside world via client.get_data() + mod_util = module_utility() + + def __init__(self, session, base_url) -> None: + self.session = session + self.base_url = base_url + + @mod_util + def get_utilities(self, *args, **kwargs) -> List: + """ + Get the list of registered utilites by name + """ + return [*self.mod_util.all] + + @mod_util + def get_factories(self, normalize, *args, **kwargs): + """ + Utility function to get the machines + from the ma machine API + Recommend to use 'enable_pagination':True for larger datasets + """ + url = "{}{}".format(self.base_url, ENDPOINTS["Factory"]["url_v1"]) + + kwargs = self.modify_input_params(**kwargs) + records = self._get_records_mongo_v1(url, normalize, **kwargs) + # if not isinstance(records, List): + # raise ValueError("Error - {}".format(records)) + return records + + def modify_input_params(self, **kwargs): + v1_params = [ + "cb", + "select", + "where", + "group_by", + "order_by", + "limit", + "offset", + "per_page", + "cursor", + ] + + list_of_operations = [ + "ne", + "lt", + "lte", + "gt", + "gte", + "not", + "in", + "nin", + "mod", + "all", + "exists", + "exact", + "iexact", + "contains", + "icontains", + "startswith", + "istartswith", + "endswith", + "iendswith", + "match", + ] + + # Special handling for EF type names + machine = kwargs.get("machine__source") + machine = machine[1:-1] if machine and machine[0] == "'" else machine + + machine_type = kwargs.get("machine_type") + machine_type = ( + machine_type[1:-1] + if machine_type and machine_type[0] == "'" + else machine_type + ) + + new_kwargs = {} + etime = datetime.now() + stime = etime - timedelta(days=1) + + start_key, end_key = self.get_starttime_endtime_keys(**kwargs) + + # https://37-60546292-gh.circle-artifacts.com/0/build/html/web_api/v1/datatab/index.html#get--v1-datatab-cycle + where = [] + if start_key: + starttime = kwargs.get(start_key, "") if start_key else stime + where.append( + { + "name": start_key.split("__")[0], + "op": start_key.split("__")[-1], + "value": starttime.isoformat(), + } + ) + + if end_key: + endtime = kwargs.get(end_key, "") if end_key else stime + where.append( + { + "name": end_key.split("__")[0], + "op": end_key.split("__")[-1], + "value": endtime.isoformat(), + } + ) + + if machine: + where.append({"name": "machine_source", "op": "eq", "value": machine}) + + if machine_type: + where.append({"name": "machine_type", "op": "eq", "value": machine_type}) + + for kw in kwargs: + if check_kw(kw) and kw not in v1_params: + if "__" not in kw: + where.append({"name": kw, "op": "eq", "value": kwargs[kw]}) + elif "__" in kw and kw.split("__")[-1] in list_of_operations: + key = "__".join(kw.split("__")[:-1]) + op = kw.split("__")[-1] + + if op == "val": + op = "eq" + key += "__val" + + if op != "exists": + where.append({"name": key, "op": op, "value": kwargs[kw]}) + else: + if kwargs[kw]: + where.append({"name": key, "op": "ne", "value": None}) + else: + where.append({"name": key, "op": "eq", "value": None}) + else: + where.append({"name": kw, "op": "eq", "value": kwargs[kw]}) + + if kwargs.get("_only"): + new_kwargs["select"] = [{"name": i} for i in kwargs["_only"]] + + new_kwargs["offset"] = kwargs.get("_offset", kwargs.get("offset", 1)) + new_kwargs["limit"] = kwargs.get("_limit", kwargs.get("limit", np.Inf)) + new_kwargs["where"] = where + + for p in v1_params: + if not new_kwargs.get(p) and kwargs.get(p): + new_kwargs[p] = kwargs.get(p) + + if kwargs.get("_order_by", ""): + order_key = kwargs["_order_by"].replace("_epoch", "") + if order_key.startswith("-"): + order_type = "desc" + order_key = order_key[1:] + else: + order_type = "asc" + new_kwargs["order_by"] = [{"name": order_key, "order": order_type}] + + new_kwargs = {key: json.dumps(value) for key, value in new_kwargs.items()} + return new_kwargs diff --git a/smsdk/smsdk_entities/machine/machineV1.py b/smsdk/smsdk_entities/machine/machineV1.py new file mode 100644 index 0000000..2feeba5 --- /dev/null +++ b/smsdk/smsdk_entities/machine/machineV1.py @@ -0,0 +1,179 @@ +from typing import List +import json +from datetime import datetime, timedelta + +try: + import importlib.resources as pkg_resources +except ImportError: + # Try backported to PY<37 `importlib_resources`. + import importlib_resources as pkg_resources + +import numpy as np +import pandas as pd + +from smsdk.tool_register import SmsdkEntities, smsdkentities +from smsdk.utils import module_utility, check_kw +from smsdk import config +from smsdk.ma_session import MaSession + +ENDPOINTS = json.loads(pkg_resources.read_text(config, "api_endpoints.json")) + + +@smsdkentities.register("machine_v1") +class Machine(SmsdkEntities, MaSession): + # Decorator to register a function as utility + # Only the registered utilites would be accessible + # to outside world via client.get_data() + mod_util = module_utility() + + def __init__(self, session, base_url) -> None: + self.session = session + self.base_url = base_url + + @mod_util + def get_utilities(self, *args, **kwargs) -> List: + """ + Get the list of registered utilites by name + """ + return [*self.mod_util.all] + + @mod_util + def get_machines(self, normalize, *args, **kwargs) -> pd.DataFrame: + """ + Utility function to get the machines + from the ma machine API + Recommend to use 'enable_pagination':True for larger datasets + """ + url = "{}{}".format(self.base_url, ENDPOINTS["Machine"]["url_v1"]) + + kwargs = self.modify_input_params(**kwargs) + records = self._get_records_mongo_v1(url, normalize, **kwargs) + # if not isinstance(records, List): + # raise ValueError("Error - {}".format(records)) + return records + + def modify_input_params(self, **kwargs): + v1_params = [ + "cb", + "select", + "where", + "group_by", + "order_by", + "limit", + "offset", + "per_page", + "cursor", + ] + + list_of_operations = [ + "ne", + "lt", + "lte", + "gt", + "gte", + "not", + "in", + "nin", + "mod", + "all", + "exists", + "exact", + "iexact", + "contains", + "icontains", + "startswith", + "istartswith", + "endswith", + "iendswith", + "match", + ] + + # Special handling for EF type names + machine = kwargs.get("machine__source") + machine = machine[1:-1] if machine and machine[0] == "'" else machine + + machine_type = kwargs.get("machine_type") + machine_type = ( + machine_type[1:-1] + if machine_type and machine_type[0] == "'" + else machine_type + ) + + new_kwargs = {} + etime = datetime.now() + stime = etime - timedelta(days=1) + + start_key, end_key = self.get_starttime_endtime_keys(**kwargs) + + # https://37-60546292-gh.circle-artifacts.com/0/build/html/web_api/v1/datatab/index.html#get--v1-datatab-cycle + where = [] + if start_key: + starttime = kwargs.get(start_key, "") if start_key else stime + where.append( + { + "name": start_key.split("__")[0], + "op": start_key.split("__")[-1], + "value": starttime.isoformat(), + } + ) + + if end_key: + endtime = kwargs.get(end_key, "") if end_key else stime + where.append( + { + "name": end_key.split("__")[0], + "op": end_key.split("__")[-1], + "value": endtime.isoformat(), + } + ) + + if machine: + where.append({"name": "machine_source", "op": "eq", "value": machine}) + + if machine_type: + where.append({"name": "machine_type", "op": "eq", "value": machine_type}) + + for kw in kwargs: + if check_kw(kw) and kw not in v1_params: + if "__" not in kw: + where.append({"name": kw, "op": "eq", "value": kwargs[kw]}) + elif "__" in kw and kw.split("__")[-1] in list_of_operations: + key = "__".join(kw.split("__")[:-1]) + op = kw.split("__")[-1] + + if op == "val": + op = "eq" + key += "__val" + + if op != "exists": + where.append({"name": key, "op": op, "value": kwargs[kw]}) + else: + if kwargs[kw]: + where.append({"name": key, "op": "ne", "value": None}) + else: + where.append({"name": key, "op": "eq", "value": None}) + else: + where.append({"name": kw, "op": "eq", "value": kwargs[kw]}) + + if kwargs.get("_only"): + new_kwargs["select"] = [{"name": i} for i in kwargs["_only"]] + + new_kwargs["offset"] = kwargs.get("_offset", kwargs.get("offset", 1)) + new_kwargs["limit"] = kwargs.get("_limit", kwargs.get("limit", np.Inf)) + new_kwargs["where"] = where + + for p in v1_params: + if not new_kwargs.get(p) and kwargs.get(p): + new_kwargs[p] = kwargs.get(p) + + if kwargs.get("_order_by", ""): + order_key = kwargs["_order_by"].replace("_epoch", "") + if order_key.startswith("-"): + order_type = "desc" + order_key = order_key[1:] + else: + order_type = "asc" + new_kwargs["order_by"] = [{"name": order_key, "order": order_type}] + + new_kwargs = {key: json.dumps(value) for key, value in new_kwargs.items()} + return new_kwargs diff --git a/smsdk/smsdk_entities/machine_type/machinetypeV1.py b/smsdk/smsdk_entities/machine_type/machinetypeV1.py new file mode 100644 index 0000000..2f38ae2 --- /dev/null +++ b/smsdk/smsdk_entities/machine_type/machinetypeV1.py @@ -0,0 +1,175 @@ +from typing import List +import json +from datetime import datetime, timedelta + +try: + import importlib.resources as pkg_resources +except ImportError: + # Try backported to PY<37 `importlib_resources`. + import importlib_resources as pkg_resources + + +import numpy as np + +from smsdk.tool_register import SmsdkEntities, smsdkentities +from smsdk.utils import module_utility, check_kw +from smsdk import config +from smsdk.ma_session import MaSession + +ENDPOINTS = json.loads(pkg_resources.read_text(config, "api_endpoints.json")) + + +@smsdkentities.register("machine_type_v1") +class MachineType(SmsdkEntities, MaSession): + # Decorator to register a function as utility + # Only the registered utilites would be accessible + # to outside world via client.get_data() + mod_util = module_utility() + + def __init__(self, session, base_url) -> None: + self.session = session + self.base_url = base_url + + @mod_util + def get_utilities(self, *args, **kwargs) -> List: + return [*self.mod_util.all] + + @mod_util + def get_machine_types(self, normalize, *args, **kwargs): + """ + Utility function to get the machine types + from the ma machine API + Recommend to use 'enable_pagination':True for larger datasets + """ + kwargs = self.modify_input_params(**kwargs) + url = "{}{}".format(self.base_url, ENDPOINTS["MachineType"]["url_v1"]) + records = self._get_records_mongo_v1(url, normalize, **kwargs) + # if not isinstance(records, List): + # raise ValueError("Error - {}".format(records)) + return records + + def modify_input_params(self, **kwargs): + v1_params = [ + "cb", + "select", + "where", + "group_by", + "order_by", + "limit", + "offset", + "per_page", + "cursor", + ] + + list_of_operations = [ + "ne", + "lt", + "lte", + "gt", + "gte", + "not", + "in", + "nin", + "mod", + "all", + "exists", + "exact", + "iexact", + "contains", + "icontains", + "startswith", + "istartswith", + "endswith", + "iendswith", + "match", + ] + + # Special handling for EF type names + machine = kwargs.get("machine__source") + machine = machine[1:-1] if machine and machine[0] == "'" else machine + + machine_type = kwargs.get("machine_type") + machine_type = ( + machine_type[1:-1] + if machine_type and machine_type[0] == "'" + else machine_type + ) + + new_kwargs = {} + etime = datetime.now() + stime = etime - timedelta(days=1) + + start_key, end_key = self.get_starttime_endtime_keys(**kwargs) + + # https://37-60546292-gh.circle-artifacts.com/0/build/html/web_api/v1/datatab/index.html#get--v1-datatab-cycle + where = [] + if start_key: + starttime = kwargs.get(start_key, "") if start_key else stime + where.append( + { + "name": start_key.split("__")[0], + "op": start_key.split("__")[-1], + "value": starttime.isoformat(), + } + ) + + if end_key: + endtime = kwargs.get(end_key, "") if end_key else stime + where.append( + { + "name": end_key.split("__")[0], + "op": end_key.split("__")[-1], + "value": endtime.isoformat(), + } + ) + + if machine: + where.append({"name": "machine_source", "op": "eq", "value": machine}) + + if machine_type: + where.append({"name": "machine_type", "op": "eq", "value": machine_type}) + + for kw in kwargs: + if check_kw(kw) and kw not in v1_params: + if "__" not in kw: + where.append({"name": kw, "op": "eq", "value": kwargs[kw]}) + elif "__" in kw and kw.split("__")[-1] in list_of_operations: + key = "__".join(kw.split("__")[:-1]) + op = kw.split("__")[-1] + + if op == "val": + op = "eq" + key += "__val" + + if op != "exists": + where.append({"name": key, "op": op, "value": kwargs[kw]}) + else: + if kwargs[kw]: + where.append({"name": key, "op": "ne", "value": None}) + else: + where.append({"name": key, "op": "eq", "value": None}) + else: + where.append({"name": kw, "op": "eq", "value": kwargs[kw]}) + + if kwargs.get("_only"): + new_kwargs["select"] = [{"name": i} for i in kwargs["_only"]] + + new_kwargs["offset"] = kwargs.get("_offset", kwargs.get("offset", 1)) + new_kwargs["limit"] = kwargs.get("_limit", kwargs.get("limit", np.Inf)) + new_kwargs["where"] = where + + for p in v1_params: + if not new_kwargs.get(p) and kwargs.get(p): + new_kwargs[p] = kwargs.get(p) + + if kwargs.get("_order_by", ""): + order_key = kwargs["_order_by"].replace("_epoch", "") + if order_key.startswith("-"): + order_type = "desc" + order_key = order_key[1:] + else: + order_type = "asc" + new_kwargs["order_by"] = [{"name": order_key, "order": order_type}] + + new_kwargs = {key: json.dumps(value) for key, value in new_kwargs.items()} + return new_kwargs diff --git a/smsdk/utils.py b/smsdk/utils.py index 8b14fc1..122a03b 100644 --- a/smsdk/utils.py +++ b/smsdk/utils.py @@ -1,3 +1,21 @@ +import typing +import re + +import pandas as pd +from pandas import json_normalize + +ENCODING_MAP = { + ".": "__", + "_": "_5F", + "5": "5", + "F": "F", + "$": "_24", + "2": "2", + "4": "4", +} +ENCODING_RE = re.compile(r"_*_5F|_*_24|_*_\.__*|_*_\.|\.__*|___*|\$|\.") + + def module_utility(): """ Function to register class functions as tool functions @@ -27,3 +45,81 @@ def get_url(protocol, tenant, site_domain): """ return "{}://{}.{}".format(protocol, tenant, site_domain) + + +def check_kw(kw: str) -> bool: + """This function is used to remove code duplicacy where + it checks whether the kw is of special type of keyword supported + in v0 sdk and it also checks the type of keywprd so that it + can be parsed effectively for supportig v0 style of using sdk + """ + + for key in [ + "machine_type", + "Machine", + "machine__source", + "End Time", + "endtime", + "Start Time", + "starttime", + ]: + if kw.startswith("_") or key in kw: + return False + return True + + +def escape_replacement(m): + # type: (typing.Match) -> str + return "".join(map(ENCODING_MAP.__getitem__, m.group())) + + +def escape_mongo_field_name(field_name): + # type: (str) -> str + """ + Translation map for character encoding + || Decoded || Encoded || + | \. | __ | + | \._ | ___5F | + | _\._ | _5F___5F | + | __+ | _5F(_5F)+ | + | _5F | _5F5F | + | _+_5F | (_5F)+_5F5F | + Note: underscore is only escaped if a leading or trailing character in name or multiple + underscores are next to each other + An encoding of _ => ___ ( _ x 3) does not work because it becomes impossible to decode _. (encoded _ x 5) + See ma.tests.unit.utils.test_mongoutils.TestMongoEscapeUtils for test cases + """ + return ENCODING_RE.sub(escape_replacement, field_name) + + +def dict_to_df(data, normalize=True): + if normalize: + # special case to handle the 'stats' block + if data and "stats" in data[0]: + if isinstance(data[0]["stats"], dict): + # part stats are dict + df = json_normalize(data) + else: + # machine type stats are list + cols = [*data[0]] + cols.remove("stats") + df = json_normalize( + data, "stats", cols, record_prefix="stats.", errors="ignore" + ) + else: + try: + df = json_normalize(data) + except: + # From cases like _distinct which don't have a "normal" return format + return pd.DataFrame({"values": data}) + else: + df = pd.DataFrame(data) + + if len(df) > 0: + if "_id" in df.columns: + df.set_index("_id", inplace=True) + + if "id" in df.columns: + df.set_index("id", inplace=True) + + return df diff --git a/tests/Auth/test_auth.py b/tests/Auth/test_auth.py index 5211402..816e096 100644 --- a/tests/Auth/test_auth.py +++ b/tests/Auth/test_auth.py @@ -114,16 +114,16 @@ def json(): post=MagicMock(return_value=Response()), get=MagicMock(return_value=Response()) ) - tenant = "" + tenant = "demo" secret_id = "secret_Test" key_id = "key_test" cli = client.Client(tenant) authed = cli.auth assert authed._auth_apikey(secret_id=secret_id, key_id=key_id) is True - mocked.return_value.get.assert_called_once_with( - f"https://{tenant}.sightmachine.io/api/cycle", - params={"_only": ["_id"], "_offset": 0, "_limit": 1}, + mocked.return_value.post.assert_called_once_with( + f"https://{tenant}.sightmachine.io/v1/datatab/cycle", + json={"_limit": 1, "_only": ["_id"], "limit": 50000, "db_mode": "sql"}, ) @@ -175,9 +175,9 @@ def json(): authed = cli.auth assert authed.check_auth() is True - mocked.return_value.get.assert_called_once_with( - f"https://{tenant}.sightmachine.io/api/cycle", - params={"_only": ["_id"], "_offset": 0, "_limit": 1}, + mocked.return_value.post.assert_called_once_with( + f"https://{tenant}.sightmachine.io/v1/datatab/cycle", + json={"_limit": 1, "_only": ["_id"], "limit": 50000, "db_mode": "sql"}, ) @@ -203,7 +203,7 @@ def json(): # assert authed.check_auth() is False # mocked.return_value.get.assert_called_once_with( -# f"https://{tenant}.sightmachine.io/api/cycle", +# f"https://{tenant}.sightmachine.io/v1/datatab/cycle", # params={"_limit": 1, "_only": ["_id"]}, # ) diff --git a/tests/cycle/test_cycle.py b/tests/cycle/test_cycle.py index bd2f9c0..6cca744 100644 --- a/tests/cycle/test_cycle.py +++ b/tests/cycle/test_cycle.py @@ -1,13 +1,13 @@ import pandas as pd from requests.sessions import Session from tests.cycle.cycle_data import JSON_MACHINE_CYCLE_50 -from smsdk.smsdk_entities.cycle.cycle import Cycle +from smsdk.smsdk_entities.cycle.cycleV1 import Cycle def test_get_cycles(monkeypatch): # Setup def mockapi(self, session, endpoint, **kwargs): - if endpoint.startswith("/api/cycle"): + if endpoint.startswith("/v1/datatab/cycle"): return pd.DataFrame(JSON_MACHINE_CYCLE_50) return pd.DataFrame() @@ -16,6 +16,6 @@ def mockapi(self, session, endpoint, **kwargs): dt = Cycle(Session(), "demo") # Run - df = dt.get_cycles(Session(), "/api/cycle") + df = dt.get_cycles(Session(), "/v1/datatab/cycle") assert df.shape == (50, 29) diff --git a/tests/downtime/test_downtime.py b/tests/downtime/test_downtime.py index 9d28b21..38fb804 100644 --- a/tests/downtime/test_downtime.py +++ b/tests/downtime/test_downtime.py @@ -2,13 +2,13 @@ from requests.sessions import Session from tests.downtime.downtime_data import JSON_MACHINE_DOWNTIME_100 -from smsdk.smsdk_entities.downtime.downtime import Downtime +from smsdk.smsdk_entities.downtime.downtimeV1 import Downtime def test_get_downtime(monkeypatch): # Setup def mockapi(self, session, endpoint, **kwargs): - if endpoint.startswith("/api/downtime"): + if endpoint.startswith("/v1/datatab/downtime"): return pd.DataFrame(JSON_MACHINE_DOWNTIME_100) return pd.DataFrame() @@ -17,6 +17,6 @@ def mockapi(self, session, endpoint, **kwargs): dt = Downtime(Session(), "demo") # Run - df = dt.get_downtime(Session(), "/api/downtime") + df = dt.get_downtime(Session(), "/v1/datatab/downtime") assert df.shape == (100, 22) diff --git a/tests/factory/__init__.py b/tests/factory/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/factory/factory_data.py b/tests/factory/factory_data.py new file mode 100644 index 0000000..4e431c6 --- /dev/null +++ b/tests/factory/factory_data.py @@ -0,0 +1,242 @@ +JSON_FACTORY = [ + { + "updatetime": "2023-03-10 03:03:27.802000", + "hash": "5dfff57aca8d585f7ab98b7a9b07952df313f8790730cba4fe723ec91d116bcf", + "factory_id": "ETL3_BN", + "factory_location": "BN", + "factory_location_clean": "Busan", + "factory_partner": "ETL3", + "schema_name": "14-65", + "geo_location": {"lat": 35.13172, "lng": 129.064884}, + "place_name": "Busan, South Korea", + "shift_events": [], + "shifts": {}, + "metadata": {"timezone": "Asia/Seoul"}, + "id": "02b8ec6b72238839910e8641", + }, + { + "updatetime": "2023-03-10 03:03:27.802000", + "hash": "f7a544f4a6c311d3b6064a4e4c6965e93f9ff456f8927c2951df83b3c9660e19", + "factory_id": "ETL3_HM", + "factory_location": "HM", + "factory_location_clean": "Hamilton", + "factory_partner": "ETL3", + "schema_name": "14-65", + "geo_location": {"lat": 43.254721, "lng": -79.902764}, + "place_name": "Hamilton, Ontario, Canada", + "shift_events": [], + "shifts": {}, + "metadata": {"timezone": "America/Toronto"}, + "id": "0c1dbf1144c67c38aa951278", + }, + { + "updatetime": "2023-03-10 03:03:27.803000", + "hash": "cd0cf2c4e8fe2bb90f98c7a0852287c1aa0a1d965aa3d1a44d7d4d8ce01ee7f7", + "factory_id": "ETL3_HN", + "factory_location": "HN", + "factory_location_clean": "Hendersonville", + "factory_partner": "ETL3", + "schema_name": "14-65", + "geo_location": {"lat": 35.346065, "lng": -82.427882}, + "place_name": "Hendersonville, North Carolina, USA", + "shift_events": [], + "shifts": {}, + "metadata": {"timezone": "America/New_York"}, + "id": "272c877f92256a0557461f84", + }, + { + "updatetime": "2023-03-10 03:03:27.803000", + "hash": "c9c76446732b43e966a057d088b5f1113b76b0c580a8febae6d349cf78b0eca2", + "factory_id": "ETL3_LM", + "factory_location": "LM", + "factory_location_clean": "Lima", + "factory_partner": "ETL3", + "schema_name": "14-65", + "geo_location": {"lat": -12.0398048, "lng": -77.0264601}, + "place_name": "Lima, Peru", + "shift_events": [], + "shifts": {}, + "metadata": {"timezone": "America/Lima"}, + "id": "418efbd889bb87fb9357ce33", + }, + { + "updatetime": "2023-03-10 03:03:27.804000", + "hash": "7ee2b26dc29c7038948c0fcfddf06c2e1a7d8790e64b2e0b634f1756cf15abbc", + "factory_id": "ETL3_SC", + "factory_location": "SC", + "factory_location_clean": "Santa Catarina", + "factory_partner": "ETL3", + "schema_name": "14-65", + "geo_location": {"lat": 25.667063, "lng": -100.448557}, + "place_name": "Santa Catarina, Nuevo Leon, Mexico", + "shift_events": [], + "shifts": {}, + "metadata": {"timezone": "America/Monterrey"}, + "id": "67c367f37eef8c37e8d8ae65", + }, + { + "updatetime": "2023-03-10 03:03:27.804000", + "hash": "647ed1219541ab9dc016b30ba685bcbc65d993604165b67cb9fc8acc1b951b11", + "factory_id": "ETL3_NG", + "factory_location": "NG", + "factory_location_clean": "Nagoya", + "factory_partner": "ETL3", + "schema_name": "14-65", + "geo_location": {"lat": 35.0789846, "lng": 136.8617383}, + "place_name": "Nagoya, Chūbu, Japan", + "shift_events": [], + "shifts": {}, + "metadata": {"timezone": "Asia/Tokyo"}, + "id": "74bf178fde18c46dda65b7ce", + }, + { + "updatetime": "2023-03-10 03:03:27.804000", + "hash": "d15a99bbf2c1e3646b661a61f5cdcb1586dfb1e615143260e44226f635e8ec6f", + "factory_id": "ETL3_LL", + "factory_location": "LL", + "factory_location_clean": "Ljubljana", + "factory_partner": "ETL3", + "schema_name": "14-65", + "geo_location": {"lat": 46.0631845, "lng": 14.5613135}, + "place_name": "Ljubljana, Slovenia", + "shift_events": [], + "shifts": {}, + "metadata": {"timezone": "Europe/Ljubljana"}, + "id": "93918492156f22b831ad9fd8", + }, + { + "updatetime": "2023-03-10 03:03:27.805000", + "hash": "f79a6d6e2528b1f8ceac8e4a40e6213039426ce5d71a5cd8d497a18c9a4efecd", + "factory_id": "ETL3_SG", + "factory_location": "SG", + "factory_location_clean": "Singapore", + "factory_partner": "ETL3", + "schema_name": "14-65", + "geo_location": {"lat": 1.3428286, "lng": 103.75292}, + "place_name": "Singapore", + "shift_events": [], + "shifts": {}, + "metadata": {"timezone": "Asia/Singapore"}, + "id": "9c772744528b4e4c37d8ab26", + }, + { + "updatetime": "2023-03-10 03:03:27.805000", + "hash": "5f5394ccac57727bba10f5aa7208f6baa3aa3f2f491e96c10b1de2278b1da09d", + "factory_id": "ETL3_SW", + "factory_location": "SW", + "factory_location_clean": "Suwon", + "factory_partner": "ETL3", + "schema_name": "14-65", + "geo_location": {"lat": 37.230212, "lng": 127.025326}, + "place_name": "Suwon, South Korea", + "shift_events": [], + "shifts": {}, + "metadata": {"timezone": "Asia/Seoul"}, + "id": "a8bbae52613e817de40ddbb4", + }, + { + "updatetime": "2023-03-10 03:03:27.805000", + "hash": "e6bf7de0f1330fa899b5b1b33c2724ebf2fbcdaa81207c55f8c195a959259f89", + "factory_id": "ETL3_TL", + "factory_location": "TL", + "factory_location_clean": "Toulouse", + "factory_partner": "ETL3", + "schema_name": "14-65", + "geo_location": {"lat": 43.584058, "lng": 1.389739}, + "place_name": "Toulouse, Haute-Garonne, France", + "shift_events": [], + "shifts": {}, + "metadata": {"timezone": "Europe/Paris"}, + "id": "ab73004aa9646202df2e7861", + }, + { + "updatetime": "2023-03-10 03:03:27.806000", + "hash": "c983a6b70ed1a199711de46f37508630b3ae798b4d84df92986f869803d5ec83", + "factory_id": "ETL3_HK", + "factory_location": "HK", + "factory_location_clean": "Hong Kong", + "factory_partner": "ETL3", + "schema_name": "14-65", + "geo_location": {"lat": 22.3376639, "lng": 114.146738}, + "place_name": "Hong Kong, China", + "shift_events": [], + "shifts": {}, + "metadata": {"timezone": "Asia/Hong_Kong"}, + "id": "caa2599763d14d8830a4ec07", + }, + { + "updatetime": "2023-03-10 03:03:27.806000", + "hash": "bd9e2a9fa10597d38c8e99eac74d8c9fa5e3de4d622d77baecbc15ac74258536", + "factory_id": "ETL3_HB", + "factory_location": "HB", + "factory_location_clean": "Harbin", + "factory_partner": "ETL3", + "schema_name": "14-65", + "geo_location": {"lat": 45.779505, "lng": 126.771333}, + "place_name": "Harbin, Heilongjiang, China", + "shift_events": [], + "shifts": {}, + "metadata": {"timezone": "Asia/Harbin"}, + "id": "e4fe61f543328798b1ba3c84", + }, + { + "updatetime": "2023-03-10 03:03:27.806000", + "hash": "449740882f46a98eec44135d7affc5b0a6e871cd9fa028d3d1d9986b56ca70e6", + "factory_id": "ETL3_AB", + "factory_location": "AB", + "factory_location_clean": "Abidjan", + "factory_partner": "ETL3", + "schema_name": "14-65", + "geo_location": {"lat": 5.275944, "lng": -4.010024}, + "place_name": "Abidjan, Côte d'Ivoire", + "shift_events": [], + "shifts": {}, + "metadata": {"timezone": "Africa/Abidjan"}, + "id": "e71e2bcdb0369adace9a50d2", + }, + { + "updatetime": "2023-03-10 03:03:27.807000", + "hash": "af4ac42805c37bd2bf601906aa2f870c1ad464bc1ccf8941363ee5aa01c71c14", + "factory_id": "ETL3_BT", + "factory_location": "BT", + "factory_location_clean": "Bantam City", + "factory_partner": "ETL3", + "schema_name": "14-65", + "geo_location": {"lat": 1.078805, "lng": 104.024972}, + "place_name": "Batam City, Indonesia", + "shift_events": [], + "shifts": {}, + "metadata": {"timezone": "Asia/Jakarta"}, + "id": "e816784fa1eb6ae7d6fccc74", + }, + { + "updatetime": "2023-03-10 03:03:27.807000", + "hash": "fe7b0b96997729f84c84d6c4e03cb1bedf69d9b96146a180ff03170714f6c01e", + "factory_id": "ETL3_DC", + "factory_location": "DC", + "factory_location_clean": "Decatur", + "factory_partner": "ETL3", + "schema_name": "14-65", + "geo_location": {"lat": 39.876186, "lng": -88.91199}, + "place_name": "Decatur, Illinois, USA", + "shift_events": [], + "shifts": {}, + "metadata": {"timezone": "America/Chicago"}, + "id": "eb6b451963136f4bcaab2c66", + }, + { + "updatetime": "2023-03-10 03:03:27.807000", + "hash": "ccb5529573b10a01a64f810c5f2447088580557883e2e326fce6ab59fc489400", + "factory_id": "ETL3_CA", + "factory_location": "CA", + "factory_location_clean": "Carmel", + "factory_partner": "ETL3", + "schema_name": "14-65", + "geo_location": {"lat": 39.9663951, "lng": -86.2290612}, + "place_name": "Carmel, Indiana, USA", + "shift_events": [], + "shifts": {}, + "metadata": {"timezone": "America/New_York"}, + "id": "fdfcc948f186e34a794ed90a", + }, +] diff --git a/tests/factory/test_factory.py b/tests/factory/test_factory.py new file mode 100644 index 0000000..762dd45 --- /dev/null +++ b/tests/factory/test_factory.py @@ -0,0 +1,42 @@ +from unittest.mock import MagicMock +from mock import patch +import pandas as pd +from requests.sessions import Session +from smsdk.client import Client +from tests.factory.factory_data import JSON_FACTORY +from smsdk.smsdk_entities.factory.factoryV1 import Factory + + +def test_get_factories(monkeypatch): + # Setup + def mockapi(self, session, endpoint): + if endpoint == "/v1/obj/factory": + return pd.DataFrame(JSON_FACTORY) + return pd.DataFrame() + + monkeypatch.setattr(Factory, "get_factories", mockapi) + + dt = Factory(Session(), "demo") + + # Run + df = dt.get_factories(Session(), "/v1/obj/factory") + + # Verify + assert df.shape == (16, 13) + + cols = [ + "factory_id", + "factory_location", + "factory_location_clean", + "factory_partner", + "geo_location", + "hash", + "id", + "metadata", + "place_name", + "schema_name", + "shift_events", + "shifts", + "updatetime", + ] + assert cols == df.columns.sort_values().tolist() diff --git a/tests/machine/test_machine.py b/tests/machine/test_machine.py index 75c8e36..5a24937 100644 --- a/tests/machine/test_machine.py +++ b/tests/machine/test_machine.py @@ -4,13 +4,13 @@ from requests.sessions import Session from smsdk.client import Client from tests.machine.machine_data import JSON_MACHINE, MACHINE_TYPE -from smsdk.smsdk_entities.machine.machine import Machine +from smsdk.smsdk_entities.machine.machineV1 import Machine def test_get_machines(monkeypatch): # Setup def mockapi(self, session, endpoint): - if endpoint == "/api/machine": + if endpoint == "/v1/obj/machine": return pd.DataFrame(JSON_MACHINE) return pd.DataFrame() @@ -19,7 +19,7 @@ def mockapi(self, session, endpoint): dt = Machine(Session(), "demo") # Run - df = dt.get_machines(Session(), "/api/machine") + df = dt.get_machines(Session(), "/v1/obj/machine") # Verify assert df.shape == (27, 7) diff --git a/tests/machine_type/test_machine_type.py b/tests/machine_type/test_machine_type.py index 164459e..96a6f88 100644 --- a/tests/machine_type/test_machine_type.py +++ b/tests/machine_type/test_machine_type.py @@ -3,13 +3,13 @@ from requests.sessions import Session from smsdk.client import Client from tests.machine_type.machine_type_data import JSON_MACHINETYPE, MACHINE_TYPE_FIELDS -from smsdk.smsdk_entities.machine_type.machinetype import MachineType +from smsdk.smsdk_entities.machine_type.machinetypeV1 import MachineType def test_get_machine_types(monkeypatch): # Setup def mockapi(self, session, endpoint): - if endpoint == "/api/machinetype": + if endpoint == "/v1/obj/machine_types": return pd.DataFrame(JSON_MACHINETYPE) return pd.DataFrame() @@ -18,7 +18,7 @@ def mockapi(self, session, endpoint): dt = MachineType(Session(), "demo") # Run - df = dt.get_machine_types(Session(), "/api/machinetype") + df = dt.get_machine_types(Session(), "/v1/obj/machine_types") # Verify assert df.shape == (2, 22) diff --git a/tests/parts/test_part.py b/tests/parts/test_part.py index 1dba737..1a677e2 100644 --- a/tests/parts/test_part.py +++ b/tests/parts/test_part.py @@ -2,13 +2,13 @@ from requests.sessions import Session from tests.parts.part_data import JSON_PART -from smsdk.smsdk_entities.parts.parts import Parts +from smsdk.smsdk_entities.parts.partsV1 import Parts def test_get_parts(monkeypatch): # Setup def mockapi(self, session, endpoint, **kwargs): - if endpoint.startswith("/api/part"): + if endpoint.startswith("/v1/datatab/part"): return pd.DataFrame(JSON_PART) return pd.DataFrame() @@ -17,7 +17,7 @@ def mockapi(self, session, endpoint, **kwargs): dt = Parts(Session(), "demo") # Run - df = dt.get_parts(Session(), "/api/part") + df = dt.get_parts(Session(), "/v1/datatab/part") assert df.shape == (1, 29) cols = [