diff --git a/test/test_all.py b/test/test_all.py index db179555..37f09309 100644 --- a/test/test_all.py +++ b/test/test_all.py @@ -6,6 +6,7 @@ from wikibaseintegrator.entities import Item from wikibaseintegrator.wbi_config import config from wikibaseintegrator.wbi_enums import ActionIfExists +from wikibaseintegrator.wbi_fastrun import get_fastrun_container config['DEBUG'] = True @@ -53,7 +54,7 @@ class TestFastRun(unittest.TestCase): some basic tests for fastrun mode """ - def test_fast_run(self): + def test_fastrun(self): statements = [ datatypes.ExternalID(value='P40095', prop_nr='P352'), datatypes.ExternalID(value='YER158C', prop_nr='P705') @@ -61,35 +62,30 @@ def test_fast_run(self): frc = wbi_fastrun.FastRunContainer(base_filter=[BaseDataType(prop_nr='P352'), datatypes.Item(prop_nr='P703', value='Q27510868')], base_data_type=datatypes.BaseDataType) - fast_run_result = frc.write_required(data=statements) + fastrun_result = frc.write_required(data=statements) - if fast_run_result: + if fastrun_result: message = 'fastrun failed' else: message = 'successful fastrun' - print(fast_run_result, message) + print(fastrun_result, message) # here, fastrun should succeed, if not, test failed - if fast_run_result: + if fastrun_result: raise ValueError def test_fastrun_label(self): # tests fastrun label, description and aliases, and label in another language - fast_run_base_filter = [datatypes.Item(prop_nr='P361', value='Q18589965')] + frc = get_fastrun_container(base_filter=[datatypes.ExternalID(value='/m/02j71', prop_nr='P646')]) item = WikibaseIntegrator().item.get('Q2') - item.init_fastrun(base_filter=fast_run_base_filter) - item.init_fastrun(base_filter=fast_run_base_filter) # Test if we found the same FastRunContainer - item.claims.add(datatypes.ExternalID(value='/m/02j71', prop_nr='P646')) - - frc = wbi_fastrun.FastRunContainer(base_filter=[BaseDataType(prop_nr='P699')], base_data_type=datatypes.BaseDataType) assert item.labels.get(language='en') == "Earth" descr = item.descriptions.get(language='en') assert len(descr) > 3 assert "Terra" in item.aliases.get() - assert list(item.fast_run_container.get_language_data("Q2", 'en', 'label'))[0] == "Earth" - assert item.fast_run_container.check_language_data("Q2", ['not the Earth'], 'en', 'label') + assert list(frc.get_language_data("Q2", 'en', 'label'))[0] == item.labels.get(language='en') + assert frc.check_language_data("Q2", ['not the Earth'], 'en', 'label') assert "Terra" in item.aliases.get() assert "planet" in item.descriptions.get() diff --git a/wikibaseintegrator/entities/baseentity.py b/wikibaseintegrator/entities/baseentity.py index b0c65739..646dc171 100644 --- a/wikibaseintegrator/entities/baseentity.py +++ b/wikibaseintegrator/entities/baseentity.py @@ -1,16 +1,16 @@ from __future__ import annotations from copy import copy -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, List, Union import simplejson +from wikibaseintegrator import wbi_fastrun from wikibaseintegrator.datatypes import BaseDataType from wikibaseintegrator.models.claims import Claim, Claims from wikibaseintegrator.wbi_config import config from wikibaseintegrator.wbi_enums import ActionIfExists from wikibaseintegrator.wbi_exceptions import MWApiError, NonUniqueLabelDescriptionPairError -from wikibaseintegrator.wbi_fastrun import FastRunContainer from wikibaseintegrator.wbi_helpers import mediawiki_api_call_helper from wikibaseintegrator.wbi_login import Login @@ -19,8 +19,6 @@ class BaseEntity: - fast_run_store: List[FastRunContainer] = [] - ETYPE = 'base-entity' def __init__(self, api: 'WikibaseIntegrator' = None, lastrevid: int = None, type: str = None, id: str = None, claims: Claims = None, is_bot: bool = None, login: Login = None): @@ -38,8 +36,6 @@ def __init__(self, api: 'WikibaseIntegrator' = None, lastrevid: int = None, type self.id = id self.claims = claims or Claims() - self.fast_run_container: Optional[FastRunContainer] = None - self.debug = config['DEBUG'] def add_claims(self, claims: Union[Claim, list], action_if_exists: ActionIfExists = ActionIfExists.APPEND) -> BaseEntity: @@ -174,43 +170,8 @@ def _write(self, data: Dict = None, summary: str = None, allow_anonymous: bool = self.lastrevid = json_data['entity']['lastrevid'] return json_data['entity'] - def init_fastrun(self, base_filter: List[BaseDataType | List[BaseDataType]] = None, use_refs: bool = False, case_insensitive: bool = False) -> None: - if base_filter is None: - base_filter = [] - - if self.debug: - print('Initialize Fast Run init_fastrun') - # We search if we already have a FastRunContainer with the same parameters to re-use it - for fast_run in BaseEntity.fast_run_store: - if (fast_run.base_filter == base_filter) and (fast_run.use_refs == use_refs) and (fast_run.case_insensitive == case_insensitive) and ( - fast_run.sparql_endpoint_url == config['SPARQL_ENDPOINT_URL']): - self.fast_run_container = fast_run - self.fast_run_container.current_qid = '' - self.fast_run_container.base_data_type = BaseDataType - if self.debug: - print("Found an already existing FastRunContainer") - - if not self.fast_run_container: - if self.debug: - print("Create a new FastRunContainer") - self.fast_run_container = FastRunContainer(base_filter=base_filter, use_refs=use_refs, base_data_type=BaseDataType, case_insensitive=case_insensitive) - BaseEntity.fast_run_store.append(self.fast_run_container) - - # def fr_search(self, **kwargs: Any) -> str: - # self.init_fastrun(**kwargs) - # - # if self.fast_run_container is None: - # raise ValueError("FastRunContainer is not initialized.") - # - # self.fast_run_container.load_item(self.claims) - # - # return self.fast_run_container.current_qid - def write_required(self, base_filter: List[BaseDataType | List[BaseDataType]] = None, **kwargs: Any) -> bool: - self.init_fastrun(base_filter=base_filter, **kwargs) - - if self.fast_run_container is None: - raise ValueError("FastRunContainer is not initialized.") + fastrun_container = wbi_fastrun.get_fastrun_container(base_filter=base_filter, **kwargs) if base_filter is None: base_filter = [] @@ -220,7 +181,7 @@ def write_required(self, base_filter: List[BaseDataType | List[BaseDataType]] = if claim.mainsnak.property_number in base_filter: claims_to_check.append(claim) - return self.fast_run_container.write_required(data=claims_to_check, cqid=self.id) + return fastrun_container.write_required(data=claims_to_check, cqid=self.id) def __repr__(self): """A mixin implementing a simple __repr__.""" diff --git a/wikibaseintegrator/wbi_fastrun.py b/wikibaseintegrator/wbi_fastrun.py index f99e19d6..ac9155db 100644 --- a/wikibaseintegrator/wbi_fastrun.py +++ b/wikibaseintegrator/wbi_fastrun.py @@ -29,7 +29,6 @@ def __init__(self, base_data_type: Type[BaseDataType], mediawiki_api_url: str = self.base_filter: List[BaseDataType | List[BaseDataType]] = [] self.base_filter_string = '' self.prop_dt_map: Dict[str, str] = {} - self.current_qid = '' self.base_data_type = base_data_type self.mediawiki_api_url = mediawiki_api_url or config['MEDIAWIKI_API_URL'] @@ -114,11 +113,7 @@ def reconstruct_statements(self, qid: str) -> List[BaseDataType]: self.reconstructed_statements = reconstructed_statements return reconstructed_statements - def get_item(self, claims: List, cqid: str = None) -> str: - self.load_item(claims=claims, cqid=cqid) - return self.current_qid - - def load_item(self, claims: Union[list, Claims], cqid: str = None) -> bool: + def get_item(self, claims: Union[list, Claims], cqid: str = None) -> Optional[str]: match_sets = [] for claim in claims: # skip to next if statement has no value or no data type defined, e.g. for deletion objects @@ -162,7 +157,7 @@ def load_item(self, claims: Union[list, Claims], cqid: str = None) -> bool: # return True if not match_sets: - return True + return None if cqid: matching_qids = {cqid} @@ -174,11 +169,9 @@ def load_item(self, claims: Union[list, Claims], cqid: str = None) -> bool: if not len(matching_qids) == 1: if self.debug: print(f"no matches ({len(matching_qids)})") - return True + return None - qid = matching_qids.pop() - self.current_qid = qid - return False + return matching_qids.pop() def write_required(self, data: List[BaseDataType], action_if_exists: ActionIfExists = ActionIfExists.REPLACE, cqid: str = None) -> bool: del_props = set() @@ -190,12 +183,12 @@ def write_required(self, data: List[BaseDataType], action_if_exists: ActionIfExi for x in data: if x.mainsnak.datavalue and x.mainsnak.datatype: data_props.add(x.mainsnak.property_number) - self.load_item(data, cqid) + qid = self.get_item(data, cqid) - if not self.current_qid: + if not qid: return True - reconstructed_statements = self.reconstruct_statements(self.current_qid) + reconstructed_statements = self.reconstruct_statements(qid) tmp_rs = copy.deepcopy(reconstructed_statements) # handle append properties @@ -645,54 +638,29 @@ def __repr__(self) -> str: ) -# def fr_search(**kwargs: Any) -> str: -# FastRunContainer.init_fastrun(**kwargs) -# -# if self.fast_run_container is None: -# raise ValueError("FastRunContainer is not initialized.") -# -# self.fast_run_container.load_item(self.claims) -# -# return self.fast_run_container.current_qid - - -# def freezeargs(func): -# """Transform mutable dictionnary -# Into immutable -# Useful to be compatible with cache -# """ -# -# @wraps(func) -# def wrapped(*args: Any, **kwargs: Any) -> Any: -# args = tuple(frozendict(arg) if isinstance(arg, dict) else arg for arg in args) -# kwargs = {k: frozendict(v) if isinstance(v, dict) else v for k, v in kwargs.items()} -# return func(*args, **kwargs) -# -# return wrapped - - def get_fastrun_container(base_filter: List[BaseDataType | List[BaseDataType]] = None, use_refs: bool = False, case_insensitive: bool = False) -> FastRunContainer: if base_filter is None: base_filter = [] + if config['DEBUG']: + print('Initialize Fast Run get_fastrun_container') + + # We search if we already have a FastRunContainer with the same parameters to re-use it fastrun_container = search_fastrun_store(base_filter=base_filter, use_refs=use_refs, case_insensitive=case_insensitive) - fastrun_container.current_qid = '' - fastrun_container.base_data_type = BaseDataType return fastrun_container -# @freezeargs -# @lru_cache() def search_fastrun_store(base_filter: List[BaseDataType | List[BaseDataType]] = None, use_refs: bool = False, case_insensitive: bool = False) -> FastRunContainer: - for c in fastrun_store: - if (c.base_filter == base_filter) and (c.use_refs == use_refs) and (c.case_insensitive == case_insensitive) and ( - c.sparql_endpoint_url == config['SPARQL_ENDPOINT_URL']): - return c + for fastrun in fastrun_store: + if (fastrun.base_filter == base_filter) and (fastrun.use_refs == use_refs) and (fastrun.case_insensitive == case_insensitive) and ( + fastrun.sparql_endpoint_url == config['SPARQL_ENDPOINT_URL']): + return fastrun # In case nothing was found in the fastrun_store if config['DEBUG']: print("Create a new FastRunContainer") - fastrun_container = FastRunContainer(base_filter=base_filter, use_refs=use_refs, base_data_type=BaseDataType, case_insensitive=case_insensitive) + + fastrun_container = FastRunContainer(base_data_type=BaseDataType, base_filter=base_filter, use_refs=use_refs, case_insensitive=case_insensitive) fastrun_store.append(fastrun_container) return fastrun_container