diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eafd6e2f..52cfda4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ concurrency: jobs: test: name: Test - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 services: redis: image: redis @@ -35,6 +35,7 @@ jobs: - name: Install dependencies run: | + sudo apt update sudo apt-get install -y libkrb5-dev pip install -U setuptools pip wheel pip install -e .[cpphash,redis,uwsgi] diff --git a/CHANGES.txt b/CHANGES.txt index 52688577..d60d05ef 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,7 @@ +10.3.0 (Jun 17, 2025) +- Added support for rule-based segments. These segments determine membership at runtime by evaluating their configured rules against the user attributes provided to the SDK. +- Added support for feature flag prerequisites. This allows customers to define dependency conditions between flags, which are evaluated before any allowlists or targeting rules. + 10.2.0 (Jan 17, 2025) - Added support for the new impressions tracking toggle available on feature flags, both respecting the setting and including the new field being returned on SplitView type objects. Read more in our docs. diff --git a/setup.py b/setup.py index 10fa308f..1e1928fc 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ 'flake8', 'pytest==7.0.1', 'pytest-mock==3.11.1', - 'coverage', + 'coverage==7.0.0', 'pytest-cov==4.1.0', 'importlib-metadata==6.7', 'tomli==1.2.3', @@ -17,7 +17,8 @@ 'pytest-asyncio==0.21.0', 'aiohttp>=3.8.4', 'aiofiles>=23.1.0', - 'requests-kerberos>=0.15.0' + 'requests-kerberos>=0.15.0', + 'urllib3==2.0.7' ] INSTALL_REQUIRES = [ diff --git a/splitio/api/client.py b/splitio/api/client.py index 5db1cadb..c9032e0e 100644 --- a/splitio/api/client.py +++ b/splitio/api/client.py @@ -92,6 +92,25 @@ def proxy_headers(self, proxy): class HttpClientBase(object, metaclass=abc.ABCMeta): """HttpClient wrapper template.""" + def __init__(self, timeout=None, sdk_url=None, events_url=None, auth_url=None, telemetry_url=None): + """ + Class constructor. + + :param timeout: How many milliseconds to wait until the server responds. + :type timeout: int + :param sdk_url: Optional alternative sdk URL. + :type sdk_url: str + :param events_url: Optional alternative events URL. + :type events_url: str + :param auth_url: Optional alternative auth URL. + :type auth_url: str + :param telemetry_url: Optional alternative telemetry URL. + :type telemetry_url: str + """ + _LOGGER.debug("Initializing httpclient") + self._timeout = timeout/1000 if timeout else None # Convert ms to seconds. + self._urls = _construct_urls(sdk_url, events_url, auth_url, telemetry_url) + @abc.abstractmethod def get(self, server, path, apikey): """http get request""" @@ -113,6 +132,9 @@ def set_telemetry_data(self, metric_name, telemetry_runtime_producer): self._telemetry_runtime_producer = telemetry_runtime_producer self._metric_name = metric_name + def is_sdk_endpoint_overridden(self): + return self._urls['sdk'] != SDK_URL + def _get_headers(self, extra_headers, sdk_key): headers = _build_basic_headers(sdk_key) if extra_headers is not None: @@ -154,10 +176,8 @@ def __init__(self, timeout=None, sdk_url=None, events_url=None, auth_url=None, t :param telemetry_url: Optional alternative telemetry URL. :type telemetry_url: str """ - _LOGGER.debug("Initializing httpclient") - self._timeout = timeout/1000 if timeout else None # Convert ms to seconds. - self._urls = _construct_urls(sdk_url, events_url, auth_url, telemetry_url) - + HttpClientBase.__init__(self, timeout, sdk_url, events_url, auth_url, telemetry_url) + def get(self, server, path, sdk_key, query=None, extra_headers=None): # pylint: disable=too-many-arguments """ Issue a get request. @@ -187,7 +207,11 @@ def get(self, server, path, sdk_key, query=None, extra_headers=None): # pylint: self._record_telemetry(response.status_code, get_current_epoch_time_ms() - start) return HttpResponse(response.status_code, response.text, response.headers) - except Exception as exc: # pylint: disable=broad-except + except requests.exceptions.ChunkedEncodingError as exc: + _LOGGER.error("IncompleteRead exception detected: %s", exc) + return HttpResponse(400, "", {}) + + except Exception as exc: # pylint: disable=broad-except raise HttpClientException(_EXC_MSG.format(source='request')) from exc def post(self, server, path, sdk_key, body, query=None, extra_headers=None): # pylint: disable=too-many-arguments @@ -241,8 +265,7 @@ def __init__(self, timeout=None, sdk_url=None, events_url=None, auth_url=None, t :param telemetry_url: Optional alternative telemetry URL. :type telemetry_url: str """ - self._timeout = timeout/1000 if timeout else None # Convert ms to seconds. - self._urls = _construct_urls(sdk_url, events_url, auth_url, telemetry_url) + HttpClientBase.__init__(self, timeout, sdk_url, events_url, auth_url, telemetry_url) self._session = aiohttp.ClientSession() async def get(self, server, path, apikey, query=None, extra_headers=None): # pylint: disable=too-many-arguments @@ -281,6 +304,10 @@ async def get(self, server, path, apikey, query=None, extra_headers=None): # py await self._record_telemetry(response.status, get_current_epoch_time_ms() - start) return HttpResponse(response.status, body, response.headers) + except aiohttp.ClientPayloadError as exc: + _LOGGER.error("ContentLengthError exception detected: %s", exc) + return HttpResponse(400, "", {}) + except aiohttp.ClientError as exc: # pylint: disable=broad-except raise HttpClientException(_EXC_MSG.format(source='aiohttp')) from exc diff --git a/splitio/api/commons.py b/splitio/api/commons.py index 2ca75595..9dda1ee0 100644 --- a/splitio/api/commons.py +++ b/splitio/api/commons.py @@ -57,7 +57,7 @@ def record_telemetry(status_code, elapsed, metric_name, telemetry_runtime_produc class FetchOptions(object): """Fetch Options object.""" - def __init__(self, cache_control_headers=False, change_number=None, sets=None, spec=SPEC_VERSION): + def __init__(self, cache_control_headers=False, change_number=None, rbs_change_number=None, sets=None, spec=SPEC_VERSION): """ Class constructor. @@ -72,6 +72,7 @@ def __init__(self, cache_control_headers=False, change_number=None, sets=None, s """ self._cache_control_headers = cache_control_headers self._change_number = change_number + self._rbs_change_number = rbs_change_number self._sets = sets self._spec = spec @@ -85,6 +86,11 @@ def change_number(self): """Return change number.""" return self._change_number + @property + def rbs_change_number(self): + """Return change number.""" + return self._rbs_change_number + @property def sets(self): """Return sets.""" @@ -103,14 +109,19 @@ def __eq__(self, other): if self._change_number != other._change_number: return False + if self._rbs_change_number != other._rbs_change_number: + return False + if self._sets != other._sets: return False + if self._spec != other._spec: return False + return True -def build_fetch(change_number, fetch_options, metadata): +def build_fetch(change_number, fetch_options, metadata, rbs_change_number=None): """ Build fetch with new flags if that is the case. @@ -123,11 +134,16 @@ def build_fetch(change_number, fetch_options, metadata): :param metadata: Metadata Headers. :type metadata: dict + :param rbs_change_number: Last known timestamp of a rule based segment modification. + :type rbs_change_number: int + :return: Objects for fetch :rtype: dict, dict """ query = {'s': fetch_options.spec} if fetch_options.spec is not None else {} query['since'] = change_number + if rbs_change_number is not None: + query['rbSince'] = rbs_change_number extra_headers = metadata if fetch_options is None: return query, extra_headers diff --git a/splitio/api/splits.py b/splitio/api/splits.py index 692fde3b..771100fc 100644 --- a/splitio/api/splits.py +++ b/splitio/api/splits.py @@ -4,14 +4,18 @@ import json from splitio.api import APIException, headers_from_metadata -from splitio.api.commons import build_fetch +from splitio.api.commons import build_fetch, FetchOptions from splitio.api.client import HttpClientException from splitio.models.telemetry import HTTPExceptionsAndLatencies +from splitio.util.time import utctime_ms +from splitio.spec import SPEC_VERSION +from splitio.sync import util _LOGGER = logging.getLogger(__name__) +_SPEC_1_1 = "1.1" +_PROXY_CHECK_INTERVAL_MILLISECONDS_SS = 24 * 60 * 60 * 1000 - -class SplitsAPI(object): # pylint: disable=too-few-public-methods +class SplitsAPIBase(object): # pylint: disable=too-few-public-methods """Class that uses an httpClient to communicate with the splits API.""" def __init__(self, client, sdk_key, sdk_metadata, telemetry_runtime_producer): @@ -30,14 +34,51 @@ def __init__(self, client, sdk_key, sdk_metadata, telemetry_runtime_producer): self._metadata = headers_from_metadata(sdk_metadata) self._telemetry_runtime_producer = telemetry_runtime_producer self._client.set_telemetry_data(HTTPExceptionsAndLatencies.SPLIT, self._telemetry_runtime_producer) + self._spec_version = SPEC_VERSION + self._last_proxy_check_timestamp = 0 + self.clear_storage = False + self._old_spec_since = None + + def _check_last_proxy_check_timestamp(self, since): + if self._spec_version == _SPEC_1_1 and ((utctime_ms() - self._last_proxy_check_timestamp) >= _PROXY_CHECK_INTERVAL_MILLISECONDS_SS): + _LOGGER.info("Switching to new Feature flag spec (%s) and fetching.", SPEC_VERSION); + self._spec_version = SPEC_VERSION + self._old_spec_since = since + + def _check_old_spec_since(self, change_number): + if self._spec_version == _SPEC_1_1 and self._old_spec_since is not None: + since = self._old_spec_since + self._old_spec_since = None + return since + return change_number + + +class SplitsAPI(SplitsAPIBase): # pylint: disable=too-few-public-methods + """Class that uses an httpClient to communicate with the splits API.""" + + def __init__(self, client, sdk_key, sdk_metadata, telemetry_runtime_producer): + """ + Class constructor. + + :param client: HTTP Client responsble for issuing calls to the backend. + :type client: HttpClient + :param sdk_key: User sdk_key token. + :type sdk_key: string + :param sdk_metadata: SDK version & machine name & IP. + :type sdk_metadata: splitio.client.util.SdkMetadata + """ + SplitsAPIBase.__init__(self, client, sdk_key, sdk_metadata, telemetry_runtime_producer) - def fetch_splits(self, change_number, fetch_options): + def fetch_splits(self, change_number, rbs_change_number, fetch_options): """ Fetch feature flags from backend. :param change_number: Last known timestamp of a split modification. :type change_number: int + :param rbs_change_number: Last known timestamp of a rule based segment modification. + :type rbs_change_number: int + :param fetch_options: Fetch options for getting feature flag definitions. :type fetch_options: splitio.api.commons.FetchOptions @@ -45,7 +86,14 @@ def fetch_splits(self, change_number, fetch_options): :rtype: dict """ try: - query, extra_headers = build_fetch(change_number, fetch_options, self._metadata) + self._check_last_proxy_check_timestamp(change_number) + change_number = self._check_old_spec_since(change_number) + + if self._spec_version == _SPEC_1_1: + fetch_options = FetchOptions(fetch_options.cache_control_headers, fetch_options.change_number, + None, fetch_options.sets, self._spec_version) + rbs_change_number = None + query, extra_headers = build_fetch(change_number, fetch_options, self._metadata, rbs_change_number) response = self._client.get( 'sdk', 'splitChanges', @@ -54,19 +102,32 @@ def fetch_splits(self, change_number, fetch_options): query=query, ) if 200 <= response.status_code < 300: + if self._spec_version == _SPEC_1_1: + return util.convert_to_new_spec(json.loads(response.body)) + + self.clear_storage = self._last_proxy_check_timestamp != 0 + self._last_proxy_check_timestamp = 0 return json.loads(response.body) else: if response.status_code == 414: _LOGGER.error('Error fetching feature flags; the amount of flag sets provided are too big, causing uri length error.') + + if self._client.is_sdk_endpoint_overridden() and response.status_code == 400 and self._spec_version == SPEC_VERSION: + _LOGGER.warning('Detected proxy response error, changing spec version from %s to %s and re-fetching.', self._spec_version, _SPEC_1_1) + self._spec_version = _SPEC_1_1 + self._last_proxy_check_timestamp = utctime_ms() + return self.fetch_splits(change_number, None, FetchOptions(fetch_options.cache_control_headers, fetch_options.change_number, + None, fetch_options.sets, self._spec_version)) + raise APIException(response.body, response.status_code) + except HttpClientException as exc: _LOGGER.error('Error fetching feature flags because an exception was raised by the HTTPClient') _LOGGER.debug('Error: ', exc_info=True) raise APIException('Feature flags not fetched correctly.') from exc - -class SplitsAPIAsync(object): # pylint: disable=too-few-public-methods +class SplitsAPIAsync(SplitsAPIBase): # pylint: disable=too-few-public-methods """Class that uses an httpClient to communicate with the splits API.""" def __init__(self, client, sdk_key, sdk_metadata, telemetry_runtime_producer): @@ -80,18 +141,17 @@ def __init__(self, client, sdk_key, sdk_metadata, telemetry_runtime_producer): :param sdk_metadata: SDK version & machine name & IP. :type sdk_metadata: splitio.client.util.SdkMetadata """ - self._client = client - self._sdk_key = sdk_key - self._metadata = headers_from_metadata(sdk_metadata) - self._telemetry_runtime_producer = telemetry_runtime_producer - self._client.set_telemetry_data(HTTPExceptionsAndLatencies.SPLIT, self._telemetry_runtime_producer) + SplitsAPIBase.__init__(self, client, sdk_key, sdk_metadata, telemetry_runtime_producer) - async def fetch_splits(self, change_number, fetch_options): + async def fetch_splits(self, change_number, rbs_change_number, fetch_options): """ Fetch feature flags from backend. :param change_number: Last known timestamp of a split modification. :type change_number: int + + :param rbs_change_number: Last known timestamp of a rule based segment modification. + :type rbs_change_number: int :param fetch_options: Fetch options for getting feature flag definitions. :type fetch_options: splitio.api.commons.FetchOptions @@ -100,7 +160,14 @@ async def fetch_splits(self, change_number, fetch_options): :rtype: dict """ try: - query, extra_headers = build_fetch(change_number, fetch_options, self._metadata) + self._check_last_proxy_check_timestamp(change_number) + change_number = self._check_old_spec_since(change_number) + if self._spec_version == _SPEC_1_1: + fetch_options = FetchOptions(fetch_options.cache_control_headers, fetch_options.change_number, + None, fetch_options.sets, self._spec_version) + rbs_change_number = None + + query, extra_headers = build_fetch(change_number, fetch_options, self._metadata, rbs_change_number) response = await self._client.get( 'sdk', 'splitChanges', @@ -109,12 +176,26 @@ async def fetch_splits(self, change_number, fetch_options): query=query, ) if 200 <= response.status_code < 300: + if self._spec_version == _SPEC_1_1: + return util.convert_to_new_spec(json.loads(response.body)) + + self.clear_storage = self._last_proxy_check_timestamp != 0 + self._last_proxy_check_timestamp = 0 return json.loads(response.body) else: if response.status_code == 414: _LOGGER.error('Error fetching feature flags; the amount of flag sets provided are too big, causing uri length error.') + + if self._client.is_sdk_endpoint_overridden() and response.status_code == 400 and self._spec_version == SPEC_VERSION: + _LOGGER.warning('Detected proxy response error, changing spec version from %s to %s and re-fetching.', self._spec_version, _SPEC_1_1) + self._spec_version = _SPEC_1_1 + self._last_proxy_check_timestamp = utctime_ms() + return await self.fetch_splits(change_number, None, FetchOptions(fetch_options.cache_control_headers, fetch_options.change_number, + None, fetch_options.sets, self._spec_version)) + raise APIException(response.body, response.status_code) + except HttpClientException as exc: _LOGGER.error('Error fetching feature flags because an exception was raised by the HTTPClient') _LOGGER.debug('Error: ', exc_info=True) diff --git a/splitio/client/client.py b/splitio/client/client.py index d4c37fa4..8e71030e 100644 --- a/splitio/client/client.py +++ b/splitio/client/client.py @@ -201,7 +201,7 @@ def __init__(self, factory, recorder, labels_enabled=True): :rtype: Client """ ClientBase.__init__(self, factory, recorder, labels_enabled) - self._context_factory = EvaluationDataFactory(factory._get_storage('splits'), factory._get_storage('segments')) + self._context_factory = EvaluationDataFactory(factory._get_storage('splits'), factory._get_storage('segments'), factory._get_storage('rule_based_segments')) def destroy(self): """ @@ -668,7 +668,7 @@ def __init__(self, factory, recorder, labels_enabled=True): :rtype: Client """ ClientBase.__init__(self, factory, recorder, labels_enabled) - self._context_factory = AsyncEvaluationDataFactory(factory._get_storage('splits'), factory._get_storage('segments')) + self._context_factory = AsyncEvaluationDataFactory(factory._get_storage('splits'), factory._get_storage('segments'), factory._get_storage('rule_based_segments')) async def destroy(self): """ diff --git a/splitio/client/factory.py b/splitio/client/factory.py index bb402bb5..f6070243 100644 --- a/splitio/client/factory.py +++ b/splitio/client/factory.py @@ -23,14 +23,17 @@ from splitio.storage.inmemmory import InMemorySplitStorage, InMemorySegmentStorage, \ InMemoryImpressionStorage, InMemoryEventStorage, InMemoryTelemetryStorage, LocalhostTelemetryStorage, \ InMemorySplitStorageAsync, InMemorySegmentStorageAsync, InMemoryImpressionStorageAsync, \ - InMemoryEventStorageAsync, InMemoryTelemetryStorageAsync, LocalhostTelemetryStorageAsync + InMemoryEventStorageAsync, InMemoryTelemetryStorageAsync, LocalhostTelemetryStorageAsync, \ + InMemoryRuleBasedSegmentStorage, InMemoryRuleBasedSegmentStorageAsync from splitio.storage.adapters import redis from splitio.storage.redis import RedisSplitStorage, RedisSegmentStorage, RedisImpressionsStorage, \ RedisEventsStorage, RedisTelemetryStorage, RedisSplitStorageAsync, RedisEventsStorageAsync,\ - RedisSegmentStorageAsync, RedisImpressionsStorageAsync, RedisTelemetryStorageAsync + RedisSegmentStorageAsync, RedisImpressionsStorageAsync, RedisTelemetryStorageAsync, \ + RedisRuleBasedSegmentsStorage, RedisRuleBasedSegmentsStorageAsync from splitio.storage.pluggable import PluggableEventsStorage, PluggableImpressionsStorage, PluggableSegmentStorage, \ PluggableSplitStorage, PluggableTelemetryStorage, PluggableTelemetryStorageAsync, PluggableEventsStorageAsync, \ - PluggableImpressionsStorageAsync, PluggableSegmentStorageAsync, PluggableSplitStorageAsync + PluggableImpressionsStorageAsync, PluggableSegmentStorageAsync, PluggableSplitStorageAsync, \ + PluggableRuleBasedSegmentsStorage, PluggableRuleBasedSegmentsStorageAsync # APIs from splitio.api.client import HttpClient, HttpClientAsync, HttpClientKerberos @@ -543,6 +546,7 @@ def _build_in_memory_factory(api_key, cfg, sdk_url=None, events_url=None, # pyl storages = { 'splits': InMemorySplitStorage(cfg['flagSetsFilter'] if cfg['flagSetsFilter'] is not None else []), 'segments': InMemorySegmentStorage(), + 'rule_based_segments': InMemoryRuleBasedSegmentStorage(), 'impressions': InMemoryImpressionStorage(cfg['impressionsQueueSize'], telemetry_runtime_producer), 'events': InMemoryEventStorage(cfg['eventsQueueSize'], telemetry_runtime_producer), } @@ -559,8 +563,8 @@ def _build_in_memory_factory(api_key, cfg, sdk_url=None, events_url=None, # pyl imp_strategy, none_strategy, telemetry_runtime_producer) synchronizers = SplitSynchronizers( - SplitSynchronizer(apis['splits'], storages['splits']), - SegmentSynchronizer(apis['segments'], storages['splits'], storages['segments']), + SplitSynchronizer(apis['splits'], storages['splits'], storages['rule_based_segments']), + SegmentSynchronizer(apis['segments'], storages['splits'], storages['segments'], storages['rule_based_segments']), ImpressionSynchronizer(apis['impressions'], storages['impressions'], cfg['impressionsBulkSize']), EventSynchronizer(apis['events'], storages['events'], cfg['eventsBulkSize']), @@ -671,6 +675,7 @@ async def _build_in_memory_factory_async(api_key, cfg, sdk_url=None, events_url= storages = { 'splits': InMemorySplitStorageAsync(cfg['flagSetsFilter'] if cfg['flagSetsFilter'] is not None else []), 'segments': InMemorySegmentStorageAsync(), + 'rule_based_segments': InMemoryRuleBasedSegmentStorageAsync(), 'impressions': InMemoryImpressionStorageAsync(cfg['impressionsQueueSize'], telemetry_runtime_producer), 'events': InMemoryEventStorageAsync(cfg['eventsQueueSize'], telemetry_runtime_producer), } @@ -687,8 +692,8 @@ async def _build_in_memory_factory_async(api_key, cfg, sdk_url=None, events_url= imp_strategy, none_strategy, telemetry_runtime_producer) synchronizers = SplitSynchronizers( - SplitSynchronizerAsync(apis['splits'], storages['splits']), - SegmentSynchronizerAsync(apis['segments'], storages['splits'], storages['segments']), + SplitSynchronizerAsync(apis['splits'], storages['splits'], storages['rule_based_segments']), + SegmentSynchronizerAsync(apis['segments'], storages['splits'], storages['segments'], storages['rule_based_segments']), ImpressionSynchronizerAsync(apis['impressions'], storages['impressions'], cfg['impressionsBulkSize']), EventSynchronizerAsync(apis['events'], storages['events'], cfg['eventsBulkSize']), @@ -756,6 +761,7 @@ def _build_redis_factory(api_key, cfg): storages = { 'splits': RedisSplitStorage(redis_adapter, cache_enabled, cache_ttl, []), 'segments': RedisSegmentStorage(redis_adapter), + 'rule_based_segments': RedisRuleBasedSegmentsStorage(redis_adapter), 'impressions': RedisImpressionsStorage(redis_adapter, sdk_metadata), 'events': RedisEventsStorage(redis_adapter, sdk_metadata), 'telemetry': RedisTelemetryStorage(redis_adapter, sdk_metadata) @@ -839,6 +845,7 @@ async def _build_redis_factory_async(api_key, cfg): storages = { 'splits': RedisSplitStorageAsync(redis_adapter, cache_enabled, cache_ttl), 'segments': RedisSegmentStorageAsync(redis_adapter), + 'rule_based_segments': RedisRuleBasedSegmentsStorageAsync(redis_adapter), 'impressions': RedisImpressionsStorageAsync(redis_adapter, sdk_metadata), 'events': RedisEventsStorageAsync(redis_adapter, sdk_metadata), 'telemetry': await RedisTelemetryStorageAsync.create(redis_adapter, sdk_metadata) @@ -922,6 +929,7 @@ def _build_pluggable_factory(api_key, cfg): storages = { 'splits': PluggableSplitStorage(pluggable_adapter, storage_prefix, []), 'segments': PluggableSegmentStorage(pluggable_adapter, storage_prefix), + 'rule_based_segments': PluggableRuleBasedSegmentsStorage(pluggable_adapter, storage_prefix), 'impressions': PluggableImpressionsStorage(pluggable_adapter, sdk_metadata, storage_prefix), 'events': PluggableEventsStorage(pluggable_adapter, sdk_metadata, storage_prefix), 'telemetry': PluggableTelemetryStorage(pluggable_adapter, sdk_metadata, storage_prefix) @@ -1003,6 +1011,7 @@ async def _build_pluggable_factory_async(api_key, cfg): storages = { 'splits': PluggableSplitStorageAsync(pluggable_adapter, storage_prefix), 'segments': PluggableSegmentStorageAsync(pluggable_adapter, storage_prefix), + 'rule_based_segments': PluggableRuleBasedSegmentsStorageAsync(pluggable_adapter, storage_prefix), 'impressions': PluggableImpressionsStorageAsync(pluggable_adapter, sdk_metadata, storage_prefix), 'events': PluggableEventsStorageAsync(pluggable_adapter, sdk_metadata, storage_prefix), 'telemetry': await PluggableTelemetryStorageAsync.create(pluggable_adapter, sdk_metadata, storage_prefix) @@ -1081,6 +1090,7 @@ def _build_localhost_factory(cfg): storages = { 'splits': InMemorySplitStorage(cfg['flagSetsFilter'] if cfg['flagSetsFilter'] is not None else []), 'segments': InMemorySegmentStorage(), # not used, just to avoid possible future errors. + 'rule_based_segments': InMemoryRuleBasedSegmentStorage(), 'impressions': LocalhostImpressionsStorage(), 'events': LocalhostEventsStorage(), } @@ -1088,6 +1098,7 @@ def _build_localhost_factory(cfg): synchronizers = SplitSynchronizers( LocalSplitSynchronizer(cfg['splitFile'], storages['splits'], + storages['rule_based_segments'], localhost_mode), LocalSegmentSynchronizer(cfg['segmentDirectory'], storages['splits'], storages['segments']), None, None, None, @@ -1151,6 +1162,7 @@ async def _build_localhost_factory_async(cfg): storages = { 'splits': InMemorySplitStorageAsync(), 'segments': InMemorySegmentStorageAsync(), # not used, just to avoid possible future errors. + 'rule_based_segments': InMemoryRuleBasedSegmentStorageAsync(), 'impressions': LocalhostImpressionsStorageAsync(), 'events': LocalhostEventsStorageAsync(), } @@ -1158,6 +1170,7 @@ async def _build_localhost_factory_async(cfg): synchronizers = SplitSynchronizers( LocalSplitSynchronizerAsync(cfg['splitFile'], storages['splits'], + storages['rule_based_segments'], localhost_mode), LocalSegmentSynchronizerAsync(cfg['segmentDirectory'], storages['splits'], storages['segments']), None, None, None, diff --git a/splitio/engine/evaluator.py b/splitio/engine/evaluator.py index f913ebba..26875a68 100644 --- a/splitio/engine/evaluator.py +++ b/splitio/engine/evaluator.py @@ -6,10 +6,13 @@ from splitio.models.grammar.condition import ConditionType from splitio.models.grammar.matchers.misc import DependencyMatcher from splitio.models.grammar.matchers.keys import UserDefinedSegmentMatcher +from splitio.models.grammar.matchers import RuleBasedSegmentMatcher +from splitio.models.grammar.matchers.prerequisites import PrerequisitesMatcher +from splitio.models.rule_based_segments import SegmentType from splitio.optional.loaders import asyncio CONTROL = 'control' -EvaluationContext = namedtuple('EvaluationContext', ['flags', 'segment_memberships']) +EvaluationContext = namedtuple('EvaluationContext', ['flags', 'segment_memberships', 'rbs_segments']) _LOGGER = logging.getLogger(__name__) @@ -54,13 +57,9 @@ def eval_with_context(self, key, bucketing, feature_name, attrs, ctx): label = Label.KILLED _treatment = feature.default_treatment else: - treatment, label = self._treatment_for_flag(feature, key, bucketing, attrs, ctx) - if treatment is None: - label = Label.NO_CONDITION_MATCHED - _treatment = feature.default_treatment - else: - _treatment = treatment - + label, _treatment = self._check_prerequisites(feature, bucketing, key, attrs, ctx, label, _treatment) + label, _treatment = self._get_treatment(feature, bucketing, key, attrs, ctx, label, _treatment) + return { 'treatment': _treatment, 'configurations': feature.get_configurations_for(_treatment) if feature else None, @@ -71,6 +70,30 @@ def eval_with_context(self, key, bucketing, feature_name, attrs, ctx): 'impressions_disabled': feature.impressions_disabled if feature else None } + def _get_treatment(self, feature, bucketing, key, attrs, ctx, label, _treatment): + if _treatment == CONTROL: + treatment, label = self._treatment_for_flag(feature, key, bucketing, attrs, ctx) + if treatment is None: + label = Label.NO_CONDITION_MATCHED + _treatment = feature.default_treatment + else: + _treatment = treatment + + return label, _treatment + + def _check_prerequisites(self, feature, bucketing, key, attrs, ctx, label, _treatment): + if feature.prerequisites is not None: + prerequisites_matcher = PrerequisitesMatcher(feature.prerequisites) + if not prerequisites_matcher.match(key, attrs, { + 'evaluator': self, + 'bucketing_key': bucketing, + 'ec': ctx}): + label = Label.PREREQUISITES_NOT_MET + _treatment = feature.default_treatment + + return label, _treatment + + def _treatment_for_flag(self, flag, key, bucketing, attributes, ctx): """ ... @@ -98,9 +121,10 @@ def _treatment_for_flag(self, flag, key, bucketing, attributes, ctx): class EvaluationDataFactory: - def __init__(self, split_storage, segment_storage): + def __init__(self, split_storage, segment_storage, rbs_segment_storage): self._flag_storage = split_storage self._segment_storage = segment_storage + self._rbs_segment_storage = rbs_segment_storage def context_for(self, key, feature_names): """ @@ -112,30 +136,31 @@ def context_for(self, key, feature_names): :rtype: EvaluationContext """ pending = set(feature_names) + pending_rbs = set() splits = {} + rb_segments = {} pending_memberships = set() - while pending: + while pending or pending_rbs: fetched = self._flag_storage.fetch_many(list(pending)) - features = filter_missing(fetched) - splits.update(features) - pending = set() - for feature in features.values(): - cf, cs = get_dependencies(feature) - pending.update(filter(lambda f: f not in splits, cf)) - pending_memberships.update(cs) - - return EvaluationContext(splits, { - segment: self._segment_storage.segment_contains(segment, key) - for segment in pending_memberships - }) - - + fetched_rbs = self._rbs_segment_storage.fetch_many(list(pending_rbs)) + features, rbsegments, splits, rb_segments = update_objects(fetched, fetched_rbs, splits, rb_segments) + pending, pending_memberships, pending_rbs = get_pending_objects(features, splits, rbsegments, rb_segments, pending_memberships) + + return EvaluationContext( + splits, + { segment: self._segment_storage.segment_contains(segment, key) + for segment in pending_memberships + }, + rb_segments + ) + class AsyncEvaluationDataFactory: - def __init__(self, split_storage, segment_storage): + def __init__(self, split_storage, segment_storage, rbs_segment_storage): self._flag_storage = split_storage self._segment_storage = segment_storage - + self._rbs_segment_storage = rbs_segment_storage + async def context_for(self, key, feature_names): """ Recursively iterate & fetch all data required to evaluate these flags. @@ -146,41 +171,79 @@ async def context_for(self, key, feature_names): :rtype: EvaluationContext """ pending = set(feature_names) + pending_rbs = set() splits = {} + rb_segments = {} pending_memberships = set() - while pending: + while pending or pending_rbs: fetched = await self._flag_storage.fetch_many(list(pending)) - features = filter_missing(fetched) - splits.update(features) - pending = set() - for feature in features.values(): - cf, cs = get_dependencies(feature) - pending.update(filter(lambda f: f not in splits, cf)) - pending_memberships.update(cs) + fetched_rbs = await self._rbs_segment_storage.fetch_many(list(pending_rbs)) + features, rbsegments, splits, rb_segments = update_objects(fetched, fetched_rbs, splits, rb_segments) + pending, pending_memberships, pending_rbs = get_pending_objects(features, splits, rbsegments, rb_segments, pending_memberships) segment_names = list(pending_memberships) segment_memberships = await asyncio.gather(*[ self._segment_storage.segment_contains(segment, key) for segment in segment_names ]) - - return EvaluationContext(splits, dict(zip(segment_names, segment_memberships))) - - -def get_dependencies(feature): + + return EvaluationContext( + splits, + dict(zip(segment_names, segment_memberships)), + rb_segments + ) + +def get_dependencies(object): """ :rtype: tuple(list, list) """ feature_names = [] segment_names = [] - for condition in feature.conditions: + rbs_segment_names = [] + for condition in object.conditions: for matcher in condition.matchers: + if isinstance(matcher,RuleBasedSegmentMatcher): + rbs_segment_names.append(matcher._rbs_segment_name) if isinstance(matcher,UserDefinedSegmentMatcher): segment_names.append(matcher._segment_name) elif isinstance(matcher, DependencyMatcher): feature_names.append(matcher._split_name) - return feature_names, segment_names + return feature_names, segment_names, rbs_segment_names def filter_missing(features): return {k: v for (k, v) in features.items() if v is not None} + +def get_pending_objects(features, splits, rbsegments, rb_segments, pending_memberships): + pending = set() + pending_rbs = set() + for feature in features.values(): + cf, cs, crbs = get_dependencies(feature) + cf.extend(get_prerequisites(feature)) + pending.update(filter(lambda f: f not in splits, cf)) + pending_memberships.update(cs) + pending_rbs.update(filter(lambda f: f not in rb_segments, crbs)) + + for rb_segment in rbsegments.values(): + cf, cs, crbs = get_dependencies(rb_segment) + pending.update(filter(lambda f: f not in splits, cf)) + pending_memberships.update(cs) + for excluded_segment in rb_segment.excluded.get_excluded_segments(): + if excluded_segment.type == SegmentType.STANDARD: + pending_memberships.add(excluded_segment.name) + else: + pending_rbs.update(filter(lambda f: f not in rb_segments, [excluded_segment.name])) + pending_rbs.update(filter(lambda f: f not in rb_segments, crbs)) + + return pending, pending_memberships, pending_rbs + +def update_objects(fetched, fetched_rbs, splits, rb_segments): + features = filter_missing(fetched) + rbsegments = filter_missing(fetched_rbs) + splits.update(features) + rb_segments.update(rbsegments) + + return features, rbsegments, splits, rb_segments + +def get_prerequisites(feature): + return [prerequisite.feature_flag_name for prerequisite in feature.prerequisites] diff --git a/splitio/models/grammar/condition.py b/splitio/models/grammar/condition.py index 778c7867..79fdb928 100644 --- a/splitio/models/grammar/condition.py +++ b/splitio/models/grammar/condition.py @@ -119,10 +119,12 @@ def from_raw(raw_condition): :return: A condition object. :rtype: Condition """ - parsed_partitions = [ - partitions.from_raw(raw_partition) - for raw_partition in raw_condition['partitions'] - ] + parsed_partitions = [] + if raw_condition.get("partitions") is not None: + parsed_partitions = [ + partitions.from_raw(raw_partition) + for raw_partition in raw_condition['partitions'] + ] matcher_objects = [matchers.from_raw(x) for x in raw_condition['matcherGroup']['matchers']] diff --git a/splitio/models/grammar/matchers/__init__.py b/splitio/models/grammar/matchers/__init__.py index 34006e8b..def75626 100644 --- a/splitio/models/grammar/matchers/__init__.py +++ b/splitio/models/grammar/matchers/__init__.py @@ -10,6 +10,7 @@ from splitio.models.grammar.matchers.misc import BooleanMatcher, DependencyMatcher from splitio.models.grammar.matchers.semver import EqualToSemverMatcher, GreaterThanOrEqualToSemverMatcher, LessThanOrEqualToSemverMatcher, \ BetweenSemverMatcher, InListSemverMatcher +from splitio.models.grammar.matchers.rule_based_segment import RuleBasedSegmentMatcher MATCHER_TYPE_ALL_KEYS = 'ALL_KEYS' @@ -34,6 +35,7 @@ MATCHER_LESS_THAN_OR_EQUAL_TO_SEMVER = 'LESS_THAN_OR_EQUAL_TO_SEMVER' MATCHER_BETWEEN_SEMVER = 'BETWEEN_SEMVER' MATCHER_INLIST_SEMVER = 'IN_LIST_SEMVER' +MATCHER_IN_RULE_BASED_SEGMENT = 'IN_RULE_BASED_SEGMENT' _MATCHER_BUILDERS = { @@ -58,7 +60,8 @@ MATCHER_GREATER_THAN_OR_EQUAL_TO_SEMVER: GreaterThanOrEqualToSemverMatcher, MATCHER_LESS_THAN_OR_EQUAL_TO_SEMVER: LessThanOrEqualToSemverMatcher, MATCHER_BETWEEN_SEMVER: BetweenSemverMatcher, - MATCHER_INLIST_SEMVER: InListSemverMatcher + MATCHER_INLIST_SEMVER: InListSemverMatcher, + MATCHER_IN_RULE_BASED_SEGMENT: RuleBasedSegmentMatcher } def from_raw(raw_matcher): diff --git a/splitio/models/grammar/matchers/prerequisites.py b/splitio/models/grammar/matchers/prerequisites.py new file mode 100644 index 00000000..799df5c4 --- /dev/null +++ b/splitio/models/grammar/matchers/prerequisites.py @@ -0,0 +1,38 @@ +"""Prerequisites matcher classes.""" + +class PrerequisitesMatcher(object): + + def __init__(self, prerequisites): + """ + Build a PrerequisitesMatcher. + + :param prerequisites: prerequisites + :type raw_matcher: List of Prerequisites + """ + self._prerequisites = prerequisites + + def match(self, key, attributes=None, context=None): + """ + Evaluate user input against a matcher and return whether the match is successful. + + :param key: User key. + :type key: str. + :param attributes: Custom user attributes. + :type attributes: dict. + :param context: Evaluation context + :type context: dict + + :returns: Wheter the match is successful. + :rtype: bool + """ + if self._prerequisites == None: + return True + + evaluator = context.get('evaluator') + bucketing_key = context.get('bucketing_key') + for prerequisite in self._prerequisites: + result = evaluator.eval_with_context(key, bucketing_key, prerequisite.feature_flag_name, attributes, context['ec']) + if result['treatment'] not in prerequisite.treatments: + return False + + return True \ No newline at end of file diff --git a/splitio/models/grammar/matchers/rule_based_segment.py b/splitio/models/grammar/matchers/rule_based_segment.py new file mode 100644 index 00000000..6e4c8023 --- /dev/null +++ b/splitio/models/grammar/matchers/rule_based_segment.py @@ -0,0 +1,72 @@ +"""Rule based segment matcher classes.""" +from splitio.models.grammar.matchers.base import Matcher +from splitio.models.rule_based_segments import SegmentType + +class RuleBasedSegmentMatcher(Matcher): + + def _build(self, raw_matcher): + """ + Build an RuleBasedSegmentMatcher. + + :param raw_matcher: raw matcher as fetched from splitChanges response. + :type raw_matcher: dict + """ + self._rbs_segment_name = raw_matcher['userDefinedSegmentMatcherData']['segmentName'] + + def _match(self, key, attributes=None, context=None): + """ + Evaluate user input against a matcher and return whether the match is successful. + + :param key: User key. + :type key: str. + :param attributes: Custom user attributes. + :type attributes: dict. + :param context: Evaluation context + :type context: dict + + :returns: Wheter the match is successful. + :rtype: bool + """ + if self._rbs_segment_name == None: + return False + + rb_segment = context['ec'].rbs_segments.get(self._rbs_segment_name) + + if key in rb_segment.excluded.get_excluded_keys(): + return False + + if self._match_dep_rb_segments(rb_segment.excluded.get_excluded_segments(), key, attributes, context): + return False + + return self._match_conditions(rb_segment.conditions, key, attributes, context) + + def _add_matcher_specific_properties_to_json(self): + """Return UserDefinedSegment specific properties.""" + return { + 'userDefinedSegmentMatcherData': { + 'segmentName': self._rbs_segment_name + } + } + + def _match_conditions(self, rbs_segment_conditions, key, attributes, context): + for parsed_condition in rbs_segment_conditions: + if parsed_condition.matches(key, attributes, context): + return True + + return False + + def _match_dep_rb_segments(self, excluded_rb_segments, key, attributes, context): + for excluded_rb_segment in excluded_rb_segments: + if excluded_rb_segment.type == SegmentType.STANDARD: + if context['ec'].segment_memberships[excluded_rb_segment.name]: + return True + else: + excluded_segment = context['ec'].rbs_segments.get(excluded_rb_segment.name) + if key in excluded_segment.excluded.get_excluded_keys(): + return False + + if self._match_dep_rb_segments(excluded_segment.excluded.get_excluded_segments(), key, attributes, context) \ + or self._match_conditions(excluded_segment.conditions, key, attributes, context): + return True + + return False diff --git a/splitio/models/impressions.py b/splitio/models/impressions.py index 9bdfb3a9..9224d15b 100644 --- a/splitio/models/impressions.py +++ b/splitio/models/impressions.py @@ -60,3 +60,8 @@ class Label(object): # pylint: disable=too-few-public-methods # Treatment: control # Label: not ready NOT_READY = 'not ready' + + # Condition: Prerequisites not met + # Treatment: Default treatment + # Label: prerequisites not met + PREREQUISITES_NOT_MET = "prerequisites not met" diff --git a/splitio/models/rule_based_segments.py b/splitio/models/rule_based_segments.py new file mode 100644 index 00000000..f7bf3f4d --- /dev/null +++ b/splitio/models/rule_based_segments.py @@ -0,0 +1,195 @@ +"""RuleBasedSegment module.""" + +from enum import Enum +import logging + +from splitio.models import MatcherNotFoundException +from splitio.models.splits import _DEFAULT_CONDITIONS_TEMPLATE +from splitio.models.grammar import condition +from splitio.models.splits import Status + +_LOGGER = logging.getLogger(__name__) + +class SegmentType(Enum): + """Segment type.""" + + STANDARD = "standard" + RULE_BASED = "rule-based" + +class RuleBasedSegment(object): + """RuleBasedSegment object class.""" + + def __init__(self, name, traffic_type_name, change_number, status, conditions, excluded): + """ + Class constructor. + + :param name: Segment name. + :type name: str + :param traffic_type_name: traffic type name. + :type traffic_type_name: str + :param change_number: change number. + :type change_number: str + :param status: status. + :type status: str + :param conditions: List of conditions belonging to the segment. + :type conditions: List + :param excluded: excluded objects. + :type excluded: Excluded + """ + self._name = name + self._traffic_type_name = traffic_type_name + self._change_number = change_number + self._conditions = conditions + self._excluded = excluded + try: + self._status = Status(status) + except ValueError: + self._status = Status.ARCHIVED + + @property + def name(self): + """Return segment name.""" + return self._name + + @property + def traffic_type_name(self): + """Return traffic type name.""" + return self._traffic_type_name + + @property + def change_number(self): + """Return change number.""" + return self._change_number + + @property + def status(self): + """Return status.""" + return self._status + + @property + def conditions(self): + """Return conditions.""" + return self._conditions + + @property + def excluded(self): + """Return excluded.""" + return self._excluded + + def to_json(self): + """Return a JSON representation of this rule based segment.""" + return { + 'changeNumber': self.change_number, + 'trafficTypeName': self.traffic_type_name, + 'name': self.name, + 'status': self.status.value, + 'conditions': [c.to_json() for c in self.conditions], + 'excluded': self.excluded.to_json() + } + + def get_condition_segment_names(self): + segments = set() + for condition in self._conditions: + for matcher in condition.matchers: + if matcher._matcher_type == 'IN_SEGMENT': + segments.add(matcher.to_json()['userDefinedSegmentMatcherData']['segmentName']) + return segments + +def from_raw(raw_rule_based_segment): + """ + Parse a Rule based segment from a JSON portion of splitChanges. + + :param raw_rule_based_segment: JSON object extracted from a splitChange's response + :type raw_rule_based_segment: dict + + :return: A parsed RuleBasedSegment object capable of performing evaluations. + :rtype: RuleBasedSegment + """ + try: + conditions = [condition.from_raw(c) for c in raw_rule_based_segment['conditions']] + except MatcherNotFoundException as e: + _LOGGER.error(str(e)) + _LOGGER.debug("Using default conditions template for feature flag: %s", raw_rule_based_segment['name']) + conditions = [condition.from_raw(_DEFAULT_CONDITIONS_TEMPLATE)] + + if raw_rule_based_segment.get('excluded') == None: + raw_rule_based_segment['excluded'] = {'keys': [], 'segments': []} + + if raw_rule_based_segment['excluded'].get('keys') == None: + raw_rule_based_segment['excluded']['keys'] = [] + + if raw_rule_based_segment['excluded'].get('segments') == None: + raw_rule_based_segment['excluded']['segments'] = [] + + return RuleBasedSegment( + raw_rule_based_segment['name'], + raw_rule_based_segment['trafficTypeName'], + raw_rule_based_segment['changeNumber'], + raw_rule_based_segment['status'], + conditions, + Excluded(raw_rule_based_segment['excluded']['keys'], raw_rule_based_segment['excluded']['segments']) + ) + +class Excluded(object): + + def __init__(self, keys, segments): + """ + Class constructor. + + :param keys: List of excluded keys in a rule based segment. + :type keys: List + :param segments: List of excluded segments in a rule based segment. + :type segments: List + """ + self._keys = keys + self._segments = [ExcludedSegment(segment['name'], segment['type']) for segment in segments] + + def get_excluded_keys(self): + """Return excluded keys.""" + return self._keys + + def get_excluded_segments(self): + """Return excluded segments""" + return self._segments + + def get_excluded_standard_segments(self): + """Return excluded segments""" + to_return = [] + for segment in self._segments: + if segment.type == SegmentType.STANDARD: + to_return.append(segment.name) + return to_return + + def to_json(self): + """Return a JSON representation of this object.""" + return { + 'keys': self._keys, + 'segments': self._segments + } + +class ExcludedSegment(object): + + def __init__(self, name, type): + """ + Class constructor. + + :param name: rule based segment name + :type name: str + :param type: segment type + :type type: str + """ + self._name = name + try: + self._type = SegmentType(type) + except ValueError: + self._type = SegmentType.STANDARD + + @property + def name(self): + """Return name.""" + return self._name + + @property + def type(self): + """Return type.""" + return self._type diff --git a/splitio/models/splits.py b/splitio/models/splits.py index 92a277c4..47e69284 100644 --- a/splitio/models/splits.py +++ b/splitio/models/splits.py @@ -10,7 +10,7 @@ SplitView = namedtuple( 'SplitView', - ['name', 'traffic_type', 'killed', 'treatments', 'change_number', 'configs', 'default_treatment', 'sets', 'impressions_disabled'] + ['name', 'traffic_type', 'killed', 'treatments', 'change_number', 'configs', 'default_treatment', 'sets', 'impressions_disabled', 'prerequisites'] ) _DEFAULT_CONDITIONS_TEMPLATE = { @@ -40,7 +40,28 @@ "label": "targeting rule type unsupported by sdk" } +class Prerequisites(object): + """Prerequisites.""" + def __init__(self, feature_flag_name, treatments): + self._feature_flag_name = feature_flag_name + self._treatments = treatments + + @property + def feature_flag_name(self): + """Return featur eflag name.""" + return self._feature_flag_name + @property + def treatments(self): + """Return treatments.""" + return self._treatments + + def to_json(self): + to_return = [] + for feature_flag_name in self._feature_flag_name: + to_return.append({"n": feature_flag_name, "ts": [treatment for treatment in self._treatments]}) + + return to_return class Status(Enum): """Split status.""" @@ -74,7 +95,8 @@ def __init__( # pylint: disable=too-many-arguments traffic_allocation_seed=None, configurations=None, sets=None, - impressions_disabled=None + impressions_disabled=None, + prerequisites = None ): """ Class constructor. @@ -99,6 +121,8 @@ def __init__( # pylint: disable=too-many-arguments :type sets: list :pram impressions_disabled: track impressions flag :type impressions_disabled: boolean + :pram prerequisites: prerequisites + :type prerequisites: List of Preqreuisites """ self._name = name self._seed = seed @@ -129,6 +153,7 @@ def __init__( # pylint: disable=too-many-arguments self._configurations = configurations self._sets = set(sets) if sets is not None else set() self._impressions_disabled = impressions_disabled if impressions_disabled is not None else False + self._prerequisites = prerequisites if prerequisites is not None else [] @property def name(self): @@ -194,6 +219,11 @@ def sets(self): def impressions_disabled(self): """Return impressions_disabled of the split.""" return self._impressions_disabled + + @property + def prerequisites(self): + """Return prerequisites of the split.""" + return self._prerequisites def get_configurations_for(self, treatment): """Return the mapping of treatments to configurations.""" @@ -224,7 +254,8 @@ def to_json(self): 'conditions': [c.to_json() for c in self.conditions], 'configurations': self._configurations, 'sets': list(self._sets), - 'impressionsDisabled': self._impressions_disabled + 'impressionsDisabled': self._impressions_disabled, + 'prerequisites': [prerequisite.to_json() for prerequisite in self._prerequisites] } def to_split_view(self): @@ -243,7 +274,8 @@ def to_split_view(self): self._configurations if self._configurations is not None else {}, self._default_treatment, list(self._sets) if self._sets is not None else [], - self._impressions_disabled + self._impressions_disabled, + self._prerequisites ) def local_kill(self, default_treatment, change_number): @@ -300,5 +332,13 @@ def from_raw(raw_split): traffic_allocation_seed=raw_split.get('trafficAllocationSeed'), configurations=raw_split.get('configurations'), sets=set(raw_split.get('sets')) if raw_split.get('sets') is not None else [], - impressions_disabled=raw_split.get('impressionsDisabled') if raw_split.get('impressionsDisabled') is not None else False + impressions_disabled=raw_split.get('impressionsDisabled') if raw_split.get('impressionsDisabled') is not None else False, + prerequisites=from_raw_prerequisites(raw_split.get('prerequisites')) if raw_split.get('prerequisites') is not None else [] ) + +def from_raw_prerequisites(raw_prerequisites): + to_return = [] + for prerequisite in raw_prerequisites: + to_return.append(Prerequisites(prerequisite['n'], prerequisite['ts'])) + + return to_return \ No newline at end of file diff --git a/splitio/models/telemetry.py b/splitio/models/telemetry.py index f734cf67..c9715da4 100644 --- a/splitio/models/telemetry.py +++ b/splitio/models/telemetry.py @@ -140,6 +140,7 @@ class OperationMode(Enum): class UpdateFromSSE(Enum): """Update from sse constants""" SPLIT_UPDATE = 'sp' + RBS_UPDATE = 'rbs' def get_latency_bucket_index(micros): """ diff --git a/splitio/push/parser.py b/splitio/push/parser.py index 098221e1..79b410e3 100644 --- a/splitio/push/parser.py +++ b/splitio/push/parser.py @@ -28,6 +28,7 @@ class UpdateType(Enum): SPLIT_UPDATE = 'SPLIT_UPDATE' SPLIT_KILL = 'SPLIT_KILL' SEGMENT_UPDATE = 'SEGMENT_UPDATE' + RB_SEGMENT_UPDATE = 'RB_SEGMENT_UPDATE' class ControlType(Enum): @@ -329,7 +330,7 @@ def __init__(self, channel, timestamp, change_number, previous_change_number, fe """Class constructor.""" BaseUpdate.__init__(self, channel, timestamp, change_number) self._previous_change_number = previous_change_number - self._feature_flag_definition = feature_flag_definition + self._object_definition = feature_flag_definition self._compression = compression @property @@ -352,13 +353,13 @@ def previous_change_number(self): # pylint:disable=no-self-use return self._previous_change_number @property - def feature_flag_definition(self): # pylint:disable=no-self-use + def object_definition(self): # pylint:disable=no-self-use """ Return feature flag definition :returns: The new feature flag definition :rtype: str """ - return self._feature_flag_definition + return self._object_definition @property def compression(self): # pylint:disable=no-self-use @@ -451,6 +452,56 @@ def __str__(self): """Return string representation.""" return "SegmentChange - changeNumber=%d, name=%s" % (self.change_number, self.segment_name) +class RBSChangeUpdate(BaseUpdate): + """rbs Change notification.""" + + def __init__(self, channel, timestamp, change_number, previous_change_number, rbs_definition, compression): + """Class constructor.""" + BaseUpdate.__init__(self, channel, timestamp, change_number) + self._previous_change_number = previous_change_number + self._object_definition = rbs_definition + self._compression = compression + + @property + def update_type(self): # pylint:disable=no-self-use + """ + Return the message type. + + :returns: The type of this parsed Update. + :rtype: UpdateType + """ + return UpdateType.RB_SEGMENT_UPDATE + + @property + def previous_change_number(self): # pylint:disable=no-self-use + """ + Return previous change number + :returns: The previous change number + :rtype: int + """ + return self._previous_change_number + + @property + def object_definition(self): # pylint:disable=no-self-use + """ + Return rbs definition + :returns: The new rbs definition + :rtype: str + """ + return self._object_definition + + @property + def compression(self): # pylint:disable=no-self-use + """ + Return previous compression type + :returns: The compression type + :rtype: int + """ + return self._compression + + def __str__(self): + """Return string representation.""" + return "RBSChange - changeNumber=%d" % (self.change_number) class ControlMessage(BaseMessage): """Control notification.""" @@ -503,6 +554,9 @@ def _parse_update(channel, timestamp, data): if update_type == UpdateType.SPLIT_UPDATE and change_number is not None: return SplitChangeUpdate(channel, timestamp, change_number, data.get('pcn'), data.get('d'), data.get('c')) + if update_type == UpdateType.RB_SEGMENT_UPDATE and change_number is not None: + return RBSChangeUpdate(channel, timestamp, change_number, data.get('pcn'), data.get('d'), data.get('c')) + elif update_type == UpdateType.SPLIT_KILL and change_number is not None: return SplitKillUpdate(channel, timestamp, change_number, data['splitName'], data['defaultTreatment']) diff --git a/splitio/push/processor.py b/splitio/push/processor.py index e8de95c8..41d796c7 100644 --- a/splitio/push/processor.py +++ b/splitio/push/processor.py @@ -35,12 +35,13 @@ def __init__(self, synchronizer, telemetry_runtime_producer): self._feature_flag_queue = Queue() self._segments_queue = Queue() self._synchronizer = synchronizer - self._feature_flag_worker = SplitWorker(synchronizer.synchronize_splits, synchronizer.synchronize_segment, self._feature_flag_queue, synchronizer.split_sync.feature_flag_storage, synchronizer.segment_storage, telemetry_runtime_producer) + self._feature_flag_worker = SplitWorker(synchronizer.synchronize_splits, synchronizer.synchronize_segment, self._feature_flag_queue, synchronizer.split_sync.feature_flag_storage, synchronizer.segment_storage, telemetry_runtime_producer, synchronizer.split_sync.rule_based_segment_storage) self._segments_worker = SegmentWorker(synchronizer.synchronize_segment, self._segments_queue) self._handlers = { UpdateType.SPLIT_UPDATE: self._handle_feature_flag_update, UpdateType.SPLIT_KILL: self._handle_feature_flag_kill, - UpdateType.SEGMENT_UPDATE: self._handle_segment_change + UpdateType.SEGMENT_UPDATE: self._handle_segment_change, + UpdateType.RB_SEGMENT_UPDATE: self._handle_feature_flag_update } def _handle_feature_flag_update(self, event): @@ -119,12 +120,13 @@ def __init__(self, synchronizer, telemetry_runtime_producer): self._feature_flag_queue = asyncio.Queue() self._segments_queue = asyncio.Queue() self._synchronizer = synchronizer - self._feature_flag_worker = SplitWorkerAsync(synchronizer.synchronize_splits, synchronizer.synchronize_segment, self._feature_flag_queue, synchronizer.split_sync.feature_flag_storage, synchronizer.segment_storage, telemetry_runtime_producer) + self._feature_flag_worker = SplitWorkerAsync(synchronizer.synchronize_splits, synchronizer.synchronize_segment, self._feature_flag_queue, synchronizer.split_sync.feature_flag_storage, synchronizer.segment_storage, telemetry_runtime_producer, synchronizer.split_sync.rule_based_segment_storage) self._segments_worker = SegmentWorkerAsync(synchronizer.synchronize_segment, self._segments_queue) self._handlers = { UpdateType.SPLIT_UPDATE: self._handle_feature_flag_update, UpdateType.SPLIT_KILL: self._handle_feature_flag_kill, - UpdateType.SEGMENT_UPDATE: self._handle_segment_change + UpdateType.SEGMENT_UPDATE: self._handle_segment_change, + UpdateType.RB_SEGMENT_UPDATE: self._handle_feature_flag_update } async def _handle_feature_flag_update(self, event): diff --git a/splitio/push/workers.py b/splitio/push/workers.py index 5161d15d..e0dd8369 100644 --- a/splitio/push/workers.py +++ b/splitio/push/workers.py @@ -9,11 +9,13 @@ from enum import Enum from splitio.models.splits import from_raw +from splitio.models.rule_based_segments import from_raw as rbs_from_raw from splitio.models.telemetry import UpdateFromSSE from splitio.push import SplitStorageException from splitio.push.parser import UpdateType from splitio.optional.loaders import asyncio -from splitio.util.storage_helper import update_feature_flag_storage, update_feature_flag_storage_async +from splitio.util.storage_helper import update_feature_flag_storage, update_feature_flag_storage_async, \ + update_rule_based_segment_storage, update_rule_based_segment_storage_async _LOGGER = logging.getLogger(__name__) @@ -25,14 +27,16 @@ class CompressionMode(Enum): ZLIB_COMPRESSION = 2 _compression_handlers = { - CompressionMode.NO_COMPRESSION: lambda event: base64.b64decode(event.feature_flag_definition), - CompressionMode.GZIP_COMPRESSION: lambda event: gzip.decompress(base64.b64decode(event.feature_flag_definition)).decode('utf-8'), - CompressionMode.ZLIB_COMPRESSION: lambda event: zlib.decompress(base64.b64decode(event.feature_flag_definition)).decode('utf-8'), + CompressionMode.NO_COMPRESSION: lambda event: base64.b64decode(event.object_definition), + CompressionMode.GZIP_COMPRESSION: lambda event: gzip.decompress(base64.b64decode(event.object_definition)).decode('utf-8'), + CompressionMode.ZLIB_COMPRESSION: lambda event: zlib.decompress(base64.b64decode(event.object_definition)).decode('utf-8'), } class WorkerBase(object, metaclass=abc.ABCMeta): """Worker template.""" + _fetching_segment = "Fetching new segment {segment_name}" + @abc.abstractmethod def is_running(self): """Return whether the working is running.""" @@ -45,10 +49,19 @@ def start(self): def stop(self): """Stop worker.""" - def _get_feature_flag_definition(self, event): - """return feature flag definition in event.""" + def _get_object_definition(self, event): + """return feature flag or rule based segment definition in event.""" cm = CompressionMode(event.compression) # will throw if the number is not defined in compression mode return _compression_handlers[cm](event) + + def _get_referenced_rbs(self, feature_flag): + referenced_rbs = set() + for condition in feature_flag.conditions: + for matcher in condition.matchers: + raw_matcher = matcher.to_json() + if raw_matcher['matcherType'] == 'IN_RULE_BASED_SEGMENT': + referenced_rbs.add(raw_matcher['userDefinedSegmentMatcherData']['segmentName']) + return referenced_rbs class SegmentWorker(WorkerBase): """Segment Worker for processing updates.""" @@ -173,7 +186,7 @@ class SplitWorker(WorkerBase): _centinel = object() - def __init__(self, synchronize_feature_flag, synchronize_segment, feature_flag_queue, feature_flag_storage, segment_storage, telemetry_runtime_producer): + def __init__(self, synchronize_feature_flag, synchronize_segment, feature_flag_queue, feature_flag_storage, segment_storage, telemetry_runtime_producer, rule_based_segment_storage): """ Class constructor. @@ -189,6 +202,8 @@ def __init__(self, synchronize_feature_flag, synchronize_segment, feature_flag_q :type segment_storage: splitio.storage.inmemory.InMemorySegmentStorage :param telemetry_runtime_producer: Telemetry runtime producer instance :type telemetry_runtime_producer: splitio.engine.telemetry.TelemetryRuntimeProducer + :param rule_based_segment_storage: Rule based segment Storage. + :type rule_based_segment_storage: splitio.storage.InMemoryRuleBasedStorage """ self._feature_flag_queue = feature_flag_queue self._handler = synchronize_feature_flag @@ -198,6 +213,7 @@ def __init__(self, synchronize_feature_flag, synchronize_segment, feature_flag_q self._feature_flag_storage = feature_flag_storage self._segment_storage = segment_storage self._telemetry_runtime_producer = telemetry_runtime_producer + self._rule_based_segment_storage = rule_based_segment_storage def is_running(self): """Return whether the working is running.""" @@ -206,25 +222,43 @@ def is_running(self): def _apply_iff_if_needed(self, event): if not self._check_instant_ff_update(event): return False - try: - new_feature_flag = from_raw(json.loads(self._get_feature_flag_definition(event))) - segment_list = update_feature_flag_storage(self._feature_flag_storage, [new_feature_flag], event.change_number) - for segment_name in segment_list: - if self._segment_storage.get(segment_name) is None: - _LOGGER.debug('Fetching new segment %s', segment_name) - self._segment_handler(segment_name, event.change_number) - - self._telemetry_runtime_producer.record_update_from_sse(UpdateFromSSE.SPLIT_UPDATE) + if event.update_type == UpdateType.SPLIT_UPDATE: + new_feature_flag = from_raw(json.loads(self._get_object_definition(event))) + segment_list = update_feature_flag_storage(self._feature_flag_storage, [new_feature_flag], event.change_number) + for segment_name in segment_list: + if self._segment_storage.get(segment_name) is None: + _LOGGER.debug(self._fetching_segment.format(segment_name=segment_name)) + self._segment_handler(segment_name, event.change_number) + + referenced_rbs = self._get_referenced_rbs(new_feature_flag) + self._fetch_rbs_segment_if_needed(referenced_rbs, event) + self._telemetry_runtime_producer.record_update_from_sse(UpdateFromSSE.SPLIT_UPDATE) + else: + new_rbs = rbs_from_raw(json.loads(self._get_object_definition(event))) + segment_list = update_rule_based_segment_storage(self._rule_based_segment_storage, [new_rbs], event.change_number) + for segment_name in segment_list: + if self._segment_storage.get(segment_name) is None: + _LOGGER.debug(self._fetching_segment.format(segment_name=segment_name)) + self._segment_handler(segment_name, event.change_number) + self._telemetry_runtime_producer.record_update_from_sse(UpdateFromSSE.RBS_UPDATE) return True - + except Exception as e: raise SplitStorageException(e) + def _fetch_rbs_segment_if_needed(self, referenced_rbs, event): + if len(referenced_rbs) > 0 and not self._rule_based_segment_storage.contains(referenced_rbs): + _LOGGER.debug('Fetching new rule based segment(s) %s', referenced_rbs) + self._handler(None, event.change_number) + def _check_instant_ff_update(self, event): if event.update_type == UpdateType.SPLIT_UPDATE and event.compression is not None and event.previous_change_number == self._feature_flag_storage.get_change_number(): return True + if event.update_type == UpdateType.RB_SEGMENT_UPDATE and event.compression is not None and event.previous_change_number == self._rule_based_segment_storage.get_change_number(): + return True + return False def _run(self): @@ -235,12 +269,16 @@ def _run(self): break if event == self._centinel: continue + _LOGGER.debug('Processing feature flag update %d', event.change_number) try: if self._apply_iff_if_needed(event): continue - - sync_result = self._handler(event.change_number) + + till = None + rbs_till = None + till, rbs_till = self._check_update_type(till, rbs_till, event) + sync_result = self._handler(till, rbs_till) if not sync_result.success and sync_result.error_code is not None and sync_result.error_code == 414: _LOGGER.error("URI too long exception caught, sync failed") @@ -254,6 +292,14 @@ def _run(self): _LOGGER.error('Exception raised in feature flag synchronization') _LOGGER.debug('Exception information: ', exc_info=True) + def _check_update_type(self, till, rbs_till, event): + if event.update_type == UpdateType.SPLIT_UPDATE: + till = event.change_number + else: + rbs_till = event.change_number + + return till, rbs_till + def start(self): """Start worker.""" if self.is_running(): @@ -279,7 +325,7 @@ class SplitWorkerAsync(WorkerBase): _centinel = object() - def __init__(self, synchronize_feature_flag, synchronize_segment, feature_flag_queue, feature_flag_storage, segment_storage, telemetry_runtime_producer): + def __init__(self, synchronize_feature_flag, synchronize_segment, feature_flag_queue, feature_flag_storage, segment_storage, telemetry_runtime_producer, rule_based_segment_storage): """ Class constructor. @@ -295,6 +341,8 @@ def __init__(self, synchronize_feature_flag, synchronize_segment, feature_flag_q :type segment_storage: splitio.storage.inmemory.InMemorySegmentStorage :param telemetry_runtime_producer: Telemetry runtime producer instance :type telemetry_runtime_producer: splitio.engine.telemetry.TelemetryRuntimeProducer + :param rule_based_segment_storage: Rule based segment Storage. + :type rule_based_segment_storage: splitio.storage.InMemoryRuleBasedStorage """ self._feature_flag_queue = feature_flag_queue self._handler = synchronize_feature_flag @@ -303,7 +351,8 @@ def __init__(self, synchronize_feature_flag, synchronize_segment, feature_flag_q self._feature_flag_storage = feature_flag_storage self._segment_storage = segment_storage self._telemetry_runtime_producer = telemetry_runtime_producer - + self._rule_based_segment_storage = rule_based_segment_storage + def is_running(self): """Return whether the working is running.""" return self._running @@ -312,23 +361,42 @@ async def _apply_iff_if_needed(self, event): if not await self._check_instant_ff_update(event): return False try: - new_feature_flag = from_raw(json.loads(self._get_feature_flag_definition(event))) - segment_list = await update_feature_flag_storage_async(self._feature_flag_storage, [new_feature_flag], event.change_number) - for segment_name in segment_list: - if await self._segment_storage.get(segment_name) is None: - _LOGGER.debug('Fetching new segment %s', segment_name) - await self._segment_handler(segment_name, event.change_number) - - await self._telemetry_runtime_producer.record_update_from_sse(UpdateFromSSE.SPLIT_UPDATE) + if event.update_type == UpdateType.SPLIT_UPDATE: + new_feature_flag = from_raw(json.loads(self._get_object_definition(event))) + segment_list = await update_feature_flag_storage_async(self._feature_flag_storage, [new_feature_flag], event.change_number) + for segment_name in segment_list: + if await self._segment_storage.get(segment_name) is None: + _LOGGER.debug(self._fetching_segment.format(segment_name=segment_name)) + await self._segment_handler(segment_name, event.change_number) + + referenced_rbs = self._get_referenced_rbs(new_feature_flag) + await self._fetch_rbs_segment_if_needed(referenced_rbs, event) + await self._telemetry_runtime_producer.record_update_from_sse(UpdateFromSSE.SPLIT_UPDATE) + else: + new_rbs = rbs_from_raw(json.loads(self._get_object_definition(event))) + segment_list = await update_rule_based_segment_storage_async(self._rule_based_segment_storage, [new_rbs], event.change_number) + for segment_name in segment_list: + if await self._segment_storage.get(segment_name) is None: + _LOGGER.debug(self._fetching_segment.format(segment_name=segment_name)) + await self._segment_handler(segment_name, event.change_number) + await self._telemetry_runtime_producer.record_update_from_sse(UpdateFromSSE.RBS_UPDATE) return True except Exception as e: raise SplitStorageException(e) + async def _fetch_rbs_segment_if_needed(self, referenced_rbs, event): + if len(referenced_rbs) > 0 and not await self._rule_based_segment_storage.contains(referenced_rbs): + _LOGGER.debug('Fetching new rule based segment(s) %s', referenced_rbs) + await self._handler(None, event.change_number) async def _check_instant_ff_update(self, event): if event.update_type == UpdateType.SPLIT_UPDATE and event.compression is not None and event.previous_change_number == await self._feature_flag_storage.get_change_number(): return True + + if event.update_type == UpdateType.RB_SEGMENT_UPDATE and event.compression is not None and event.previous_change_number == await self._rule_based_segment_storage.get_change_number(): + return True + return False async def _run(self): @@ -343,7 +411,13 @@ async def _run(self): try: if await self._apply_iff_if_needed(event): continue - await self._handler(event.change_number) + till = None + rbs_till = None + if event.update_type == UpdateType.SPLIT_UPDATE: + till = event.change_number + else: + rbs_till = event.change_number + await self._handler(till, rbs_till) except SplitStorageException as e: # pylint: disable=broad-except _LOGGER.error('Exception Updating Feature Flag') _LOGGER.debug('Exception information: ', exc_info=True) diff --git a/splitio/spec.py b/splitio/spec.py index 1388fcda..cd7588e0 100644 --- a/splitio/spec.py +++ b/splitio/spec.py @@ -1 +1 @@ -SPEC_VERSION = '1.1' +SPEC_VERSION = '1.3' diff --git a/splitio/storage/__init__.py b/splitio/storage/__init__.py index cd3bf1a0..079ee863 100644 --- a/splitio/storage/__init__.py +++ b/splitio/storage/__init__.py @@ -354,4 +354,75 @@ def intersect(self, flag_sets): if not isinstance(flag_sets, set) or len(flag_sets) == 0: return False - return any(self.flag_sets.intersection(flag_sets)) \ No newline at end of file + return any(self.flag_sets.intersection(flag_sets)) + +class RuleBasedSegmentsStorage(object, metaclass=abc.ABCMeta): + """SplitRule based segment storage interface implemented as an abstract class.""" + + @abc.abstractmethod + def get(self, segment_name): + """ + Retrieve a rule based segment. + + :param segment_name: Name of the segment to fetch. + :type segment_name: str + + :rtype: str + """ + pass + + @abc.abstractmethod + def update(self, to_add, to_delete, new_change_number): + """ + Update rule based segment.. + + :param to_add: List of rule based segment. to add + :type to_add: list[splitio.models.rule_based_segments.RuleBasedSegment] + :param to_delete: List of rule based segment. to delete + :type to_delete: list[splitio.models.rule_based_segments.RuleBasedSegment] + :param new_change_number: New change number. + :type new_change_number: int + """ + pass + + @abc.abstractmethod + def get_change_number(self): + """ + Retrieve latest rule based segment change number. + + :rtype: int + """ + pass + + @abc.abstractmethod + def contains(self, segment_names): + """ + Return whether the segments exists in rule based segment in cache. + + :param segment_names: segment name to validate. + :type segment_names: str + + :return: True if segment names exists. False otherwise. + :rtype: bool + """ + pass + + @abc.abstractmethod + def get_segment_names(self): + """ + Retrieve a list of all excluded segments names. + + :return: List of segment names. + :rtype: list(str) + """ + pass + + @abc.abstractmethod + def get_large_segment_names(self): + """ + Retrieve a list of all excluded large segments names. + + :return: List of segment names. + :rtype: list(str) + """ + pass \ No newline at end of file diff --git a/splitio/storage/inmemmory.py b/splitio/storage/inmemmory.py index e4cf3da3..e1740b72 100644 --- a/splitio/storage/inmemmory.py +++ b/splitio/storage/inmemmory.py @@ -7,7 +7,7 @@ from splitio.models.segments import Segment from splitio.models.telemetry import HTTPErrors, HTTPLatencies, MethodExceptions, MethodLatencies, LastSynchronization, StreamingEvents, TelemetryConfig, TelemetryCounters, CounterConstants, \ HTTPErrorsAsync, HTTPLatenciesAsync, MethodExceptionsAsync, MethodLatenciesAsync, LastSynchronizationAsync, StreamingEventsAsync, TelemetryConfigAsync, TelemetryCountersAsync -from splitio.storage import FlagSetsFilter, SplitStorage, SegmentStorage, ImpressionStorage, EventStorage, TelemetryStorage +from splitio.storage import FlagSetsFilter, SplitStorage, SegmentStorage, ImpressionStorage, EventStorage, TelemetryStorage, RuleBasedSegmentsStorage from splitio.optional.loaders import asyncio MAX_SIZE_BYTES = 5 * 1024 * 1024 @@ -107,6 +107,259 @@ def remove_flag_set(self, flag_sets, feature_flag_name, should_filter): if self.flag_set_exist(flag_set) and len(self.get_flag_set(flag_set)) == 0 and not should_filter: self._remove_flag_set(flag_set) +class InMemoryRuleBasedSegmentStorage(RuleBasedSegmentsStorage): + """InMemory implementation of a feature flag storage base.""" + + def __init__(self): + """Constructor.""" + self._lock = threading.RLock() + self._rule_based_segments = {} + self._change_number = -1 + + def clear(self): + """ + Clear storage + """ + with self._lock: + self._rule_based_segments = {} + self._change_number = -1 + + def get(self, segment_name): + """ + Retrieve a rule based segment. + + :param segment_name: Name of the segment to fetch. + :type segment_name: str + + :rtype: splitio.models.rule_based_segments.RuleBasedSegment + """ + with self._lock: + return self._rule_based_segments.get(segment_name) + + def update(self, to_add, to_delete, new_change_number): + """ + Update rule based segment. + + :param to_add: List of rule based segment. to add + :type to_add: list[splitio.models.rule_based_segments.RuleBasedSegment] + :param to_delete: List of rule based segment. to delete + :type to_delete: list[splitio.models.rule_based_segments.RuleBasedSegment] + :param new_change_number: New change number. + :type new_change_number: int + """ + [self._put(add_segment) for add_segment in to_add] + [self._remove(delete_segment) for delete_segment in to_delete] + self._set_change_number(new_change_number) + + def _put(self, rule_based_segment): + """ + Store a rule based segment. + + :param rule_based_segment: RuleBasedSegment object. + :type rule_based_segment: splitio.models.rule_based_segments.RuleBasedSegment + """ + with self._lock: + self._rule_based_segments[rule_based_segment.name] = rule_based_segment + + def _remove(self, segment_name): + """ + Remove a rule based segment. + + :param segment_name: Name of the rule based segment to remove. + :type segment_name: str + + :return: True if the rule based segment was found and removed. False otherwise. + :rtype: bool + """ + with self._lock: + rule_based_segment = self._rule_based_segments.get(segment_name) + if not rule_based_segment: + _LOGGER.warning("Tried to delete nonexistant Rule based segment %s. Skipping", segment_name) + return False + + self._rule_based_segments.pop(segment_name) + return True + + def get_change_number(self): + """ + Retrieve latest rule based segment change number. + + :rtype: int + """ + with self._lock: + return self._change_number + + def _set_change_number(self, new_change_number): + """ + Set the latest change number. + + :param new_change_number: New change number. + :type new_change_number: int + """ + with self._lock: + self._change_number = new_change_number + + def get_segment_names(self): + """ + Retrieve a list of all rule based segments names. + + :return: List of segment names. + :rtype: list(str) + """ + with self._lock: + return list(self._rule_based_segments.keys()) + + def get_large_segment_names(self): + """ + Retrieve a list of all excluded large segments names. + + :return: List of segment names. + :rtype: list(str) + """ + pass + + def contains(self, segment_names): + """ + Return whether the segment exists in storage + + :param segment_names: rule based segment name + :type segment_names: str + + :return: True if the segment exists. False otherwise. + :rtype: bool + """ + with self._lock: + return set(segment_names).issubset(self._rule_based_segments.keys()) + + def fetch_many(self, segment_names): + return {rb_segment_name: self.get(rb_segment_name) for rb_segment_name in segment_names} + +class InMemoryRuleBasedSegmentStorageAsync(RuleBasedSegmentsStorage): + """InMemory implementation of a feature flag storage base.""" + def __init__(self): + """Constructor.""" + self._lock = asyncio.Lock() + self._rule_based_segments = {} + self._change_number = -1 + + async def clear(self): + """ + Clear storage + """ + async with self._lock: + self._rule_based_segments = {} + self._change_number = -1 + + async def get(self, segment_name): + """ + Retrieve a rule based segment. + + :param segment_name: Name of the segment to fetch. + :type segment_name: str + + :rtype: splitio.models.rule_based_segments.RuleBasedSegment + """ + async with self._lock: + return self._rule_based_segments.get(segment_name) + + async def update(self, to_add, to_delete, new_change_number): + """ + Update rule based segment. + + :param to_add: List of rule based segment. to add + :type to_add: list[splitio.models.rule_based_segments.RuleBasedSegment] + :param to_delete: List of rule based segment. to delete + :type to_delete: list[splitio.models.rule_based_segments.RuleBasedSegment] + :param new_change_number: New change number. + :type new_change_number: int + """ + [await self._put(add_segment) for add_segment in to_add] + [await self._remove(delete_segment) for delete_segment in to_delete] + await self._set_change_number(new_change_number) + + async def _put(self, rule_based_segment): + """ + Store a rule based segment. + + :param rule_based_segment: RuleBasedSegment object. + :type rule_based_segment: splitio.models.rule_based_segments.RuleBasedSegment + """ + async with self._lock: + self._rule_based_segments[rule_based_segment.name] = rule_based_segment + + async def _remove(self, segment_name): + """ + Remove a rule based segment. + + :param segment_name: Name of the rule based segment to remove. + :type segment_name: str + + :return: True if the rule based segment was found and removed. False otherwise. + :rtype: bool + """ + async with self._lock: + rule_based_segment = self._rule_based_segments.get(segment_name) + if not rule_based_segment: + _LOGGER.warning("Tried to delete nonexistant Rule based segment %s. Skipping", segment_name) + return False + + self._rule_based_segments.pop(segment_name) + return True + + async def get_change_number(self): + """ + Retrieve latest rule based segment change number. + + :rtype: int + """ + async with self._lock: + return self._change_number + + async def _set_change_number(self, new_change_number): + """ + Set the latest change number. + + :param new_change_number: New change number. + :type new_change_number: int + """ + async with self._lock: + self._change_number = new_change_number + + async def get_segment_names(self): + """ + Retrieve a list of all excluded segments names. + + :return: List of segment names. + :rtype: list(str) + """ + async with self._lock: + return list(self._rule_based_segments.keys()) + + async def get_large_segment_names(self): + """ + Retrieve a list of all excluded large segments names. + + :return: List of segment names. + :rtype: list(str) + """ + pass + + async def contains(self, segment_names): + """ + Return whether the segment exists in storage + + :param segment_names: rule based segment name + :type segment_names: str + + :return: True if the segment exists. False otherwise. + :rtype: bool + """ + async with self._lock: + return set(segment_names).issubset(self._rule_based_segments.keys()) + + async def fetch_many(self, segment_names): + return {rb_segment_name: await self.get(rb_segment_name) for rb_segment_name in segment_names} + class InMemorySplitStorageBase(SplitStorage): """InMemory implementation of a feature flag storage base.""" @@ -235,6 +488,16 @@ def __init__(self, flag_sets=[]): self.flag_set = FlagSets(flag_sets) self.flag_set_filter = FlagSetsFilter(flag_sets) + def clear(self): + """ + Clear storage + """ + with self._lock: + self._feature_flags = {} + self._change_number = -1 + self._traffic_types = Counter() + self.flag_set = FlagSets(self.flag_set_filter.flag_sets) + def get(self, feature_flag_name): """ Retrieve a feature flag. @@ -441,6 +704,16 @@ def __init__(self, flag_sets=[]): self.flag_set = FlagSets(flag_sets) self.flag_set_filter = FlagSetsFilter(flag_sets) + async def clear(self): + """ + Clear storage + """ + async with self._lock: + self._feature_flags = {} + self._change_number = -1 + self._traffic_types = Counter() + self.flag_set = FlagSets(self.flag_set_filter.flag_sets) + async def get(self, feature_flag_name): """ Retrieve a feature flag. diff --git a/splitio/storage/pluggable.py b/splitio/storage/pluggable.py index 7f0a5287..36b27d7d 100644 --- a/splitio/storage/pluggable.py +++ b/splitio/storage/pluggable.py @@ -5,15 +5,295 @@ import threading from splitio.optional.loaders import asyncio -from splitio.models import splits, segments +from splitio.models import splits, segments, rule_based_segments from splitio.models.impressions import Impression from splitio.models.telemetry import MethodExceptions, MethodLatencies, TelemetryConfig, MAX_TAGS,\ MethodLatenciesAsync, MethodExceptionsAsync, TelemetryConfigAsync -from splitio.storage import FlagSetsFilter, SplitStorage, SegmentStorage, ImpressionStorage, EventStorage, TelemetryStorage +from splitio.storage import FlagSetsFilter, SplitStorage, SegmentStorage, ImpressionStorage, EventStorage, TelemetryStorage, RuleBasedSegmentsStorage from splitio.util.storage_helper import get_valid_flag_sets, combine_valid_flag_sets _LOGGER = logging.getLogger(__name__) +class PluggableRuleBasedSegmentsStorageBase(RuleBasedSegmentsStorage): + """Pluggable storage for rule based segments.""" + + _TILL_LENGTH = 4 + + def __init__(self, pluggable_adapter, prefix=None): + """ + Class constructor. + + :param redis_client: Redis client or compliant interface. + :type redis_client: splitio.storage.adapters.redis.RedisAdapter + """ + self._pluggable_adapter = pluggable_adapter + self._prefix = "SPLITIO.rbsegment.{segment_name}" + self._rb_segments_till_prefix = "SPLITIO.rbsegments.till" + self._rb_segment_name_length = 18 + if prefix is not None: + self._rb_segment_name_length += len(prefix) + 1 + self._prefix = prefix + "." + self._prefix + self._rb_segments_till_prefix = prefix + "." + self._rb_segments_till_prefix + + def get(self, segment_name): + """ + Retrieve a rule based segment. + + :param segment_name: Name of the segment to fetch. + :type segment_name: str + + :rtype: str + """ + pass + + def get_change_number(self): + """ + Retrieve latest rule based segment change number. + + :rtype: int + """ + pass + + def contains(self, segment_names): + """ + Return whether the segments exists in rule based segment in cache. + + :param segment_names: segment name to validate. + :type segment_names: str + + :return: True if segment names exists. False otherwise. + :rtype: bool + """ + pass + + def get_segment_names(self): + """ + Retrieve a list of all excluded segments names. + + :return: List of segment names. + :rtype: list(str) + """ + pass + + def update(self, to_add, to_delete, new_change_number): + """ + Update rule based segment.. + + :param to_add: List of rule based segment. to add + :type to_add: list[splitio.models.rule_based_segments.RuleBasedSegment] + :param to_delete: List of rule based segment. to delete + :type to_delete: list[splitio.models.rule_based_segments.RuleBasedSegment] + :param new_change_number: New change number. + :type new_change_number: int + """ + raise NotImplementedError('Only redis-consumer mode is supported.') + + def get_large_segment_names(self): + """ + Retrieve a list of all excluded large segments names. + + :return: List of segment names. + :rtype: list(str) + """ + pass + +class PluggableRuleBasedSegmentsStorage(PluggableRuleBasedSegmentsStorageBase): + """Pluggable storage for rule based segments.""" + + def __init__(self, pluggable_adapter, prefix=None): + """ + Class constructor. + + :param redis_client: Redis client or compliant interface. + :type redis_client: splitio.storage.adapters.redis.RedisAdapter + """ + PluggableRuleBasedSegmentsStorageBase.__init__(self, pluggable_adapter, prefix) + + def get(self, segment_name): + """ + Retrieve a rule based segment. + + :param segment_name: Name of the segment to fetch. + :type segment_name: str + + :rtype: str + """ + try: + rb_segment = self._pluggable_adapter.get(self._prefix.format(segment_name=segment_name)) + if not rb_segment: + return None + + return rule_based_segments.from_raw(rb_segment) + + except Exception: + _LOGGER.error('Error getting rule based segment from storage') + _LOGGER.debug('Error: ', exc_info=True) + return None + + def get_change_number(self): + """ + Retrieve latest rule based segment change number. + + :rtype: int + """ + try: + return self._pluggable_adapter.get(self._rb_segments_till_prefix) + + except Exception: + _LOGGER.error('Error getting change number in rule based segment storage') + _LOGGER.debug('Error: ', exc_info=True) + return None + + def contains(self, segment_names): + """ + Return whether the segments exists in rule based segment in cache. + + :param segment_names: segment name to validate. + :type segment_names: str + + :return: True if segment names exists. False otherwise. + :rtype: bool + """ + return set(segment_names).issubset(self.get_segment_names()) + + def get_segment_names(self): + """ + Retrieve a list of all rule based segments names. + + :return: List of segment names. + :rtype: list(str) + """ + try: + _LOGGER.error(self._rb_segment_name_length) + _LOGGER.error(self._prefix) + _LOGGER.error(self._prefix[:self._rb_segment_name_length]) + keys = [] + for key in self._pluggable_adapter.get_keys_by_prefix(self._prefix[:self._rb_segment_name_length]): + if key[-self._TILL_LENGTH:] != 'till': + keys.append(key[len(self._prefix[:self._rb_segment_name_length]):]) + return keys + + except Exception: + _LOGGER.error('Error getting rule based segments names from storage') + _LOGGER.debug('Error: ', exc_info=True) + return None + + def fetch_many(self, rb_segment_names): + """ + Retrieve rule based segments. + + :param rb_segment_names: Names of the rule based segments to fetch. + :type rb_segment_names: list(str) + + :return: A dict with rule based segment objects parsed from queue. + :rtype: dict(rb_segment_names, splitio.models.rile_based_segment.RuleBasedSegment) + """ + try: + prefix_added = [self._prefix.format(segment_name=rb_segment_name) for rb_segment_name in rb_segment_names] + return {rb_segment['name']: rule_based_segments.from_raw(rb_segment) for rb_segment in self._pluggable_adapter.get_many(prefix_added)} + + except Exception: + _LOGGER.error('Error getting rule based segments from storage') + _LOGGER.debug('Error: ', exc_info=True) + return None + +class PluggableRuleBasedSegmentsStorageAsync(PluggableRuleBasedSegmentsStorageBase): + """Pluggable storage for rule based segments.""" + + def __init__(self, pluggable_adapter, prefix=None): + """ + Class constructor. + + :param redis_client: Redis client or compliant interface. + :type redis_client: splitio.storage.adapters.redis.RedisAdapter + """ + PluggableRuleBasedSegmentsStorageBase.__init__(self, pluggable_adapter, prefix) + + async def get(self, segment_name): + """ + Retrieve a rule based segment. + + :param segment_name: Name of the segment to fetch. + :type segment_name: str + + :rtype: str + """ + try: + rb_segment = await self._pluggable_adapter.get(self._prefix.format(segment_name=segment_name)) + if not rb_segment: + return None + + return rule_based_segments.from_raw(rb_segment) + + except Exception: + _LOGGER.error('Error getting rule based segment from storage') + _LOGGER.debug('Error: ', exc_info=True) + return None + + async def get_change_number(self): + """ + Retrieve latest rule based segment change number. + + :rtype: int + """ + try: + return await self._pluggable_adapter.get(self._rb_segments_till_prefix) + + except Exception: + _LOGGER.error('Error getting change number in rule based segment storage') + _LOGGER.debug('Error: ', exc_info=True) + return None + + async def contains(self, segment_names): + """ + Return whether the segments exists in rule based segment in cache. + + :param segment_names: segment name to validate. + :type segment_names: str + + :return: True if segment names exists. False otherwise. + :rtype: bool + """ + return set(segment_names).issubset(await self.get_segment_names()) + + async def get_segment_names(self): + """ + Retrieve a list of all rule based segments names. + + :return: List of segment names. + :rtype: list(str) + """ + try: + keys = [] + for key in await self._pluggable_adapter.get_keys_by_prefix(self._prefix[:self._rb_segment_name_length]): + if key[-self._TILL_LENGTH:] != 'till': + keys.append(key[len(self._prefix[:self._rb_segment_name_length]):]) + return keys + + except Exception: + _LOGGER.error('Error getting rule based segments names from storage') + _LOGGER.debug('Error: ', exc_info=True) + return None + + async def fetch_many(self, rb_segment_names): + """ + Retrieve rule based segments. + + :param rb_segment_names: Names of the rule based segments to fetch. + :type rb_segment_names: list(str) + + :return: A dict with rule based segment objects parsed from queue. + :rtype: dict(rb_segment_names, splitio.models.rile_based_segment.RuleBasedSegment) + """ + try: + prefix_added = [self._prefix.format(segment_name=rb_segment_name) for rb_segment_name in rb_segment_names] + return {rb_segment['name']: rule_based_segments.from_raw(rb_segment) for rb_segment in await self._pluggable_adapter.get_many(prefix_added)} + + except Exception: + _LOGGER.error('Error getting rule based segments from storage') + _LOGGER.debug('Error: ', exc_info=True) + return None + class PluggableSplitStorageBase(SplitStorage): """InMemory implementation of a feature flag storage.""" @@ -90,7 +370,7 @@ def update(self, to_add, to_delete, new_change_number): :param new_change_number: New change number. :type new_change_number: int """ -# pass + pass # try: # split = self.get(feature_flag_name) # if not split: diff --git a/splitio/storage/redis.py b/splitio/storage/redis.py index 982e0213..09ddee29 100644 --- a/splitio/storage/redis.py +++ b/splitio/storage/redis.py @@ -4,10 +4,10 @@ import threading from splitio.models.impressions import Impression -from splitio.models import splits, segments +from splitio.models import splits, segments, rule_based_segments from splitio.models.telemetry import TelemetryConfig, TelemetryConfigAsync from splitio.storage import SplitStorage, SegmentStorage, ImpressionStorage, EventStorage, \ - ImpressionPipelinedStorage, TelemetryStorage, FlagSetsFilter + ImpressionPipelinedStorage, TelemetryStorage, FlagSetsFilter, RuleBasedSegmentsStorage from splitio.storage.adapters.redis import RedisAdapterException from splitio.storage.adapters.cache_trait import decorate as add_cache, DEFAULT_MAX_AGE from splitio.storage.adapters.cache_trait import LocalMemoryCache, LocalMemoryCacheAsync @@ -16,8 +16,296 @@ _LOGGER = logging.getLogger(__name__) MAX_TAGS = 10 +class RedisRuleBasedSegmentsStorage(RuleBasedSegmentsStorage): + """Redis-based storage for rule based segments.""" + + _RB_SEGMENT_KEY = 'SPLITIO.rbsegment.{segment_name}' + _RB_SEGMENT_TILL_KEY = 'SPLITIO.rbsegments.till' + + def __init__(self, redis_client): + """ + Class constructor. + + :param redis_client: Redis client or compliant interface. + :type redis_client: splitio.storage.adapters.redis.RedisAdapter + """ + self._redis = redis_client + self._pipe = self._redis.pipeline + + def _get_key(self, segment_name): + """ + Use the provided feature_flag_name to build the appropriate redis key. + + :param feature_flag_name: Name of the feature flag to interact with in redis. + :type feature_flag_name: str + + :return: Redis key. + :rtype: str. + """ + return self._RB_SEGMENT_KEY.format(segment_name=segment_name) + + def get(self, segment_name): + """ + Retrieve a rule based segment. + + :param segment_name: Name of the segment to fetch. + :type segment_name: str + + :rtype: str + """ + try: + raw = self._redis.get(self._get_key(segment_name)) + _LOGGER.debug("Fetchting rule based segment [%s] from redis" % segment_name) + _LOGGER.debug(raw) + return rule_based_segments.from_raw(json.loads(raw)) if raw is not None else None + + except RedisAdapterException: + _LOGGER.error('Error fetching rule based segment from storage') + _LOGGER.debug('Error: ', exc_info=True) + return None + + def update(self, to_add, to_delete, new_change_number): + """ + Update rule based segment.. + + :param to_add: List of rule based segment. to add + :type to_add: list[splitio.models.rule_based_segments.RuleBasedSegment] + :param to_delete: List of rule based segment. to delete + :type to_delete: list[splitio.models.rule_based_segments.RuleBasedSegment] + :param new_change_number: New change number. + :type new_change_number: int + """ + raise NotImplementedError('Only redis-consumer mode is supported.') + + def get_change_number(self): + """ + Retrieve latest rule based segment change number. + + :rtype: int + """ + try: + stored_value = self._redis.get(self._RB_SEGMENT_TILL_KEY) + _LOGGER.debug("Fetching rule based segment Change Number from redis: %s" % stored_value) + return json.loads(stored_value) if stored_value is not None else None + + except RedisAdapterException: + _LOGGER.error('Error fetching rule based segment change number from storage') + _LOGGER.debug('Error: ', exc_info=True) + return None + + def contains(self, segment_names): + """ + Return whether the segments exists in rule based segment in cache. + + :param segment_names: segment name to validate. + :type segment_names: str + + :return: True if segment names exists. False otherwise. + :rtype: bool + """ + return set(segment_names).issubset(self.get_segment_names()) + + def get_segment_names(self): + """ + Retrieve a list of all rule based segments names. + + :return: List of segment names. + :rtype: list(str) + """ + try: + keys = self._redis.keys(self._get_key('*')) + _LOGGER.debug("Fetchting rule based segments names from redis: %s" % keys) + return [key.replace(self._get_key(''), '') for key in keys] + + except RedisAdapterException: + _LOGGER.error('Error fetching rule based segments names from storage') + _LOGGER.debug('Error: ', exc_info=True) + return [] + + def get_large_segment_names(self): + """ + Retrieve a list of all excluded large segments names. + + :return: List of segment names. + :rtype: list(str) + """ + pass + + def fetch_many(self, segment_names): + """ + Retrieve rule based segment. + + :param segment_names: Names of the rule based segments to fetch. + :type segment_names: list(str) + + :return: A dict with rule based segment objects parsed from redis. + :rtype: dict(segment_name, splitio.models.rule_based_segment.RuleBasedSegment) + """ + to_return = dict() + try: + keys = [self._get_key(segment_name) for segment_name in segment_names] + raw_rbs_segments = self._redis.mget(keys) + _LOGGER.debug("Fetchting rule based segment [%s] from redis" % segment_names) + _LOGGER.debug(raw_rbs_segments) + for i in range(len(raw_rbs_segments)): + rbs_segment = None + try: + rbs_segment = rule_based_segments.from_raw(json.loads(raw_rbs_segments[i])) + except (ValueError, TypeError): + _LOGGER.error('Could not parse rule based segment.') + _LOGGER.debug("Raw rule based segment that failed parsing attempt: %s", raw_rbs_segments[i]) + to_return[segment_names[i]] = rbs_segment + except RedisAdapterException: + _LOGGER.error('Error fetching rule based segments from storage') + _LOGGER.debug('Error: ', exc_info=True) + return to_return + +class RedisRuleBasedSegmentsStorageAsync(RuleBasedSegmentsStorage): + """Redis-based storage for rule based segments.""" + + _RB_SEGMENT_KEY = 'SPLITIO.rbsegment.{segment_name}' + _RB_SEGMENT_TILL_KEY = 'SPLITIO.rbsegments.till' + + def __init__(self, redis_client): + """ + Class constructor. + + :param redis_client: Redis client or compliant interface. + :type redis_client: splitio.storage.adapters.redis.RedisAdapter + """ + self._redis = redis_client + self._pipe = self._redis.pipeline + + def _get_key(self, segment_name): + """ + Use the provided feature_flag_name to build the appropriate redis key. + + :param feature_flag_name: Name of the feature flag to interact with in redis. + :type feature_flag_name: str + + :return: Redis key. + :rtype: str. + """ + return self._RB_SEGMENT_KEY.format(segment_name=segment_name) + + async def get(self, segment_name): + """ + Retrieve a rule based segment. + + :param segment_name: Name of the segment to fetch. + :type segment_name: str + + :rtype: str + """ + try: + raw = await self._redis.get(self._get_key(segment_name)) + _LOGGER.debug("Fetchting rule based segment [%s] from redis" % segment_name) + _LOGGER.debug(raw) + return rule_based_segments.from_raw(json.loads(raw)) if raw is not None else None + + except RedisAdapterException: + _LOGGER.error('Error fetching rule based segment from storage') + _LOGGER.debug('Error: ', exc_info=True) + return None + + async def update(self, to_add, to_delete, new_change_number): + """ + Update rule based segment.. + + :param to_add: List of rule based segment. to add + :type to_add: list[splitio.models.rule_based_segments.RuleBasedSegment] + :param to_delete: List of rule based segment. to delete + :type to_delete: list[splitio.models.rule_based_segments.RuleBasedSegment] + :param new_change_number: New change number. + :type new_change_number: int + """ + raise NotImplementedError('Only redis-consumer mode is supported.') + + async def get_change_number(self): + """ + Retrieve latest rule based segment change number. + + :rtype: int + """ + try: + stored_value = await self._redis.get(self._RB_SEGMENT_TILL_KEY) + _LOGGER.debug("Fetching rule based segment Change Number from redis: %s" % stored_value) + return json.loads(stored_value) if stored_value is not None else None + + except RedisAdapterException: + _LOGGER.error('Error fetching rule based segment change number from storage') + _LOGGER.debug('Error: ', exc_info=True) + return None + + async def contains(self, segment_names): + """ + Return whether the segments exists in rule based segment in cache. + + :param segment_names: segment name to validate. + :type segment_names: str + + :return: True if segment names exists. False otherwise. + :rtype: bool + """ + return set(segment_names).issubset(await self.get_segment_names()) + + async def get_segment_names(self): + """ + Retrieve a list of all rule based segments names. + + :return: List of segment names. + :rtype: list(str) + """ + try: + keys = await self._redis.keys(self._get_key('*')) + _LOGGER.debug("Fetchting rule based segments names from redis: %s" % keys) + return [key.replace(self._get_key(''), '') for key in keys] + + except RedisAdapterException: + _LOGGER.error('Error fetching rule based segments names from storage') + _LOGGER.debug('Error: ', exc_info=True) + return [] + + async def get_large_segment_names(self): + """ + Retrieve a list of all excluded large segments names. + + :return: List of segment names. + :rtype: list(str) + """ + pass + + async def fetch_many(self, segment_names): + """ + Retrieve rule based segment. + + :param segment_names: Names of the rule based segments to fetch. + :type segment_names: list(str) + + :return: A dict with rule based segment objects parsed from redis. + :rtype: dict(segment_name, splitio.models.rule_based_segment.RuleBasedSegment) + """ + to_return = dict() + try: + keys = [self._get_key(segment_name) for segment_name in segment_names] + raw_rbs_segments = await self._redis.mget(keys) + _LOGGER.debug("Fetchting rule based segment [%s] from redis" % segment_names) + _LOGGER.debug(raw_rbs_segments) + for i in range(len(raw_rbs_segments)): + rbs_segment = None + try: + rbs_segment = rule_based_segments.from_raw(json.loads(raw_rbs_segments[i])) + except (ValueError, TypeError): + _LOGGER.error('Could not parse rule based segment.') + _LOGGER.debug("Raw rule based segment that failed parsing attempt: %s", raw_rbs_segments[i]) + to_return[segment_names[i]] = rbs_segment + except RedisAdapterException: + _LOGGER.error('Error fetching rule based segments from storage') + _LOGGER.debug('Error: ', exc_info=True) + return to_return + class RedisSplitStorageBase(SplitStorage): - """Redis-based storage base for s.""" + """Redis-based storage base for feature flags.""" _FEATURE_FLAG_KEY = 'SPLITIO.split.{feature_flag_name}' _FEATURE_FLAG_TILL_KEY = 'SPLITIO.splits.till' diff --git a/splitio/sync/segment.py b/splitio/sync/segment.py index 59d9fad8..a87759e1 100644 --- a/splitio/sync/segment.py +++ b/splitio/sync/segment.py @@ -10,6 +10,7 @@ from splitio.util.backoff import Backoff from splitio.optional.loaders import asyncio, aiofiles from splitio.sync import util +from splitio.util.storage_helper import get_standard_segment_names_in_rbs_storage, get_standard_segment_names_in_rbs_storage_async from splitio.optional.loaders import asyncio _LOGGER = logging.getLogger(__name__) @@ -22,7 +23,7 @@ class SegmentSynchronizer(object): - def __init__(self, segment_api, feature_flag_storage, segment_storage): + def __init__(self, segment_api, feature_flag_storage, segment_storage, rule_based_segment_storage): """ Class constructor. @@ -39,6 +40,7 @@ def __init__(self, segment_api, feature_flag_storage, segment_storage): self._api = segment_api self._feature_flag_storage = feature_flag_storage self._segment_storage = segment_storage + self._rule_based_segment_storage = rule_based_segment_storage self._worker_pool = workerpool.WorkerPool(_MAX_WORKERS, self.synchronize_segment) self._worker_pool.start() self._backoff = Backoff( @@ -181,9 +183,12 @@ def synchronize_segments(self, segment_names = None, dont_wait = False): :rtype: bool """ if segment_names is None: - segment_names = self._feature_flag_storage.get_segment_names() + segment_names = set(self._feature_flag_storage.get_segment_names()) + segment_names.update(get_standard_segment_names_in_rbs_storage(self._rule_based_segment_storage)) for segment_name in segment_names: + _LOGGER.debug("Adding segment name to sync worker") + _LOGGER.debug(segment_name) self._worker_pool.submit_work(segment_name) if (dont_wait): return True @@ -204,7 +209,7 @@ def segment_exist_in_storage(self, segment_name): class SegmentSynchronizerAsync(object): - def __init__(self, segment_api, feature_flag_storage, segment_storage): + def __init__(self, segment_api, feature_flag_storage, segment_storage, rule_based_segment_storage): """ Class constructor. @@ -221,6 +226,7 @@ def __init__(self, segment_api, feature_flag_storage, segment_storage): self._api = segment_api self._feature_flag_storage = feature_flag_storage self._segment_storage = segment_storage + self._rule_based_segment_storage = rule_based_segment_storage self._worker_pool = workerpool.WorkerPoolAsync(_MAX_WORKERS, self.synchronize_segment) self._worker_pool.start() self._backoff = Backoff( @@ -364,7 +370,8 @@ async def synchronize_segments(self, segment_names = None, dont_wait = False): :rtype: bool """ if segment_names is None: - segment_names = await self._feature_flag_storage.get_segment_names() + segment_names = set(await self._feature_flag_storage.get_segment_names()) + segment_names.update(await get_standard_segment_names_in_rbs_storage_async(self._rule_based_segment_storage)) self._jobs = await self._worker_pool.submit_work(segment_names) if (dont_wait): diff --git a/splitio/sync/split.py b/splitio/sync/split.py index 7bb13117..c1b5aa39 100644 --- a/splitio/sync/split.py +++ b/splitio/sync/split.py @@ -10,10 +10,12 @@ from splitio.api import APIException, APIUriException from splitio.api.commons import FetchOptions from splitio.client.input_validator import validate_flag_sets -from splitio.models import splits +from splitio.models import splits, rule_based_segments from splitio.util.backoff import Backoff from splitio.util.time import get_current_epoch_time_ms -from splitio.util.storage_helper import update_feature_flag_storage, update_feature_flag_storage_async +from splitio.util.storage_helper import update_feature_flag_storage, update_feature_flag_storage_async, \ + update_rule_based_segment_storage, update_rule_based_segment_storage_async + from splitio.sync import util from splitio.optional.loaders import asyncio, aiofiles @@ -32,7 +34,7 @@ class SplitSynchronizerBase(object): """Feature Flag changes synchronizer.""" - def __init__(self, feature_flag_api, feature_flag_storage): + def __init__(self, feature_flag_api, feature_flag_storage, rule_based_segment_storage): """ Class constructor. @@ -41,9 +43,13 @@ def __init__(self, feature_flag_api, feature_flag_storage): :param feature_flag_storage: Feature Flag Storage. :type feature_flag_storage: splitio.storage.InMemorySplitStorage + + :param rule_based_segment_storage: Rule based segment Storage. + :type rule_based_segment_storage: splitio.storage.InMemoryRuleBasedStorage """ self._api = feature_flag_api self._feature_flag_storage = feature_flag_storage + self._rule_based_segment_storage = rule_based_segment_storage self._backoff = Backoff( _ON_DEMAND_FETCH_BACKOFF_BASE, _ON_DEMAND_FETCH_BACKOFF_MAX_WAIT) @@ -53,6 +59,11 @@ def feature_flag_storage(self): """Return Feature_flag storage object""" return self._feature_flag_storage + @property + def rule_based_segment_storage(self): + """Return rule base segment storage object""" + return self._rule_based_segment_storage + def _get_config_sets(self): """ Get all filter flag sets cnverrted to string, if no filter flagsets exist return None @@ -64,10 +75,16 @@ def _get_config_sets(self): return ','.join(self._feature_flag_storage.flag_set_filter.sorted_flag_sets) + def _check_exit_conditions(self, till, rbs_till, change_number, rbs_change_number): + return (till is not None and till < change_number) or (rbs_till is not None and rbs_till < rbs_change_number) + + def _check_return_conditions(self, feature_flag_changes): + return feature_flag_changes.get('ff')['t'] == feature_flag_changes.get('ff')['s'] and feature_flag_changes.get('rbs')['t'] == feature_flag_changes.get('rbs')['s'] + class SplitSynchronizer(SplitSynchronizerBase): """Feature Flag changes synchronizer.""" - def __init__(self, feature_flag_api, feature_flag_storage): + def __init__(self, feature_flag_api, feature_flag_storage, rule_based_segment_storage): """ Class constructor. @@ -76,10 +93,13 @@ def __init__(self, feature_flag_api, feature_flag_storage): :param feature_flag_storage: Feature Flag Storage. :type feature_flag_storage: splitio.storage.InMemorySplitStorage + + :param rule_based_segment_storage: Rule based segment Storage. + :type rule_based_segment_storage: splitio.storage.InMemoryRuleBasedStorage """ - SplitSynchronizerBase.__init__(self, feature_flag_api, feature_flag_storage) + SplitSynchronizerBase.__init__(self, feature_flag_api, feature_flag_storage, rule_based_segment_storage) - def _fetch_until(self, fetch_options, till=None): + def _fetch_until(self, fetch_options, till=None, rbs_till=None): """ Hit endpoint, update storage and return when since==till. @@ -89,6 +109,9 @@ def _fetch_until(self, fetch_options, till=None): :param till: Passed till from Streaming. :type till: int + :param rbs_till: Passed rbs till from Streaming. + :type rbs_till: int + :return: last change number :rtype: int """ @@ -97,12 +120,17 @@ def _fetch_until(self, fetch_options, till=None): change_number = self._feature_flag_storage.get_change_number() if change_number is None: change_number = -1 - if till is not None and till < change_number: + + rbs_change_number = self._rule_based_segment_storage.get_change_number() + if rbs_change_number is None: + rbs_change_number = -1 + + if self._check_exit_conditions(till, rbs_till, change_number, rbs_change_number): # the passed till is less than change_number, no need to perform updates - return change_number, segment_list + return change_number, rbs_change_number, segment_list try: - feature_flag_changes = self._api.fetch_splits(change_number, fetch_options) + feature_flag_changes = self._api.fetch_splits(change_number, rbs_change_number, fetch_options) except APIException as exc: if exc._status_code is not None and exc._status_code == 414: _LOGGER.error('Exception caught: the amount of flag sets provided are big causing uri length error.') @@ -112,17 +140,18 @@ def _fetch_until(self, fetch_options, till=None): _LOGGER.error('Exception raised while fetching feature flags') _LOGGER.debug('Exception information: ', exc_info=True) raise exc - fetched_feature_flags = [(splits.from_raw(feature_flag)) for feature_flag in feature_flag_changes.get('splits', [])] - segment_list = update_feature_flag_storage(self._feature_flag_storage, fetched_feature_flags, feature_flag_changes['till']) - if feature_flag_changes['till'] == feature_flag_changes['since']: - return feature_flag_changes['till'], segment_list - - fetched_feature_flags = [(splits.from_raw(feature_flag)) for feature_flag in feature_flag_changes.get('splits', [])] - segment_list = update_feature_flag_storage(self._feature_flag_storage, fetched_feature_flags, feature_flag_changes['till']) - if feature_flag_changes['till'] == feature_flag_changes['since']: - return feature_flag_changes['till'], segment_list - - def _attempt_feature_flag_sync(self, fetch_options, till=None): + + fetched_rule_based_segments = [(rule_based_segments.from_raw(rule_based_segment)) for rule_based_segment in feature_flag_changes.get('rbs').get('d', [])] + rbs_segment_list = update_rule_based_segment_storage(self._rule_based_segment_storage, fetched_rule_based_segments, feature_flag_changes.get('rbs')['t'], self._api.clear_storage) + + fetched_feature_flags = [(splits.from_raw(feature_flag)) for feature_flag in feature_flag_changes.get('ff').get('d', [])] + segment_list.update(update_feature_flag_storage(self._feature_flag_storage, fetched_feature_flags, feature_flag_changes.get('ff')['t'], self._api.clear_storage)) + segment_list.update(rbs_segment_list) + + if self._check_return_conditions(feature_flag_changes): + return feature_flag_changes.get('ff')['t'], feature_flag_changes.get('rbs')['t'], segment_list + + def _attempt_feature_flag_sync(self, fetch_options, till=None, rbs_till=None): """ Hit endpoint, update storage and return True if sync is complete. @@ -132,6 +161,9 @@ def _attempt_feature_flag_sync(self, fetch_options, till=None): :param till: Passed till from Streaming. :type till: int + :param rbs_till: Passed rbs till from Streaming. + :type rbs_till: int + :return: Flags to check if it should perform bypass or operation ended :rtype: bool, int, int """ @@ -140,13 +172,13 @@ def _attempt_feature_flag_sync(self, fetch_options, till=None): remaining_attempts = _ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES while True: remaining_attempts -= 1 - change_number, segment_list = self._fetch_until(fetch_options, till) + change_number, rbs_change_number, segment_list = self._fetch_until(fetch_options, till, rbs_till) final_segment_list.update(segment_list) - if till is None or till <= change_number: - return True, remaining_attempts, change_number, final_segment_list + if (till is None or till <= change_number) and (rbs_till is None or rbs_till <= rbs_change_number): + return True, remaining_attempts, change_number, rbs_change_number, final_segment_list elif remaining_attempts <= 0: - return False, remaining_attempts, change_number, final_segment_list + return False, remaining_attempts, change_number, rbs_change_number, final_segment_list how_long = self._backoff.get() time.sleep(how_long) @@ -163,25 +195,28 @@ def _get_config_sets(self): return ','.join(self._feature_flag_storage.flag_set_filter.sorted_flag_sets) - def synchronize_splits(self, till=None): + def synchronize_splits(self, till=None, rbs_till=None): """ Hit endpoint, update storage and return True if sync is complete. :param till: Passed till from Streaming. :type till: int + + :param rbs_till: Passed rbs till from Streaming. + :type rbs_till: int """ final_segment_list = set() fetch_options = FetchOptions(True, sets=self._get_config_sets()) # Set Cache-Control to no-cache - successful_sync, remaining_attempts, change_number, segment_list = self._attempt_feature_flag_sync(fetch_options, - till) + successful_sync, remaining_attempts, change_number, rbs_change_number, segment_list = self._attempt_feature_flag_sync(fetch_options, + till, rbs_till) final_segment_list.update(segment_list) attempts = _ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES - remaining_attempts if successful_sync: # succedeed sync _LOGGER.debug('Refresh completed in %d attempts.', attempts) return final_segment_list - with_cdn_bypass = FetchOptions(True, change_number, sets=self._get_config_sets()) # Set flag for bypassing CDN - without_cdn_successful_sync, remaining_attempts, change_number, segment_list = self._attempt_feature_flag_sync(with_cdn_bypass, till) + with_cdn_bypass = FetchOptions(True, change_number, rbs_change_number, sets=self._get_config_sets()) # Set flag for bypassing CDN + without_cdn_successful_sync, remaining_attempts, change_number, rbs_change_number, segment_list = self._attempt_feature_flag_sync(with_cdn_bypass, till, rbs_till) final_segment_list.update(segment_list) without_cdn_attempts = _ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES - remaining_attempts if without_cdn_successful_sync: @@ -208,7 +243,7 @@ def kill_split(self, feature_flag_name, default_treatment, change_number): class SplitSynchronizerAsync(SplitSynchronizerBase): """Feature Flag changes synchronizer async.""" - def __init__(self, feature_flag_api, feature_flag_storage): + def __init__(self, feature_flag_api, feature_flag_storage, rule_based_segment_storage): """ Class constructor. @@ -217,10 +252,13 @@ def __init__(self, feature_flag_api, feature_flag_storage): :param feature_flag_storage: Feature Flag Storage. :type feature_flag_storage: splitio.storage.InMemorySplitStorage + + :param rule_based_segment_storage: Rule based segment Storage. + :type rule_based_segment_storage: splitio.storage.InMemoryRuleBasedStorage """ - SplitSynchronizerBase.__init__(self, feature_flag_api, feature_flag_storage) + SplitSynchronizerBase.__init__(self, feature_flag_api, feature_flag_storage, rule_based_segment_storage) - async def _fetch_until(self, fetch_options, till=None): + async def _fetch_until(self, fetch_options, till=None, rbs_till=None): """ Hit endpoint, update storage and return when since==till. @@ -230,6 +268,9 @@ async def _fetch_until(self, fetch_options, till=None): :param till: Passed till from Streaming. :type till: int + :param rbs_till: Passed rbs till from Streaming. + :type rbs_till: int + :return: last change number :rtype: int """ @@ -238,12 +279,17 @@ async def _fetch_until(self, fetch_options, till=None): change_number = await self._feature_flag_storage.get_change_number() if change_number is None: change_number = -1 - if till is not None and till < change_number: + + rbs_change_number = await self._rule_based_segment_storage.get_change_number() + if rbs_change_number is None: + rbs_change_number = -1 + + if self._check_exit_conditions(till, rbs_till, change_number, rbs_change_number): # the passed till is less than change_number, no need to perform updates - return change_number, segment_list + return change_number, rbs_change_number, segment_list try: - feature_flag_changes = await self._api.fetch_splits(change_number, fetch_options) + feature_flag_changes = await self._api.fetch_splits(change_number, rbs_change_number, fetch_options) except APIException as exc: if exc._status_code is not None and exc._status_code == 414: _LOGGER.error('Exception caught: the amount of flag sets provided are big causing uri length error.') @@ -254,12 +300,17 @@ async def _fetch_until(self, fetch_options, till=None): _LOGGER.debug('Exception information: ', exc_info=True) raise exc - fetched_feature_flags = [(splits.from_raw(feature_flag)) for feature_flag in feature_flag_changes.get('splits', [])] - segment_list = await update_feature_flag_storage_async(self._feature_flag_storage, fetched_feature_flags, feature_flag_changes['till']) - if feature_flag_changes['till'] == feature_flag_changes['since']: - return feature_flag_changes['till'], segment_list + fetched_rule_based_segments = [(rule_based_segments.from_raw(rule_based_segment)) for rule_based_segment in feature_flag_changes.get('rbs').get('d', [])] + rbs_segment_list = await update_rule_based_segment_storage_async(self._rule_based_segment_storage, fetched_rule_based_segments, feature_flag_changes.get('rbs')['t'], self._api.clear_storage) + + fetched_feature_flags = [(splits.from_raw(feature_flag)) for feature_flag in feature_flag_changes.get('ff').get('d', [])] + segment_list = await update_feature_flag_storage_async(self._feature_flag_storage, fetched_feature_flags, feature_flag_changes.get('ff')['t'], self._api.clear_storage) + segment_list.update(rbs_segment_list) + + if self._check_return_conditions(feature_flag_changes): + return feature_flag_changes.get('ff')['t'], feature_flag_changes.get('rbs')['t'], segment_list - async def _attempt_feature_flag_sync(self, fetch_options, till=None): + async def _attempt_feature_flag_sync(self, fetch_options, till=None, rbs_till=None): """ Hit endpoint, update storage and return True if sync is complete. @@ -269,6 +320,9 @@ async def _attempt_feature_flag_sync(self, fetch_options, till=None): :param till: Passed till from Streaming. :type till: int + :param rbs_till: Passed rbs till from Streaming. + :type rbs_till: int + :return: Flags to check if it should perform bypass or operation ended :rtype: bool, int, int """ @@ -277,36 +331,39 @@ async def _attempt_feature_flag_sync(self, fetch_options, till=None): remaining_attempts = _ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES while True: remaining_attempts -= 1 - change_number, segment_list = await self._fetch_until(fetch_options, till) + change_number, rbs_change_number, segment_list = await self._fetch_until(fetch_options, till, rbs_till) final_segment_list.update(segment_list) - if till is None or till <= change_number: - return True, remaining_attempts, change_number, final_segment_list + if (till is None or till <= change_number) and (rbs_till is None or rbs_till <= rbs_change_number): + return True, remaining_attempts, change_number, rbs_change_number, final_segment_list elif remaining_attempts <= 0: - return False, remaining_attempts, change_number, final_segment_list + return False, remaining_attempts, change_number, rbs_change_number, final_segment_list how_long = self._backoff.get() await asyncio.sleep(how_long) - async def synchronize_splits(self, till=None): + async def synchronize_splits(self, till=None, rbs_till=None): """ Hit endpoint, update storage and return True if sync is complete. :param till: Passed till from Streaming. :type till: int + + :param rbs_till: Passed rbs till from Streaming. + :type rbs_till: int """ final_segment_list = set() fetch_options = FetchOptions(True, sets=self._get_config_sets()) # Set Cache-Control to no-cache - successful_sync, remaining_attempts, change_number, segment_list = await self._attempt_feature_flag_sync(fetch_options, - till) + successful_sync, remaining_attempts, change_number, rbs_change_number, segment_list = await self._attempt_feature_flag_sync(fetch_options, + till, rbs_till) final_segment_list.update(segment_list) attempts = _ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES - remaining_attempts if successful_sync: # succedeed sync _LOGGER.debug('Refresh completed in %d attempts.', attempts) return final_segment_list - with_cdn_bypass = FetchOptions(True, change_number, sets=self._get_config_sets()) # Set flag for bypassing CDN - without_cdn_successful_sync, remaining_attempts, change_number, segment_list = await self._attempt_feature_flag_sync(with_cdn_bypass, till) + with_cdn_bypass = FetchOptions(True, change_number, rbs_change_number, sets=self._get_config_sets()) # Set flag for bypassing CDN + without_cdn_successful_sync, remaining_attempts, change_number, rbs_change_number, segment_list = await self._attempt_feature_flag_sync(with_cdn_bypass, till, rbs_till) final_segment_list.update(segment_list) without_cdn_attempts = _ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES - remaining_attempts if without_cdn_successful_sync: @@ -342,6 +399,25 @@ class LocalSplitSynchronizerBase(object): """Localhost mode feature_flag base synchronizer.""" _DEFAULT_FEATURE_FLAG_TILL = -1 + _DEFAULT_RB_SEGMENT_TILL = -1 + + def __init__(self, filename, feature_flag_storage, rule_based_segment_storage, localhost_mode=LocalhostMode.LEGACY): + """ + Class constructor. + + :param filename: File to parse feature flags from. + :type filename: str + :param feature_flag_storage: Feature flag Storage. + :type feature_flag_storage: splitio.storage.InMemorySplitStorage + :param localhost_mode: mode for localhost either JSON, YAML or LEGACY. + :type localhost_mode: splitio.sync.split.LocalhostMode + """ + self._filename = filename + self._feature_flag_storage = feature_flag_storage + self._rule_based_segment_storage = rule_based_segment_storage + self._localhost_mode = localhost_mode + self._current_ff_sha = "-1" + self._current_rbs_sha = "-1" @staticmethod def _make_feature_flag(feature_flag_name, conditions, configs=None): @@ -363,7 +439,8 @@ def _make_feature_flag(feature_flag_name, conditions, configs=None): 'defaultTreatment': 'control', 'algo': 2, 'conditions': conditions, - 'configurations': configs + 'configurations': configs, + 'prerequisites': [] }) @staticmethod @@ -406,10 +483,10 @@ def _make_whitelist_condition(whitelist, treatment): 'combiner': 'AND' } } - - def _sanitize_feature_flag(self, parsed): + + def _sanitize_json_elements(self, parsed): """ - implement Sanitization if neded. + Sanitize all json elements. :param parsed: feature flags, till and since elements dict :type parsed: Dict @@ -417,14 +494,14 @@ def _sanitize_feature_flag(self, parsed): :return: sanitized structure dict :rtype: Dict """ - parsed = self._sanitize_json_elements(parsed) - parsed['splits'] = self._sanitize_feature_flag_elements(parsed['splits']) - + parsed = self._satitize_json_section(parsed, 'ff') + parsed = self._satitize_json_section(parsed, 'rbs') + return parsed - def _sanitize_json_elements(self, parsed): + def _satitize_json_section(self, parsed, section_name): """ - Sanitize all json elements. + Sanitize specific json section. :param parsed: feature flags, till and since elements dict :type parsed: Dict @@ -432,15 +509,17 @@ def _sanitize_json_elements(self, parsed): :return: sanitized structure dict :rtype: Dict """ - if 'splits' not in parsed: - parsed['splits'] = [] - if 'till' not in parsed or parsed['till'] is None or parsed['till'] < -1: - parsed['till'] = -1 - if 'since' not in parsed or parsed['since'] is None or parsed['since'] < -1 or parsed['since'] > parsed['till']: - parsed['since'] = parsed['till'] + if section_name not in parsed: + parsed['ff'] = {"t": -1, "s": -1, "d": []} + if 'd' not in parsed[section_name]: + parsed[section_name]['d'] = [] + if 't' not in parsed[section_name] or parsed[section_name]['t'] is None or parsed[section_name]['t'] < -1: + parsed[section_name]['t'] = -1 + if 's' not in parsed[section_name] or parsed[section_name]['s'] is None or parsed[section_name]['s'] < -1 or parsed[section_name]['s'] > parsed[section_name]['t']: + parsed[section_name]['s'] = parsed[section_name]['t'] return parsed - + def _sanitize_feature_flag_elements(self, parsed_feature_flags): """ Sanitize all feature flags elements. @@ -470,9 +549,36 @@ def _sanitize_feature_flag_elements(self, parsed_feature_flags): if 'sets' not in feature_flag: feature_flag['sets'] = [] feature_flag['sets'] = validate_flag_sets(feature_flag['sets'], 'Localhost Validator') + if 'prerequisites' not in feature_flag: + feature_flag['prerequisites'] = [] sanitized_feature_flags.append(feature_flag) return sanitized_feature_flags + def _sanitize_rb_segment_elements(self, parsed_rb_segments): + """ + Sanitize all rule based segments elements. + + :param parsed_rb_segments: rule based segments array + :type parsed_rb_segments: [Dict] + + :return: sanitized structure dict + :rtype: [Dict] + """ + sanitized_rb_segments = [] + for rb_segment in parsed_rb_segments: + if 'name' not in rb_segment or rb_segment['name'].strip() == '': + _LOGGER.warning("A rule based segment in json file does not have (Name) or property is empty, skipping.") + continue + + for element in [('trafficTypeName', 'user', None, None, None, None), + ('status', splits.Status.ACTIVE.value, None, None, [e.value for e in splits.Status], None), + ('changeNumber', 0, 0, None, None, None)]: + rb_segment = util._sanitize_object_element(rb_segment, 'rule based segment', element[0], element[1], lower_value=element[2], upper_value=element[3], in_list=element[4], not_in_list=element[5]) + rb_segment = self._sanitize_condition(rb_segment) + rb_segment = self._remove_partition(rb_segment) + sanitized_rb_segments.append(rb_segment) + return sanitized_rb_segments + def _sanitize_condition(self, feature_flag): """ Sanitize feature flag and ensure a condition type ROLLOUT and matcher exist with ALL_KEYS elements. @@ -521,9 +627,18 @@ def _sanitize_condition(self, feature_flag): { "treatment": "off", "size": 100 } ], "label": "default rule" - }) + }) return feature_flag + + def _remove_partition(self, rb_segment): + sanitized = [] + for condition in rb_segment['conditions']: + if 'partition' in condition: + del condition['partition'] + sanitized.append(condition) + rb_segment['conditions'] = sanitized + return rb_segment @classmethod def _convert_yaml_to_feature_flag(cls, parsed): @@ -547,11 +662,14 @@ def _convert_yaml_to_feature_flag(cls, parsed): to_return[feature_flag_name] = cls._make_feature_flag(feature_flag_name, whitelist + all_keys, configs) return to_return + def _check_exit_conditions(self, storage_cn, parsed_till, default_till): + if storage_cn > parsed_till and parsed_till != default_till: + return True class LocalSplitSynchronizer(LocalSplitSynchronizerBase): """Localhost mode feature_flag synchronizer.""" - def __init__(self, filename, feature_flag_storage, localhost_mode=LocalhostMode.LEGACY): + def __init__(self, filename, feature_flag_storage, rule_based_segment_storage, localhost_mode=LocalhostMode.LEGACY): """ Class constructor. @@ -562,11 +680,8 @@ def __init__(self, filename, feature_flag_storage, localhost_mode=LocalhostMode. :param localhost_mode: mode for localhost either JSON, YAML or LEGACY. :type localhost_mode: splitio.sync.split.LocalhostMode """ - self._filename = filename - self._feature_flag_storage = feature_flag_storage - self._localhost_mode = localhost_mode - self._current_json_sha = "-1" - + LocalSplitSynchronizerBase.__init__(self, filename, feature_flag_storage, rule_based_segment_storage, localhost_mode) + @classmethod def _read_feature_flags_from_legacy_file(cls, filename): """ @@ -656,23 +771,34 @@ def _synchronize_json(self): :rtype: [str] """ try: - fetched, till = self._read_feature_flags_from_json_file(self._filename) + parsed = self._read_feature_flags_from_json_file(self._filename) segment_list = set() - fecthed_sha = util._get_sha(json.dumps(fetched)) - if fecthed_sha == self._current_json_sha: + fecthed_ff_sha = util._get_sha(json.dumps(parsed['ff'])) + fecthed_rbs_sha = util._get_sha(json.dumps(parsed['rbs'])) + + if fecthed_ff_sha == self._current_ff_sha and fecthed_rbs_sha == self._current_rbs_sha: return [] - self._current_json_sha = fecthed_sha - if self._feature_flag_storage.get_change_number() > till and till != self._DEFAULT_FEATURE_FLAG_TILL: + self._current_ff_sha = fecthed_ff_sha + self._current_rbs_sha = fecthed_rbs_sha + + if self._check_exit_conditions(self._feature_flag_storage.get_change_number(), parsed['ff']['t'], self._DEFAULT_FEATURE_FLAG_TILL) \ + and self._check_exit_conditions(self._rule_based_segment_storage.get_change_number(), parsed['rbs']['t'], self._DEFAULT_RB_SEGMENT_TILL): return [] - fetched_feature_flags = [splits.from_raw(feature_flag) for feature_flag in fetched] - segment_list = update_feature_flag_storage(self._feature_flag_storage, fetched_feature_flags, till) + if not self._check_exit_conditions(self._feature_flag_storage.get_change_number(), parsed['ff']['t'], self._DEFAULT_FEATURE_FLAG_TILL): + fetched_feature_flags = [splits.from_raw(feature_flag) for feature_flag in parsed['ff']['d']] + segment_list = update_feature_flag_storage(self._feature_flag_storage, fetched_feature_flags, parsed['ff']['t']) + + if not self._check_exit_conditions(self._rule_based_segment_storage.get_change_number(), parsed['rbs']['t'], self._DEFAULT_RB_SEGMENT_TILL): + fetched_rb_segments = [rule_based_segments.from_raw(rb_segment) for rb_segment in parsed['rbs']['d']] + segment_list.update(update_rule_based_segment_storage(self._rule_based_segment_storage, fetched_rb_segments, parsed['rbs']['t'])) + return segment_list except Exception as exc: _LOGGER.debug('Exception: ', exc_info=True) - raise ValueError("Error reading feature flags from json.") from exc + raise ValueError("Error reading feature flags from json.") from exc def _read_feature_flags_from_json_file(self, filename): """ @@ -687,17 +813,24 @@ def _read_feature_flags_from_json_file(self, filename): try: with open(filename, 'r') as flo: parsed = json.load(flo) - santitized = self._sanitize_feature_flag(parsed) - return santitized['splits'], santitized['till'] + + # check if spec version is old + if parsed.get('splits'): + parsed = util.convert_to_new_spec(parsed) + + santitized = self._sanitize_json_elements(parsed) + santitized['ff']['d'] = self._sanitize_feature_flag_elements(santitized['ff']['d']) + santitized['rbs']['d'] = self._sanitize_rb_segment_elements(santitized['rbs']['d']) + return santitized + except Exception as exc: _LOGGER.debug('Exception: ', exc_info=True) raise ValueError("Error parsing file %s. Make sure it's readable." % filename) from exc - class LocalSplitSynchronizerAsync(LocalSplitSynchronizerBase): """Localhost mode async feature_flag synchronizer.""" - def __init__(self, filename, feature_flag_storage, localhost_mode=LocalhostMode.LEGACY): + def __init__(self, filename, feature_flag_storage, rule_based_segment_storage, localhost_mode=LocalhostMode.LEGACY): """ Class constructor. @@ -708,10 +841,7 @@ def __init__(self, filename, feature_flag_storage, localhost_mode=LocalhostMode. :param localhost_mode: mode for localhost either JSON, YAML or LEGACY. :type localhost_mode: splitio.sync.split.LocalhostMode """ - self._filename = filename - self._feature_flag_storage = feature_flag_storage - self._localhost_mode = localhost_mode - self._current_json_sha = "-1" + LocalSplitSynchronizerBase.__init__(self, filename, feature_flag_storage, rule_based_segment_storage, localhost_mode) @classmethod async def _read_feature_flags_from_legacy_file(cls, filename): @@ -803,18 +933,29 @@ async def _synchronize_json(self): :rtype: [str] """ try: - fetched, till = await self._read_feature_flags_from_json_file(self._filename) + parsed = await self._read_feature_flags_from_json_file(self._filename) segment_list = set() - fecthed_sha = util._get_sha(json.dumps(fetched)) - if fecthed_sha == self._current_json_sha: + fecthed_ff_sha = util._get_sha(json.dumps(parsed['ff'])) + fecthed_rbs_sha = util._get_sha(json.dumps(parsed['rbs'])) + + if fecthed_ff_sha == self._current_ff_sha and fecthed_rbs_sha == self._current_rbs_sha: return [] - self._current_json_sha = fecthed_sha - if await self._feature_flag_storage.get_change_number() > till and till != self._DEFAULT_FEATURE_FLAG_TILL: + self._current_ff_sha = fecthed_ff_sha + self._current_rbs_sha = fecthed_rbs_sha + + if self._check_exit_conditions(await self._feature_flag_storage.get_change_number(), parsed['ff']['t'], self._DEFAULT_FEATURE_FLAG_TILL) \ + and self._check_exit_conditions(await self._rule_based_segment_storage.get_change_number(), parsed['rbs']['t'], self._DEFAULT_RB_SEGMENT_TILL): return [] - fetched_feature_flags = [splits.from_raw(feature_flag) for feature_flag in fetched] - segment_list = await update_feature_flag_storage_async(self._feature_flag_storage, fetched_feature_flags, till) + if not self._check_exit_conditions(await self._feature_flag_storage.get_change_number(), parsed['ff']['t'], self._DEFAULT_FEATURE_FLAG_TILL): + fetched_feature_flags = [splits.from_raw(feature_flag) for feature_flag in parsed['ff']['d']] + segment_list = await update_feature_flag_storage_async(self._feature_flag_storage, fetched_feature_flags, parsed['ff']['t']) + + if not self._check_exit_conditions(await self._rule_based_segment_storage.get_change_number(), parsed['rbs']['t'], self._DEFAULT_RB_SEGMENT_TILL): + fetched_rb_segments = [rule_based_segments.from_raw(rb_segment) for rb_segment in parsed['rbs']['d']] + segment_list.update(await update_rule_based_segment_storage_async(self._rule_based_segment_storage, fetched_rb_segments, parsed['rbs']['t'])) + return segment_list except Exception as exc: @@ -834,8 +975,15 @@ async def _read_feature_flags_from_json_file(self, filename): try: async with aiofiles.open(filename, 'r') as flo: parsed = json.loads(await flo.read()) - santitized = self._sanitize_feature_flag(parsed) - return santitized['splits'], santitized['till'] + + # check if spec version is old + if parsed.get('splits'): + parsed = util.convert_to_new_spec(parsed) + + santitized = self._sanitize_json_elements(parsed) + santitized['ff']['d'] = self._sanitize_feature_flag_elements(santitized['ff']['d']) + santitized['rbs']['d'] = self._sanitize_rb_segment_elements(santitized['rbs']['d']) + return santitized except Exception as exc: _LOGGER.debug('Exception: ', exc_info=True) raise ValueError("Error parsing file %s. Make sure it's readable." % filename) from exc diff --git a/splitio/sync/util.py b/splitio/sync/util.py index 07ec5f24..cd32d2c2 100644 --- a/splitio/sync/util.py +++ b/splitio/sync/util.py @@ -62,3 +62,7 @@ def _sanitize_object_element(object, object_name, element_name, default_value, l _LOGGER.debug("Sanitized element [%s] to '%s' in %s: %s.", element_name, default_value, object_name, object['name']) return object + +def convert_to_new_spec(body): + return {"ff": {"d": body["splits"], "s": body["since"], "t": body["till"]}, + "rbs": {"d": [], "s": -1, "t": -1}} diff --git a/splitio/util/storage_helper.py b/splitio/util/storage_helper.py index 8476cec2..81fdef65 100644 --- a/splitio/util/storage_helper.py +++ b/splitio/util/storage_helper.py @@ -1,11 +1,11 @@ """Storage Helper.""" import logging - from splitio.models import splits +from splitio.models import rule_based_segments _LOGGER = logging.getLogger(__name__) -def update_feature_flag_storage(feature_flag_storage, feature_flags, change_number): +def update_feature_flag_storage(feature_flag_storage, feature_flags, change_number, clear_storage=False): """ Update feature flag storage from given list of feature flags while checking the flag set logic @@ -22,6 +22,9 @@ def update_feature_flag_storage(feature_flag_storage, feature_flags, change_numb segment_list = set() to_add = [] to_delete = [] + if clear_storage: + feature_flag_storage.clear() + for feature_flag in feature_flags: if feature_flag_storage.flag_set_filter.intersect(feature_flag.sets) and feature_flag.status == splits.Status.ACTIVE: to_add.append(feature_flag) @@ -33,7 +36,54 @@ def update_feature_flag_storage(feature_flag_storage, feature_flags, change_numb feature_flag_storage.update(to_add, to_delete, change_number) return segment_list -async def update_feature_flag_storage_async(feature_flag_storage, feature_flags, change_number): +def update_rule_based_segment_storage(rule_based_segment_storage, rule_based_segments, change_number, clear_storage=False): + """ + Update rule based segment storage from given list of rule based segments + + :param rule_based_segment_storage: rule based segment storage instance + :type rule_based_segment_storage: splitio.storage.RuleBasedSegmentStorage + :param rule_based_segments: rule based segment instance to validate. + :type rule_based_segments: splitio.models.rule_based_segments.RuleBasedSegment + :param: last change number + :type: int + + :return: segments list from excluded segments list + :rtype: list(str) + """ + if clear_storage: + rule_based_segment_storage.clear() + + segment_list = set() + to_add = [] + to_delete = [] + for rule_based_segment in rule_based_segments: + if rule_based_segment.status == splits.Status.ACTIVE: + to_add.append(rule_based_segment) + segment_list.update(set(rule_based_segment.excluded.get_excluded_standard_segments())) + segment_list.update(rule_based_segment.get_condition_segment_names()) + else: + if rule_based_segment_storage.get(rule_based_segment.name) is not None: + to_delete.append(rule_based_segment.name) + + rule_based_segment_storage.update(to_add, to_delete, change_number) + return segment_list + +def get_standard_segment_names_in_rbs_storage(rule_based_segment_storage): + """ + Retrieve a list of all standard segments names. + + :return: Set of segment names. + :rtype: Set(str) + """ + segment_list = set() + for rb_segment in rule_based_segment_storage.get_segment_names(): + rb_segment_obj = rule_based_segment_storage.get(rb_segment) + segment_list.update(set(rb_segment_obj.excluded.get_excluded_standard_segments())) + segment_list.update(rb_segment_obj.get_condition_segment_names()) + + return segment_list + +async def update_feature_flag_storage_async(feature_flag_storage, feature_flags, change_number, clear_storage=False): """ Update feature flag storage from given list of feature flags while checking the flag set logic @@ -47,6 +97,9 @@ async def update_feature_flag_storage_async(feature_flag_storage, feature_flags, :return: segments list from feature flags list :rtype: list(str) """ + if clear_storage: + await feature_flag_storage.clear() + segment_list = set() to_add = [] to_delete = [] @@ -61,6 +114,54 @@ async def update_feature_flag_storage_async(feature_flag_storage, feature_flags, await feature_flag_storage.update(to_add, to_delete, change_number) return segment_list +async def update_rule_based_segment_storage_async(rule_based_segment_storage, rule_based_segments, change_number, clear_storage=False): + """ + Update rule based segment storage from given list of rule based segments + + :param rule_based_segment_storage: rule based segment storage instance + :type rule_based_segment_storage: splitio.storage.RuleBasedSegmentStorage + :param rule_based_segments: rule based segment instance to validate. + :type rule_based_segments: splitio.models.rule_based_segments.RuleBasedSegment + :param: last change number + :type: int + + :return: segments list from excluded segments list + :rtype: list(str) + """ + if clear_storage: + await rule_based_segment_storage.clear() + + segment_list = set() + to_add = [] + to_delete = [] + for rule_based_segment in rule_based_segments: + if rule_based_segment.status == splits.Status.ACTIVE: + to_add.append(rule_based_segment) + segment_list.update(set(rule_based_segment.excluded.get_excluded_standard_segments())) + segment_list.update(rule_based_segment.get_condition_segment_names()) + else: + if await rule_based_segment_storage.get(rule_based_segment.name) is not None: + to_delete.append(rule_based_segment.name) + + await rule_based_segment_storage.update(to_add, to_delete, change_number) + return segment_list + +async def get_standard_segment_names_in_rbs_storage_async(rule_based_segment_storage): + """ + Retrieve a list of all standard segments names. + + :return: Set of segment names. + :rtype: Set(str) + """ + segment_list = set() + segment_names = await rule_based_segment_storage.get_segment_names() + for rb_segment in segment_names: + rb_segment_obj = await rule_based_segment_storage.get(rb_segment) + segment_list.update(set(rb_segment_obj.excluded.get_excluded_standard_segments())) + segment_list.update(rb_segment_obj.get_condition_segment_names()) + + return segment_list + def get_valid_flag_sets(flag_sets, flag_set_filter): """ Check each flag set in given array, return it if exist in a given config flag set array, if config array is empty return all diff --git a/splitio/version.py b/splitio/version.py index e8137101..8d2afd7b 100644 --- a/splitio/version.py +++ b/splitio/version.py @@ -1 +1 @@ -__version__ = '10.2.0' \ No newline at end of file +__version__ = '10.3.0' \ No newline at end of file diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index a842bd36..175977a2 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -34,7 +34,7 @@ def test_auth(self, mocker): call_made = httpclient.get.mock_calls[0] # validate positional arguments - assert call_made[1] == ('auth', 'v2/auth?s=1.1', 'some_api_key') + assert call_made[1] == ('auth', 'v2/auth?s=1.3', 'some_api_key') # validate key-value args (headers) assert call_made[2]['extra_headers'] == { @@ -89,7 +89,7 @@ async def get(verb, url, key, extra_headers): # validate positional arguments assert self.verb == 'auth' - assert self.url == 'v2/auth?s=1.1' + assert self.url == 'v2/auth?s=1.3' assert self.key == 'some_api_key' assert self.headers == { 'SplitSDKVersion': 'python-%s' % __version__, diff --git a/tests/api/test_segments_api.py b/tests/api/test_segments_api.py index 73e3efe7..8681be59 100644 --- a/tests/api/test_segments_api.py +++ b/tests/api/test_segments_api.py @@ -16,7 +16,7 @@ def test_fetch_segment_changes(self, mocker): httpclient.get.return_value = client.HttpResponse(200, '{"prop1": "value1"}', {}) segment_api = segments.SegmentsAPI(httpclient, 'some_api_key', SdkMetadata('1.0', 'some', '1.2.3.4'), mocker.Mock()) - response = segment_api.fetch_segment('some_segment', 123, FetchOptions(None, None, None, None)) + response = segment_api.fetch_segment('some_segment', 123, FetchOptions(None, None, None, None, None)) assert response['prop1'] == 'value1' assert httpclient.get.mock_calls == [mocker.call('sdk', 'segmentChanges/some_segment', 'some_api_key', extra_headers={ @@ -27,7 +27,7 @@ def test_fetch_segment_changes(self, mocker): query={'since': 123})] httpclient.reset_mock() - response = segment_api.fetch_segment('some_segment', 123, FetchOptions(True, None, None, None)) + response = segment_api.fetch_segment('some_segment', 123, FetchOptions(True, None, None, None, None)) assert response['prop1'] == 'value1' assert httpclient.get.mock_calls == [mocker.call('sdk', 'segmentChanges/some_segment', 'some_api_key', extra_headers={ @@ -39,7 +39,7 @@ def test_fetch_segment_changes(self, mocker): query={'since': 123})] httpclient.reset_mock() - response = segment_api.fetch_segment('some_segment', 123, FetchOptions(True, 123, None, None)) + response = segment_api.fetch_segment('some_segment', 123, FetchOptions(True, 123, None, None, None)) assert response['prop1'] == 'value1' assert httpclient.get.mock_calls == [mocker.call('sdk', 'segmentChanges/some_segment', 'some_api_key', extra_headers={ @@ -83,7 +83,7 @@ async def get(verb, url, key, query, extra_headers): return client.HttpResponse(200, '{"prop1": "value1"}', {}) httpclient.get = get - response = await segment_api.fetch_segment('some_segment', 123, FetchOptions(None, None, None, None)) + response = await segment_api.fetch_segment('some_segment', 123, FetchOptions(None, None, None, None, None)) assert response['prop1'] == 'value1' assert self.verb == 'sdk' assert self.url == 'segmentChanges/some_segment' @@ -96,7 +96,7 @@ async def get(verb, url, key, query, extra_headers): assert self.query == {'since': 123} httpclient.reset_mock() - response = await segment_api.fetch_segment('some_segment', 123, FetchOptions(True, None, None, None)) + response = await segment_api.fetch_segment('some_segment', 123, FetchOptions(True, None, None, None, None)) assert response['prop1'] == 'value1' assert self.verb == 'sdk' assert self.url == 'segmentChanges/some_segment' @@ -110,7 +110,7 @@ async def get(verb, url, key, query, extra_headers): assert self.query == {'since': 123} httpclient.reset_mock() - response = await segment_api.fetch_segment('some_segment', 123, FetchOptions(True, 123, None, None)) + response = await segment_api.fetch_segment('some_segment', 123, FetchOptions(True, 123, None, None, None)) assert response['prop1'] == 'value1' assert self.verb == 'sdk' assert self.url == 'segmentChanges/some_segment' @@ -128,6 +128,6 @@ def raise_exception(*args, **kwargs): raise client.HttpClientException('some_message') httpclient.get = raise_exception with pytest.raises(APIException) as exc_info: - response = await segment_api.fetch_segment('some_segment', 123, FetchOptions(None, None, None, None)) + response = await segment_api.fetch_segment('some_segment', 123, FetchOptions(None, None, None, None, None)) assert exc_info.type == APIException assert exc_info.value.message == 'some_message' diff --git a/tests/api/test_splits_api.py b/tests/api/test_splits_api.py index d1d276b7..c9aeee8b 100644 --- a/tests/api/test_splits_api.py +++ b/tests/api/test_splits_api.py @@ -2,6 +2,7 @@ import pytest import unittest.mock as mock +import time from splitio.api import splits, client, APIException from splitio.api.commons import FetchOptions @@ -16,7 +17,7 @@ def test_fetch_split_changes(self, mocker): httpclient.get.return_value = client.HttpResponse(200, '{"prop1": "value1"}', {}) split_api = splits.SplitsAPI(httpclient, 'some_api_key', SdkMetadata('1.0', 'some', '1.2.3.4'), mocker.Mock()) - response = split_api.fetch_splits(123, FetchOptions(False, None, 'set1,set2')) + response = split_api.fetch_splits(123, -1, FetchOptions(False, None, None, 'set1,set2')) assert response['prop1'] == 'value1' assert httpclient.get.mock_calls == [mocker.call('sdk', 'splitChanges', 'some_api_key', extra_headers={ @@ -24,10 +25,10 @@ def test_fetch_split_changes(self, mocker): 'SplitSDKMachineIP': '1.2.3.4', 'SplitSDKMachineName': 'some' }, - query={'s': '1.1', 'since': 123, 'sets': 'set1,set2'})] + query={'s': '1.3', 'since': 123, 'rbSince': -1, 'sets': 'set1,set2'})] httpclient.reset_mock() - response = split_api.fetch_splits(123, FetchOptions(True, 123, 'set3')) + response = split_api.fetch_splits(123, 1, FetchOptions(True, 123, None,'set3')) assert response['prop1'] == 'value1' assert httpclient.get.mock_calls == [mocker.call('sdk', 'splitChanges', 'some_api_key', extra_headers={ @@ -36,10 +37,10 @@ def test_fetch_split_changes(self, mocker): 'SplitSDKMachineName': 'some', 'Cache-Control': 'no-cache' }, - query={'s': '1.1', 'since': 123, 'till': 123, 'sets': 'set3'})] + query={'s': '1.3', 'since': 123, 'rbSince': 1, 'till': 123, 'sets': 'set3'})] httpclient.reset_mock() - response = split_api.fetch_splits(123, FetchOptions(True, 123, 'set3')) + response = split_api.fetch_splits(123, 122, FetchOptions(True, 123, None, 'set3')) assert response['prop1'] == 'value1' assert httpclient.get.mock_calls == [mocker.call('sdk', 'splitChanges', 'some_api_key', extra_headers={ @@ -48,18 +49,115 @@ def test_fetch_split_changes(self, mocker): 'SplitSDKMachineName': 'some', 'Cache-Control': 'no-cache' }, - query={'s': '1.1', 'since': 123, 'till': 123, 'sets': 'set3'})] + query={'s': '1.3', 'since': 123, 'rbSince': 122, 'till': 123, 'sets': 'set3'})] httpclient.reset_mock() def raise_exception(*args, **kwargs): raise client.HttpClientException('some_message') httpclient.get.side_effect = raise_exception with pytest.raises(APIException) as exc_info: - response = split_api.fetch_splits(123, FetchOptions()) + response = split_api.fetch_splits(123, 12, FetchOptions()) assert exc_info.type == APIException assert exc_info.value.message == 'some_message' + def test_old_spec(self, mocker): + """Test old split changes fetching API call.""" + httpclient = mocker.Mock(spec=client.HttpClient) + self.counter = 0 + self.query = [] + def get(sdk, splitChanges, sdk_key, extra_headers, query): + self.counter += 1 + self.query.append(query) + if self.counter == 1: + return client.HttpResponse(400, 'error', {}) + if self.counter == 2: + return client.HttpResponse(200, '{"splits": [], "since": 123, "till": 456}', {}) + + httpclient.get = get + split_api = splits.SplitsAPI(httpclient, 'some_api_key', SdkMetadata('1.0', 'some', '1.2.3.4'), mocker.Mock()) + + httpclient.is_sdk_endpoint_overridden.return_value = False + try: + response = split_api.fetch_splits(123, -1, FetchOptions(False, None, None, None)) + except Exception as e: + print(e) + + # no attempt to fetch old spec + assert self.query == [{'s': '1.3', 'since': 123, 'rbSince': -1}] + + httpclient.is_sdk_endpoint_overridden.return_value = True + self.query = [] + self.counter = 0 + response = split_api.fetch_splits(123, -1, FetchOptions(False, None, None, None)) + assert response == {"ff": {"d": [], "s": 123, "t": 456}, "rbs": {"d": [], "s": -1, "t": -1}} + assert self.query == [{'s': '1.3', 'since': 123, 'rbSince': -1}, {'s': '1.1', 'since': 123}] + assert not split_api.clear_storage + + def test_switch_to_new_spec(self, mocker): + """Test old split changes fetching API call.""" + httpclient = mocker.Mock(spec=client.HttpClient) + self.counter = 0 + self.query = [] + def get(sdk, splitChanges, sdk_key, extra_headers, query): + self.counter += 1 + self.query.append(query) + if self.counter == 1: + return client.HttpResponse(400, 'error', {}) + if self.counter == 2: + return client.HttpResponse(200, '{"splits": [], "since": 123, "till": 456}', {}) + if self.counter == 3: + return client.HttpResponse(200, '{"ff": {"d": [], "s": 123, "t": 456}, "rbs": {"d": [], "s": 123, "t": -1}}', {}) + + httpclient.is_sdk_endpoint_overridden.return_value = True + httpclient.get = get + split_api = splits.SplitsAPI(httpclient, 'some_api_key', SdkMetadata('1.0', 'some', '1.2.3.4'), mocker.Mock()) + response = split_api.fetch_splits(123, -1, FetchOptions(False, None, None, None)) + assert response == {"ff": {"d": [], "s": 123, "t": 456}, "rbs": {"d": [], "s": -1, "t": -1}} + assert self.query == [{'s': '1.3', 'since': 123, 'rbSince': -1}, {'s': '1.1', 'since': 123}] + assert not split_api.clear_storage + time.sleep(1) + splits._PROXY_CHECK_INTERVAL_MILLISECONDS_SS = 10 + response = split_api.fetch_splits(123, -1, FetchOptions(False, None, None, None)) + assert self.query[2] == {'s': '1.3', 'since': 123, 'rbSince': -1} + assert response == {"ff": {"d": [], "s": 123, "t": 456}, "rbs": {"d": [], "s": 123, "t": -1}} + assert split_api.clear_storage + + def test_using_old_spec_since(self, mocker): + """Test using old_spec_since variable.""" + httpclient = mocker.Mock(spec=client.HttpClient) + self.counter = 0 + self.query = [] + def get(sdk, splitChanges, sdk_key, extra_headers, query): + self.counter += 1 + self.query.append(query) + if self.counter == 1: + return client.HttpResponse(400, 'error', {}) + if self.counter == 2: + return client.HttpResponse(200, '{"splits": [], "since": 123, "till": 456}', {}) + if self.counter == 3: + return client.HttpResponse(400, 'error', {}) + if self.counter == 4: + return client.HttpResponse(200, '{"splits": [], "since": 456, "till": 456}', {}) + + httpclient.is_sdk_endpoint_overridden.return_value = True + httpclient.get = get + split_api = splits.SplitsAPI(httpclient, 'some_api_key', SdkMetadata('1.0', 'some', '1.2.3.4'), mocker.Mock()) + response = split_api.fetch_splits(123, -1, FetchOptions(False, None, None, None)) + assert response == {"ff": {"d": [], "s": 123, "t": 456}, "rbs": {"d": [], "s": -1, "t": -1}} + assert self.query == [{'s': '1.3', 'since': 123, 'rbSince': -1}, {'s': '1.1', 'since': 123}] + assert not split_api.clear_storage + + time.sleep(1) + splits._PROXY_CHECK_INTERVAL_MILLISECONDS_SS = 10 + + response = split_api.fetch_splits(456, -1, FetchOptions(False, None, None, None)) + time.sleep(1) + splits._PROXY_CHECK_INTERVAL_MILLISECONDS_SS = 1000000 + assert self.query[2] == {'s': '1.3', 'since': 456, 'rbSince': -1} + assert self.query[3] == {'s': '1.1', 'since': 456} + assert response == {"ff": {"d": [], "s": 456, "t": 456}, "rbs": {"d": [], "s": -1, "t": -1}} + class SplitAPIAsyncTests(object): """Split async API test cases.""" @@ -82,7 +180,7 @@ async def get(verb, url, key, query, extra_headers): return client.HttpResponse(200, '{"prop1": "value1"}', {}) httpclient.get = get - response = await split_api.fetch_splits(123, FetchOptions(False, None, 'set1,set2')) + response = await split_api.fetch_splits(123, -1, FetchOptions(False, None, None, 'set1,set2')) assert response['prop1'] == 'value1' assert self.verb == 'sdk' assert self.url == 'splitChanges' @@ -92,10 +190,10 @@ async def get(verb, url, key, query, extra_headers): 'SplitSDKMachineIP': '1.2.3.4', 'SplitSDKMachineName': 'some' } - assert self.query == {'s': '1.1', 'since': 123, 'sets': 'set1,set2'} + assert self.query == {'s': '1.3', 'since': 123, 'rbSince': -1, 'sets': 'set1,set2'} httpclient.reset_mock() - response = await split_api.fetch_splits(123, FetchOptions(True, 123, 'set3')) + response = await split_api.fetch_splits(123, 1, FetchOptions(True, 123, None, 'set3')) assert response['prop1'] == 'value1' assert self.verb == 'sdk' assert self.url == 'splitChanges' @@ -106,10 +204,10 @@ async def get(verb, url, key, query, extra_headers): 'SplitSDKMachineName': 'some', 'Cache-Control': 'no-cache' } - assert self.query == {'s': '1.1', 'since': 123, 'till': 123, 'sets': 'set3'} + assert self.query == {'s': '1.3', 'since': 123, 'rbSince': 1, 'till': 123, 'sets': 'set3'} httpclient.reset_mock() - response = await split_api.fetch_splits(123, FetchOptions(True, 123)) + response = await split_api.fetch_splits(123, 122, FetchOptions(True, 123, None)) assert response['prop1'] == 'value1' assert self.verb == 'sdk' assert self.url == 'splitChanges' @@ -120,13 +218,115 @@ async def get(verb, url, key, query, extra_headers): 'SplitSDKMachineName': 'some', 'Cache-Control': 'no-cache' } - assert self.query == {'s': '1.1', 'since': 123, 'till': 123} + assert self.query == {'s': '1.3', 'since': 123, 'rbSince': 122, 'till': 123} httpclient.reset_mock() def raise_exception(*args, **kwargs): raise client.HttpClientException('some_message') httpclient.get = raise_exception with pytest.raises(APIException) as exc_info: - response = await split_api.fetch_splits(123, FetchOptions()) + response = await split_api.fetch_splits(123, 12, FetchOptions()) assert exc_info.type == APIException assert exc_info.value.message == 'some_message' + + @pytest.mark.asyncio + async def test_old_spec(self, mocker): + """Test old split changes fetching API call.""" + httpclient = mocker.Mock(spec=client.HttpClientAsync) + self.counter = 0 + self.query = [] + async def get(sdk, splitChanges, sdk_key, extra_headers, query): + self.counter += 1 + self.query.append(query) + if self.counter == 1: + return client.HttpResponse(400, 'error', {}) + if self.counter == 2: + return client.HttpResponse(200, '{"splits": [], "since": 123, "till": 456}', {}) + + httpclient.is_sdk_endpoint_overridden.return_value = True + httpclient.get = get + split_api = splits.SplitsAPIAsync(httpclient, 'some_api_key', SdkMetadata('1.0', 'some', '1.2.3.4'), mocker.Mock()) + + httpclient.is_sdk_endpoint_overridden.return_value = False + try: + response = await split_api.fetch_splits(123, -1, FetchOptions(False, None, None, None)) + except Exception as e: + print(e) + + # no attempt to fetch old spec + assert self.query == [{'s': '1.3', 'since': 123, 'rbSince': -1}] + + httpclient.is_sdk_endpoint_overridden.return_value = True + self.query = [] + self.counter = 0 + response = await split_api.fetch_splits(123, -1, FetchOptions(False, None, None, None)) + assert response == {"ff": {"d": [], "s": 123, "t": 456}, "rbs": {"d": [], "s": -1, "t": -1}} + assert self.query == [{'s': '1.3', 'since': 123, 'rbSince': -1}, {'s': '1.1', 'since': 123}] + assert not split_api.clear_storage + + @pytest.mark.asyncio + async def test_switch_to_new_spec(self, mocker): + """Test old split changes fetching API call.""" + httpclient = mocker.Mock(spec=client.HttpClientAsync) + self.counter = 0 + self.query = [] + async def get(sdk, splitChanges, sdk_key, extra_headers, query): + self.counter += 1 + self.query.append(query) + if self.counter == 1: + return client.HttpResponse(400, 'error', {}) + if self.counter == 2: + return client.HttpResponse(200, '{"splits": [], "since": 123, "till": 456}', {}) + if self.counter == 3: + return client.HttpResponse(200, '{"ff": {"d": [], "s": 123, "t": 456}, "rbs": {"d": [], "s": 123, "t": -1}}', {}) + + httpclient.is_sdk_endpoint_overridden.return_value = True + httpclient.get = get + split_api = splits.SplitsAPIAsync(httpclient, 'some_api_key', SdkMetadata('1.0', 'some', '1.2.3.4'), mocker.Mock()) + response = await split_api.fetch_splits(123, -1, FetchOptions(False, None, None, None)) + assert response == {"ff": {"d": [], "s": 123, "t": 456}, "rbs": {"d": [], "s": -1, "t": -1}} + assert self.query == [{'s': '1.3', 'since': 123, 'rbSince': -1}, {'s': '1.1', 'since': 123}] + assert not split_api.clear_storage + + time.sleep(1) + splits._PROXY_CHECK_INTERVAL_MILLISECONDS_SS = 10 + response = await split_api.fetch_splits(123, -1, FetchOptions(False, None, None, None)) + assert self.query[2] == {'s': '1.3', 'since': 123, 'rbSince': -1} + assert response == {"ff": {"d": [], "s": 123, "t": 456}, "rbs": {"d": [], "s": 123, "t": -1}} + assert split_api.clear_storage + + @pytest.mark.asyncio + async def test_using_old_spec_since(self, mocker): + """Test using old_spec_since variable.""" + httpclient = mocker.Mock(spec=client.HttpClient) + self.counter = 0 + self.query = [] + async def get(sdk, splitChanges, sdk_key, extra_headers, query): + self.counter += 1 + self.query.append(query) + if self.counter == 1: + return client.HttpResponse(400, 'error', {}) + if self.counter == 2: + return client.HttpResponse(200, '{"splits": [], "since": 123, "till": 456}', {}) + if self.counter == 3: + return client.HttpResponse(400, 'error', {}) + if self.counter == 4: + return client.HttpResponse(200, '{"splits": [], "since": 456, "till": 456}', {}) + + httpclient.is_sdk_endpoint_overridden.return_value = True + httpclient.get = get + split_api = splits.SplitsAPIAsync(httpclient, 'some_api_key', SdkMetadata('1.0', 'some', '1.2.3.4'), mocker.Mock()) + response = await split_api.fetch_splits(123, -1, FetchOptions(False, None, None, None)) + assert response == {"ff": {"d": [], "s": 123, "t": 456}, "rbs": {"d": [], "s": -1, "t": -1}} + assert self.query == [{'s': '1.3', 'since': 123, 'rbSince': -1}, {'s': '1.1', 'since': 123}] + assert not split_api.clear_storage + + time.sleep(1) + splits._PROXY_CHECK_INTERVAL_MILLISECONDS_SS = 10 + + response = await split_api.fetch_splits(456, -1, FetchOptions(False, None, None, None)) + time.sleep(1) + splits._PROXY_CHECK_INTERVAL_MILLISECONDS_SS = 1000000 + assert self.query[2] == {'s': '1.3', 'since': 456, 'rbSince': -1} + assert self.query[3] == {'s': '1.1', 'since': 456} + assert response == {"ff": {"d": [], "s": 456, "t": 456}, "rbs": {"d": [], "s": -1, "t": -1}} diff --git a/tests/client/test_client.py b/tests/client/test_client.py index 48a0fba2..49b6ba7a 100644 --- a/tests/client/test_client.py +++ b/tests/client/test_client.py @@ -11,10 +11,11 @@ from splitio.client.factory import SplitFactory, Status as FactoryStatus, SplitFactoryAsync from splitio.models.impressions import Impression, Label from splitio.models.events import Event, EventWrapper -from splitio.storage import EventStorage, ImpressionStorage, SegmentStorage, SplitStorage +from splitio.storage import EventStorage, ImpressionStorage, SegmentStorage, SplitStorage, RuleBasedSegmentsStorage from splitio.storage.inmemmory import InMemorySplitStorage, InMemorySegmentStorage, \ InMemoryImpressionStorage, InMemoryTelemetryStorage, InMemorySplitStorageAsync, \ - InMemoryImpressionStorageAsync, InMemorySegmentStorageAsync, InMemoryTelemetryStorageAsync, InMemoryEventStorageAsync + InMemoryImpressionStorageAsync, InMemorySegmentStorageAsync, InMemoryTelemetryStorageAsync, InMemoryEventStorageAsync, \ + InMemoryRuleBasedSegmentStorage, InMemoryRuleBasedSegmentStorageAsync from splitio.models.splits import Split, Status, from_raw from splitio.engine.impressions.impressions import Manager as ImpressionManager from splitio.engine.impressions.manager import Counter as ImpressionsCounter @@ -35,6 +36,7 @@ def test_get_treatment(self, mocker): telemetry_producer = TelemetryStorageProducer(telemetry_storage) split_storage = InMemorySplitStorage() segment_storage = InMemorySegmentStorage() + rb_segment_storage = InMemoryRuleBasedSegmentStorage() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorage(10, telemetry_runtime_producer) event_storage = mocker.Mock(spec=EventStorage) @@ -55,6 +57,7 @@ def synchronize_config(*_): factory = SplitFactory(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -70,7 +73,7 @@ def synchronize_config(*_): type(factory).ready = ready_property factory.block_until_ready(5) - split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0])], [], -1) + split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0])], [], -1) client = Client(factory, recorder, True) client._evaluator = mocker.Mock(spec=Evaluator) client._evaluator.eval_with_context.return_value = { @@ -110,6 +113,7 @@ def test_get_treatment_with_config(self, mocker): telemetry_producer = TelemetryStorageProducer(telemetry_storage) split_storage = InMemorySplitStorage() segment_storage = InMemorySegmentStorage() + rb_segment_storage = InMemoryRuleBasedSegmentStorage() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorage(10, telemetry_runtime_producer) impmanager = ImpressionManager(StrategyDebugMode(), StrategyNoneMode(), telemetry_runtime_producer) @@ -123,6 +127,7 @@ def test_get_treatment_with_config(self, mocker): {'splits': split_storage, 'segments': segment_storage, 'impressions': impression_storage, + 'rule_based_segments': rb_segment_storage, 'events': event_storage}, mocker.Mock(), recorder, @@ -140,7 +145,7 @@ def synchronize_config(*_): mocker.patch('splitio.client.client.utctime_ms', new=lambda: 1000) mocker.patch('splitio.client.client.get_latency_bucket_index', new=lambda x: 5) - split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0])], [], -1) + split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0])], [], -1) client = Client(factory, recorder, True) client._evaluator = mocker.Mock(spec=Evaluator) client._evaluator.eval_with_context.return_value = { @@ -185,11 +190,12 @@ def test_get_treatments(self, mocker): telemetry_producer = TelemetryStorageProducer(telemetry_storage) split_storage = InMemorySplitStorage() segment_storage = InMemorySegmentStorage() + rb_segment_storage = InMemoryRuleBasedSegmentStorage() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorage(10, telemetry_runtime_producer) impmanager = ImpressionManager(StrategyDebugMode(), StrategyNoneMode(), telemetry_runtime_producer) event_storage = mocker.Mock(spec=EventStorage) - split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0]), from_raw(splits_json['splitChange1_1']['splits'][1])], [], -1) + split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0]), from_raw(splits_json['splitChange1_1']['ff']['d'][1])], [], -1) destroyed_property = mocker.PropertyMock() destroyed_property.return_value = False @@ -198,6 +204,7 @@ def test_get_treatments(self, mocker): factory = SplitFactory(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -263,11 +270,12 @@ def test_get_treatments_by_flag_set(self, mocker): telemetry_producer = TelemetryStorageProducer(telemetry_storage) split_storage = InMemorySplitStorage() segment_storage = InMemorySegmentStorage() + rb_segment_storage = InMemoryRuleBasedSegmentStorage() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorage(10, telemetry_runtime_producer) impmanager = ImpressionManager(StrategyDebugMode(), StrategyNoneMode(), telemetry_runtime_producer) event_storage = mocker.Mock(spec=EventStorage) - split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0]), from_raw(splits_json['splitChange1_1']['splits'][1])], [], -1) + split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0]), from_raw(splits_json['splitChange1_1']['ff']['d'][1])], [], -1) destroyed_property = mocker.PropertyMock() destroyed_property.return_value = False @@ -276,6 +284,7 @@ def test_get_treatments_by_flag_set(self, mocker): factory = SplitFactory(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -340,11 +349,12 @@ def test_get_treatments_by_flag_sets(self, mocker): telemetry_producer = TelemetryStorageProducer(telemetry_storage) split_storage = InMemorySplitStorage() segment_storage = InMemorySegmentStorage() + rb_segment_storage = InMemoryRuleBasedSegmentStorage() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorage(10, telemetry_runtime_producer) impmanager = ImpressionManager(StrategyDebugMode(), StrategyNoneMode(), telemetry_runtime_producer) event_storage = mocker.Mock(spec=EventStorage) - split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0]), from_raw(splits_json['splitChange1_1']['splits'][1])], [], -1) + split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0]), from_raw(splits_json['splitChange1_1']['ff']['d'][1])], [], -1) destroyed_property = mocker.PropertyMock() destroyed_property.return_value = False @@ -353,6 +363,7 @@ def test_get_treatments_by_flag_sets(self, mocker): factory = SplitFactory(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -417,11 +428,12 @@ def test_get_treatments_with_config(self, mocker): telemetry_producer = TelemetryStorageProducer(telemetry_storage) split_storage = InMemorySplitStorage() segment_storage = InMemorySegmentStorage() + rb_segment_storage = InMemoryRuleBasedSegmentStorage() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorage(10, telemetry_runtime_producer) impmanager = ImpressionManager(StrategyDebugMode(), StrategyNoneMode(), telemetry_runtime_producer) event_storage = mocker.Mock(spec=EventStorage) - split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0]), from_raw(splits_json['splitChange1_1']['splits'][1])], [], -1) + split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0]), from_raw(splits_json['splitChange1_1']['ff']['d'][1])], [], -1) destroyed_property = mocker.PropertyMock() destroyed_property.return_value = False @@ -429,6 +441,7 @@ def test_get_treatments_with_config(self, mocker): factory = SplitFactory(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -498,11 +511,12 @@ def test_get_treatments_with_config_by_flag_set(self, mocker): telemetry_producer = TelemetryStorageProducer(telemetry_storage) split_storage = InMemorySplitStorage() segment_storage = InMemorySegmentStorage() + rb_segment_storage = InMemoryRuleBasedSegmentStorage() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorage(10, telemetry_runtime_producer) impmanager = ImpressionManager(StrategyDebugMode(), StrategyNoneMode(), telemetry_runtime_producer) event_storage = mocker.Mock(spec=EventStorage) - split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0]), from_raw(splits_json['splitChange1_1']['splits'][1])], [], -1) + split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0]), from_raw(splits_json['splitChange1_1']['ff']['d'][1])], [], -1) destroyed_property = mocker.PropertyMock() destroyed_property.return_value = False @@ -510,6 +524,7 @@ def test_get_treatments_with_config_by_flag_set(self, mocker): factory = SplitFactory(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -576,11 +591,12 @@ def test_get_treatments_with_config_by_flag_sets(self, mocker): telemetry_producer = TelemetryStorageProducer(telemetry_storage) split_storage = InMemorySplitStorage() segment_storage = InMemorySegmentStorage() + rb_segment_storage = InMemoryRuleBasedSegmentStorage() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorage(10, telemetry_runtime_producer) impmanager = ImpressionManager(StrategyDebugMode(), StrategyNoneMode(), telemetry_runtime_producer) event_storage = mocker.Mock(spec=EventStorage) - split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0]), from_raw(splits_json['splitChange1_1']['splits'][1])], [], -1) + split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0]), from_raw(splits_json['splitChange1_1']['ff']['d'][1])], [], -1) destroyed_property = mocker.PropertyMock() destroyed_property.return_value = False @@ -588,6 +604,7 @@ def test_get_treatments_with_config_by_flag_sets(self, mocker): factory = SplitFactory(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -654,6 +671,7 @@ def test_impression_toggle_optimized(self, mocker): telemetry_producer = TelemetryStorageProducer(telemetry_storage) split_storage = InMemorySplitStorage() segment_storage = InMemorySegmentStorage() + rb_segment_storage = InMemoryRuleBasedSegmentStorage() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorage(10, telemetry_runtime_producer) event_storage = mocker.Mock(spec=EventStorage) @@ -673,6 +691,7 @@ def synchronize_config(*_): factory = SplitFactory(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -687,9 +706,9 @@ def synchronize_config(*_): factory.block_until_ready(5) split_storage.update([ - from_raw(splits_json['splitChange1_1']['splits'][0]), - from_raw(splits_json['splitChange1_1']['splits'][1]), - from_raw(splits_json['splitChange1_1']['splits'][2]) + from_raw(splits_json['splitChange1_1']['ff']['d'][0]), + from_raw(splits_json['splitChange1_1']['ff']['d'][1]), + from_raw(splits_json['splitChange1_1']['ff']['d'][2]) ], [], -1) client = Client(factory, recorder, True) assert client.get_treatment('some_key', 'SPLIT_1') == 'off' @@ -716,6 +735,7 @@ def test_impression_toggle_debug(self, mocker): telemetry_producer = TelemetryStorageProducer(telemetry_storage) split_storage = InMemorySplitStorage() segment_storage = InMemorySegmentStorage() + rb_segment_storage = InMemoryRuleBasedSegmentStorage() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorage(10, telemetry_runtime_producer) event_storage = mocker.Mock(spec=EventStorage) @@ -735,6 +755,7 @@ def synchronize_config(*_): factory = SplitFactory(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -749,9 +770,9 @@ def synchronize_config(*_): factory.block_until_ready(5) split_storage.update([ - from_raw(splits_json['splitChange1_1']['splits'][0]), - from_raw(splits_json['splitChange1_1']['splits'][1]), - from_raw(splits_json['splitChange1_1']['splits'][2]) + from_raw(splits_json['splitChange1_1']['ff']['d'][0]), + from_raw(splits_json['splitChange1_1']['ff']['d'][1]), + from_raw(splits_json['splitChange1_1']['ff']['d'][2]) ], [], -1) client = Client(factory, recorder, True) assert client.get_treatment('some_key', 'SPLIT_1') == 'off' @@ -778,6 +799,7 @@ def test_impression_toggle_none(self, mocker): telemetry_producer = TelemetryStorageProducer(telemetry_storage) split_storage = InMemorySplitStorage() segment_storage = InMemorySegmentStorage() + rb_segment_storage = InMemoryRuleBasedSegmentStorage() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorage(10, telemetry_runtime_producer) event_storage = mocker.Mock(spec=EventStorage) @@ -797,6 +819,7 @@ def synchronize_config(*_): factory = SplitFactory(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -811,9 +834,9 @@ def synchronize_config(*_): factory.block_until_ready(5) split_storage.update([ - from_raw(splits_json['splitChange1_1']['splits'][0]), - from_raw(splits_json['splitChange1_1']['splits'][1]), - from_raw(splits_json['splitChange1_1']['splits'][2]) + from_raw(splits_json['splitChange1_1']['ff']['d'][0]), + from_raw(splits_json['splitChange1_1']['ff']['d'][1]), + from_raw(splits_json['splitChange1_1']['ff']['d'][2]) ], [], -1) client = Client(factory, recorder, True) assert client.get_treatment('some_key', 'SPLIT_1') == 'off' @@ -829,6 +852,7 @@ def test_destroy(self, mocker): """Test that destroy/destroyed calls are forwarded to the factory.""" split_storage = mocker.Mock(spec=SplitStorage) segment_storage = mocker.Mock(spec=SegmentStorage) + rb_segment_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) impression_storage = mocker.Mock(spec=ImpressionStorage) event_storage = mocker.Mock(spec=EventStorage) @@ -839,6 +863,7 @@ def test_destroy(self, mocker): factory = SplitFactory(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -863,6 +888,7 @@ def test_track(self, mocker): """Test that destroy/destroyed calls are forwarded to the factory.""" split_storage = mocker.Mock(spec=SplitStorage) segment_storage = mocker.Mock(spec=SegmentStorage) + rb_segment_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) impression_storage = mocker.Mock(spec=ImpressionStorage) event_storage = mocker.Mock(spec=EventStorage) event_storage.put.return_value = True @@ -874,6 +900,7 @@ def test_track(self, mocker): factory = SplitFactory(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -912,7 +939,8 @@ def test_evaluations_before_running_post_fork(self, mocker): impmanager = ImpressionManager(StrategyDebugMode(), StrategyNoneMode(), telemetry_runtime_producer) split_storage = InMemorySplitStorage() segment_storage = InMemorySegmentStorage() - split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0])], [], -1) + rb_segment_storage = InMemoryRuleBasedSegmentStorage() + split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0])], [], -1) destroyed_property = mocker.PropertyMock() destroyed_property.return_value = False @@ -921,6 +949,7 @@ def test_evaluations_before_running_post_fork(self, mocker): factory = SplitFactory(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': mocker.Mock()}, mocker.Mock(), @@ -991,11 +1020,13 @@ def test_telemetry_not_ready(self, mocker): impmanager = ImpressionManager(StrategyDebugMode(), StrategyNoneMode(), telemetry_runtime_producer) split_storage = InMemorySplitStorage() segment_storage = InMemorySegmentStorage() - split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0])], [], -1) + rb_segment_storage = InMemoryRuleBasedSegmentStorage() + split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0])], [], -1) recorder = StandardRecorder(impmanager, mocker.Mock(), mocker.Mock(), telemetry_producer.get_telemetry_evaluation_producer(), telemetry_producer.get_telemetry_runtime_producer()) factory = SplitFactory('localhost', {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': mocker.Mock()}, mocker.Mock(), @@ -1021,8 +1052,9 @@ def synchronize_config(*_): def test_telemetry_record_treatment_exception(self, mocker): split_storage = InMemorySplitStorage() - split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0])], [], -1) + split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0])], [], -1) segment_storage = mocker.Mock(spec=SegmentStorage) + rb_segment_storage = InMemoryRuleBasedSegmentStorage() impression_storage = mocker.Mock(spec=ImpressionStorage) event_storage = mocker.Mock(spec=EventStorage) destroyed_property = mocker.PropertyMock() @@ -1038,6 +1070,7 @@ def test_telemetry_record_treatment_exception(self, mocker): factory = SplitFactory('localhost', {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -1125,7 +1158,8 @@ def test_telemetry_method_latency(self, mocker): impmanager = ImpressionManager(StrategyDebugMode(), StrategyNoneMode(), telemetry_runtime_producer) split_storage = InMemorySplitStorage() segment_storage = InMemorySegmentStorage() - split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0])], [], -1) + rb_segment_storage = InMemoryRuleBasedSegmentStorage() + split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0])], [], -1) recorder = StandardRecorder(impmanager, event_storage, impression_storage, telemetry_producer.get_telemetry_evaluation_producer(), telemetry_producer.get_telemetry_runtime_producer()) destroyed_property = mocker.PropertyMock() destroyed_property.return_value = False @@ -1136,6 +1170,7 @@ def test_telemetry_method_latency(self, mocker): factory = SplitFactory(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -1189,6 +1224,7 @@ def stop(*_): def test_telemetry_track_exception(self, mocker): split_storage = mocker.Mock(spec=SplitStorage) segment_storage = mocker.Mock(spec=SegmentStorage) + rb_segment_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) impression_storage = mocker.Mock(spec=ImpressionStorage) event_storage = mocker.Mock(spec=EventStorage) destroyed_property = mocker.PropertyMock() @@ -1204,6 +1240,7 @@ def test_telemetry_track_exception(self, mocker): factory = SplitFactory(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -1238,12 +1275,13 @@ async def test_get_treatment_async(self, mocker): telemetry_producer = TelemetryStorageProducerAsync(telemetry_storage) split_storage = InMemorySplitStorageAsync() segment_storage = InMemorySegmentStorageAsync() + rb_segment_storage = InMemoryRuleBasedSegmentStorageAsync() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorageAsync(10, telemetry_runtime_producer) event_storage = mocker.Mock(spec=EventStorage) impmanager = ImpressionManager(StrategyDebugMode(), StrategyNoneMode(), telemetry_runtime_producer) recorder = StandardRecorderAsync(impmanager, event_storage, impression_storage, telemetry_producer.get_telemetry_evaluation_producer(), telemetry_producer.get_telemetry_runtime_producer()) - await split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0])], [], -1) + await split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0])], [], -1) destroyed_property = mocker.PropertyMock() destroyed_property.return_value = False @@ -1257,6 +1295,7 @@ async def synchronize_config(*_): factory = SplitFactoryAsync(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -1307,12 +1346,13 @@ async def test_get_treatment_with_config_async(self, mocker): telemetry_producer = TelemetryStorageProducerAsync(telemetry_storage) split_storage = InMemorySplitStorageAsync() segment_storage = InMemorySegmentStorageAsync() + rb_segment_storage = InMemoryRuleBasedSegmentStorageAsync() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorageAsync(10, telemetry_runtime_producer) event_storage = mocker.Mock(spec=EventStorage) impmanager = ImpressionManager(StrategyDebugMode(), StrategyNoneMode(), telemetry_runtime_producer) recorder = StandardRecorderAsync(impmanager, event_storage, impression_storage, telemetry_producer.get_telemetry_evaluation_producer(), telemetry_producer.get_telemetry_runtime_producer()) - await split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0])], [], -1) + await split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0])], [], -1) destroyed_property = mocker.PropertyMock() destroyed_property.return_value = False @@ -1320,6 +1360,7 @@ async def test_get_treatment_with_config_async(self, mocker): factory = SplitFactoryAsync(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -1382,12 +1423,13 @@ async def test_get_treatments_async(self, mocker): telemetry_producer = TelemetryStorageProducerAsync(telemetry_storage) split_storage = InMemorySplitStorageAsync() segment_storage = InMemorySegmentStorageAsync() + rb_segment_storage = InMemoryRuleBasedSegmentStorageAsync() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorageAsync(10, telemetry_runtime_producer) event_storage = mocker.Mock(spec=EventStorage) impmanager = ImpressionManager(StrategyDebugMode(), StrategyNoneMode(), telemetry_runtime_producer) recorder = StandardRecorderAsync(impmanager, event_storage, impression_storage, telemetry_producer.get_telemetry_evaluation_producer(), telemetry_producer.get_telemetry_runtime_producer()) - await split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0]), from_raw(splits_json['splitChange1_1']['splits'][1])], [], -1) + await split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0]), from_raw(splits_json['splitChange1_1']['ff']['d'][1])], [], -1) destroyed_property = mocker.PropertyMock() destroyed_property.return_value = False @@ -1395,6 +1437,7 @@ async def test_get_treatments_async(self, mocker): factory = SplitFactoryAsync(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -1460,12 +1503,13 @@ async def test_get_treatments_by_flag_set_async(self, mocker): telemetry_producer = TelemetryStorageProducerAsync(telemetry_storage) split_storage = InMemorySplitStorageAsync() segment_storage = InMemorySegmentStorageAsync() + rb_segment_storage = InMemoryRuleBasedSegmentStorageAsync() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorageAsync(10, telemetry_runtime_producer) event_storage = mocker.Mock(spec=EventStorage) impmanager = ImpressionManager(StrategyDebugMode(), StrategyNoneMode(), telemetry_runtime_producer) recorder = StandardRecorderAsync(impmanager, event_storage, impression_storage, telemetry_producer.get_telemetry_evaluation_producer(), telemetry_producer.get_telemetry_runtime_producer()) - await split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0]), from_raw(splits_json['splitChange1_1']['splits'][1])], [], -1) + await split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0]), from_raw(splits_json['splitChange1_1']['ff']['d'][1])], [], -1) destroyed_property = mocker.PropertyMock() destroyed_property.return_value = False @@ -1473,6 +1517,7 @@ async def test_get_treatments_by_flag_set_async(self, mocker): factory = SplitFactoryAsync(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -1538,12 +1583,13 @@ async def test_get_treatments_by_flag_sets_async(self, mocker): telemetry_producer = TelemetryStorageProducerAsync(telemetry_storage) split_storage = InMemorySplitStorageAsync() segment_storage = InMemorySegmentStorageAsync() + rb_segment_storage = InMemoryRuleBasedSegmentStorageAsync() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorageAsync(10, telemetry_runtime_producer) event_storage = mocker.Mock(spec=EventStorage) impmanager = ImpressionManager(StrategyDebugMode(), StrategyNoneMode(), telemetry_runtime_producer) recorder = StandardRecorderAsync(impmanager, event_storage, impression_storage, telemetry_producer.get_telemetry_evaluation_producer(), telemetry_producer.get_telemetry_runtime_producer()) - await split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0]), from_raw(splits_json['splitChange1_1']['splits'][1])], [], -1) + await split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0]), from_raw(splits_json['splitChange1_1']['ff']['d'][1])], [], -1) destroyed_property = mocker.PropertyMock() destroyed_property.return_value = False @@ -1551,6 +1597,7 @@ async def test_get_treatments_by_flag_sets_async(self, mocker): factory = SplitFactoryAsync(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -1616,18 +1663,20 @@ async def test_get_treatments_with_config(self, mocker): telemetry_producer = TelemetryStorageProducerAsync(telemetry_storage) split_storage = InMemorySplitStorageAsync() segment_storage = InMemorySegmentStorageAsync() + rb_segment_storage = InMemoryRuleBasedSegmentStorageAsync() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorageAsync(10, telemetry_runtime_producer) event_storage = mocker.Mock(spec=EventStorage) impmanager = ImpressionManager(StrategyDebugMode(), StrategyNoneMode(), telemetry_runtime_producer) recorder = StandardRecorderAsync(impmanager, event_storage, impression_storage, telemetry_producer.get_telemetry_evaluation_producer(), telemetry_producer.get_telemetry_runtime_producer()) - await split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0]), from_raw(splits_json['splitChange1_1']['splits'][1])], [], -1) + await split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0]), from_raw(splits_json['splitChange1_1']['ff']['d'][1])], [], -1) destroyed_property = mocker.PropertyMock() destroyed_property.return_value = False factory = SplitFactoryAsync(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -1698,18 +1747,20 @@ async def test_get_treatments_with_config_by_flag_set(self, mocker): telemetry_producer = TelemetryStorageProducerAsync(telemetry_storage) split_storage = InMemorySplitStorageAsync() segment_storage = InMemorySegmentStorageAsync() + rb_segment_storage = InMemoryRuleBasedSegmentStorageAsync() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorageAsync(10, telemetry_runtime_producer) event_storage = mocker.Mock(spec=EventStorage) impmanager = ImpressionManager(StrategyDebugMode(), StrategyNoneMode(), telemetry_runtime_producer) recorder = StandardRecorderAsync(impmanager, event_storage, impression_storage, telemetry_producer.get_telemetry_evaluation_producer(), telemetry_producer.get_telemetry_runtime_producer()) - await split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0]), from_raw(splits_json['splitChange1_1']['splits'][1])], [], -1) + await split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0]), from_raw(splits_json['splitChange1_1']['ff']['d'][1])], [], -1) destroyed_property = mocker.PropertyMock() destroyed_property.return_value = False factory = SplitFactoryAsync(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -1780,18 +1831,20 @@ async def test_get_treatments_with_config_by_flag_sets(self, mocker): telemetry_producer = TelemetryStorageProducerAsync(telemetry_storage) split_storage = InMemorySplitStorageAsync() segment_storage = InMemorySegmentStorageAsync() + rb_segment_storage = InMemoryRuleBasedSegmentStorageAsync() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorageAsync(10, telemetry_runtime_producer) event_storage = mocker.Mock(spec=EventStorage) impmanager = ImpressionManager(StrategyDebugMode(), StrategyNoneMode(), telemetry_runtime_producer) recorder = StandardRecorderAsync(impmanager, event_storage, impression_storage, telemetry_producer.get_telemetry_evaluation_producer(), telemetry_producer.get_telemetry_runtime_producer()) - await split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0]), from_raw(splits_json['splitChange1_1']['splits'][1])], [], -1) + await split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0]), from_raw(splits_json['splitChange1_1']['ff']['d'][1])], [], -1) destroyed_property = mocker.PropertyMock() destroyed_property.return_value = False factory = SplitFactoryAsync(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -1862,6 +1915,7 @@ async def test_impression_toggle_optimized(self, mocker): telemetry_producer = TelemetryStorageProducerAsync(telemetry_storage) split_storage = InMemorySplitStorageAsync() segment_storage = InMemorySegmentStorageAsync() + rb_segment_storage = InMemoryRuleBasedSegmentStorageAsync() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorageAsync(10, telemetry_runtime_producer) event_storage = mocker.Mock(spec=EventStorage) @@ -1877,6 +1931,7 @@ async def test_impression_toggle_optimized(self, mocker): factory = SplitFactoryAsync(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -1890,9 +1945,9 @@ async def test_impression_toggle_optimized(self, mocker): await factory.block_until_ready(5) await split_storage.update([ - from_raw(splits_json['splitChange1_1']['splits'][0]), - from_raw(splits_json['splitChange1_1']['splits'][1]), - from_raw(splits_json['splitChange1_1']['splits'][2]) + from_raw(splits_json['splitChange1_1']['ff']['d'][0]), + from_raw(splits_json['splitChange1_1']['ff']['d'][1]), + from_raw(splits_json['splitChange1_1']['ff']['d'][2]) ], [], -1) client = ClientAsync(factory, recorder, True) treatment = await client.get_treatment('some_key', 'SPLIT_1') @@ -1923,6 +1978,7 @@ async def test_impression_toggle_debug(self, mocker): telemetry_producer = TelemetryStorageProducerAsync(telemetry_storage) split_storage = InMemorySplitStorageAsync() segment_storage = InMemorySegmentStorageAsync() + rb_segment_storage = InMemoryRuleBasedSegmentStorageAsync() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorageAsync(10, telemetry_runtime_producer) event_storage = mocker.Mock(spec=EventStorage) @@ -1938,6 +1994,7 @@ async def test_impression_toggle_debug(self, mocker): factory = SplitFactoryAsync(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -1951,9 +2008,9 @@ async def test_impression_toggle_debug(self, mocker): await factory.block_until_ready(5) await split_storage.update([ - from_raw(splits_json['splitChange1_1']['splits'][0]), - from_raw(splits_json['splitChange1_1']['splits'][1]), - from_raw(splits_json['splitChange1_1']['splits'][2]) + from_raw(splits_json['splitChange1_1']['ff']['d'][0]), + from_raw(splits_json['splitChange1_1']['ff']['d'][1]), + from_raw(splits_json['splitChange1_1']['ff']['d'][2]) ], [], -1) client = ClientAsync(factory, recorder, True) assert await client.get_treatment('some_key', 'SPLIT_1') == 'off' @@ -1981,6 +2038,7 @@ async def test_impression_toggle_none(self, mocker): telemetry_producer = TelemetryStorageProducerAsync(telemetry_storage) split_storage = InMemorySplitStorageAsync() segment_storage = InMemorySegmentStorageAsync() + rb_segment_storage = InMemoryRuleBasedSegmentStorageAsync() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorageAsync(10, telemetry_runtime_producer) event_storage = mocker.Mock(spec=EventStorage) @@ -1996,6 +2054,7 @@ async def test_impression_toggle_none(self, mocker): factory = SplitFactoryAsync(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -2009,9 +2068,9 @@ async def test_impression_toggle_none(self, mocker): await factory.block_until_ready(5) await split_storage.update([ - from_raw(splits_json['splitChange1_1']['splits'][0]), - from_raw(splits_json['splitChange1_1']['splits'][1]), - from_raw(splits_json['splitChange1_1']['splits'][2]) + from_raw(splits_json['splitChange1_1']['ff']['d'][0]), + from_raw(splits_json['splitChange1_1']['ff']['d'][1]), + from_raw(splits_json['splitChange1_1']['ff']['d'][2]) ], [], -1) client = ClientAsync(factory, recorder, True) assert await client.get_treatment('some_key', 'SPLIT_1') == 'off' @@ -2027,6 +2086,7 @@ async def test_track_async(self, mocker): """Test that destroy/destroyed calls are forwarded to the factory.""" split_storage = InMemorySplitStorageAsync() segment_storage = mocker.Mock(spec=SegmentStorage) + rb_segment_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) impression_storage = mocker.Mock(spec=ImpressionStorage) event_storage = mocker.Mock(spec=EventStorage) self.events = [] @@ -2042,6 +2102,7 @@ async def put(event): factory = SplitFactoryAsync(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -2076,15 +2137,17 @@ async def test_telemetry_not_ready_async(self, mocker): telemetry_producer = TelemetryStorageProducerAsync(telemetry_storage) split_storage = InMemorySplitStorageAsync() segment_storage = InMemorySegmentStorageAsync() + rb_segment_storage = InMemoryRuleBasedSegmentStorageAsync() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorageAsync(10, telemetry_runtime_producer) event_storage = InMemoryEventStorageAsync(10, telemetry_runtime_producer) impmanager = ImpressionManager(StrategyDebugMode(), StrategyNoneMode(), telemetry_runtime_producer) recorder = StandardRecorderAsync(impmanager, event_storage, impression_storage, telemetry_producer.get_telemetry_evaluation_producer(), telemetry_producer.get_telemetry_runtime_producer()) - await split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0])], [], -1) + await split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0])], [], -1) factory = SplitFactoryAsync('localhost', {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': mocker.Mock()}, mocker.Mock(), @@ -2117,12 +2180,13 @@ async def test_telemetry_record_treatment_exception_async(self, mocker): telemetry_producer = TelemetryStorageProducerAsync(telemetry_storage) split_storage = InMemorySplitStorageAsync() segment_storage = InMemorySegmentStorageAsync() + rb_segment_storage = InMemoryRuleBasedSegmentStorageAsync() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorageAsync(10, telemetry_runtime_producer) event_storage = InMemoryEventStorageAsync(10, telemetry_runtime_producer) impmanager = ImpressionManager(StrategyDebugMode(), StrategyNoneMode(), telemetry_runtime_producer) recorder = StandardRecorderAsync(impmanager, event_storage, impression_storage, telemetry_producer.get_telemetry_evaluation_producer(), telemetry_producer.get_telemetry_runtime_producer()) - await split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0])], [], -1) + await split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0])], [], -1) destroyed_property = mocker.PropertyMock() destroyed_property.return_value = False @@ -2132,6 +2196,7 @@ async def test_telemetry_record_treatment_exception_async(self, mocker): factory = SplitFactoryAsync(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -2189,12 +2254,13 @@ async def test_telemetry_method_latency_async(self, mocker): telemetry_producer = TelemetryStorageProducerAsync(telemetry_storage) split_storage = InMemorySplitStorageAsync() segment_storage = InMemorySegmentStorageAsync() + rb_segment_storage = InMemoryRuleBasedSegmentStorageAsync() telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() impression_storage = InMemoryImpressionStorageAsync(10, telemetry_runtime_producer) event_storage = InMemoryEventStorageAsync(10, telemetry_runtime_producer) impmanager = ImpressionManager(StrategyDebugMode(), StrategyNoneMode(), telemetry_runtime_producer) recorder = StandardRecorderAsync(impmanager, event_storage, impression_storage, telemetry_producer.get_telemetry_evaluation_producer(), telemetry_producer.get_telemetry_runtime_producer()) - await split_storage.update([from_raw(splits_json['splitChange1_1']['splits'][0])], [], -1) + await split_storage.update([from_raw(splits_json['splitChange1_1']['ff']['d'][0])], [], -1) destroyed_property = mocker.PropertyMock() destroyed_property.return_value = False @@ -2204,6 +2270,7 @@ async def test_telemetry_method_latency_async(self, mocker): factory = SplitFactoryAsync(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), @@ -2260,6 +2327,7 @@ async def synchronize_config(*_): async def test_telemetry_track_exception_async(self, mocker): split_storage = InMemorySplitStorageAsync() segment_storage = mocker.Mock(spec=SegmentStorage) + rb_segment_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) impression_storage = mocker.Mock(spec=ImpressionStorage) destroyed_property = mocker.PropertyMock() destroyed_property.return_value = False @@ -2275,6 +2343,7 @@ async def test_telemetry_track_exception_async(self, mocker): factory = SplitFactoryAsync(mocker.Mock(), {'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': impression_storage, 'events': event_storage}, mocker.Mock(), diff --git a/tests/client/test_input_validator.py b/tests/client/test_input_validator.py index 5afecdd4..0659ee43 100644 --- a/tests/client/test_input_validator.py +++ b/tests/client/test_input_validator.py @@ -6,9 +6,9 @@ from splitio.client.client import CONTROL, Client, _LOGGER as _logger, ClientAsync from splitio.client.manager import SplitManager, SplitManagerAsync from splitio.client.key import Key -from splitio.storage import SplitStorage, EventStorage, ImpressionStorage, SegmentStorage +from splitio.storage import SplitStorage, EventStorage, ImpressionStorage, SegmentStorage, RuleBasedSegmentsStorage from splitio.storage.inmemmory import InMemoryTelemetryStorage, InMemoryTelemetryStorageAsync, \ - InMemorySplitStorage, InMemorySplitStorageAsync + InMemorySplitStorage, InMemorySplitStorageAsync, InMemoryRuleBasedSegmentStorage, InMemoryRuleBasedSegmentStorageAsync from splitio.models.splits import Split from splitio.client import input_validator from splitio.recorder.recorder import StandardRecorder, StandardRecorderAsync @@ -28,8 +28,11 @@ def test_get_treatment(self, mocker): conditions_mock = mocker.PropertyMock() conditions_mock.return_value = [] type(split_mock).conditions = conditions_mock + type(split_mock).prerequisites = [] storage_mock = mocker.Mock(spec=SplitStorage) storage_mock.fetch_many.return_value = {'some_feature': split_mock} + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorage) + rbs_storage.fetch_many.return_value = {} impmanager = mocker.Mock(spec=ImpressionManager) telemetry_storage = InMemoryTelemetryStorage() @@ -40,6 +43,7 @@ def test_get_treatment(self, mocker): { 'splits': storage_mock, 'segments': mocker.Mock(spec=SegmentStorage), + 'rule_based_segments': rbs_storage, 'impressions': mocker.Mock(spec=ImpressionStorage), 'events': mocker.Mock(spec=EventStorage), }, @@ -261,12 +265,15 @@ def test_get_treatment_with_config(self, mocker): conditions_mock = mocker.PropertyMock() conditions_mock.return_value = [] type(split_mock).conditions = conditions_mock + type(split_mock).prerequisites = [] def _configs(treatment): return '{"some": "property"}' if treatment == 'default_treatment' else None split_mock.get_configurations_for.side_effect = _configs storage_mock = mocker.Mock(spec=SplitStorage) storage_mock.fetch_many.return_value = {'some_feature': split_mock} + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorage) + rbs_storage.fetch_many.return_value = {} impmanager = mocker.Mock(spec=ImpressionManager) telemetry_storage = InMemoryTelemetryStorage() @@ -277,6 +284,7 @@ def _configs(treatment): { 'splits': storage_mock, 'segments': mocker.Mock(spec=SegmentStorage), + 'rule_based_segments': rbs_storage, 'impressions': mocker.Mock(spec=ImpressionStorage), 'events': mocker.Mock(spec=EventStorage), }, @@ -551,6 +559,7 @@ def test_track(self, mocker): { 'splits': split_storage_mock, 'segments': mocker.Mock(spec=SegmentStorage), + 'rule_based_segments': mocker.Mock(spec=RuleBasedSegmentsStorage), 'impressions': mocker.Mock(spec=ImpressionStorage), 'events': events_storage_mock, }, @@ -812,10 +821,15 @@ def test_get_treatments(self, mocker): conditions_mock = mocker.PropertyMock() conditions_mock.return_value = [] type(split_mock).conditions = conditions_mock + type(split_mock).prerequisites = [] + storage_mock = mocker.Mock(spec=SplitStorage) storage_mock.fetch_many.return_value = { 'some_feature': split_mock } + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorage) + rbs_storage.fetch_many.return_value = {} + impmanager = mocker.Mock(spec=ImpressionManager) telemetry_storage = InMemoryTelemetryStorage() telemetry_producer = TelemetryStorageProducer(telemetry_storage) @@ -825,6 +839,7 @@ def test_get_treatments(self, mocker): { 'splits': storage_mock, 'segments': mocker.Mock(spec=SegmentStorage), + 'rule_based_segments': rbs_storage, 'impressions': mocker.Mock(spec=ImpressionStorage), 'events': mocker.Mock(spec=EventStorage), }, @@ -954,11 +969,14 @@ def test_get_treatments_with_config(self, mocker): conditions_mock = mocker.PropertyMock() conditions_mock.return_value = [] type(split_mock).conditions = conditions_mock + type(split_mock).prerequisites = [] storage_mock = mocker.Mock(spec=SplitStorage) storage_mock.fetch_many.return_value = { 'some_feature': split_mock } + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorage) + rbs_storage.fetch_many.return_value = {} impmanager = mocker.Mock(spec=ImpressionManager) telemetry_storage = InMemoryTelemetryStorage() @@ -969,6 +987,7 @@ def test_get_treatments_with_config(self, mocker): { 'splits': storage_mock, 'segments': mocker.Mock(spec=SegmentStorage), + 'rule_based_segments': rbs_storage, 'impressions': mocker.Mock(spec=ImpressionStorage), 'events': mocker.Mock(spec=EventStorage), }, @@ -1099,10 +1118,13 @@ def test_get_treatments_by_flag_set(self, mocker): conditions_mock = mocker.PropertyMock() conditions_mock.return_value = [] type(split_mock).conditions = conditions_mock + type(split_mock).prerequisites = [] storage_mock = mocker.Mock(spec=InMemorySplitStorage) storage_mock.fetch_many.return_value = { 'some_feature': split_mock } + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorage) + rbs_storage.fetch_many.return_value = {} storage_mock.get_feature_flags_by_sets.return_value = ['some_feature'] impmanager = mocker.Mock(spec=ImpressionManager) telemetry_storage = InMemoryTelemetryStorage() @@ -1113,6 +1135,7 @@ def test_get_treatments_by_flag_set(self, mocker): { 'splits': storage_mock, 'segments': mocker.Mock(spec=SegmentStorage), + 'rule_based_segments': rbs_storage, 'impressions': mocker.Mock(spec=ImpressionStorage), 'events': mocker.Mock(spec=EventStorage), }, @@ -1214,10 +1237,13 @@ def test_get_treatments_by_flag_sets(self, mocker): conditions_mock = mocker.PropertyMock() conditions_mock.return_value = [] type(split_mock).conditions = conditions_mock + type(split_mock).prerequisites = [] storage_mock = mocker.Mock(spec=InMemorySplitStorage) storage_mock.fetch_many.return_value = { 'some_feature': split_mock } + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorage) + rbs_storage.fetch_many.return_value = {} storage_mock.get_feature_flags_by_sets.return_value = ['some_feature'] impmanager = mocker.Mock(spec=ImpressionManager) telemetry_storage = InMemoryTelemetryStorage() @@ -1228,6 +1254,7 @@ def test_get_treatments_by_flag_sets(self, mocker): { 'splits': storage_mock, 'segments': mocker.Mock(spec=SegmentStorage), + 'rule_based_segments': rbs_storage, 'impressions': mocker.Mock(spec=ImpressionStorage), 'events': mocker.Mock(spec=EventStorage), }, @@ -1338,10 +1365,14 @@ def _configs(treatment): conditions_mock = mocker.PropertyMock() conditions_mock.return_value = [] type(split_mock).conditions = conditions_mock + type(split_mock).prerequisites = [] storage_mock = mocker.Mock(spec=InMemorySplitStorage) storage_mock.fetch_many.return_value = { 'some_feature': split_mock } + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorage) + rbs_storage.fetch_many.return_value = {} + storage_mock.get_feature_flags_by_sets.return_value = ['some_feature'] impmanager = mocker.Mock(spec=ImpressionManager) @@ -1353,6 +1384,7 @@ def _configs(treatment): { 'splits': storage_mock, 'segments': mocker.Mock(spec=SegmentStorage), + 'rule_based_segments': rbs_storage, 'impressions': mocker.Mock(spec=ImpressionStorage), 'events': mocker.Mock(spec=EventStorage), }, @@ -1457,10 +1489,14 @@ def _configs(treatment): conditions_mock = mocker.PropertyMock() conditions_mock.return_value = [] type(split_mock).conditions = conditions_mock + type(split_mock).prerequisites = [] storage_mock = mocker.Mock(spec=InMemorySplitStorage) storage_mock.fetch_many.return_value = { 'some_feature': split_mock } + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorage) + rbs_storage.fetch_many.return_value = {} + storage_mock.get_feature_flags_by_sets.return_value = ['some_feature'] impmanager = mocker.Mock(spec=ImpressionManager) @@ -1472,6 +1508,7 @@ def _configs(treatment): { 'splits': storage_mock, 'segments': mocker.Mock(spec=SegmentStorage), + 'rule_based_segments': rbs_storage, 'impressions': mocker.Mock(spec=ImpressionStorage), 'events': mocker.Mock(spec=EventStorage), }, @@ -1604,12 +1641,17 @@ async def test_get_treatment(self, mocker): conditions_mock = mocker.PropertyMock() conditions_mock.return_value = [] type(split_mock).conditions = conditions_mock + type(split_mock).prerequisites = [] storage_mock = mocker.Mock(spec=SplitStorage) async def fetch_many(*_): return { 'some_feature': split_mock } storage_mock.fetch_many = fetch_many + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorageAsync) + async def fetch_many_rbs(*_): + return {} + rbs_storage.fetch_many = fetch_many_rbs async def get_change_number(*_): return 1 @@ -1624,6 +1666,7 @@ async def get_change_number(*_): { 'splits': storage_mock, 'segments': mocker.Mock(spec=SegmentStorage), + 'rule_based_segments': rbs_storage, 'impressions': mocker.Mock(spec=ImpressionStorage), 'events': mocker.Mock(spec=EventStorage), }, @@ -1856,6 +1899,7 @@ async def test_get_treatment_with_config(self, mocker): conditions_mock = mocker.PropertyMock() conditions_mock.return_value = [] type(split_mock).conditions = conditions_mock + type(split_mock).prerequisites = [] def _configs(treatment): return '{"some": "property"}' if treatment == 'default_treatment' else None @@ -1866,6 +1910,10 @@ async def fetch_many(*_): 'some_feature': split_mock } storage_mock.fetch_many = fetch_many + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorageAsync) + async def fetch_many_rbs(*_): + return {} + rbs_storage.fetch_many = fetch_many_rbs async def get_change_number(*_): return 1 @@ -1880,6 +1928,7 @@ async def get_change_number(*_): { 'splits': storage_mock, 'segments': mocker.Mock(spec=SegmentStorage), + 'rule_based_segments': rbs_storage, 'impressions': mocker.Mock(spec=ImpressionStorage), 'events': mocker.Mock(spec=EventStorage), }, @@ -2123,6 +2172,7 @@ async def put(*_): { 'splits': split_storage_mock, 'segments': mocker.Mock(spec=SegmentStorage), + 'rule_based_segments': mocker.Mock(spec=RuleBasedSegmentsStorage), 'impressions': mocker.Mock(spec=ImpressionStorage), 'events': events_storage_mock, }, @@ -2384,6 +2434,7 @@ async def test_get_treatments(self, mocker): conditions_mock = mocker.PropertyMock() conditions_mock.return_value = [] type(split_mock).conditions = conditions_mock + type(split_mock).prerequisites = [] storage_mock = mocker.Mock(spec=SplitStorage) async def get(*_): return split_mock @@ -2397,6 +2448,10 @@ async def fetch_many(*_): 'some': split_mock, } storage_mock.fetch_many = fetch_many + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorageAsync) + async def fetch_many_rbs(*_): + return {} + rbs_storage.fetch_many = fetch_many_rbs impmanager = mocker.Mock(spec=ImpressionManager) telemetry_storage = await InMemoryTelemetryStorageAsync.create() @@ -2407,6 +2462,7 @@ async def fetch_many(*_): { 'splits': storage_mock, 'segments': mocker.Mock(spec=SegmentStorage), + 'rule_based_segments': rbs_storage, 'impressions': mocker.Mock(spec=ImpressionStorage), 'events': mocker.Mock(spec=EventStorage), }, @@ -2542,6 +2598,7 @@ async def test_get_treatments_with_config(self, mocker): conditions_mock = mocker.PropertyMock() conditions_mock.return_value = [] type(split_mock).conditions = conditions_mock + type(split_mock).prerequisites = [] storage_mock = mocker.Mock(spec=SplitStorage) async def get(*_): @@ -2555,6 +2612,10 @@ async def fetch_many(*_): 'some_feature': split_mock } storage_mock.fetch_many = fetch_many + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorageAsync) + async def fetch_many_rbs(*_): + return {} + rbs_storage.fetch_many = fetch_many_rbs impmanager = mocker.Mock(spec=ImpressionManager) telemetry_storage = await InMemoryTelemetryStorageAsync.create() @@ -2565,6 +2626,7 @@ async def fetch_many(*_): { 'splits': storage_mock, 'segments': mocker.Mock(spec=SegmentStorage), + 'rule_based_segments': rbs_storage, 'impressions': mocker.Mock(spec=ImpressionStorage), 'events': mocker.Mock(spec=EventStorage), }, @@ -2700,6 +2762,7 @@ async def test_get_treatments_by_flag_set(self, mocker): conditions_mock = mocker.PropertyMock() conditions_mock.return_value = [] type(split_mock).conditions = conditions_mock + type(split_mock).prerequisites = [] storage_mock = mocker.Mock(spec=SplitStorage) async def get(*_): return split_mock @@ -2716,6 +2779,10 @@ async def fetch_many(*_): async def get_feature_flags_by_sets(*_): return ['some_feature'] storage_mock.get_feature_flags_by_sets = get_feature_flags_by_sets + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorageAsync) + async def fetch_many_rbs(*_): + return {} + rbs_storage.fetch_many = fetch_many_rbs impmanager = mocker.Mock(spec=ImpressionManager) telemetry_storage = await InMemoryTelemetryStorageAsync.create() @@ -2726,6 +2793,7 @@ async def get_feature_flags_by_sets(*_): { 'splits': storage_mock, 'segments': mocker.Mock(spec=SegmentStorage), + 'rule_based_segments': rbs_storage, 'impressions': mocker.Mock(spec=ImpressionStorage), 'events': mocker.Mock(spec=EventStorage), }, @@ -2839,6 +2907,7 @@ async def test_get_treatments_by_flag_sets(self, mocker): conditions_mock = mocker.PropertyMock() conditions_mock.return_value = [] type(split_mock).conditions = conditions_mock + type(split_mock).prerequisites = [] storage_mock = mocker.Mock(spec=SplitStorage) async def get(*_): return split_mock @@ -2852,6 +2921,11 @@ async def fetch_many(*_): 'some': split_mock, } storage_mock.fetch_many = fetch_many + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorageAsync) + async def fetch_many_rbs(*_): + return {} + rbs_storage.fetch_many = fetch_many_rbs + async def get_feature_flags_by_sets(*_): return ['some_feature'] storage_mock.get_feature_flags_by_sets = get_feature_flags_by_sets @@ -2865,6 +2939,7 @@ async def get_feature_flags_by_sets(*_): { 'splits': storage_mock, 'segments': mocker.Mock(spec=SegmentStorage), + 'rule_based_segments': rbs_storage, 'impressions': mocker.Mock(spec=ImpressionStorage), 'events': mocker.Mock(spec=EventStorage), }, @@ -2988,6 +3063,7 @@ def _configs(treatment): conditions_mock = mocker.PropertyMock() conditions_mock.return_value = [] type(split_mock).conditions = conditions_mock + type(split_mock).prerequisites = [] storage_mock = mocker.Mock(spec=SplitStorage) async def get(*_): return split_mock @@ -3001,6 +3077,10 @@ async def fetch_many(*_): 'some': split_mock, } storage_mock.fetch_many = fetch_many + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorageAsync) + async def fetch_many_rbs(*_): + return {} + rbs_storage.fetch_many = fetch_many_rbs async def get_feature_flags_by_sets(*_): return ['some_feature'] storage_mock.get_feature_flags_by_sets = get_feature_flags_by_sets @@ -3014,6 +3094,7 @@ async def get_feature_flags_by_sets(*_): { 'splits': storage_mock, 'segments': mocker.Mock(spec=SegmentStorage), + 'rule_based_segments': rbs_storage, 'impressions': mocker.Mock(spec=ImpressionStorage), 'events': mocker.Mock(spec=EventStorage), }, @@ -3130,6 +3211,7 @@ def _configs(treatment): conditions_mock = mocker.PropertyMock() conditions_mock.return_value = [] type(split_mock).conditions = conditions_mock + type(split_mock).prerequisites = [] storage_mock = mocker.Mock(spec=SplitStorage) async def get(*_): return split_mock @@ -3143,6 +3225,11 @@ async def fetch_many(*_): 'some': split_mock, } storage_mock.fetch_many = fetch_many + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorageAsync) + async def fetch_many_rbs(*_): + return {} + rbs_storage.fetch_many = fetch_many_rbs + async def get_feature_flags_by_sets(*_): return ['some_feature'] storage_mock.get_feature_flags_by_sets = get_feature_flags_by_sets @@ -3156,6 +3243,7 @@ async def get_feature_flags_by_sets(*_): { 'splits': storage_mock, 'segments': mocker.Mock(spec=SegmentStorage), + 'rule_based_segments': rbs_storage, 'impressions': mocker.Mock(spec=ImpressionStorage), 'events': mocker.Mock(spec=EventStorage), }, @@ -3312,6 +3400,7 @@ def test_split_(self, mocker): { 'splits': storage_mock, 'segments': mocker.Mock(spec=SegmentStorage), + 'rule_based_segments': mocker.Mock(spec=RuleBasedSegmentsStorage), 'impressions': mocker.Mock(spec=ImpressionStorage), 'events': mocker.Mock(spec=EventStorage), }, @@ -3388,6 +3477,7 @@ async def get(*_): { 'splits': storage_mock, 'segments': mocker.Mock(spec=SegmentStorage), + 'rule_based_segments': mocker.Mock(spec=RuleBasedSegmentsStorage), 'impressions': mocker.Mock(spec=ImpressionStorage), 'events': mocker.Mock(spec=EventStorage), }, diff --git a/tests/client/test_localhost.py b/tests/client/test_localhost.py index 280e79f9..598d6100 100644 --- a/tests/client/test_localhost.py +++ b/tests/client/test_localhost.py @@ -6,7 +6,7 @@ from splitio.sync.split import LocalSplitSynchronizer from splitio.models.splits import Split from splitio.models.grammar.matchers import AllKeysMatcher -from splitio.storage import SplitStorage +from splitio.storage import SplitStorage, RuleBasedSegmentsStorage class LocalHostStoragesTests(object): @@ -112,10 +112,10 @@ def test_update_splits(self, mocker): parse_yaml.return_value = {} storage_mock = mocker.Mock(spec=SplitStorage) storage_mock.get_split_names.return_value = [] - + rbs = mocker.Mock(spec=RuleBasedSegmentsStorage) parse_legacy.reset_mock() parse_yaml.reset_mock() - sync = LocalSplitSynchronizer('something', storage_mock) + sync = LocalSplitSynchronizer('something', storage_mock, rbs) sync._read_feature_flags_from_legacy_file = parse_legacy sync._read_feature_flags_from_yaml_file = parse_yaml sync.synchronize_splits() @@ -124,7 +124,7 @@ def test_update_splits(self, mocker): parse_legacy.reset_mock() parse_yaml.reset_mock() - sync = LocalSplitSynchronizer('something.yaml', storage_mock) + sync = LocalSplitSynchronizer('something.yaml', storage_mock, rbs) sync._read_feature_flags_from_legacy_file = parse_legacy sync._read_feature_flags_from_yaml_file = parse_yaml sync.synchronize_splits() @@ -133,7 +133,7 @@ def test_update_splits(self, mocker): parse_legacy.reset_mock() parse_yaml.reset_mock() - sync = LocalSplitSynchronizer('something.yml', storage_mock) + sync = LocalSplitSynchronizer('something.yml', storage_mock, rbs) sync._read_feature_flags_from_legacy_file = parse_legacy sync._read_feature_flags_from_yaml_file = parse_yaml sync.synchronize_splits() @@ -142,7 +142,7 @@ def test_update_splits(self, mocker): parse_legacy.reset_mock() parse_yaml.reset_mock() - sync = LocalSplitSynchronizer('something.YAML', storage_mock) + sync = LocalSplitSynchronizer('something.YAML', storage_mock, rbs) sync._read_feature_flags_from_legacy_file = parse_legacy sync._read_feature_flags_from_yaml_file = parse_yaml sync.synchronize_splits() @@ -151,7 +151,7 @@ def test_update_splits(self, mocker): parse_legacy.reset_mock() parse_yaml.reset_mock() - sync = LocalSplitSynchronizer('yaml', storage_mock) + sync = LocalSplitSynchronizer('yaml', storage_mock, rbs) sync._read_feature_flags_from_legacy_file = parse_legacy sync._read_feature_flags_from_yaml_file = parse_yaml sync.synchronize_splits() diff --git a/tests/client/test_manager.py b/tests/client/test_manager.py index ae856f9a..19e1bbb0 100644 --- a/tests/client/test_manager.py +++ b/tests/client/test_manager.py @@ -26,8 +26,8 @@ def test_manager_calls(self, mocker): factory.ready = True manager = SplitManager(factory) - split1 = splits.from_raw(splits_json["splitChange1_1"]["splits"][0]) - split2 = splits.from_raw(splits_json["splitChange1_3"]["splits"][0]) + split1 = splits.from_raw(splits_json["splitChange1_1"]['ff']['d'][0]) + split2 = splits.from_raw(splits_json["splitChange1_3"]['ff']['d'][0]) storage.update([split1, split2], [], -1) manager._storage = storage @@ -98,8 +98,8 @@ async def test_manager_calls(self, mocker): factory.ready = True manager = SplitManagerAsync(factory) - split1 = splits.from_raw(splits_json["splitChange1_1"]["splits"][0]) - split2 = splits.from_raw(splits_json["splitChange1_3"]["splits"][0]) + split1 = splits.from_raw(splits_json["splitChange1_1"]['ff']['d'][0]) + split2 = splits.from_raw(splits_json["splitChange1_3"]['ff']['d'][0]) await storage.update([split1, split2], [], -1) manager._storage = storage diff --git a/tests/engine/files/rule_base_segments.json b/tests/engine/files/rule_base_segments.json new file mode 100644 index 00000000..70b64b32 --- /dev/null +++ b/tests/engine/files/rule_base_segments.json @@ -0,0 +1,62 @@ +{"ff": {"d": [], "t": -1, "s": -1}, +"rbs": {"t": -1, "s": -1, "d": + [{ + "changeNumber": 5, + "name": "dependent_rbs", + "status": "ACTIVE", + "trafficTypeName": "user", + "excluded":{"keys":["mauro@split.io","gaston@split.io"],"segments":[]}, + "conditions": [ + { + "conditionType": "WHITELIST", + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "keySelector": { + "trafficType": "user", + "attribute": "email" + }, + "matcherType": "ENDS_WITH", + "negate": false, + "whitelistMatcherData": { + "whitelist": [ + "@split.io" + ] + } + } + ] + } + } + ]}, + { + "changeNumber": 5, + "name": "sample_rule_based_segment", + "status": "ACTIVE", + "trafficTypeName": "user", + "excluded": { + "keys": [], + "segments": [] + }, + "conditions": [ + { + "conditionType": "ROLLOUT", + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "keySelector": { + "trafficType": "user" + }, + "matcherType": "IN_RULE_BASED_SEGMENT", + "negate": false, + "userDefinedSegmentMatcherData": { + "segmentName": "dependent_rbs" + } + } + ] + } + } + ] + }] +}} diff --git a/tests/engine/files/rule_base_segments2.json b/tests/engine/files/rule_base_segments2.json new file mode 100644 index 00000000..2f77ecd5 --- /dev/null +++ b/tests/engine/files/rule_base_segments2.json @@ -0,0 +1,67 @@ +{"ff": {"d": [], "t": -1, "s": -1}, +"rbs": {"t": -1, "s": -1, "d": [ + { + "changeNumber": 5, + "name": "sample_rule_based_segment", + "status": "ACTIVE", + "trafficTypeName": "user", + "excluded":{ + "keys":["mauro@split.io","gaston@split.io"], + "segments":[{"type":"rule-based", "name":"no_excludes"}] + }, + "conditions": [ + { + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "keySelector": { + "trafficType": "user", + "attribute": "email" + }, + "matcherType": "STARTS_WITH", + "negate": false, + "whitelistMatcherData": { + "whitelist": [ + "bilal" + ] + } + } + ] + } + } + ] + }, + { + "changeNumber": 5, + "name": "no_excludes", + "status": "ACTIVE", + "trafficTypeName": "user", + "excluded":{ + "keys":["bilal2@split.io"], + "segments":[] + }, + "conditions": [ + { + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "keySelector": { + "trafficType": "user", + "attribute": "email" + }, + "matcherType": "ENDS_WITH", + "negate": false, + "whitelistMatcherData": { + "whitelist": [ + "@split.io" + ] + } + } + ] + } + } + ] + } +]}} diff --git a/tests/engine/files/rule_base_segments3.json b/tests/engine/files/rule_base_segments3.json new file mode 100644 index 00000000..f738f3f7 --- /dev/null +++ b/tests/engine/files/rule_base_segments3.json @@ -0,0 +1,35 @@ +{"ff": {"d": [], "t": -1, "s": -1}, +"rbs": {"t": -1, "s": -1, "d": [ + { + "changeNumber": 5, + "name": "sample_rule_based_segment", + "status": "ACTIVE", + "trafficTypeName": "user", + "excluded":{ + "keys":["mauro@split.io","gaston@split.io"], + "segments":[{"type":"standard", "name":"segment1"}] + }, + "conditions": [ + { + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "keySelector": { + "trafficType": "user", + "attribute": "email" + }, + "matcherType": "ENDS_WITH", + "negate": false, + "whitelistMatcherData": { + "whitelist": [ + "@split.io" + ] + } + } + ] + } + } + ] + } +]}} diff --git a/tests/engine/test_evaluator.py b/tests/engine/test_evaluator.py index 67c7387d..3ec7e136 100644 --- a/tests/engine/test_evaluator.py +++ b/tests/engine/test_evaluator.py @@ -1,13 +1,113 @@ """Evaluator tests module.""" +import json import logging +import os import pytest +import copy -from splitio.models.splits import Split +from splitio.models.splits import Split, Status, from_raw, Prerequisites +from splitio.models import segments from splitio.models.grammar.condition import Condition, ConditionType from splitio.models.impressions import Label +from splitio.models.grammar import condition +from splitio.models import rule_based_segments from splitio.engine import evaluator, splitters from splitio.engine.evaluator import EvaluationContext +from splitio.storage.inmemmory import InMemorySplitStorage, InMemorySegmentStorage, InMemoryRuleBasedSegmentStorage, \ + InMemorySplitStorageAsync, InMemorySegmentStorageAsync, InMemoryRuleBasedSegmentStorageAsync +from splitio.engine.evaluator import EvaluationDataFactory, AsyncEvaluationDataFactory +rbs_raw = { + "changeNumber": 123, + "name": "sample_rule_based_segment", + "status": "ACTIVE", + "trafficTypeName": "user", + "excluded":{ + "keys":["mauro@split.io","gaston@split.io"], + "segments":[] + }, + "conditions": [ + { + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "keySelector": { + "trafficType": "user", + "attribute": "email" + }, + "matcherType": "ENDS_WITH", + "negate": False, + "whitelistMatcherData": { + "whitelist": [ + "@split.io" + ] + } + } + ] + } + } + ] +} + +split_conditions = [ + condition.from_raw({ + "conditionType": "ROLLOUT", + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "keySelector": { + "trafficType": "user" + }, + "matcherType": "IN_RULE_BASED_SEGMENT", + "negate": False, + "userDefinedSegmentMatcherData": { + "segmentName": "sample_rule_based_segment" + } + } + ] + }, + "partitions": [ + { + "treatment": "on", + "size": 100 + }, + { + "treatment": "off", + "size": 0 + } + ], + "label": "in rule based segment sample_rule_based_segment" + }), + condition.from_raw({ + "conditionType": "ROLLOUT", + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "keySelector": { + "trafficType": "user" + }, + "matcherType": "ALL_KEYS", + "negate": False + } + ] + }, + "partitions": [ + { + "treatment": "on", + "size": 0 + }, + { + "treatment": "off", + "size": 100 + } + ], + "label": "default rule" + }) +] + class EvaluatorTests(object): """Test evaluator behavior.""" @@ -18,7 +118,7 @@ def _build_evaluator_with_mocks(self, mocker): e = evaluator.Evaluator(splitter_mock) evaluator._LOGGER = logger_mock return e - + def test_evaluate_treatment_killed_split(self, mocker): """Test that a killed split returns the default treatment.""" e = self._build_evaluator_with_mocks(mocker) @@ -27,7 +127,9 @@ def test_evaluate_treatment_killed_split(self, mocker): mocked_split.killed = True mocked_split.change_number = 123 mocked_split.get_configurations_for.return_value = '{"some_property": 123}' - ctx = EvaluationContext(flags={'some': mocked_split}, segment_memberships=set()) + mocked_split.prerequisites = [] + + ctx = EvaluationContext(flags={'some': mocked_split}, segment_memberships=set(), rbs_segments={}) result = e.eval_with_context('some_key', 'some_bucketing_key', 'some', {}, ctx) assert result['treatment'] == 'off' assert result['configurations'] == '{"some_property": 123}' @@ -45,7 +147,9 @@ def test_evaluate_treatment_ok(self, mocker): mocked_split.killed = False mocked_split.change_number = 123 mocked_split.get_configurations_for.return_value = '{"some_property": 123}' - ctx = EvaluationContext(flags={'some': mocked_split}, segment_memberships=set()) + mocked_split.prerequisites = [] + + ctx = EvaluationContext(flags={'some': mocked_split}, segment_memberships=set(), rbs_segments={}) result = e.eval_with_context('some_key', 'some_bucketing_key', 'some', {}, ctx) assert result['treatment'] == 'on' assert result['configurations'] == '{"some_property": 123}' @@ -54,7 +158,6 @@ def test_evaluate_treatment_ok(self, mocker): assert mocked_split.get_configurations_for.mock_calls == [mocker.call('on')] assert result['impressions_disabled'] == mocked_split.impressions_disabled - def test_evaluate_treatment_ok_no_config(self, mocker): """Test that a killed split returns the default treatment.""" e = self._build_evaluator_with_mocks(mocker) @@ -65,7 +168,9 @@ def test_evaluate_treatment_ok_no_config(self, mocker): mocked_split.killed = False mocked_split.change_number = 123 mocked_split.get_configurations_for.return_value = None - ctx = EvaluationContext(flags={'some': mocked_split}, segment_memberships=set()) + mocked_split.prerequisites = [] + + ctx = EvaluationContext(flags={'some': mocked_split}, segment_memberships=set(), rbs_segments={}) result = e.eval_with_context('some_key', 'some_bucketing_key', 'some', {}, ctx) assert result['treatment'] == 'on' assert result['configurations'] == None @@ -84,6 +189,7 @@ def test_evaluate_treatments(self, mocker): mocked_split.killed = False mocked_split.change_number = 123 mocked_split.get_configurations_for.return_value = '{"some_property": 123}' + mocked_split.prerequisites = [] mocked_split2 = mocker.Mock(spec=Split) mocked_split2.name = 'feature4' @@ -91,8 +197,9 @@ def test_evaluate_treatments(self, mocker): mocked_split2.killed = False mocked_split2.change_number = 123 mocked_split2.get_configurations_for.return_value = None + mocked_split2.prerequisites = [] - ctx = EvaluationContext(flags={'feature2': mocked_split, 'feature4': mocked_split2}, segment_memberships=set()) + ctx = EvaluationContext(flags={'feature2': mocked_split, 'feature4': mocked_split2}, segment_memberships=set(), rbs_segments={}) results = e.eval_many_with_context('some_key', 'some_bucketing_key', ['feature2', 'feature4'], {}, ctx) result = results['feature4'] assert result['configurations'] == None @@ -115,7 +222,9 @@ def test_get_gtreatment_for_split_no_condition_matches(self, mocker): mocked_split.change_number = '123' mocked_split.conditions = [] mocked_split.get_configurations_for = None - ctx = EvaluationContext(flags={'some': mocked_split}, segment_memberships=set()) + mocked_split.prerequisites = [] + + ctx = EvaluationContext(flags={'some': mocked_split}, segment_memberships=set(), rbs_segments={}) assert e._treatment_for_flag(mocked_split, 'some_key', 'some_bucketing', {}, ctx) == ( 'off', Label.NO_CONDITION_MATCHED @@ -132,6 +241,334 @@ def test_get_gtreatment_for_split_non_rollout(self, mocker): mocked_split = mocker.Mock(spec=Split) mocked_split.killed = False mocked_split.conditions = [mocked_condition_1] - treatment, label = e._treatment_for_flag(mocked_split, 'some_key', 'some_bucketing', {}, EvaluationContext(None, None)) + mocked_split.prerequisites = [] + + treatment, label = e._treatment_for_flag(mocked_split, 'some_key', 'some_bucketing', {}, EvaluationContext(None, None, None)) assert treatment == 'on' assert label == 'some_label' + + def test_evaluate_treatment_with_rule_based_segment(self, mocker): + """Test that a non-killed split returns the appropriate treatment.""" + e = evaluator.Evaluator(splitters.Splitter()) + + mocked_split = Split('some', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False, []) + + ctx = EvaluationContext(flags={'some': mocked_split}, segment_memberships=set(), rbs_segments={'sample_rule_based_segment': rule_based_segments.from_raw(rbs_raw)}) + result = e.eval_with_context('bilal@split.io', 'bilal@split.io', 'some', {'email': 'bilal@split.io'}, ctx) + assert result['treatment'] == 'on' + + def test_evaluate_treatment_with_rbs_in_condition(self): + e = evaluator.Evaluator(splitters.Splitter()) + splits_storage = InMemorySplitStorage() + rbs_storage = InMemoryRuleBasedSegmentStorage() + segment_storage = InMemorySegmentStorage() + evaluation_facctory = EvaluationDataFactory(splits_storage, segment_storage, rbs_storage) + + rbs_segments = os.path.join(os.path.dirname(__file__), 'files', 'rule_base_segments.json') + with open(rbs_segments, 'r') as flo: + data = json.loads(flo.read()) + + mocked_split = Split('some', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False, []) + rbs = rule_based_segments.from_raw(data["rbs"]["d"][0]) + rbs2 = rule_based_segments.from_raw(data["rbs"]["d"][1]) + rbs_storage.update([rbs, rbs2], [], 12) + splits_storage.update([mocked_split], [], 12) + + ctx = evaluation_facctory.context_for('bilal@split.io', ['some']) + assert e.eval_with_context('bilal@split.io', 'bilal@split.io', 'some', {'email': 'bilal@split.io'}, ctx)['treatment'] == "on" + + ctx = evaluation_facctory.context_for('mauro@split.io', ['some']) + assert e.eval_with_context('mauro@split.io', 'mauro@split.io', 'some', {'email': 'mauro@split.io'}, ctx)['treatment'] == "off" + + def test_using_segment_in_excluded(self): + rbs_segments = os.path.join(os.path.dirname(__file__), 'files', 'rule_base_segments3.json') + with open(rbs_segments, 'r') as flo: + data = json.loads(flo.read()) + e = evaluator.Evaluator(splitters.Splitter()) + splits_storage = InMemorySplitStorage() + rbs_storage = InMemoryRuleBasedSegmentStorage() + segment_storage = InMemorySegmentStorage() + evaluation_facctory = EvaluationDataFactory(splits_storage, segment_storage, rbs_storage) + + mocked_split = Split('some', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False, []) + rbs = rule_based_segments.from_raw(data["rbs"]["d"][0]) + rbs_storage.update([rbs], [], 12) + splits_storage.update([mocked_split], [], 12) + segment = segments.from_raw({'name': 'segment1', 'added': ['pato@split.io'], 'removed': [], 'till': 123}) + segment_storage.put(segment) + + ctx = evaluation_facctory.context_for('bilal@split.io', ['some']) + assert e.eval_with_context('bilal@split.io', 'bilal@split.io', 'some', {'email': 'bilal@split.io'}, ctx)['treatment'] == "on" + ctx = evaluation_facctory.context_for('mauro@split.io', ['some']) + assert e.eval_with_context('mauro@split.io', 'mauro@split.io', 'some', {'email': 'mauro@split.io'}, ctx)['treatment'] == "off" + ctx = evaluation_facctory.context_for('pato@split.io', ['some']) + assert e.eval_with_context('pato@split.io', 'pato@split.io', 'some', {'email': 'pato@split.io'}, ctx)['treatment'] == "off" + + def test_using_rbs_in_excluded(self): + rbs_segments = os.path.join(os.path.dirname(__file__), 'files', 'rule_base_segments2.json') + with open(rbs_segments, 'r') as flo: + data = json.loads(flo.read()) + e = evaluator.Evaluator(splitters.Splitter()) + splits_storage = InMemorySplitStorage() + rbs_storage = InMemoryRuleBasedSegmentStorage() + segment_storage = InMemorySegmentStorage() + evaluation_facctory = EvaluationDataFactory(splits_storage, segment_storage, rbs_storage) + + mocked_split = Split('some', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False, []) + rbs = rule_based_segments.from_raw(data["rbs"]["d"][0]) + rbs2 = rule_based_segments.from_raw(data["rbs"]["d"][1]) + rbs_storage.update([rbs, rbs2], [], 12) + splits_storage.update([mocked_split], [], 12) + + ctx = evaluation_facctory.context_for('bilal@split.io', ['some']) + assert e.eval_with_context('bilal@split.io', 'bilal@split.io', 'some', {'email': 'bilal@split.io'}, ctx)['treatment'] == "off" + ctx = evaluation_facctory.context_for('bilal', ['some']) + assert e.eval_with_context('bilal', 'bilal', 'some', {'email': 'bilal'}, ctx)['treatment'] == "on" + ctx = evaluation_facctory.context_for('bilal2@split.io', ['some']) + assert e.eval_with_context('bilal2@split.io', 'bilal2@split.io', 'some', {'email': 'bilal2@split.io'}, ctx)['treatment'] == "on" + + def test_prerequisites(self): + splits_load = os.path.join(os.path.dirname(__file__), '../models/grammar/files', 'splits_prereq.json') + with open(splits_load, 'r') as flo: + data = json.loads(flo.read()) + e = evaluator.Evaluator(splitters.Splitter()) + splits_storage = InMemorySplitStorage() + rbs_storage = InMemoryRuleBasedSegmentStorage() + segment_storage = InMemorySegmentStorage() + evaluation_facctory = EvaluationDataFactory(splits_storage, segment_storage, rbs_storage) + + rbs = rule_based_segments.from_raw(data["rbs"]["d"][0]) + split1 = from_raw(data["ff"]["d"][0]) + split2 = from_raw(data["ff"]["d"][1]) + split3 = from_raw(data["ff"]["d"][2]) + split4 = from_raw(data["ff"]["d"][3]) + rbs_storage.update([rbs], [], 12) + splits_storage.update([split1, split2, split3, split4], [], 12) + segment = segments.from_raw({'name': 'segment-test', 'added': ['pato@split.io'], 'removed': [], 'till': 123}) + segment_storage.put(segment) + + ctx = evaluation_facctory.context_for('bilal@split.io', ['test_prereq']) + assert e.eval_with_context('bilal@split.io', 'bilal@split.io', 'test_prereq', {'email': 'bilal@split.io'}, ctx)['treatment'] == "on" + assert e.eval_with_context('bilal@split.io', 'bilal@split.io', 'test_prereq', {}, ctx)['treatment'] == "def_treatment" + + ctx = evaluation_facctory.context_for('mauro@split.io', ['test_prereq']) + assert e.eval_with_context('mauro@split.io', 'mauro@split.io', 'test_prereq', {'email': 'mauro@split.io'}, ctx)['treatment'] == "def_treatment" + + ctx = evaluation_facctory.context_for('pato@split.io', ['test_prereq']) + assert e.eval_with_context('pato@split.io', 'pato@split.io', 'test_prereq', {'email': 'pato@split.io'}, ctx)['treatment'] == "def_treatment" + + ctx = evaluation_facctory.context_for('nico@split.io', ['test_prereq']) + assert e.eval_with_context('nico@split.io', 'nico@split.io', 'test_prereq', {'email': 'nico@split.io'}, ctx)['treatment'] == "on" + + ctx = evaluation_facctory.context_for('bilal@split.io', ['prereq_chain']) + assert e.eval_with_context('bilal@split.io', 'bilal@split.io', 'prereq_chain', {'email': 'bilal@split.io'}, ctx)['treatment'] == "on_whitelist" + + ctx = evaluation_facctory.context_for('nico@split.io', ['prereq_chain']) + assert e.eval_with_context('nico@split.io', 'nico@split.io', 'test_prereq', {'email': 'nico@split.io'}, ctx)['treatment'] == "on" + + ctx = evaluation_facctory.context_for('pato@split.io', ['prereq_chain']) + assert e.eval_with_context('pato@split.io', 'pato@split.io', 'prereq_chain', {'email': 'pato@split.io'}, ctx)['treatment'] == "on_default" + + ctx = evaluation_facctory.context_for('mauro@split.io', ['prereq_chain']) + assert e.eval_with_context('mauro@split.io', 'mauro@split.io', 'prereq_chain', {'email': 'mauro@split.io'}, ctx)['treatment'] == "on_default" + + @pytest.mark.asyncio + async def test_evaluate_treatment_with_rbs_in_condition_async(self): + e = evaluator.Evaluator(splitters.Splitter()) + splits_storage = InMemorySplitStorageAsync() + rbs_storage = InMemoryRuleBasedSegmentStorageAsync() + segment_storage = InMemorySegmentStorageAsync() + evaluation_facctory = AsyncEvaluationDataFactory(splits_storage, segment_storage, rbs_storage) + + rbs_segments = os.path.join(os.path.dirname(__file__), 'files', 'rule_base_segments.json') + with open(rbs_segments, 'r') as flo: + data = json.loads(flo.read()) + + mocked_split = Split('some', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False) + rbs = rule_based_segments.from_raw(data["rbs"]["d"][0]) + rbs2 = rule_based_segments.from_raw(data["rbs"]["d"][1]) + await rbs_storage.update([rbs, rbs2], [], 12) + await splits_storage.update([mocked_split], [], 12) + + ctx = await evaluation_facctory.context_for('bilal@split.io', ['some']) + assert e.eval_with_context('bilal@split.io', 'bilal@split.io', 'some', {'email': 'bilal@split.io'}, ctx)['treatment'] == "on" + ctx = await evaluation_facctory.context_for('mauro@split.io', ['some']) + assert e.eval_with_context('mauro@split.io', 'mauro@split.io', 'some', {'email': 'mauro@split.io'}, ctx)['treatment'] == "off" + + @pytest.mark.asyncio + async def test_using_segment_in_excluded_async(self): + rbs_segments = os.path.join(os.path.dirname(__file__), 'files', 'rule_base_segments3.json') + with open(rbs_segments, 'r') as flo: + data = json.loads(flo.read()) + e = evaluator.Evaluator(splitters.Splitter()) + splits_storage = InMemorySplitStorageAsync() + rbs_storage = InMemoryRuleBasedSegmentStorageAsync() + segment_storage = InMemorySegmentStorageAsync() + evaluation_facctory = AsyncEvaluationDataFactory(splits_storage, segment_storage, rbs_storage) + + mocked_split = Split('some', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False) + rbs = rule_based_segments.from_raw(data["rbs"]["d"][0]) + await rbs_storage.update([rbs], [], 12) + await splits_storage.update([mocked_split], [], 12) + segment = segments.from_raw({'name': 'segment1', 'added': ['pato@split.io'], 'removed': [], 'till': 123}) + await segment_storage.put(segment) + + ctx = await evaluation_facctory.context_for('bilal@split.io', ['some']) + assert e.eval_with_context('bilal@split.io', 'bilal@split.io', 'some', {'email': 'bilal@split.io'}, ctx)['treatment'] == "on" + ctx = await evaluation_facctory.context_for('mauro@split.io', ['some']) + assert e.eval_with_context('mauro@split.io', 'mauro@split.io', 'some', {'email': 'mauro@split.io'}, ctx)['treatment'] == "off" + ctx = await evaluation_facctory.context_for('pato@split.io', ['some']) + assert e.eval_with_context('pato@split.io', 'pato@split.io', 'some', {'email': 'pato@split.io'}, ctx)['treatment'] == "off" + + @pytest.mark.asyncio + async def test_using_rbs_in_excluded_async(self): + rbs_segments = os.path.join(os.path.dirname(__file__), 'files', 'rule_base_segments2.json') + with open(rbs_segments, 'r') as flo: + data = json.loads(flo.read()) + e = evaluator.Evaluator(splitters.Splitter()) + splits_storage = InMemorySplitStorageAsync() + rbs_storage = InMemoryRuleBasedSegmentStorageAsync() + segment_storage = InMemorySegmentStorageAsync() + evaluation_facctory = AsyncEvaluationDataFactory(splits_storage, segment_storage, rbs_storage) + + mocked_split = Split('some', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False) + rbs = rule_based_segments.from_raw(data["rbs"]["d"][0]) + rbs2 = rule_based_segments.from_raw(data["rbs"]["d"][1]) + await rbs_storage.update([rbs, rbs2], [], 12) + await splits_storage.update([mocked_split], [], 12) + + ctx = await evaluation_facctory.context_for('bilal@split.io', ['some']) + assert e.eval_with_context('bilal@split.io', 'bilal@split.io', 'some', {'email': 'bilal@split.io'}, ctx)['treatment'] == "off" + ctx = await evaluation_facctory.context_for('bilal', ['some']) + assert e.eval_with_context('bilal', 'bilal', 'some', {'email': 'bilal'}, ctx)['treatment'] == "on" + ctx = await evaluation_facctory.context_for('bilal2@split.io', ['some']) + assert e.eval_with_context('bilal2@split.io', 'bilal2@split.io', 'some', {'email': 'bilal2@split.io'}, ctx)['treatment'] == "on" + + @pytest.mark.asyncio + async def test_prerequisites(self): + splits_load = os.path.join(os.path.dirname(__file__), '../models/grammar/files', 'splits_prereq.json') + with open(splits_load, 'r') as flo: + data = json.loads(flo.read()) + e = evaluator.Evaluator(splitters.Splitter()) + splits_storage = InMemorySplitStorageAsync() + rbs_storage = InMemoryRuleBasedSegmentStorageAsync() + segment_storage = InMemorySegmentStorageAsync() + evaluation_facctory = AsyncEvaluationDataFactory(splits_storage, segment_storage, rbs_storage) + + rbs = rule_based_segments.from_raw(data["rbs"]["d"][0]) + split1 = from_raw(data["ff"]["d"][0]) + split2 = from_raw(data["ff"]["d"][1]) + split3 = from_raw(data["ff"]["d"][2]) + split4 = from_raw(data["ff"]["d"][3]) + await rbs_storage.update([rbs], [], 12) + await splits_storage.update([split1, split2, split3, split4], [], 12) + segment = segments.from_raw({'name': 'segment-test', 'added': ['pato@split.io'], 'removed': [], 'till': 123}) + await segment_storage.put(segment) + + ctx = await evaluation_facctory.context_for('bilal@split.io', ['test_prereq']) + assert e.eval_with_context('bilal@split.io', 'bilal@split.io', 'test_prereq', {'email': 'bilal@split.io'}, ctx)['treatment'] == "on" + assert e.eval_with_context('bilal@split.io', 'bilal@split.io', 'test_prereq', {}, ctx)['treatment'] == "def_treatment" + + ctx = await evaluation_facctory.context_for('mauro@split.io', ['test_prereq']) + assert e.eval_with_context('mauro@split.io', 'mauro@split.io', 'test_prereq', {'email': 'mauro@split.io'}, ctx)['treatment'] == "def_treatment" + + ctx = await evaluation_facctory.context_for('pato@split.io', ['test_prereq']) + assert e.eval_with_context('pato@split.io', 'pato@split.io', 'test_prereq', {'email': 'pato@split.io'}, ctx)['treatment'] == "def_treatment" + + ctx = await evaluation_facctory.context_for('nico@split.io', ['test_prereq']) + assert e.eval_with_context('nico@split.io', 'nico@split.io', 'test_prereq', {'email': 'nico@split.io'}, ctx)['treatment'] == "on" + + ctx = await evaluation_facctory.context_for('bilal@split.io', ['prereq_chain']) + assert e.eval_with_context('bilal@split.io', 'bilal@split.io', 'prereq_chain', {'email': 'bilal@split.io'}, ctx)['treatment'] == "on_whitelist" + + ctx = await evaluation_facctory.context_for('nico@split.io', ['prereq_chain']) + assert e.eval_with_context('nico@split.io', 'nico@split.io', 'test_prereq', {'email': 'nico@split.io'}, ctx)['treatment'] == "on" + + ctx = await evaluation_facctory.context_for('pato@split.io', ['prereq_chain']) + assert e.eval_with_context('pato@split.io', 'pato@split.io', 'prereq_chain', {'email': 'pato@split.io'}, ctx)['treatment'] == "on_default" + + ctx = await evaluation_facctory.context_for('mauro@split.io', ['prereq_chain']) + assert e.eval_with_context('mauro@split.io', 'mauro@split.io', 'prereq_chain', {'email': 'mauro@split.io'}, ctx)['treatment'] == "on_default" + +class EvaluationDataFactoryTests(object): + """Test evaluation factory class.""" + + def test_get_context(self): + """Test context.""" + mocked_split = Split('some', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False, [Prerequisites('split2', ['on'])]) + split2 = Split('split2', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False, []) + flag_storage = InMemorySplitStorage([]) + segment_storage = InMemorySegmentStorage() + rbs_segment_storage = InMemoryRuleBasedSegmentStorage() + flag_storage.update([mocked_split, split2], [], -1) + rbs = copy.deepcopy(rbs_raw) + rbs['conditions'].append( + {"matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "matcherType": "IN_SEGMENT", + "negate": False, + "userDefinedSegmentMatcherData": { + "segmentName": "employees" + }, + "whitelistMatcherData": None + } + ] + }, + }) + rbs = rule_based_segments.from_raw(rbs) + rbs_segment_storage.update([rbs], [], -1) + + eval_factory = EvaluationDataFactory(flag_storage, segment_storage, rbs_segment_storage) + ec = eval_factory.context_for('bilal@split.io', ['some']) + assert ec.rbs_segments == {'sample_rule_based_segment': rbs} + assert ec.segment_memberships == {"employees": False} + assert ec.flags.get("split2").name == "split2" + + segment_storage.update("employees", {"mauro@split.io"}, {}, 1234) + ec = eval_factory.context_for('mauro@split.io', ['some']) + assert ec.rbs_segments == {'sample_rule_based_segment': rbs} + assert ec.segment_memberships == {"employees": True} + +class EvaluationDataFactoryAsyncTests(object): + """Test evaluation factory class.""" + + @pytest.mark.asyncio + async def test_get_context(self): + """Test context.""" + mocked_split = Split('some', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False, [Prerequisites('split2', ['on'])]) + split2 = Split('split2', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False, []) + flag_storage = InMemorySplitStorageAsync([]) + segment_storage = InMemorySegmentStorageAsync() + rbs_segment_storage = InMemoryRuleBasedSegmentStorageAsync() + await flag_storage.update([mocked_split, split2], [], -1) + rbs = copy.deepcopy(rbs_raw) + rbs['conditions'].append( + {"matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "matcherType": "IN_SEGMENT", + "negate": False, + "userDefinedSegmentMatcherData": { + "segmentName": "employees" + }, + "whitelistMatcherData": None + } + ] + }, + }) + rbs = rule_based_segments.from_raw(rbs) + await rbs_segment_storage.update([rbs], [], -1) + + eval_factory = AsyncEvaluationDataFactory(flag_storage, segment_storage, rbs_segment_storage) + ec = await eval_factory.context_for('bilal@split.io', ['some']) + assert ec.rbs_segments == {'sample_rule_based_segment': rbs} + assert ec.segment_memberships == {"employees": False} + assert ec.flags.get("split2").name == "split2" + + await segment_storage.update("employees", {"mauro@split.io"}, {}, 1234) + ec = await eval_factory.context_for('mauro@split.io', ['some']) + assert ec.rbs_segments == {'sample_rule_based_segment': rbs} + assert ec.segment_memberships == {"employees": True} diff --git a/tests/helpers/mockserver.py b/tests/helpers/mockserver.py index 71cd186b..8d41cfd2 100644 --- a/tests/helpers/mockserver.py +++ b/tests/helpers/mockserver.py @@ -3,12 +3,13 @@ from collections import namedtuple import queue import threading +import pytest from http.server import HTTPServer, BaseHTTPRequestHandler Request = namedtuple('Request', ['method', 'path', 'headers', 'body']) - +OLD_SPEC = False class SSEMockServer(object): """SSE server for testing purposes.""" @@ -102,19 +103,22 @@ class SplitMockServer(object): protocol_version = 'HTTP/1.1' def __init__(self, split_changes=None, segment_changes=None, req_queue=None, - auth_response=None): + auth_response=None, old_spec=False): """ Consruct a mock server. :param changes: mapping of changeNumbers to splitChanges responses :type changes: dict """ + global OLD_SPEC + OLD_SPEC = old_spec split_changes = split_changes if split_changes is not None else {} segment_changes = segment_changes if segment_changes is not None else {} self._server = HTTPServer(('localhost', 0), lambda *xs: SDKHandler(split_changes, segment_changes, *xs, req_queue=req_queue, - auth_response=auth_response)) + auth_response=auth_response, + )) self._server_thread = threading.Thread(target=self._blocking_run, name="SplitMockServer", daemon=True) self._done_event = threading.Event() @@ -148,7 +152,7 @@ def __init__(self, split_changes, segment_changes, *args, **kwargs): self._req_queue = kwargs.get('req_queue') self._auth_response = kwargs.get('auth_response') self._split_changes = split_changes - self._segment_changes = segment_changes + self._segment_changes = segment_changes BaseHTTPRequestHandler.__init__(self, *args) def _parse_qs(self): @@ -180,6 +184,15 @@ def _handle_segment_changes(self): self.wfile.write(json.dumps(to_send).encode('utf-8')) def _handle_split_changes(self): + global OLD_SPEC + if OLD_SPEC: + self.send_response(400) + self.send_header("Content-type", "application/json") + self.end_headers() + self.wfile.write('{}'.encode('utf-8')) + OLD_SPEC = False + return + qstring = self._parse_qs() since = int(qstring.get('since', -1)) to_send = self._split_changes.get(since) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index ee2475df..845e8c72 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -1,49 +1,55 @@ -split11 = {"splits": [ - {"trafficTypeName": "user", "name": "SPLIT_2","trafficAllocation": 100,"trafficAllocationSeed": 1057590779, "seed": -113875324, "status": "ACTIVE","killed": False, "defaultTreatment": "off", "changeNumber": 1675443569027,"algo": 2, "configurations": {},"conditions": [{"conditionType": "ROLLOUT","matcherGroup": {"combiner": "AND","matchers": [{"keySelector": { "trafficType": "user", "attribute": None },"matcherType": "ALL_KEYS","negate": False,"userDefinedSegmentMatcherData": None,"whitelistMatcherData": None,"unaryNumericMatcherData": None,"betweenMatcherData": None,"booleanMatcherData": None,"dependencyMatcherData": None,"stringMatcherData": None}]},"partitions": [{ "treatment": "on", "size": 100 },{ "treatment": "off", "size": 0 }],"label": "default rule"}], "sets": ["set_1"], "impressionsDisabled": False}, +import copy + +rbsegments_json = [{"changeNumber": 12, "name": "some_segment", "status": "ACTIVE","trafficTypeName": "user","excluded":{"keys":[],"segments":[]},"conditions": []}] + +split11 = {"ff": {"t": 1675443569027, "s": -1, "d": [ + {"trafficTypeName": "user", "name": "SPLIT_2","trafficAllocation": 100,"trafficAllocationSeed": 1057590779, "seed": -113875324, "status": "ACTIVE","killed": False, "defaultTreatment": "off", "changeNumber": 1675443569027,"algo": 2, "configurations": {},"conditions": [{"conditionType": "ROLLOUT","matcherGroup": {"combiner": "AND","matchers": [{"keySelector": { "trafficType": "user", "attribute": None },"matcherType": "ALL_KEYS","negate": False,"userDefinedSegmentMatcherData": None,"whitelistMatcherData": None,"unaryNumericMatcherData": None,"betweenMatcherData": None,"booleanMatcherData": None,"dependencyMatcherData": None,"stringMatcherData": None}]},"partitions": [{ "treatment": "on", "size": 100 },{ "treatment": "off", "size": 0 }],"label": "default rule"}], "sets": ["set_1"], "impressionsDisabled": False, 'prerequisites': []}, {"trafficTypeName": "user", "name": "SPLIT_1", "trafficAllocation": 100, "trafficAllocationSeed": -1780071202,"seed": -1442762199, "status": "ACTIVE","killed": False, "defaultTreatment": "off", "changeNumber": 1675443537882,"algo": 2, "configurations": {},"conditions": [{"conditionType": "ROLLOUT", "matcherGroup": {"combiner": "AND","matchers": [{"keySelector": { "trafficType": "user", "attribute": None },"matcherType": "ALL_KEYS","negate": False,"userDefinedSegmentMatcherData": None,"whitelistMatcherData": None,"unaryNumericMatcherData": None,"betweenMatcherData": None,"booleanMatcherData": None,"dependencyMatcherData": None,"stringMatcherData": None}]},"partitions": [{ "treatment": "on", "size": 0 },{ "treatment": "off", "size": 100 }],"label": "default rule"}], "sets": ["set_1", "set_2"]}, {"trafficTypeName": "user", "name": "SPLIT_3","trafficAllocation": 100,"trafficAllocationSeed": 1057590779, "seed": -113875324, "status": "ACTIVE","killed": False, "defaultTreatment": "off", "changeNumber": 1675443569027,"algo": 2, "configurations": {},"conditions": [{"conditionType": "ROLLOUT","matcherGroup": {"combiner": "AND","matchers": [{"keySelector": { "trafficType": "user", "attribute": None },"matcherType": "ALL_KEYS","negate": False,"userDefinedSegmentMatcherData": None,"whitelistMatcherData": None,"unaryNumericMatcherData": None,"betweenMatcherData": None,"booleanMatcherData": None,"dependencyMatcherData": None,"stringMatcherData": None}]},"partitions": [{ "treatment": "on", "size": 100 },{ "treatment": "off", "size": 0 }],"label": "default rule"}], "sets": ["set_1"], "impressionsDisabled": True} - ],"since": -1,"till": 1675443569027} -split12 = {"splits": [{"trafficTypeName": "user","name": "SPLIT_2","trafficAllocation": 100,"trafficAllocationSeed": 1057590779,"seed": -113875324,"status": "ACTIVE","killed": True,"defaultTreatment": "off","changeNumber": 1675443767288,"algo": 2,"configurations": {},"conditions": [{"conditionType": "ROLLOUT","matcherGroup": {"combiner": "AND","matchers": [{"keySelector": { "trafficType": "user", "attribute": None },"matcherType": "ALL_KEYS","negate": False,"userDefinedSegmentMatcherData": None,"whitelistMatcherData": None,"unaryNumericMatcherData": None,"betweenMatcherData": None,"booleanMatcherData": None,"dependencyMatcherData": None,"stringMatcherData": None}]},"partitions": [{ "treatment": "on", "size": 100 },{ "treatment": "off", "size": 0 }],"label": "default rule"}]}],"since": 1675443569027,"till": 167544376728} -split13 = {"splits": [ + ]}, "rbs": {"t": -1, "s": -1, "d": rbsegments_json}} +split12 = {"ff": {"s": 1675443569027,"t": 1675443767284, "d": [{"trafficTypeName": "user","name": "SPLIT_2","trafficAllocation": 100,"trafficAllocationSeed": 1057590779,"seed": -113875324,"status": "ACTIVE","killed": True,"defaultTreatment": "off","changeNumber": 1675443767288,"algo": 2,"configurations": {},"conditions": [{"conditionType": "ROLLOUT","matcherGroup": {"combiner": "AND","matchers": [{"keySelector": { "trafficType": "user", "attribute": None },"matcherType": "ALL_KEYS","negate": False,"userDefinedSegmentMatcherData": None,"whitelistMatcherData": None,"unaryNumericMatcherData": None,"betweenMatcherData": None,"booleanMatcherData": None,"dependencyMatcherData": None,"stringMatcherData": None}]},"partitions": [{ "treatment": "on", "size": 100 },{ "treatment": "off", "size": 0 }],"label": "default rule"}]}]}, "rbs": {"t": -1, "s": -1, "d": rbsegments_json}} +split13 = {"ff": {"s": 1675443767288,"t": 1675443984594, "d": [ {"trafficTypeName": "user","name": "SPLIT_1","trafficAllocation": 100,"trafficAllocationSeed": -1780071202,"seed": -1442762199,"status": "ARCHIVED","killed": False,"defaultTreatment": "off","changeNumber": 1675443984594,"algo": 2,"configurations": {},"conditions": [{"conditionType": "ROLLOUT","matcherGroup": {"combiner": "AND","matchers": [{"keySelector": { "trafficType": "user", "attribute": None },"matcherType": "ALL_KEYS","negate": False,"userDefinedSegmentMatcherData": None,"whitelistMatcherData": None,"unaryNumericMatcherData": None,"betweenMatcherData": None,"booleanMatcherData": None,"dependencyMatcherData": None,"stringMatcherData": None}]},"partitions": [{ "treatment": "on", "size": 0 },{ "treatment": "off", "size": 100 }],"label": "default rule"}]}, {"trafficTypeName": "user","name": "SPLIT_2","trafficAllocation": 100,"trafficAllocationSeed": 1057590779,"seed": -113875324,"status": "ACTIVE","killed": False,"defaultTreatment": "off","changeNumber": 1675443954220,"algo": 2,"configurations": {},"conditions": [{"conditionType": "ROLLOUT","matcherGroup": {"combiner": "AND","matchers": [{"keySelector": { "trafficType": "user", "attribute": None },"matcherType": "ALL_KEYS","negate": False,"userDefinedSegmentMatcherData": None,"whitelistMatcherData": None,"unaryNumericMatcherData": None,"betweenMatcherData": None,"booleanMatcherData": None,"dependencyMatcherData": None,"stringMatcherData": None}]},"partitions": [{ "treatment": "on", "size": 100 },{ "treatment": "off", "size": 0 }],"label": "default rule"}]} - ],"since": 1675443767288,"till": 1675443984594} - -split41 = split11 -split42 = split12 -split43 = split13 + ]}, "rbs": {"t": -1, "s": -1, "d": rbsegments_json}} -split41["since"] = None -split41["till"] = None -split42["since"] = None -split42["till"] = None -split43["since"] = None -split43["till"] = None -split61 = split11 -split62 = split12 -split63 = split13 +split41 = {"ff": {"t": None, "s": None, "d": split11['ff']['d']}, "rbs": {"t": -1, "s": -1, "d": rbsegments_json}} +split42 = {"ff": {"t": None, "s": None, "d": split12['ff']['d']}, "rbs": {"t": -1, "s": -1, "d": rbsegments_json}} +split43 = {"ff": {"t": None, "s": None, "d": split13['ff']['d']}, "rbs": {"t": -1, "s": -1, "d": rbsegments_json}} -split61["since"] = -1 -split61["till"] = -1 -split62["since"] = -1 -split62["till"] = -1 -split63["since"] = -1 -split63["till"] = -1 +split61 = {"ff": {"t": -1, "s": -1, "d": split11['ff']['d']}, "rbs": {"t": -1, "s": -1, "d": rbsegments_json}} +split62 = {"ff": {"t": -1, "s": -1, "d": split12['ff']['d']}, "rbs": {"t": -1, "s": -1, "d": rbsegments_json}} +split63 = {"ff": {"t": -1, "s": -1, "d": split13['ff']['d']}, "rbs": {"t": -1, "s": -1, "d": rbsegments_json}} splits_json = { "splitChange1_1": split11, "splitChange1_2": split12, "splitChange1_3": split13, - "splitChange2_1": {"splits": [{"name": "SPLIT_1","status": "ACTIVE","killed": False,"defaultTreatment": "off","configurations": {},"conditions": []}]}, - "splitChange3_1": {"splits": [{"trafficTypeName": "user","name": "SPLIT_2","trafficAllocation": 100,"trafficAllocationSeed": 1057590779,"seed": -113875324,"status": "ACTIVE","killed": False,"defaultTreatment": "off","changeNumber": 1675443569027,"algo": 2,"configurations": {},"conditions": [{"conditionType": "ROLLOUT","matcherGroup": {"combiner": "AND","matchers": [{"keySelector": { "trafficType": "user", "attribute": None },"matcherType": "ALL_KEYS","negate": False,"userDefinedSegmentMatcherData": None,"whitelistMatcherData": None,"unaryNumericMatcherData": None,"betweenMatcherData": None,"booleanMatcherData": None,"dependencyMatcherData": None,"stringMatcherData": None}]},"partitions": [{ "treatment": "on", "size": 100 },{ "treatment": "off", "size": 0 }],"label": "default rule"}]}],"since": -1,"till": 1675443569027}, - "splitChange3_2": {"splits": [{"trafficTypeName": "user","name": "SPLIT_2","trafficAllocation": 100,"trafficAllocationSeed": 1057590779,"seed": -113875324,"status": "ACTIVE","killed": True,"defaultTreatment": "off","changeNumber": 1675443767288,"algo": 2,"configurations": {},"conditions": [{"conditionType": "ROLLOUT","matcherGroup": {"combiner": "AND","matchers": [{"keySelector": { "trafficType": "user", "attribute": None },"matcherType": "ALL_KEYS","negate": False,"userDefinedSegmentMatcherData": None,"whitelistMatcherData": None,"unaryNumericMatcherData": None,"betweenMatcherData": None,"booleanMatcherData": None,"dependencyMatcherData": None,"stringMatcherData": None}]},"partitions": [{ "treatment": "on", "size": 100 },{ "treatment": "off", "size": 0 }],"label": "default rule"}]}],"since": 1675443569027,"till": 1675443569027}, + "splitChange2_1": {"ff": {"t": -1, "s": -1, "d": [{"name": "SPLIT_1","status": "ACTIVE","killed": False,"defaultTreatment": "off","configurations": {},"conditions": []}]}, "rbs": {"t": -1, "s": -1, "d": rbsegments_json}}, + "splitChange3_1": {"ff": {"t": -1, "s": -1, "d": [{"trafficTypeName": "user","name": "SPLIT_2","trafficAllocation": 100,"trafficAllocationSeed": 1057590779,"seed": -113875324,"status": "ACTIVE","killed": False,"defaultTreatment": "off","changeNumber": 1675443569027,"algo": 2,"configurations": {},"conditions": [{"conditionType": "ROLLOUT","matcherGroup": {"combiner": "AND","matchers": [{"keySelector": { "trafficType": "user", "attribute": None },"matcherType": "ALL_KEYS","negate": False,"userDefinedSegmentMatcherData": None,"whitelistMatcherData": None,"unaryNumericMatcherData": None,"betweenMatcherData": None,"booleanMatcherData": None,"dependencyMatcherData": None,"stringMatcherData": None}]},"partitions": [{ "treatment": "on", "size": 100 },{ "treatment": "off", "size": 0 }],"label": "default rule"}]}],"s": -1,"t": 1675443569027}, "rbs": {"t": -1, "s": -1, "d": rbsegments_json}}, + "splitChange3_2": {"ff": {"t": -1, "s": -1, "d": [{"trafficTypeName": "user","name": "SPLIT_2","trafficAllocation": 100,"trafficAllocationSeed": 1057590779,"seed": -113875324,"status": "ACTIVE","killed": True,"defaultTreatment": "off","changeNumber": 1675443767288,"algo": 2,"configurations": {},"conditions": [{"conditionType": "ROLLOUT","matcherGroup": {"combiner": "AND","matchers": [{"keySelector": { "trafficType": "user", "attribute": None },"matcherType": "ALL_KEYS","negate": False,"userDefinedSegmentMatcherData": None,"whitelistMatcherData": None,"unaryNumericMatcherData": None,"betweenMatcherData": None,"booleanMatcherData": None,"dependencyMatcherData": None,"stringMatcherData": None}]},"partitions": [{ "treatment": "on", "size": 100 },{ "treatment": "off", "size": 0 }],"label": "default rule"}]}],"s": 1675443569027,"t": 1675443569027}, "rbs": {"t": -1, "s": -1, "d": rbsegments_json}}, "splitChange4_1": split41, "splitChange4_2": split42, "splitChange4_3": split43, - "splitChange5_1": {"splits": [{"trafficTypeName": "user","name": "SPLIT_2","trafficAllocation": 100,"trafficAllocationSeed": 1057590779,"seed": -113875324,"status": "ACTIVE","killed": False,"defaultTreatment": "off","changeNumber": 1675443569027,"algo": 2,"configurations": {},"conditions": [{"conditionType": "ROLLOUT","matcherGroup": {"combiner": "AND","matchers": [{"keySelector": { "trafficType": "user", "attribute": None },"matcherType": "ALL_KEYS","negate": False,"userDefinedSegmentMatcherData": None,"whitelistMatcherData": None,"unaryNumericMatcherData": None,"betweenMatcherData": None,"booleanMatcherData": None,"dependencyMatcherData": None,"stringMatcherData": None}]},"partitions": [{ "treatment": "on", "size": 100 },{ "treatment": "off", "size": 0 }],"label": "default rule"}]}],"since": -1,"till": 1675443569027}, - "splitChange5_2": {"splits": [{"trafficTypeName": "user","name": "SPLIT_2","trafficAllocation": 100,"trafficAllocationSeed": 1057590779,"seed": -113875324,"status": "ACTIVE","killed": True,"defaultTreatment": "off","changeNumber": 1675443767288,"algo": 2,"configurations": {},"conditions": [{"conditionType": "ROLLOUT","matcherGroup": {"combiner": "AND","matchers": [{"keySelector": { "trafficType": "user", "attribute": None },"matcherType": "ALL_KEYS","negate": False,"userDefinedSegmentMatcherData": None,"whitelistMatcherData": None,"unaryNumericMatcherData": None,"betweenMatcherData": None,"booleanMatcherData": None,"dependencyMatcherData": None,"stringMatcherData": None}]},"partitions": [{ "treatment": "on", "size": 100 },{ "treatment": "off", "size": 0 }],"label": "default rule"}]}],"since": 1675443569026,"till": 1675443569026}, + "splitChange5_1": {"ff": {"t": -1, "s": -1, "d": [{"trafficTypeName": "user","name": "SPLIT_2","trafficAllocation": 100,"trafficAllocationSeed": 1057590779,"seed": -113875324,"status": "ACTIVE","killed": False,"defaultTreatment": "off","changeNumber": 1675443569027,"algo": 2,"configurations": {},"conditions": [{"conditionType": "ROLLOUT","matcherGroup": {"combiner": "AND","matchers": [{"keySelector": { "trafficType": "user", "attribute": None },"matcherType": "ALL_KEYS","negate": False,"userDefinedSegmentMatcherData": None,"whitelistMatcherData": None,"unaryNumericMatcherData": None,"betweenMatcherData": None,"booleanMatcherData": None,"dependencyMatcherData": None,"stringMatcherData": None}]},"partitions": [{ "treatment": "on", "size": 100 },{ "treatment": "off", "size": 0 }],"label": "default rule"}]}],"s": -1,"t": 1675443569027}, "rbs": {"t": -1, "s": -1, "d": rbsegments_json}}, + "splitChange5_2": {"ff": {"t": -1, "s": -1, "d": [{"trafficTypeName": "user","name": "SPLIT_2","trafficAllocation": 100,"trafficAllocationSeed": 1057590779,"seed": -113875324,"status": "ACTIVE","killed": True,"defaultTreatment": "off","changeNumber": 1675443767288,"algo": 2,"configurations": {},"conditions": [{"conditionType": "ROLLOUT","matcherGroup": {"combiner": "AND","matchers": [{"keySelector": { "trafficType": "user", "attribute": None },"matcherType": "ALL_KEYS","negate": False,"userDefinedSegmentMatcherData": None,"whitelistMatcherData": None,"unaryNumericMatcherData": None,"betweenMatcherData": None,"booleanMatcherData": None,"dependencyMatcherData": None,"stringMatcherData": None}]},"partitions": [{ "treatment": "on", "size": 100 },{ "treatment": "off", "size": 0 }],"label": "default rule"}]}],"s": 1675443569026,"t": 1675443569026}, "rbs": {"t": -1, "s": -1, "d": rbsegments_json}}, "splitChange6_1": split61, "splitChange6_2": split62, "splitChange6_3": split63, -} + "splitChange7_1": {"ff": { + "t": -1, + "s": -1, + "d": [{"changeNumber": 10,"trafficTypeName": "user","name": "rbs_feature_flag","trafficAllocation": 100,"trafficAllocationSeed": 1828377380,"seed": -286617921,"status": "ACTIVE","killed": False,"defaultTreatment": "off","algo": 2, + "conditions": [{"conditionType": "ROLLOUT","matcherGroup": {"combiner": "AND","matchers": [{"keySelector": {"trafficType": "user"},"matcherType": "IN_RULE_BASED_SEGMENT","negate": False,"userDefinedSegmentMatcherData": {"segmentName": "sample_rule_based_segment"}}]},"partitions": [{"treatment": "on","size": 100},{"treatment": "off","size": 0}],"label": "in rule based segment sample_rule_based_segment"},{"conditionType": "ROLLOUT","matcherGroup": {"combiner": "AND","matchers": [{"keySelector": {"trafficType": "user"},"matcherType": "ALL_KEYS","negate": False}]},"partitions": [{"treatment": "on","size": 0},{"treatment": "off","size": 100}],"label": "default rule"}], + "configurations": {}, + "sets": [], + "impressionsDisabled": False + }] + }, "rbs": { + "t": 1675259356568, + "s": -1, + "d": [{"changeNumber": 5,"name": "sample_rule_based_segment","status": "ACTIVE","trafficTypeName": "user","excluded":{"keys":["mauro@split.io","gaston@split.io"],"segments":[]}, + "conditions": [{"matcherGroup": {"combiner": "AND","matchers": [{"keySelector": {"trafficType": "user","attribute": "email"},"matcherType": "ENDS_WITH","negate": False,"whitelistMatcherData": {"whitelist": ["@split.io"]}}]}}]} + ]}} +} \ No newline at end of file diff --git a/tests/integration/files/splitChanges.json b/tests/integration/files/splitChanges.json index 9125481d..84f7c2cd 100644 --- a/tests/integration/files/splitChanges.json +++ b/tests/integration/files/splitChanges.json @@ -1,5 +1,6 @@ { - "splits": [ + "ff": { + "d": [ { "orgId": null, "environment": null, @@ -22,7 +23,8 @@ "userDefinedSegmentMatcherData": null, "whitelistMatcherData": { "whitelist": [ - "whitelisted_user" + "whitelisted_user", + "user1234" ] } } @@ -321,8 +323,154 @@ } ], "sets": [] + }, + { + "changeNumber": 10, + "trafficTypeName": "user", + "name": "rbs_feature_flag", + "trafficAllocation": 100, + "trafficAllocationSeed": 1828377380, + "seed": -286617921, + "status": "ACTIVE", + "killed": false, + "defaultTreatment": "off", + "algo": 2, + "conditions": [ + { + "conditionType": "ROLLOUT", + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "keySelector": { + "trafficType": "user" + }, + "matcherType": "IN_RULE_BASED_SEGMENT", + "negate": false, + "userDefinedSegmentMatcherData": { + "segmentName": "sample_rule_based_segment" + } + } + ] + }, + "partitions": [ + { + "treatment": "on", + "size": 100 + }, + { + "treatment": "off", + "size": 0 + } + ], + "label": "in rule based segment sample_rule_based_segment" + }, + { + "conditionType": "ROLLOUT", + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "keySelector": { + "trafficType": "user" + }, + "matcherType": "ALL_KEYS", + "negate": false + } + ] + }, + "partitions": [ + { + "treatment": "on", + "size": 0 + }, + { + "treatment": "off", + "size": 100 + } + ], + "label": "default rule" + } + ], + "configurations": {}, + "sets": [], + "impressionsDisabled": false + }, + { + "orgId": null, + "environment": null, + "trafficTypeId": null, + "trafficTypeName": null, + "name": "prereq_feature", + "seed": 1699838640, + "status": "ACTIVE", + "killed": false, + "changeNumber": 123, + "defaultTreatment": "off_default", + "conditions": [ + { + "conditionType": "ROLLOUT", + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "matcherType": "ALL_KEYS", + "negate": false, + "userDefinedSegmentMatcherData": null, + "whitelistMatcherData": null + } + ] + }, + "partitions": [ + { + "treatment": "on", + "size": 100 + }, + { + "treatment": "off", + "size": 0 + } + ] + } + ], + "sets": [], + "prerequisites": [ + {"n": "regex_test", "ts": ["on"]}, + {"n": "whitelist_feature", "ts": ["off"]} + ] } ], - "since": -1, - "till": 1457726098069 -} + "s": -1, + "t": 1457726098069 +}, "rbs": {"t": -1, "s": -1, "d": [{ + "changeNumber": 123, + "name": "sample_rule_based_segment", + "status": "ACTIVE", + "trafficTypeName": "user", + "excluded":{ + "keys":["mauro@split.io","gaston@split.io"], + "segments":[] + }, + "conditions": [ + { + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "keySelector": { + "trafficType": "user", + "attribute": "email" + }, + "matcherType": "ENDS_WITH", + "negate": false, + "whitelistMatcherData": { + "whitelist": [ + "@split.io" + ] + } + } + ] + } + } + ] +}]}} diff --git a/tests/integration/files/split_changes.json b/tests/integration/files/split_changes.json index 6084b108..f0708043 100644 --- a/tests/integration/files/split_changes.json +++ b/tests/integration/files/split_changes.json @@ -1,5 +1,6 @@ { - "splits": [ + "ff": { + "d": [ { "orgId": null, "environment": null, @@ -323,6 +324,7 @@ "sets": [] } ], - "since": -1, - "till": 1457726098069 + "s": -1, + "t": 1457726098069 +}, "rbs": {"t": -1, "s": -1, "d": []} } diff --git a/tests/integration/files/split_changes_temp.json b/tests/integration/files/split_changes_temp.json index 162c0b17..64575226 100644 --- a/tests/integration/files/split_changes_temp.json +++ b/tests/integration/files/split_changes_temp.json @@ -1 +1 @@ -{"splits": [{"trafficTypeName": "user", "name": "SPLIT_1", "trafficAllocation": 100, "trafficAllocationSeed": -1780071202, "seed": -1442762199, "status": "ARCHIVED", "killed": false, "defaultTreatment": "off", "changeNumber": 1675443984594, "algo": 2, "configurations": {}, "conditions": [{"conditionType": "ROLLOUT", "matcherGroup": {"combiner": "AND", "matchers": [{"keySelector": {"trafficType": "user", "attribute": null}, "matcherType": "ALL_KEYS", "negate": false, "userDefinedSegmentMatcherData": null, "whitelistMatcherData": null, "unaryNumericMatcherData": null, "betweenMatcherData": null, "booleanMatcherData": null, "dependencyMatcherData": null, "stringMatcherData": null}]}, "partitions": [{"treatment": "on", "size": 0}, {"treatment": "off", "size": 100}], "label": "default rule"}]}, {"trafficTypeName": "user", "name": "SPLIT_2", "trafficAllocation": 100, "trafficAllocationSeed": 1057590779, "seed": -113875324, "status": "ACTIVE", "killed": false, "defaultTreatment": "off", "changeNumber": 1675443954220, "algo": 2, "configurations": {}, "conditions": [{"conditionType": "ROLLOUT", "matcherGroup": {"combiner": "AND", "matchers": [{"keySelector": {"trafficType": "user", "attribute": null}, "matcherType": "ALL_KEYS", "negate": false, "userDefinedSegmentMatcherData": null, "whitelistMatcherData": null, "unaryNumericMatcherData": null, "betweenMatcherData": null, "booleanMatcherData": null, "dependencyMatcherData": null, "stringMatcherData": null}]}, "partitions": [{"treatment": "on", "size": 100}, {"treatment": "off", "size": 0}], "label": "default rule"}]}], "since": -1, "till": -1} \ No newline at end of file +{"ff": {"t": -1, "s": -1, "d": [{"changeNumber": 10, "trafficTypeName": "user", "name": "rbs_feature_flag", "trafficAllocation": 100, "trafficAllocationSeed": 1828377380, "seed": -286617921, "status": "ACTIVE", "killed": false, "defaultTreatment": "off", "algo": 2, "conditions": [{"conditionType": "ROLLOUT", "matcherGroup": {"combiner": "AND", "matchers": [{"keySelector": {"trafficType": "user"}, "matcherType": "IN_RULE_BASED_SEGMENT", "negate": false, "userDefinedSegmentMatcherData": {"segmentName": "sample_rule_based_segment"}}]}, "partitions": [{"treatment": "on", "size": 100}, {"treatment": "off", "size": 0}], "label": "in rule based segment sample_rule_based_segment"}, {"conditionType": "ROLLOUT", "matcherGroup": {"combiner": "AND", "matchers": [{"keySelector": {"trafficType": "user"}, "matcherType": "ALL_KEYS", "negate": false}]}, "partitions": [{"treatment": "on", "size": 0}, {"treatment": "off", "size": 100}], "label": "default rule"}], "configurations": {}, "sets": [], "impressionsDisabled": false}]}, "rbs": {"t": 1675259356568, "s": -1, "d": [{"changeNumber": 5, "name": "sample_rule_based_segment", "status": "ACTIVE", "trafficTypeName": "user", "excluded": {"keys": ["mauro@split.io", "gaston@split.io"], "segments": []}, "conditions": [{"matcherGroup": {"combiner": "AND", "matchers": [{"keySelector": {"trafficType": "user", "attribute": "email"}, "matcherType": "ENDS_WITH", "negate": false, "whitelistMatcherData": {"whitelist": ["@split.io"]}}]}}]}]}} \ No newline at end of file diff --git a/tests/integration/files/split_old_spec.json b/tests/integration/files/split_old_spec.json new file mode 100644 index 00000000..0d7edf86 --- /dev/null +++ b/tests/integration/files/split_old_spec.json @@ -0,0 +1,328 @@ +{ + "splits": [ + { + "orgId": null, + "environment": null, + "trafficTypeId": null, + "trafficTypeName": null, + "name": "whitelist_feature", + "seed": -1222652054, + "status": "ACTIVE", + "killed": false, + "changeNumber": 123, + "defaultTreatment": "off", + "conditions": [ + { + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "matcherType": "WHITELIST", + "negate": false, + "userDefinedSegmentMatcherData": null, + "whitelistMatcherData": { + "whitelist": [ + "whitelisted_user" + ] + } + } + ] + }, + "partitions": [ + { + "treatment": "on", + "size": 100 + } + ] + }, + { + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "matcherType": "ALL_KEYS", + "negate": false, + "userDefinedSegmentMatcherData": null, + "whitelistMatcherData": null + } + ] + }, + "partitions": [ + { + "treatment": "on", + "size": 0 + }, + { + "treatment": "off", + "size": 100 + } + ] + } + ], + "sets": ["set1", "set2"] + }, + { + "orgId": null, + "environment": null, + "trafficTypeId": null, + "trafficTypeName": null, + "name": "all_feature", + "seed": 1699838640, + "status": "ACTIVE", + "killed": false, + "changeNumber": 123, + "defaultTreatment": "off", + "conditions": [ + { + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "matcherType": "ALL_KEYS", + "negate": false, + "userDefinedSegmentMatcherData": null, + "whitelistMatcherData": null + } + ] + }, + "partitions": [ + { + "treatment": "on", + "size": 100 + }, + { + "treatment": "off", + "size": 0 + } + ] + } + ], + "sets": ["set4"] + }, + { + "orgId": null, + "environment": null, + "trafficTypeId": null, + "trafficTypeName": null, + "name": "killed_feature", + "seed": -480091424, + "status": "ACTIVE", + "killed": true, + "changeNumber": 123, + "defaultTreatment": "defTreatment", + "configurations": { + "off": "{\"size\":15,\"test\":20}", + "defTreatment": "{\"size\":15,\"defTreatment\":true}" + }, + "conditions": [ + { + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "matcherType": "ALL_KEYS", + "negate": false, + "userDefinedSegmentMatcherData": null, + "whitelistMatcherData": null + } + ] + }, + "partitions": [ + { + "treatment": "defTreatment", + "size": 100 + }, + { + "treatment": "off", + "size": 0 + } + ] + } + ], + "sets": ["set3"] + }, + { + "orgId": null, + "environment": null, + "trafficTypeId": null, + "trafficTypeName": null, + "name": "sample_feature", + "seed": 1548363147, + "status": "ACTIVE", + "killed": false, + "changeNumber": 123, + "defaultTreatment": "off", + "configurations": { + "on": "{\"size\":15,\"test\":20}" + }, + "conditions": [ + { + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "matcherType": "IN_SEGMENT", + "negate": false, + "userDefinedSegmentMatcherData": { + "segmentName": "employees" + }, + "whitelistMatcherData": null + } + ] + }, + "partitions": [ + { + "treatment": "on", + "size": 100 + } + ] + }, + { + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "matcherType": "IN_SEGMENT", + "negate": false, + "userDefinedSegmentMatcherData": { + "segmentName": "human_beigns" + }, + "whitelistMatcherData": null + } + ] + }, + "partitions": [ + { + "treatment": "on", + "size": 30 + }, + { + "treatment": "off", + "size": 70 + } + ] + } + ], + "sets": ["set1"] + }, + { + "orgId": null, + "environment": null, + "trafficTypeId": null, + "trafficTypeName": null, + "name": "dependency_test", + "seed": 1222652054, + "status": "ACTIVE", + "killed": false, + "changeNumber": 123, + "defaultTreatment": "off", + "conditions": [ + { + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "matcherType": "IN_SPLIT_TREATMENT", + "negate": false, + "userDefinedSegmentMatcherData": null, + "dependencyMatcherData": { + "split": "all_feature", + "treatments": ["on"] + } + } + ] + }, + "partitions": [ + { + "treatment": "on", + "size": 0 + }, + { + "treatment": "off", + "size": 100 + } + ] + } + ], + "sets": [] + }, + { + "orgId": null, + "environment": null, + "trafficTypeId": null, + "trafficTypeName": null, + "name": "regex_test", + "seed": 1222652051, + "status": "ACTIVE", + "killed": false, + "changeNumber": 123, + "defaultTreatment": "off", + "conditions": [ + { + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "matcherType": "MATCHES_STRING", + "negate": false, + "userDefinedSegmentMatcherData": null, + "stringMatcherData": "abc[0-9]" + } + ] + }, + "partitions": [ + { + "treatment": "on", + "size": 100 + }, + { + "treatment": "off", + "size": 0 + } + ] + } + ], + "sets": [] + }, + { + "orgId": null, + "environment": null, + "trafficTypeId": null, + "trafficTypeName": null, + "name": "boolean_test", + "status": "ACTIVE", + "killed": false, + "changeNumber": 123, + "seed": 12321809, + "defaultTreatment": "off", + "conditions": [ + { + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "matcherType": "EQUAL_TO_BOOLEAN", + "negate": false, + "userDefinedSegmentMatcherData": null, + "booleanMatcherData": true + } + ] + }, + "partitions": [ + { + "treatment": "on", + "size": 100 + }, + { + "treatment": "off", + "size": 0 + } + ] + } + ], + "sets": [] + } + ], + "since": -1, + "till": 1457726098069 +} \ No newline at end of file diff --git a/tests/integration/test_client_e2e.py b/tests/integration/test_client_e2e.py index 94a11624..f50869cf 100644 --- a/tests/integration/test_client_e2e.py +++ b/tests/integration/test_client_e2e.py @@ -1,5 +1,6 @@ """Client integration tests.""" # pylint: disable=protected-access,line-too-long,no-self-use +from asyncio import Queue import json import os import threading @@ -15,15 +16,17 @@ from splitio.storage.inmemmory import InMemoryEventStorage, InMemoryImpressionStorage, \ InMemorySegmentStorage, InMemorySplitStorage, InMemoryTelemetryStorage, InMemorySplitStorageAsync,\ InMemoryEventStorageAsync, InMemoryImpressionStorageAsync, InMemorySegmentStorageAsync, \ - InMemoryTelemetryStorageAsync + InMemoryTelemetryStorageAsync, InMemoryRuleBasedSegmentStorage, InMemoryRuleBasedSegmentStorageAsync from splitio.storage.redis import RedisEventsStorage, RedisImpressionsStorage, \ RedisSplitStorage, RedisSegmentStorage, RedisTelemetryStorage, RedisEventsStorageAsync,\ - RedisImpressionsStorageAsync, RedisSegmentStorageAsync, RedisSplitStorageAsync, RedisTelemetryStorageAsync + RedisImpressionsStorageAsync, RedisSegmentStorageAsync, RedisSplitStorageAsync, RedisTelemetryStorageAsync, \ + RedisRuleBasedSegmentsStorage, RedisRuleBasedSegmentsStorageAsync from splitio.storage.pluggable import PluggableEventsStorage, PluggableImpressionsStorage, PluggableSegmentStorage, \ PluggableTelemetryStorage, PluggableSplitStorage, PluggableEventsStorageAsync, PluggableImpressionsStorageAsync, \ - PluggableSegmentStorageAsync, PluggableSplitStorageAsync, PluggableTelemetryStorageAsync + PluggableSegmentStorageAsync, PluggableSplitStorageAsync, PluggableTelemetryStorageAsync, \ + PluggableRuleBasedSegmentsStorage, PluggableRuleBasedSegmentsStorageAsync from splitio.storage.adapters.redis import build, RedisAdapter, RedisAdapterAsync, build_async -from splitio.models import splits, segments +from splitio.models import splits, segments, rule_based_segments from splitio.engine.impressions.impressions import Manager as ImpressionsManager, ImpressionsMode from splitio.engine.impressions import set_classes, set_classes_async from splitio.engine.impressions.strategies import StrategyDebugMode, StrategyOptimizedMode, StrategyNoneMode @@ -39,6 +42,7 @@ from splitio.sync.synchronizer import PluggableSynchronizer, PluggableSynchronizerAsync from splitio.sync.telemetry import RedisTelemetrySubmitter, RedisTelemetrySubmitterAsync +from tests.helpers.mockserver import SplitMockServer from tests.integration import splits_json from tests.storage.test_pluggable import StorageMockAdapter, StorageMockAdapterAsync @@ -97,7 +101,7 @@ def _validate_last_events(client, *to_validate): as_tup_set = set((i.key, i.traffic_type_name, i.event_type_id, i.value, str(i.properties)) for i in events) assert as_tup_set == set(to_validate) -def _get_treatment(factory): +def _get_treatment(factory, skip_rbs=False): """Test client.get_treatment().""" try: client = factory.client() @@ -154,6 +158,29 @@ def _get_treatment(factory): if not isinstance(factory._recorder._impressions_manager._strategy, StrategyNoneMode): _validate_last_impressions(client, ('regex_test', 'abc4', 'on')) + if skip_rbs: + return + + # test rule based segment matcher + assert client.get_treatment('bilal@split.io', 'rbs_feature_flag', {'email': 'bilal@split.io'}) == 'on' + if not isinstance(factory._recorder._impressions_manager._strategy, StrategyNoneMode): + _validate_last_impressions(client, ('rbs_feature_flag', 'bilal@split.io', 'on')) + + # test rule based segment matcher + assert client.get_treatment('mauro@split.io', 'rbs_feature_flag', {'email': 'mauro@split.io'}) == 'off' + if not isinstance(factory._recorder._impressions_manager._strategy, StrategyNoneMode): + _validate_last_impressions(client, ('rbs_feature_flag', 'mauro@split.io', 'off')) + + # test prerequisites matcher + assert client.get_treatment('abc4', 'prereq_feature') == 'on' + if not isinstance(factory._recorder._impressions_manager._strategy, StrategyNoneMode): + _validate_last_impressions(client, ('prereq_feature', 'abc4', 'on')) + + # test prerequisites matcher + assert client.get_treatment('user1234', 'prereq_feature') == 'off_default' + if not isinstance(factory._recorder._impressions_manager._strategy, StrategyNoneMode): + _validate_last_impressions(client, ('prereq_feature', 'user1234', 'off_default')) + def _get_treatment_with_config(factory): """Test client.get_treatment_with_config().""" try: @@ -407,7 +434,7 @@ def _track(factory): ('user1', 'user', 'conversion', 1, "{'prop1': 'value1'}") ) -def _manager_methods(factory): +def _manager_methods(factory, skip_rbs=False): """Test manager.split/splits.""" try: manager = factory.manager() @@ -438,8 +465,13 @@ def _manager_methods(factory): assert result.change_number == 123 assert result.configs['on'] == '{"size":15,"test":20}' - assert len(manager.split_names()) == 7 - assert len(manager.splits()) == 7 + if skip_rbs: + assert len(manager.split_names()) == 7 + assert len(manager.splits()) == 7 + return + + assert len(manager.split_names()) == 9 + assert len(manager.splits()) == 9 class InMemoryDebugIntegrationTests(object): """Inmemory storage-based integration tests.""" @@ -448,13 +480,17 @@ def setup_method(self): """Prepare storages with test data.""" split_storage = InMemorySplitStorage() segment_storage = InMemorySegmentStorage() + rb_segment_storage = InMemoryRuleBasedSegmentStorage() split_fn = os.path.join(os.path.dirname(__file__), 'files', 'splitChanges.json') with open(split_fn, 'r') as flo: data = json.loads(flo.read()) - for split in data['splits']: + for split in data['ff']['d']: split_storage.update([splits.from_raw(split)], [], 0) + for rbs in data['rbs']['d']: + rb_segment_storage.update([rule_based_segments.from_raw(rbs)], [], 0) + segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentEmployeesChanges.json') with open(segment_fn, 'r') as flo: data = json.loads(flo.read()) @@ -473,6 +509,7 @@ def setup_method(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': InMemoryImpressionStorage(5000, telemetry_runtime_producer), 'events': InMemoryEventStorage(5000, telemetry_runtime_producer), } @@ -604,13 +641,16 @@ def setup_method(self): """Prepare storages with test data.""" split_storage = InMemorySplitStorage() segment_storage = InMemorySegmentStorage() - + rb_segment_storage = InMemoryRuleBasedSegmentStorage() split_fn = os.path.join(os.path.dirname(__file__), 'files', 'splitChanges.json') with open(split_fn, 'r') as flo: data = json.loads(flo.read()) - for split in data['splits']: + for split in data['ff']['d']: split_storage.update([splits.from_raw(split)], [], 0) + for rbs in data['rbs']['d']: + rb_segment_storage.update([rule_based_segments.from_raw(rbs)], [], 0) + segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentEmployeesChanges.json') with open(segment_fn, 'r') as flo: data = json.loads(flo.read()) @@ -629,6 +669,7 @@ def setup_method(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': InMemoryImpressionStorage(5000, telemetry_runtime_producer), 'events': InMemoryEventStorage(5000, telemetry_runtime_producer), } @@ -724,6 +765,159 @@ def test_track(self): """Test client.track().""" _track(self.factory) +class InMemoryOldSpecIntegrationTests(object): + """Inmemory storage-based integration tests.""" + + def setup_method(self): + """Prepare storages with test data.""" + + split_fn = os.path.join(os.path.dirname(__file__), 'files', 'split_old_spec.json') + with open(split_fn, 'r') as flo: + data = json.loads(flo.read()) + + split_changes = { + -1: data, + 1457726098069: {"splits": [], "till": 1457726098069, "since": 1457726098069} + } + + segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentEmployeesChanges.json') + with open(segment_fn, 'r') as flo: + segment_employee = json.loads(flo.read()) + + segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentHumanBeignsChanges.json') + with open(segment_fn, 'r') as flo: + segment_human = json.loads(flo.read()) + + segment_changes = { + ("employees", -1): segment_employee, + ("employees", 1457474612832): {"name": "employees","added": [],"removed": [],"since": 1457474612832,"till": 1457474612832}, + ("human_beigns", -1): segment_human, + ("human_beigns", 1457102183278): {"name": "employees","added": [],"removed": [],"since": 1457102183278,"till": 1457102183278}, + } + + split_backend_requests = Queue() + self.split_backend = SplitMockServer(split_changes, segment_changes, split_backend_requests, + {'auth_response': {'pushEnabled': False}}, True) + self.split_backend.start() + + kwargs = { + 'sdk_api_base_url': 'http://localhost:%d/api' % self.split_backend.port(), + 'events_api_base_url': 'http://localhost:%d/api' % self.split_backend.port(), + 'auth_api_base_url': 'http://localhost:%d/api' % self.split_backend.port(), + 'config': {'connectTimeout': 10000, 'streamingEnabled': False, 'impressionsMode': 'debug'} + } + + self.factory = get_factory('some_apikey', **kwargs) + self.factory.block_until_ready(1) + assert self.factory.ready + + def teardown_method(self): + """Shut down the factory.""" + event = threading.Event() + self.factory.destroy(event) + event.wait() + self.split_backend.stop() + time.sleep(1) + + def test_get_treatment(self): + """Test client.get_treatment().""" + _get_treatment(self.factory, True) + + def test_get_treatment_with_config(self): + """Test client.get_treatment_with_config().""" + _get_treatment_with_config(self.factory) + + def test_get_treatments(self): + _get_treatments(self.factory) + # testing multiple splitNames + client = self.factory.client() + result = client.get_treatments('invalidKey', [ + 'all_feature', + 'killed_feature', + 'invalid_feature', + 'sample_feature' + ]) + assert len(result) == 4 + assert result['all_feature'] == 'on' + assert result['killed_feature'] == 'defTreatment' + assert result['invalid_feature'] == 'control' + assert result['sample_feature'] == 'off' + _validate_last_impressions( + client, + ('all_feature', 'invalidKey', 'on'), + ('killed_feature', 'invalidKey', 'defTreatment'), + ('sample_feature', 'invalidKey', 'off') + ) + + def test_get_treatments_with_config(self): + """Test client.get_treatments_with_config().""" + _get_treatments_with_config(self.factory) + # testing multiple splitNames + client = self.factory.client() + result = client.get_treatments_with_config('invalidKey', [ + 'all_feature', + 'killed_feature', + 'invalid_feature', + 'sample_feature' + ]) + assert len(result) == 4 + assert result['all_feature'] == ('on', None) + assert result['killed_feature'] == ('defTreatment', '{"size":15,"defTreatment":true}') + assert result['invalid_feature'] == ('control', None) + assert result['sample_feature'] == ('off', None) + _validate_last_impressions( + client, + ('all_feature', 'invalidKey', 'on'), + ('killed_feature', 'invalidKey', 'defTreatment'), + ('sample_feature', 'invalidKey', 'off'), + ) + + def test_get_treatments_by_flag_set(self): + """Test client.get_treatments_by_flag_set().""" + _get_treatments_by_flag_set(self.factory) + + def test_get_treatments_by_flag_sets(self): + """Test client.get_treatments_by_flag_sets().""" + _get_treatments_by_flag_sets(self.factory) + client = self.factory.client() + result = client.get_treatments_by_flag_sets('user1', ['set1', 'set2', 'set4']) + assert len(result) == 3 + assert result == {'sample_feature': 'on', + 'whitelist_feature': 'off', + 'all_feature': 'on' + } + _validate_last_impressions(client, ('sample_feature', 'user1', 'on'), + ('whitelist_feature', 'user1', 'off'), + ('all_feature', 'user1', 'on') + ) + + def test_get_treatments_with_config_by_flag_set(self): + """Test client.get_treatments_with_config_by_flag_set().""" + _get_treatments_with_config_by_flag_set(self.factory) + + def test_get_treatments_with_config_by_flag_sets(self): + """Test client.get_treatments_with_config_by_flag_sets().""" + _get_treatments_with_config_by_flag_sets(self.factory) + client = self.factory.client() + result = client.get_treatments_with_config_by_flag_sets('user1', ['set1', 'set2', 'set4']) + assert len(result) == 3 + assert result == {'sample_feature': ('on', '{"size":15,"test":20}'), + 'whitelist_feature': ('off', None), + 'all_feature': ('on', None) + } + _validate_last_impressions(client, ('sample_feature', 'user1', 'on'), + ('whitelist_feature', 'user1', 'off'), + ('all_feature', 'user1', 'on') + ) + + def test_track(self): + """Test client.track().""" + _track(self.factory) + + def test_manager_methods(self): + """Test manager.split/splits.""" + _manager_methods(self.factory, True) + class RedisIntegrationTests(object): """Redis storage-based integration tests.""" @@ -733,16 +927,20 @@ def setup_method(self): redis_client = build(DEFAULT_CONFIG.copy()) split_storage = RedisSplitStorage(redis_client) segment_storage = RedisSegmentStorage(redis_client) + rb_segment_storage = RedisRuleBasedSegmentsStorage(redis_client) split_fn = os.path.join(os.path.dirname(__file__), 'files', 'splitChanges.json') with open(split_fn, 'r') as flo: data = json.loads(flo.read()) - for split in data['splits']: + for split in data['ff']['d']: redis_client.set(split_storage._get_key(split['name']), json.dumps(split)) if split.get('sets') is not None: for flag_set in split.get('sets'): redis_client.sadd(split_storage._get_flag_set_key(flag_set), split['name']) - redis_client.set(split_storage._FEATURE_FLAG_TILL_KEY, data['till']) + redis_client.set(split_storage._FEATURE_FLAG_TILL_KEY, data['ff']['t']) + + for rbs in data['rbs']['d']: + redis_client.set(rb_segment_storage._get_key(rbs['name']), json.dumps(rbs)) segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentEmployeesChanges.json') with open(segment_fn, 'r') as flo: @@ -763,6 +961,7 @@ def setup_method(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': RedisImpressionsStorage(redis_client, metadata), 'events': RedisEventsStorage(redis_client, metadata), } @@ -899,7 +1098,10 @@ def teardown_method(self): "SPLITIO.split.set.set1", "SPLITIO.split.set.set2", "SPLITIO.split.set.set3", - "SPLITIO.split.set.set4" + "SPLITIO.split.set.set4", + "SPLITIO.split.rbs_feature_flag", + "SPLITIO.rbsegments.till", + "SPLITIO.rbsegments.sample_rule_based_segment" ] redis_client = RedisAdapter(StrictRedis()) @@ -915,13 +1117,17 @@ def setup_method(self): redis_client = build(DEFAULT_CONFIG.copy()) split_storage = RedisSplitStorage(redis_client, True) segment_storage = RedisSegmentStorage(redis_client) + rb_segment_storage = RedisRuleBasedSegmentsStorage(redis_client) split_fn = os.path.join(os.path.dirname(__file__), 'files', 'splitChanges.json') with open(split_fn, 'r') as flo: data = json.loads(flo.read()) - for split in data['splits']: + for split in data['ff']['d']: redis_client.set(split_storage._get_key(split['name']), json.dumps(split)) - redis_client.set(split_storage._FEATURE_FLAG_TILL_KEY, data['till']) + redis_client.set(split_storage._FEATURE_FLAG_TILL_KEY, data['ff']['t']) + + for rbs in data['rbs']['d']: + redis_client.set(rb_segment_storage._get_key(rbs['name']), json.dumps(rbs)) segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentEmployeesChanges.json') with open(segment_fn, 'r') as flo: @@ -943,6 +1149,7 @@ def setup_method(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': RedisImpressionsStorage(redis_client, metadata), 'events': RedisEventsStorage(redis_client, metadata), } @@ -1069,6 +1276,12 @@ def test_localhost_json_e2e(self): assert client.get_treatment("key", "SPLIT_1", None) == 'control' assert client.get_treatment("key", "SPLIT_2", None) == 'on' + # rule based segment test + self._update_temp_file(splits_json['splitChange7_1']) + self._synchronize_now() + assert client.get_treatment('bilal@split.io', 'rbs_feature_flag', {'email': 'bilal@split.io'}) == 'on' + assert client.get_treatment('mauro@split.io', 'rbs_feature_flag', {'email': 'mauro@split.io'}) == 'off' + def _update_temp_file(self, json_body): f = open(os.path.join(os.path.dirname(__file__), 'files','split_changes_temp.json'), 'w') f.write(json.dumps(json_body)) @@ -1106,7 +1319,6 @@ def test_incorrect_file_e2e(self): factory.destroy(event) event.wait() - def test_localhost_e2e(self): """Instantiate a client with a YAML file and issue get_treatment() calls.""" filename = os.path.join(os.path.dirname(__file__), 'files', 'file2.yaml') @@ -1136,7 +1348,6 @@ def test_localhost_e2e(self): factory.destroy(event) event.wait() - class PluggableIntegrationTests(object): """Pluggable storage-based integration tests.""" @@ -1146,6 +1357,7 @@ def setup_method(self): self.pluggable_storage_adapter = StorageMockAdapter() split_storage = PluggableSplitStorage(self.pluggable_storage_adapter) segment_storage = PluggableSegmentStorage(self.pluggable_storage_adapter) + rb_segment_storage = PluggableRuleBasedSegmentsStorage(self.pluggable_storage_adapter) telemetry_pluggable_storage = PluggableTelemetryStorage(self.pluggable_storage_adapter, metadata) telemetry_producer = TelemetryStorageProducer(telemetry_pluggable_storage) @@ -1155,6 +1367,7 @@ def setup_method(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': PluggableImpressionsStorage(self.pluggable_storage_adapter, metadata), 'events': PluggableEventsStorage(self.pluggable_storage_adapter, metadata), 'telemetry': telemetry_pluggable_storage @@ -1178,12 +1391,15 @@ def setup_method(self): split_fn = os.path.join(os.path.dirname(__file__), 'files', 'splitChanges.json') with open(split_fn, 'r') as flo: data = json.loads(flo.read()) - for split in data['splits']: + for split in data['ff']['d']: self.pluggable_storage_adapter.set(split_storage._prefix.format(feature_flag_name=split['name']), split) if split.get('sets') is not None: for flag_set in split.get('sets'): self.pluggable_storage_adapter.push_items(split_storage._flag_set_prefix.format(flag_set=flag_set), split['name']) - self.pluggable_storage_adapter.set(split_storage._feature_flag_till_prefix, data['till']) + self.pluggable_storage_adapter.set(split_storage._feature_flag_till_prefix, data['ff']['t']) + + for rbs in data['rbs']['d']: + self.pluggable_storage_adapter.set(rb_segment_storage._prefix.format(segment_name=rbs['name']), rbs) segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentEmployeesChanges.json') with open(segment_fn, 'r') as flo: @@ -1319,7 +1535,10 @@ def teardown_method(self): "SPLITIO.split.set.set1", "SPLITIO.split.set.set2", "SPLITIO.split.set.set3", - "SPLITIO.split.set.set4" + "SPLITIO.split.set.set4", + "SPLITIO.split.rbs_feature_flag", + "SPLITIO.rbsegments.till", + "SPLITIO.rbsegments.sample_rule_based_segment" ] for key in keys_to_delete: self.pluggable_storage_adapter.delete(key) @@ -1333,6 +1552,7 @@ def setup_method(self): self.pluggable_storage_adapter = StorageMockAdapter() split_storage = PluggableSplitStorage(self.pluggable_storage_adapter) segment_storage = PluggableSegmentStorage(self.pluggable_storage_adapter) + rb_segment_storage = PluggableRuleBasedSegmentsStorage(self.pluggable_storage_adapter) telemetry_pluggable_storage = PluggableTelemetryStorage(self.pluggable_storage_adapter, metadata) telemetry_producer = TelemetryStorageProducer(telemetry_pluggable_storage) @@ -1342,6 +1562,7 @@ def setup_method(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': PluggableImpressionsStorage(self.pluggable_storage_adapter, metadata), 'events': PluggableEventsStorage(self.pluggable_storage_adapter, metadata), 'telemetry': telemetry_pluggable_storage @@ -1365,12 +1586,15 @@ def setup_method(self): split_fn = os.path.join(os.path.dirname(__file__), 'files', 'splitChanges.json') with open(split_fn, 'r') as flo: data = json.loads(flo.read()) - for split in data['splits']: + for split in data['ff']['d']: if split.get('sets') is not None: for flag_set in split.get('sets'): self.pluggable_storage_adapter.push_items(split_storage._flag_set_prefix.format(flag_set=flag_set), split['name']) self.pluggable_storage_adapter.set(split_storage._prefix.format(feature_flag_name=split['name']), split) - self.pluggable_storage_adapter.set(split_storage._feature_flag_till_prefix, data['till']) + self.pluggable_storage_adapter.set(split_storage._feature_flag_till_prefix, data['ff']['t']) + + for rbs in data['rbs']['d']: + self.pluggable_storage_adapter.set(rb_segment_storage._prefix.format(segment_name=rbs['name']), rbs) segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentEmployeesChanges.json') with open(segment_fn, 'r') as flo: @@ -1483,7 +1707,10 @@ def teardown_method(self): "SPLITIO.split.set.set1", "SPLITIO.split.set.set2", "SPLITIO.split.set.set3", - "SPLITIO.split.set.set4" + "SPLITIO.split.set.set4", + "SPLITIO.split.rbs_feature_flag", + "SPLITIO.rbsegments.till", + "SPLITIO.rbsegments.sample_rule_based_segment" ] for key in keys_to_delete: self.pluggable_storage_adapter.delete(key) @@ -1497,7 +1724,7 @@ def setup_method(self): self.pluggable_storage_adapter = StorageMockAdapter() split_storage = PluggableSplitStorage(self.pluggable_storage_adapter) segment_storage = PluggableSegmentStorage(self.pluggable_storage_adapter) - + rb_segment_storage = PluggableRuleBasedSegmentsStorage(self.pluggable_storage_adapter) telemetry_pluggable_storage = PluggableTelemetryStorage(self.pluggable_storage_adapter, metadata) telemetry_producer = TelemetryStorageProducer(telemetry_pluggable_storage) telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() @@ -1506,6 +1733,7 @@ def setup_method(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': PluggableImpressionsStorage(self.pluggable_storage_adapter, metadata), 'events': PluggableEventsStorage(self.pluggable_storage_adapter, metadata), 'telemetry': telemetry_pluggable_storage @@ -1552,12 +1780,15 @@ def setup_method(self): split_fn = os.path.join(os.path.dirname(__file__), 'files', 'splitChanges.json') with open(split_fn, 'r') as flo: data = json.loads(flo.read()) - for split in data['splits']: + for split in data['ff']['d']: if split.get('sets') is not None: for flag_set in split.get('sets'): self.pluggable_storage_adapter.push_items(split_storage._flag_set_prefix.format(flag_set=flag_set), split['name']) self.pluggable_storage_adapter.set(split_storage._prefix.format(feature_flag_name=split['name']), split) - self.pluggable_storage_adapter.set(split_storage._feature_flag_till_prefix, data['till']) + self.pluggable_storage_adapter.set(split_storage._feature_flag_till_prefix, data['ff']['t']) + + for rbs in data['rbs']['d']: + self.pluggable_storage_adapter.set(rb_segment_storage._prefix.format(segment_name=rbs['name']), rbs) segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentEmployeesChanges.json') with open(segment_fn, 'r') as flo: @@ -1668,9 +1899,9 @@ def test_optimized(self): split_storage = InMemorySplitStorage() segment_storage = InMemorySegmentStorage() - split_storage.update([splits.from_raw(splits_json['splitChange1_1']['splits'][0]), - splits.from_raw(splits_json['splitChange1_1']['splits'][1]), - splits.from_raw(splits_json['splitChange1_1']['splits'][2]) + split_storage.update([splits.from_raw(splits_json['splitChange1_1']['ff']['d'][0]), + splits.from_raw(splits_json['splitChange1_1']['ff']['d'][1]), + splits.from_raw(splits_json['splitChange1_1']['ff']['d'][2]) ], [], -1) telemetry_storage = InMemoryTelemetryStorage() @@ -1681,6 +1912,7 @@ def test_optimized(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': InMemoryRuleBasedSegmentStorage(), 'impressions': InMemoryImpressionStorage(5000, telemetry_runtime_producer), 'events': InMemoryEventStorage(5000, telemetry_runtime_producer), } @@ -1722,9 +1954,9 @@ def test_debug(self): split_storage = InMemorySplitStorage() segment_storage = InMemorySegmentStorage() - split_storage.update([splits.from_raw(splits_json['splitChange1_1']['splits'][0]), - splits.from_raw(splits_json['splitChange1_1']['splits'][1]), - splits.from_raw(splits_json['splitChange1_1']['splits'][2]) + split_storage.update([splits.from_raw(splits_json['splitChange1_1']['ff']['d'][0]), + splits.from_raw(splits_json['splitChange1_1']['ff']['d'][1]), + splits.from_raw(splits_json['splitChange1_1']['ff']['d'][2]) ], [], -1) telemetry_storage = InMemoryTelemetryStorage() @@ -1735,6 +1967,7 @@ def test_debug(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': InMemoryRuleBasedSegmentStorage(), 'impressions': InMemoryImpressionStorage(5000, telemetry_runtime_producer), 'events': InMemoryEventStorage(5000, telemetry_runtime_producer), } @@ -1776,9 +2009,9 @@ def test_none(self): split_storage = InMemorySplitStorage() segment_storage = InMemorySegmentStorage() - split_storage.update([splits.from_raw(splits_json['splitChange1_1']['splits'][0]), - splits.from_raw(splits_json['splitChange1_1']['splits'][1]), - splits.from_raw(splits_json['splitChange1_1']['splits'][2]) + split_storage.update([splits.from_raw(splits_json['splitChange1_1']['ff']['d'][0]), + splits.from_raw(splits_json['splitChange1_1']['ff']['d'][1]), + splits.from_raw(splits_json['splitChange1_1']['ff']['d'][2]) ], [], -1) telemetry_storage = InMemoryTelemetryStorage() @@ -1789,6 +2022,7 @@ def test_none(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': InMemoryRuleBasedSegmentStorage(), 'impressions': InMemoryImpressionStorage(5000, telemetry_runtime_producer), 'events': InMemoryEventStorage(5000, telemetry_runtime_producer), } @@ -1838,9 +2072,9 @@ def test_optimized(self): split_storage = RedisSplitStorage(redis_client, True) segment_storage = RedisSegmentStorage(redis_client) - redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['splits'][0]['name']), json.dumps(splits_json['splitChange1_1']['splits'][0])) - redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['splits'][1]['name']), json.dumps(splits_json['splitChange1_1']['splits'][1])) - redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['splits'][2]['name']), json.dumps(splits_json['splitChange1_1']['splits'][2])) + redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['ff']['d'][0]['name']), json.dumps(splits_json['splitChange1_1']['ff']['d'][0])) + redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['ff']['d'][1]['name']), json.dumps(splits_json['splitChange1_1']['ff']['d'][1])) + redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['ff']['d'][2]['name']), json.dumps(splits_json['splitChange1_1']['ff']['d'][2])) redis_client.set(split_storage._FEATURE_FLAG_TILL_KEY, -1) telemetry_redis_storage = RedisTelemetryStorage(redis_client, metadata) @@ -1851,6 +2085,7 @@ def test_optimized(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': RedisRuleBasedSegmentsStorage(redis_client), 'impressions': RedisImpressionsStorage(redis_client, metadata), 'events': RedisEventsStorage(redis_client, metadata), } @@ -1901,9 +2136,9 @@ def test_debug(self): split_storage = RedisSplitStorage(redis_client, True) segment_storage = RedisSegmentStorage(redis_client) - redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['splits'][0]['name']), json.dumps(splits_json['splitChange1_1']['splits'][0])) - redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['splits'][1]['name']), json.dumps(splits_json['splitChange1_1']['splits'][1])) - redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['splits'][2]['name']), json.dumps(splits_json['splitChange1_1']['splits'][2])) + redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['ff']['d'][0]['name']), json.dumps(splits_json['splitChange1_1']['ff']['d'][0])) + redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['ff']['d'][1]['name']), json.dumps(splits_json['splitChange1_1']['ff']['d'][1])) + redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['ff']['d'][2]['name']), json.dumps(splits_json['splitChange1_1']['ff']['d'][2])) redis_client.set(split_storage._FEATURE_FLAG_TILL_KEY, -1) telemetry_redis_storage = RedisTelemetryStorage(redis_client, metadata) @@ -1914,6 +2149,7 @@ def test_debug(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': RedisRuleBasedSegmentsStorage(redis_client), 'impressions': RedisImpressionsStorage(redis_client, metadata), 'events': RedisEventsStorage(redis_client, metadata), } @@ -1964,9 +2200,9 @@ def test_none(self): split_storage = RedisSplitStorage(redis_client, True) segment_storage = RedisSegmentStorage(redis_client) - redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['splits'][0]['name']), json.dumps(splits_json['splitChange1_1']['splits'][0])) - redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['splits'][1]['name']), json.dumps(splits_json['splitChange1_1']['splits'][1])) - redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['splits'][2]['name']), json.dumps(splits_json['splitChange1_1']['splits'][2])) + redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['ff']['d'][0]['name']), json.dumps(splits_json['splitChange1_1']['ff']['d'][0])) + redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['ff']['d'][1]['name']), json.dumps(splits_json['splitChange1_1']['ff']['d'][1])) + redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['ff']['d'][2]['name']), json.dumps(splits_json['splitChange1_1']['ff']['d'][2])) redis_client.set(split_storage._FEATURE_FLAG_TILL_KEY, -1) telemetry_redis_storage = RedisTelemetryStorage(redis_client, metadata) @@ -1977,6 +2213,7 @@ def test_none(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': RedisRuleBasedSegmentsStorage(redis_client), 'impressions': RedisImpressionsStorage(redis_client, metadata), 'events': RedisEventsStorage(redis_client, metadata), } @@ -2046,13 +2283,17 @@ async def _setup_method(self): """Prepare storages with test data.""" split_storage = InMemorySplitStorageAsync() segment_storage = InMemorySegmentStorageAsync() + rb_segment_storage = InMemoryRuleBasedSegmentStorageAsync() split_fn = os.path.join(os.path.dirname(__file__), 'files', 'splitChanges.json') with open(split_fn, 'r') as flo: data = json.loads(flo.read()) - for split in data['splits']: + for split in data['ff']['d']: await split_storage.update([splits.from_raw(split)], [], -1) + for rbs in data['rbs']['d']: + await rb_segment_storage.update([rule_based_segments.from_raw(rbs)], [], 0) + segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentEmployeesChanges.json') with open(segment_fn, 'r') as flo: data = json.loads(flo.read()) @@ -2071,6 +2312,7 @@ async def _setup_method(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': InMemoryImpressionStorageAsync(5000, telemetry_runtime_producer), 'events': InMemoryEventStorageAsync(5000, telemetry_runtime_producer), } @@ -2212,13 +2454,16 @@ async def _setup_method(self): """Prepare storages with test data.""" split_storage = InMemorySplitStorageAsync() segment_storage = InMemorySegmentStorageAsync() - + rb_segment_storage = InMemoryRuleBasedSegmentStorageAsync() split_fn = os.path.join(os.path.dirname(__file__), 'files', 'splitChanges.json') with open(split_fn, 'r') as flo: data = json.loads(flo.read()) - for split in data['splits']: + for split in data['ff']['d']: await split_storage.update([splits.from_raw(split)], [], -1) + for rbs in data['rbs']['d']: + await rb_segment_storage.update([rule_based_segments.from_raw(rbs)], [], 0) + segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentEmployeesChanges.json') with open(segment_fn, 'r') as flo: data = json.loads(flo.read()) @@ -2237,6 +2482,7 @@ async def _setup_method(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': InMemoryImpressionStorageAsync(5000, telemetry_runtime_producer), 'events': InMemoryEventStorageAsync(5000, telemetry_runtime_producer), } @@ -2350,6 +2596,194 @@ async def test_track(self): await _track_async(self.factory) await self.factory.destroy() +class InMemoryOldSpecIntegrationAsyncTests(object): + """Inmemory storage-based integration tests.""" + + def setup_method(self): + self.setup_task = asyncio.get_event_loop().create_task(self._setup_method()) + + async def _setup_method(self): + """Prepare storages with test data.""" + + split_fn = os.path.join(os.path.dirname(__file__), 'files', 'split_old_spec.json') + with open(split_fn, 'r') as flo: + data = json.loads(flo.read()) + + split_changes = { + -1: data, + 1457726098069: {"splits": [], "till": 1457726098069, "since": 1457726098069} + } + + segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentEmployeesChanges.json') + with open(segment_fn, 'r') as flo: + segment_employee = json.loads(flo.read()) + + segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentHumanBeignsChanges.json') + with open(segment_fn, 'r') as flo: + segment_human = json.loads(flo.read()) + + segment_changes = { + ("employees", -1): segment_employee, + ("employees", 1457474612832): {"name": "employees","added": [],"removed": [],"since": 1457474612832,"till": 1457474612832}, + ("human_beigns", -1): segment_human, + ("human_beigns", 1457102183278): {"name": "employees","added": [],"removed": [],"since": 1457102183278,"till": 1457102183278}, + } + + split_backend_requests = Queue() + self.split_backend = SplitMockServer(split_changes, segment_changes, split_backend_requests, + {'auth_response': {'pushEnabled': False}}, True) + self.split_backend.start() + + kwargs = { + 'sdk_api_base_url': 'http://localhost:%d/api' % self.split_backend.port(), + 'events_api_base_url': 'http://localhost:%d/api' % self.split_backend.port(), + 'auth_api_base_url': 'http://localhost:%d/api' % self.split_backend.port(), + 'config': {'connectTimeout': 10000, 'streamingEnabled': False, 'impressionsMode': 'debug'} + } + + self.factory = await get_factory_async('some_apikey', **kwargs) + await self.factory.block_until_ready(1) + assert self.factory.ready + + @pytest.mark.asyncio + async def test_get_treatment(self): + """Test client.get_treatment().""" + await self.setup_task + await _get_treatment_async(self.factory, True) + await self.factory.destroy() + self.split_backend.stop() + + @pytest.mark.asyncio + async def test_get_treatment_with_config(self): + """Test client.get_treatment_with_config().""" + await self.setup_task + await _get_treatment_with_config_async(self.factory) + await self.factory.destroy() + self.split_backend.stop() + + @pytest.mark.asyncio + async def test_get_treatments(self): + await self.setup_task + await _get_treatments_async(self.factory) + # testing multiple splitNames + client = self.factory.client() + result = await client.get_treatments('invalidKey', [ + 'all_feature', + 'killed_feature', + 'invalid_feature', + 'sample_feature' + ]) + assert len(result) == 4 + assert result['all_feature'] == 'on' + assert result['killed_feature'] == 'defTreatment' + assert result['invalid_feature'] == 'control' + assert result['sample_feature'] == 'off' + await _validate_last_impressions_async( + client, + ('all_feature', 'invalidKey', 'on'), + ('killed_feature', 'invalidKey', 'defTreatment'), + ('sample_feature', 'invalidKey', 'off') + ) + await self.factory.destroy() + self.split_backend.stop() + + @pytest.mark.asyncio + async def test_get_treatments_with_config(self): + """Test client.get_treatments_with_config().""" + await self.setup_task + await _get_treatments_with_config_async(self.factory) + # testing multiple splitNames + client = self.factory.client() + result = await client.get_treatments_with_config('invalidKey', [ + 'all_feature', + 'killed_feature', + 'invalid_feature', + 'sample_feature' + ]) + assert len(result) == 4 + assert result['all_feature'] == ('on', None) + assert result['killed_feature'] == ('defTreatment', '{"size":15,"defTreatment":true}') + assert result['invalid_feature'] == ('control', None) + assert result['sample_feature'] == ('off', None) + await _validate_last_impressions_async( + client, + ('all_feature', 'invalidKey', 'on'), + ('killed_feature', 'invalidKey', 'defTreatment'), + ('sample_feature', 'invalidKey', 'off'), + ) + await self.factory.destroy() + self.split_backend.stop() + + @pytest.mark.asyncio + async def test_get_treatments_by_flag_set(self): + """Test client.get_treatments_by_flag_set().""" + await self.setup_task + await _get_treatments_by_flag_set_async(self.factory) + await self.factory.destroy() + self.split_backend.stop() + + @pytest.mark.asyncio + async def test_get_treatments_by_flag_sets(self): + """Test client.get_treatments_by_flag_sets().""" + await self.setup_task + await _get_treatments_by_flag_sets_async(self.factory) + client = self.factory.client() + result = await client.get_treatments_by_flag_sets('user1', ['set1', 'set2', 'set4']) + assert len(result) == 3 + assert result == {'sample_feature': 'on', + 'whitelist_feature': 'off', + 'all_feature': 'on' + } + await _validate_last_impressions_async(client, ('sample_feature', 'user1', 'on'), + ('whitelist_feature', 'user1', 'off'), + ('all_feature', 'user1', 'on') + ) + await self.factory.destroy() + self.split_backend.stop() + + @pytest.mark.asyncio + async def test_get_treatments_with_config_by_flag_set(self): + """Test client.get_treatments_with_config_by_flag_set().""" + await self.setup_task + await _get_treatments_with_config_by_flag_set_async(self.factory) + await self.factory.destroy() + self.split_backend.stop() + + @pytest.mark.asyncio + async def test_get_treatments_with_config_by_flag_sets(self): + """Test client.get_treatments_with_config_by_flag_sets().""" + await self.setup_task + await _get_treatments_with_config_by_flag_sets_async(self.factory) + client = self.factory.client() + result = await client.get_treatments_with_config_by_flag_sets('user1', ['set1', 'set2', 'set4']) + assert len(result) == 3 + assert result == {'sample_feature': ('on', '{"size":15,"test":20}'), + 'whitelist_feature': ('off', None), + 'all_feature': ('on', None) + } + await _validate_last_impressions_async(client, ('sample_feature', 'user1', 'on'), + ('whitelist_feature', 'user1', 'off'), + ('all_feature', 'user1', 'on') + ) + await self.factory.destroy() + self.split_backend.stop() + + @pytest.mark.asyncio + async def test_track(self): + """Test client.track().""" + await self.setup_task + await _track_async(self.factory) + await self.factory.destroy() + self.split_backend.stop() + + @pytest.mark.asyncio + async def test_manager_methods(self): + """Test manager.split/splits.""" + await self.setup_task + await _manager_methods_async(self.factory, True) + await self.factory.destroy() + self.split_backend.stop() + class RedisIntegrationAsyncTests(object): """Redis storage-based integration tests.""" @@ -2364,17 +2798,20 @@ async def _setup_method(self): split_storage = RedisSplitStorageAsync(redis_client) segment_storage = RedisSegmentStorageAsync(redis_client) + rb_segment_storage = RedisRuleBasedSegmentsStorageAsync(redis_client) split_fn = os.path.join(os.path.dirname(__file__), 'files', 'splitChanges.json') with open(split_fn, 'r') as flo: data = json.loads(flo.read()) - for split in data['splits']: + for split in data['ff']['d']: await redis_client.set(split_storage._get_key(split['name']), json.dumps(split)) if split.get('sets') is not None: for flag_set in split.get('sets'): await redis_client.sadd(split_storage._get_flag_set_key(flag_set), split['name']) + await redis_client.set(split_storage._FEATURE_FLAG_TILL_KEY, data['ff']['t']) - await redis_client.set(split_storage._FEATURE_FLAG_TILL_KEY, data['till']) + for rbs in data['rbs']['d']: + await redis_client.set(rb_segment_storage._get_key(rbs['name']), json.dumps(rbs)) segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentEmployeesChanges.json') with open(segment_fn, 'r') as flo: @@ -2396,6 +2833,7 @@ async def _setup_method(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': RedisImpressionsStorageAsync(redis_client, metadata), 'events': RedisEventsStorageAsync(redis_client, metadata), } @@ -2560,7 +2998,10 @@ async def _clear_cache(self, redis_client): "SPLITIO.segment.employees.till", "SPLITIO.split.whitelist_feature", "SPLITIO.telemetry.latencies", - "SPLITIO.split.dependency_test" + "SPLITIO.split.dependency_test", + "SPLITIO.split.rbs_feature_flag", + "SPLITIO.rbsegments.till", + "SPLITIO.rbsegments.sample_rule_based_segment" ] for key in keys_to_delete: await redis_client.delete(key) @@ -2579,16 +3020,20 @@ async def _setup_method(self): split_storage = RedisSplitStorageAsync(redis_client, True) segment_storage = RedisSegmentStorageAsync(redis_client) + rb_segment_storage = RedisRuleBasedSegmentsStorageAsync(redis_client) split_fn = os.path.join(os.path.dirname(__file__), 'files', 'splitChanges.json') with open(split_fn, 'r') as flo: data = json.loads(flo.read()) - for split in data['splits']: + for split in data['ff']['d']: await redis_client.set(split_storage._get_key(split['name']), json.dumps(split)) if split.get('sets') is not None: for flag_set in split.get('sets'): await redis_client.sadd(split_storage._get_flag_set_key(flag_set), split['name']) - await redis_client.set(split_storage._FEATURE_FLAG_TILL_KEY, data['till']) + await redis_client.set(split_storage._FEATURE_FLAG_TILL_KEY, data['ff']['t']) + + for rbs in data['rbs']['d']: + await redis_client.set(rb_segment_storage._get_key(rbs['name']), json.dumps(rbs)) segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentEmployeesChanges.json') with open(segment_fn, 'r') as flo: @@ -2610,6 +3055,7 @@ async def _setup_method(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': RedisImpressionsStorageAsync(redis_client, metadata), 'events': RedisEventsStorageAsync(redis_client, metadata), } @@ -2742,6 +3188,12 @@ async def test_localhost_json_e2e(self): assert await client.get_treatment("key", "SPLIT_1", None) == 'control' assert await client.get_treatment("key", "SPLIT_2", None) == 'on' + # rule based segment test + self._update_temp_file(splits_json['splitChange7_1']) + await self._synchronize_now() + assert await client.get_treatment('bilal@split.io', 'rbs_feature_flag', {'email': 'bilal@split.io'}) == 'on' + assert await client.get_treatment('mauro@split.io', 'rbs_feature_flag', {'email': 'mauro@split.io'}) == 'off' + def _update_temp_file(self, json_body): f = open(os.path.join(os.path.dirname(__file__), 'files','split_changes_temp.json'), 'w') f.write(json.dumps(json_body)) @@ -2821,6 +3273,7 @@ async def _setup_method(self): self.pluggable_storage_adapter = StorageMockAdapterAsync() split_storage = PluggableSplitStorageAsync(self.pluggable_storage_adapter, 'myprefix') segment_storage = PluggableSegmentStorageAsync(self.pluggable_storage_adapter, 'myprefix') + rb_segment_storage = PluggableRuleBasedSegmentsStorageAsync(self.pluggable_storage_adapter, 'myprefix') telemetry_pluggable_storage = await PluggableTelemetryStorageAsync.create(self.pluggable_storage_adapter, metadata, 'myprefix') telemetry_producer = TelemetryStorageProducerAsync(telemetry_pluggable_storage) @@ -2830,6 +3283,7 @@ async def _setup_method(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': PluggableImpressionsStorageAsync(self.pluggable_storage_adapter, metadata), 'events': PluggableEventsStorageAsync(self.pluggable_storage_adapter, metadata), 'telemetry': telemetry_pluggable_storage @@ -2858,11 +3312,14 @@ async def _setup_method(self): split_fn = os.path.join(os.path.dirname(__file__), 'files', 'splitChanges.json') with open(split_fn, 'r') as flo: data = json.loads(flo.read()) - for split in data['splits']: + for split in data['ff']['d']: await self.pluggable_storage_adapter.set(split_storage._prefix.format(feature_flag_name=split['name']), split) for flag_set in split.get('sets'): await self.pluggable_storage_adapter.push_items(split_storage._flag_set_prefix.format(flag_set=flag_set), split['name']) - await self.pluggable_storage_adapter.set(split_storage._feature_flag_till_prefix, data['till']) + await self.pluggable_storage_adapter.set(split_storage._feature_flag_till_prefix, data['ff']['d']) + + for rbs in data['rbs']['d']: + await self.pluggable_storage_adapter.set(rb_segment_storage._prefix.format(segment_name=rbs['name']), rbs) segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentEmployeesChanges.json') with open(segment_fn, 'r') as flo: @@ -3023,7 +3480,10 @@ async def _teardown_method(self): "SPLITIO.split.regex_test", "SPLITIO.segment.human_beigns.till", "SPLITIO.split.boolean_test", - "SPLITIO.split.dependency_test" + "SPLITIO.split.dependency_test", + "SPLITIO.split.rbs_feature_flag", + "SPLITIO.rbsegments.till", + "SPLITIO.rbsegments.sample_rule_based_segment" ] for key in keys_to_delete: @@ -3041,6 +3501,7 @@ async def _setup_method(self): self.pluggable_storage_adapter = StorageMockAdapterAsync() split_storage = PluggableSplitStorageAsync(self.pluggable_storage_adapter) segment_storage = PluggableSegmentStorageAsync(self.pluggable_storage_adapter) + rb_segment_storage = PluggableRuleBasedSegmentsStorageAsync(self.pluggable_storage_adapter, 'myprefix') telemetry_pluggable_storage = await PluggableTelemetryStorageAsync.create(self.pluggable_storage_adapter, metadata) telemetry_producer = TelemetryStorageProducerAsync(telemetry_pluggable_storage) @@ -3050,6 +3511,7 @@ async def _setup_method(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': PluggableImpressionsStorageAsync(self.pluggable_storage_adapter, metadata), 'events': PluggableEventsStorageAsync(self.pluggable_storage_adapter, metadata), 'telemetry': telemetry_pluggable_storage @@ -3080,11 +3542,14 @@ async def _setup_method(self): split_fn = os.path.join(os.path.dirname(__file__), 'files', 'splitChanges.json') with open(split_fn, 'r') as flo: data = json.loads(flo.read()) - for split in data['splits']: + for split in data['ff']['d']: await self.pluggable_storage_adapter.set(split_storage._prefix.format(feature_flag_name=split['name']), split) for flag_set in split.get('sets'): await self.pluggable_storage_adapter.push_items(split_storage._flag_set_prefix.format(flag_set=flag_set), split['name']) - await self.pluggable_storage_adapter.set(split_storage._feature_flag_till_prefix, data['till']) + await self.pluggable_storage_adapter.set(split_storage._feature_flag_till_prefix, data['ff']['t']) + + for rbs in data['rbs']['d']: + await self.pluggable_storage_adapter.set(rb_segment_storage._prefix.format(segment_name=rbs['name']), rbs) segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentEmployeesChanges.json') with open(segment_fn, 'r') as flo: @@ -3230,7 +3695,10 @@ async def _teardown_method(self): "SPLITIO.split.regex_test", "SPLITIO.segment.human_beigns.till", "SPLITIO.split.boolean_test", - "SPLITIO.split.dependency_test" + "SPLITIO.split.dependency_test", + "SPLITIO.split.rbs_feature_flag", + "SPLITIO.rbsegments.till", + "SPLITIO.rbsegments.sample_rule_based_segment" ] for key in keys_to_delete: @@ -3248,6 +3716,7 @@ async def _setup_method(self): self.pluggable_storage_adapter = StorageMockAdapterAsync() split_storage = PluggableSplitStorageAsync(self.pluggable_storage_adapter) segment_storage = PluggableSegmentStorageAsync(self.pluggable_storage_adapter) + rb_segment_storage = PluggableRuleBasedSegmentsStorageAsync(self.pluggable_storage_adapter, 'myprefix') telemetry_pluggable_storage = await PluggableTelemetryStorageAsync.create(self.pluggable_storage_adapter, metadata) telemetry_producer = TelemetryStorageProducerAsync(telemetry_pluggable_storage) @@ -3257,6 +3726,7 @@ async def _setup_method(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': PluggableImpressionsStorageAsync(self.pluggable_storage_adapter, metadata), 'events': PluggableEventsStorageAsync(self.pluggable_storage_adapter, metadata), 'telemetry': telemetry_pluggable_storage @@ -3302,11 +3772,14 @@ async def _setup_method(self): split_fn = os.path.join(os.path.dirname(__file__), 'files', 'splitChanges.json') with open(split_fn, 'r') as flo: data = json.loads(flo.read()) - for split in data['splits']: + for split in data['ff']['d']: await self.pluggable_storage_adapter.set(split_storage._prefix.format(feature_flag_name=split['name']), split) for flag_set in split.get('sets'): await self.pluggable_storage_adapter.push_items(split_storage._flag_set_prefix.format(flag_set=flag_set), split['name']) - await self.pluggable_storage_adapter.set(split_storage._feature_flag_till_prefix, data['till']) + await self.pluggable_storage_adapter.set(split_storage._feature_flag_till_prefix, data['ff']['t']) + + for rbs in data['rbs']['d']: + await self.pluggable_storage_adapter.set(rb_segment_storage._prefix.format(segment_name=rbs['name']), rbs) segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentEmployeesChanges.json') with open(segment_fn, 'r') as flo: @@ -3461,7 +3934,10 @@ async def _teardown_method(self): "SPLITIO.split.regex_test", "SPLITIO.segment.human_beigns.till", "SPLITIO.split.boolean_test", - "SPLITIO.split.dependency_test" + "SPLITIO.split.dependency_test", + "SPLITIO.split.rbs_feature_flag", + "SPLITIO.rbsegments.till", + "SPLITIO.rbsegments.sample_rule_based_segment" ] for key in keys_to_delete: @@ -3475,9 +3951,9 @@ async def test_optimized(self): split_storage = InMemorySplitStorageAsync() segment_storage = InMemorySegmentStorageAsync() - await split_storage.update([splits.from_raw(splits_json['splitChange1_1']['splits'][0]), - splits.from_raw(splits_json['splitChange1_1']['splits'][1]), - splits.from_raw(splits_json['splitChange1_1']['splits'][2]) + await split_storage.update([splits.from_raw(splits_json['splitChange1_1']['ff']['d'][0]), + splits.from_raw(splits_json['splitChange1_1']['ff']['d'][1]), + splits.from_raw(splits_json['splitChange1_1']['ff']['d'][2]) ], [], -1) telemetry_storage = await InMemoryTelemetryStorageAsync.create() @@ -3488,6 +3964,7 @@ async def test_optimized(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': InMemoryRuleBasedSegmentStorageAsync(), 'impressions': InMemoryImpressionStorageAsync(5000, telemetry_runtime_producer), 'events': InMemoryEventStorageAsync(5000, telemetry_runtime_producer), } @@ -3534,9 +4011,9 @@ async def test_debug(self): split_storage = InMemorySplitStorageAsync() segment_storage = InMemorySegmentStorageAsync() - await split_storage.update([splits.from_raw(splits_json['splitChange1_1']['splits'][0]), - splits.from_raw(splits_json['splitChange1_1']['splits'][1]), - splits.from_raw(splits_json['splitChange1_1']['splits'][2]) + await split_storage.update([splits.from_raw(splits_json['splitChange1_1']['ff']['d'][0]), + splits.from_raw(splits_json['splitChange1_1']['ff']['d'][1]), + splits.from_raw(splits_json['splitChange1_1']['ff']['d'][2]) ], [], -1) telemetry_storage = await InMemoryTelemetryStorageAsync.create() @@ -3547,6 +4024,7 @@ async def test_debug(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': InMemoryRuleBasedSegmentStorageAsync(), 'impressions': InMemoryImpressionStorageAsync(5000, telemetry_runtime_producer), 'events': InMemoryEventStorageAsync(5000, telemetry_runtime_producer), } @@ -3593,9 +4071,9 @@ async def test_none(self): split_storage = InMemorySplitStorageAsync() segment_storage = InMemorySegmentStorageAsync() - await split_storage.update([splits.from_raw(splits_json['splitChange1_1']['splits'][0]), - splits.from_raw(splits_json['splitChange1_1']['splits'][1]), - splits.from_raw(splits_json['splitChange1_1']['splits'][2]) + await split_storage.update([splits.from_raw(splits_json['splitChange1_1']['ff']['d'][0]), + splits.from_raw(splits_json['splitChange1_1']['ff']['d'][1]), + splits.from_raw(splits_json['splitChange1_1']['ff']['d'][2]) ], [], -1) telemetry_storage = await InMemoryTelemetryStorageAsync.create() @@ -3606,6 +4084,7 @@ async def test_none(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': InMemoryRuleBasedSegmentStorageAsync(), 'impressions': InMemoryImpressionStorageAsync(5000, telemetry_runtime_producer), 'events': InMemoryEventStorageAsync(5000, telemetry_runtime_producer), } @@ -3659,10 +4138,11 @@ async def test_optimized(self): redis_client = await build_async(DEFAULT_CONFIG.copy()) split_storage = RedisSplitStorageAsync(redis_client, True) segment_storage = RedisSegmentStorageAsync(redis_client) + rb_segment_storage = RedisRuleBasedSegmentsStorageAsync(redis_client) - await redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['splits'][0]['name']), json.dumps(splits_json['splitChange1_1']['splits'][0])) - await redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['splits'][1]['name']), json.dumps(splits_json['splitChange1_1']['splits'][1])) - await redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['splits'][2]['name']), json.dumps(splits_json['splitChange1_1']['splits'][2])) + await redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['ff']['d'][0]['name']), json.dumps(splits_json['splitChange1_1']['ff']['d'][0])) + await redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['ff']['d'][1]['name']), json.dumps(splits_json['splitChange1_1']['ff']['d'][1])) + await redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['ff']['d'][2]['name']), json.dumps(splits_json['splitChange1_1']['ff']['d'][2])) await redis_client.set(split_storage._FEATURE_FLAG_TILL_KEY, -1) telemetry_redis_storage = await RedisTelemetryStorageAsync.create(redis_client, metadata) @@ -3673,6 +4153,7 @@ async def test_optimized(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': RedisImpressionsStorageAsync(redis_client, metadata), 'events': RedisEventsStorageAsync(redis_client, metadata), } @@ -3726,10 +4207,11 @@ async def test_debug(self): redis_client = await build_async(DEFAULT_CONFIG.copy()) split_storage = RedisSplitStorageAsync(redis_client, True) segment_storage = RedisSegmentStorageAsync(redis_client) + rb_segment_storage = RedisRuleBasedSegmentsStorageAsync(redis_client) - await redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['splits'][0]['name']), json.dumps(splits_json['splitChange1_1']['splits'][0])) - await redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['splits'][1]['name']), json.dumps(splits_json['splitChange1_1']['splits'][1])) - await redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['splits'][2]['name']), json.dumps(splits_json['splitChange1_1']['splits'][2])) + await redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['ff']['d'][0]['name']), json.dumps(splits_json['splitChange1_1']['ff']['d'][0])) + await redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['ff']['d'][1]['name']), json.dumps(splits_json['splitChange1_1']['ff']['d'][1])) + await redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['ff']['d'][2]['name']), json.dumps(splits_json['splitChange1_1']['ff']['d'][2])) await redis_client.set(split_storage._FEATURE_FLAG_TILL_KEY, -1) telemetry_redis_storage = await RedisTelemetryStorageAsync.create(redis_client, metadata) @@ -3740,6 +4222,7 @@ async def test_debug(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': RedisImpressionsStorageAsync(redis_client, metadata), 'events': RedisEventsStorageAsync(redis_client, metadata), } @@ -3793,10 +4276,11 @@ async def test_none(self): redis_client = await build_async(DEFAULT_CONFIG.copy()) split_storage = RedisSplitStorageAsync(redis_client, True) segment_storage = RedisSegmentStorageAsync(redis_client) + rb_segment_storage = RedisRuleBasedSegmentsStorageAsync(redis_client) - await redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['splits'][0]['name']), json.dumps(splits_json['splitChange1_1']['splits'][0])) - await redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['splits'][1]['name']), json.dumps(splits_json['splitChange1_1']['splits'][1])) - await redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['splits'][2]['name']), json.dumps(splits_json['splitChange1_1']['splits'][2])) + await redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['ff']['d'][0]['name']), json.dumps(splits_json['splitChange1_1']['ff']['d'][0])) + await redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['ff']['d'][1]['name']), json.dumps(splits_json['splitChange1_1']['ff']['d'][1])) + await redis_client.set(split_storage._get_key(splits_json['splitChange1_1']['ff']['d'][2]['name']), json.dumps(splits_json['splitChange1_1']['ff']['d'][2])) await redis_client.set(split_storage._FEATURE_FLAG_TILL_KEY, -1) telemetry_redis_storage = await RedisTelemetryStorageAsync.create(redis_client, metadata) @@ -3807,6 +4291,7 @@ async def test_none(self): storages = { 'splits': split_storage, 'segments': segment_storage, + 'rule_based_segments': rb_segment_storage, 'impressions': RedisImpressionsStorageAsync(redis_client, metadata), 'events': RedisEventsStorageAsync(redis_client, metadata), } @@ -3868,7 +4353,7 @@ async def clear_cache(self): redis_client = await build_async(DEFAULT_CONFIG.copy()) for key in keys_to_delete: await redis_client.delete(key) - + async def _validate_last_impressions_async(client, *to_validate): """Validate the last N impressions are present disregarding the order.""" imp_storage = client._factory._get_storage('impressions') @@ -3924,7 +4409,7 @@ async def _validate_last_events_async(client, *to_validate): as_tup_set = set((i.key, i.traffic_type_name, i.event_type_id, i.value, str(i.properties)) for i in events) assert as_tup_set == set(to_validate) -async def _get_treatment_async(factory): +async def _get_treatment_async(factory, skip_rbs=False): """Test client.get_treatment().""" try: client = factory.client() @@ -3981,6 +4466,29 @@ async def _get_treatment_async(factory): if not isinstance(factory._recorder._impressions_manager._strategy, StrategyNoneMode): await _validate_last_impressions_async(client, ('regex_test', 'abc4', 'on')) + if skip_rbs: + return + + # test prerequisites matcher + assert await client.get_treatment('abc4', 'prereq_feature') == 'on' + if not isinstance(factory._recorder._impressions_manager._strategy, StrategyNoneMode): + await _validate_last_impressions_async(client, ('prereq_feature', 'abc4', 'on')) + + # test prerequisites matcher + assert await client.get_treatment('user1234', 'prereq_feature') == 'off_default' + if not isinstance(factory._recorder._impressions_manager._strategy, StrategyNoneMode): + await _validate_last_impressions_async(client, ('prereq_feature', 'user1234', 'off_default')) + + # test rule based segment matcher + assert await client.get_treatment('bilal@split.io', 'rbs_feature_flag', {'email': 'bilal@split.io'}) == 'on' + if not isinstance(factory._recorder._impressions_manager._strategy, StrategyNoneMode): + await _validate_last_impressions_async(client, ('rbs_feature_flag', 'bilal@split.io', 'on')) + + # test rule based segment matcher + assert await client.get_treatment('mauro@split.io', 'rbs_feature_flag', {'email': 'mauro@split.io'}) == 'off' + if not isinstance(factory._recorder._impressions_manager._strategy, StrategyNoneMode): + await _validate_last_impressions_async(client, ('rbs_feature_flag', 'mauro@split.io', 'off')) + async def _get_treatment_with_config_async(factory): """Test client.get_treatment_with_config().""" try: @@ -4234,7 +4742,7 @@ async def _track_async(factory): ('user1', 'user', 'conversion', 1, "{'prop1': 'value1'}") ) -async def _manager_methods_async(factory): +async def _manager_methods_async(factory, skip_rbs=False): """Test manager.split/splits.""" try: manager = factory.manager() @@ -4265,5 +4773,10 @@ async def _manager_methods_async(factory): assert result.change_number == 123 assert result.configs['on'] == '{"size":15,"test":20}' - assert len(await manager.split_names()) == 7 - assert len(await manager.splits()) == 7 + if skip_rbs: + assert len(await manager.split_names()) == 7 + assert len(await manager.splits()) == 7 + return + + assert len(await manager.split_names()) == 9 + assert len(await manager.splits()) == 9 diff --git a/tests/integration/test_pluggable_integration.py b/tests/integration/test_pluggable_integration.py index 844cde14..20545da5 100644 --- a/tests/integration/test_pluggable_integration.py +++ b/tests/integration/test_pluggable_integration.py @@ -23,12 +23,12 @@ def test_put_fetch(self): split_fn = os.path.join(os.path.dirname(__file__), 'files', 'split_changes.json') with open(split_fn, 'r') as flo: data = json.loads(flo.read()) - for split in data['splits']: + for split in data['ff']['d']: adapter.set(storage._prefix.format(feature_flag_name=split['name']), split) adapter.increment(storage._traffic_type_prefix.format(traffic_type_name=split['trafficTypeName']), 1) - adapter.set(storage._feature_flag_till_prefix, data['till']) + adapter.set(storage._feature_flag_till_prefix, data['ff']['t']) - split_objects = [splits.from_raw(raw) for raw in data['splits']] + split_objects = [splits.from_raw(raw) for raw in data['ff']['d']] for split_object in split_objects: raw = split_object.to_json() @@ -53,8 +53,8 @@ def test_put_fetch(self): assert len(original_condition.matchers) == len(fetched_condition.matchers) assert len(original_condition.partitions) == len(fetched_condition.partitions) - adapter.set(storage._feature_flag_till_prefix, data['till']) - assert storage.get_change_number() == data['till'] + adapter.set(storage._feature_flag_till_prefix, data['ff']['t']) + assert storage.get_change_number() == data['ff']['t'] assert storage.is_valid_traffic_type('user') is True assert storage.is_valid_traffic_type('account') is True @@ -89,12 +89,12 @@ def test_get_all(self): split_fn = os.path.join(os.path.dirname(__file__), 'files', 'split_changes.json') with open(split_fn, 'r') as flo: data = json.loads(flo.read()) - for split in data['splits']: + for split in data['ff']['d']: adapter.set(storage._prefix.format(feature_flag_name=split['name']), split) adapter.increment(storage._traffic_type_prefix.format(traffic_type_name=split['trafficTypeName']), 1) - adapter.set(storage._feature_flag_till_prefix, data['till']) + adapter.set(storage._feature_flag_till_prefix, data['ff']['t']) - split_objects = [splits.from_raw(raw) for raw in data['splits']] + split_objects = [splits.from_raw(raw) for raw in data['ff']['d']] original_splits = {split.name: split for split in split_objects} fetched_names = storage.get_split_names() fetched_splits = {split.name: split for split in storage.get_all_splits()} @@ -260,12 +260,12 @@ async def test_put_fetch(self): split_fn = os.path.join(os.path.dirname(__file__), 'files', 'split_changes.json') with open(split_fn, 'r') as flo: data = json.loads(flo.read()) - for split in data['splits']: + for split in data['ff']['d']: await adapter.set(storage._prefix.format(feature_flag_name=split['name']), split) await adapter.increment(storage._traffic_type_prefix.format(traffic_type_name=split['trafficTypeName']), 1) - await adapter.set(storage._feature_flag_till_prefix, data['till']) + await adapter.set(storage._feature_flag_till_prefix, data['ff']['t']) - split_objects = [splits.from_raw(raw) for raw in data['splits']] + split_objects = [splits.from_raw(raw) for raw in data['ff']['d']] for split_object in split_objects: raw = split_object.to_json() @@ -290,8 +290,8 @@ async def test_put_fetch(self): assert len(original_condition.matchers) == len(fetched_condition.matchers) assert len(original_condition.partitions) == len(fetched_condition.partitions) - await adapter.set(storage._feature_flag_till_prefix, data['till']) - assert await storage.get_change_number() == data['till'] + await adapter.set(storage._feature_flag_till_prefix, data['ff']['t']) + assert await storage.get_change_number() == data['ff']['t'] assert await storage.is_valid_traffic_type('user') is True assert await storage.is_valid_traffic_type('account') is True @@ -327,12 +327,12 @@ async def test_get_all(self): split_fn = os.path.join(os.path.dirname(__file__), 'files', 'split_changes.json') with open(split_fn, 'r') as flo: data = json.loads(flo.read()) - for split in data['splits']: + for split in data['ff']['d']: await adapter.set(storage._prefix.format(feature_flag_name=split['name']), split) await adapter.increment(storage._traffic_type_prefix.format(traffic_type_name=split['trafficTypeName']), 1) - await adapter.set(storage._feature_flag_till_prefix, data['till']) + await adapter.set(storage._feature_flag_till_prefix, data['ff']['t']) - split_objects = [splits.from_raw(raw) for raw in data['splits']] + split_objects = [splits.from_raw(raw) for raw in data['ff']['d']] original_splits = {split.name: split for split in split_objects} fetched_names = await storage.get_split_names() fetched_splits = {split.name: split for split in await storage.get_all_splits()} diff --git a/tests/integration/test_redis_integration.py b/tests/integration/test_redis_integration.py index e53ab4e2..4b70898b 100644 --- a/tests/integration/test_redis_integration.py +++ b/tests/integration/test_redis_integration.py @@ -28,7 +28,7 @@ def test_put_fetch(self): with open(os.path.join(os.path.dirname(__file__), 'files', 'split_changes.json'), 'r') as flo: split_changes = json.load(flo) - split_objects = [splits.from_raw(raw) for raw in split_changes['splits']] + split_objects = [splits.from_raw(raw) for raw in split_changes['ff']['d']] for split_object in split_objects: raw = split_object.to_json() adapter.set(RedisSplitStorage._FEATURE_FLAG_KEY.format(feature_flag_name=split_object.name), json.dumps(raw)) @@ -55,8 +55,8 @@ def test_put_fetch(self): assert len(original_condition.matchers) == len(fetched_condition.matchers) assert len(original_condition.partitions) == len(fetched_condition.partitions) - adapter.set(RedisSplitStorage._FEATURE_FLAG_TILL_KEY, split_changes['till']) - assert storage.get_change_number() == split_changes['till'] + adapter.set(RedisSplitStorage._FEATURE_FLAG_TILL_KEY, split_changes['ff']['t']) + assert storage.get_change_number() == split_changes['ff']['t'] assert storage.is_valid_traffic_type('user') is True assert storage.is_valid_traffic_type('account') is True @@ -93,7 +93,7 @@ def test_get_all(self): with open(os.path.join(os.path.dirname(__file__), 'files', 'split_changes.json'), 'r') as flo: split_changes = json.load(flo) - split_objects = [splits.from_raw(raw) for raw in split_changes['splits']] + split_objects = [splits.from_raw(raw) for raw in split_changes['ff']['d']] for split_object in split_objects: raw = split_object.to_json() adapter.set(RedisSplitStorage._FEATURE_FLAG_KEY.format(feature_flag_name=split_object.name), json.dumps(raw)) @@ -262,7 +262,7 @@ async def test_put_fetch(self): with open(os.path.join(os.path.dirname(__file__), 'files', 'split_changes.json'), 'r') as flo: split_changes = json.load(flo) - split_objects = [splits.from_raw(raw) for raw in split_changes['splits']] + split_objects = [splits.from_raw(raw) for raw in split_changes['ff']['d']] for split_object in split_objects: raw = split_object.to_json() await adapter.set(RedisSplitStorage._FEATURE_FLAG_KEY.format(feature_flag_name=split_object.name), json.dumps(raw)) @@ -289,8 +289,8 @@ async def test_put_fetch(self): assert len(original_condition.matchers) == len(fetched_condition.matchers) assert len(original_condition.partitions) == len(fetched_condition.partitions) - await adapter.set(RedisSplitStorageAsync._FEATURE_FLAG_TILL_KEY, split_changes['till']) - assert await storage.get_change_number() == split_changes['till'] + await adapter.set(RedisSplitStorageAsync._FEATURE_FLAG_TILL_KEY, split_changes['ff']['t']) + assert await storage.get_change_number() == split_changes['ff']['t'] assert await storage.is_valid_traffic_type('user') is True assert await storage.is_valid_traffic_type('account') is True @@ -326,7 +326,7 @@ async def test_get_all(self): with open(os.path.join(os.path.dirname(__file__), 'files', 'split_changes.json'), 'r') as flo: split_changes = json.load(flo) - split_objects = [splits.from_raw(raw) for raw in split_changes['splits']] + split_objects = [splits.from_raw(raw) for raw in split_changes['ff']['d']] for split_object in split_objects: raw = split_object.to_json() await adapter.set(RedisSplitStorageAsync._FEATURE_FLAG_KEY.format(feature_flag_name=split_object.name), json.dumps(raw)) diff --git a/tests/integration/test_streaming_e2e.py b/tests/integration/test_streaming_e2e.py index a87ef59d..764475de 100644 --- a/tests/integration/test_streaming_e2e.py +++ b/tests/integration/test_streaming_e2e.py @@ -34,15 +34,17 @@ def test_happiness(self): } split_changes = { - -1: { - 'since': -1, - 'till': 1, - 'splits': [make_simple_split('split1', 1, True, False, 'on', 'user', True)] + -1: {'ff': { + 's': -1, + 't': 1, + 'd': [make_simple_split('split1', 1, True, False, 'on', 'user', True)]}, + 'rbs': {'s': -1, 't': -1, 'd': []} }, - 1: { - 'since': 1, - 'till': 1, - 'splits': [] + 1: {'ff': { + 's': 1, + 't': 1, + 'd': []}, + 'rbs': {'s': -1, 't': -1, 'd': []} } } @@ -76,22 +78,26 @@ def test_happiness(self): assert(factory._telemetry_evaluation_producer._telemetry_storage._streaming_events._streaming_events[len(factory._telemetry_evaluation_producer._telemetry_storage._streaming_events._streaming_events)-1]._type == StreamingEventTypes.SYNC_MODE_UPDATE.value) assert(factory._telemetry_evaluation_producer._telemetry_storage._streaming_events._streaming_events[len(factory._telemetry_evaluation_producer._telemetry_storage._streaming_events._streaming_events)-1]._data == SSESyncMode.STREAMING.value) split_changes[1] = { - 'since': 1, - 'till': 2, - 'splits': [make_simple_split('split1', 2, True, False, 'off', 'user', False)] + 'ff': { + 's': 1, + 't': 2, + 'd': [make_simple_split('split1', 2, True, False, 'off', 'user', False)]}, + 'rbs': {'s': -1, 't': -1, 'd': []} } - split_changes[2] = {'since': 2, 'till': 2, 'splits': []} + split_changes[2] = {'ff': {'s': 2, 't': 2, 'd': []}, 'rbs': {'s': -1, 't': -1, 'd': []}} sse_server.publish(make_split_change_event(2)) time.sleep(1) assert factory.client().get_treatment('maldo', 'split1') == 'off' split_changes[2] = { - 'since': 2, - 'till': 3, - 'splits': [make_split_with_segment('split2', 2, True, False, - 'off', 'user', 'off', 'segment1')] - } - split_changes[3] = {'since': 3, 'till': 3, 'splits': []} + 'ff': { + 's': 2, + 't': 3, + 'd': [make_split_with_segment('split2', 2, True, False, + 'off', 'user', 'off', 'segment1')]}, + 'rbs': {'s': -1, 't': -1, 'd': []} + } + split_changes[3] = {'ff': {'s': 3, 't': 3, 'd': []}, 'rbs': {'s': -1, 't': -1, 'd': []}} segment_changes[('segment1', -1)] = { 'name': 'segment1', 'added': ['maldo'], @@ -141,49 +147,49 @@ def test_happiness(self): # Initial splits fetch req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=-1' + assert req.path == '/api/splitChanges?s=1.3&since=-1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Auth req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/v2/auth?s=1.1' + assert req.path == '/api/v2/auth?s=1.3' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after streaming connected req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Fetch after first notification req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Fetch after second notification req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=3' + assert req.path == '/api/splitChanges?s=1.3&since=3&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Segment change notification @@ -222,12 +228,14 @@ def test_occupancy_flicker(self): } split_changes = { - -1: { - 'since': -1, - 'till': 1, - 'splits': [make_simple_split('split1', 1, True, False, 'off', 'user', True)] + -1: {'ff': { + 's': -1, + 't': 1, + 'd': [make_simple_split('split1', 1, True, False, 'off', 'user', True)]}, + 'rbs': {'s': -1, 't': -1, 'd': []} }, - 1: {'since': 1, 'till': 1, 'splits': []} + 1: {'ff': {'s': 1, 't': 1, 'd': []}, + 'rbs': {'s': -1, 't': -1, 'd': []}} } segment_changes = {} @@ -266,11 +274,12 @@ def test_occupancy_flicker(self): # After dropping occupancy, the sdk should switch to polling # and perform a syncAll that gets this change split_changes[1] = { - 'since': 1, - 'till': 2, - 'splits': [make_simple_split('split1', 2, True, False, 'off', 'user', False)] + 'ff': {'s': 1, + 't': 2, + 'd': [make_simple_split('split1', 2, True, False, 'off', 'user', False)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[2] = {'since': 2, 'till': 2, 'splits': []} + split_changes[2] = {'ff': {'s': 2, 't': 2, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_occupancy('control_pri', 0)) sse_server.publish(make_occupancy('control_sec', 0)) @@ -282,11 +291,12 @@ def test_occupancy_flicker(self): # We restore occupancy, and it should be fetched by the # sync all after streaming is restored. split_changes[2] = { - 'since': 2, - 'till': 3, - 'splits': [make_simple_split('split1', 3, True, False, 'off', 'user', True)] + 'ff': {'s': 2, + 't': 3, + 'd': [make_simple_split('split1', 3, True, False, 'off', 'user', True)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[3] = {'since': 3, 'till': 3, 'splits': []} + split_changes[3] = {'ff': {'s': 3, 't': 3, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_occupancy('control_pri', 1)) time.sleep(2) @@ -295,22 +305,24 @@ def test_occupancy_flicker(self): # Now we make another change and send an event so it's propagated split_changes[3] = { - 'since': 3, - 'till': 4, - 'splits': [make_simple_split('split1', 4, True, False, 'off', 'user', False)] + 'ff': {'s': 3, + 't': 4, + 'd': [make_simple_split('split1', 4, True, False, 'off', 'user', False)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[4] = {'since': 4, 'till': 4, 'splits': []} + split_changes[4] = {'ff': {'s': 4, 't': 4, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_split_change_event(4)) time.sleep(2) assert factory.client().get_treatment('maldo', 'split1') == 'off' # Kill the split split_changes[4] = { - 'since': 4, - 'till': 5, - 'splits': [make_simple_split('split1', 5, True, True, 'frula', 'user', False)] + 'ff': {'s': 4, + 't': 5, + 'd': [make_simple_split('split1', 5, True, True, 'frula', 'user', False)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[5] = {'since': 5, 'till': 5, 'splits': []} + split_changes[5] = {'ff': {'s': 5, 't': 5, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_split_kill_event('split1', 'frula', 5)) time.sleep(2) assert factory.client().get_treatment('maldo', 'split1') == 'frula' @@ -342,73 +354,73 @@ def test_occupancy_flicker(self): # Initial splits fetch req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=-1' + assert req.path == '/api/splitChanges?s=1.3&since=-1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Auth req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/v2/auth?s=1.1' + assert req.path == '/api/v2/auth?s=1.3' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after streaming connected req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Fetch after first notification req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Fetch after second notification req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=3' + assert req.path == '/api/splitChanges?s=1.3&since=3&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=3' + assert req.path == '/api/splitChanges?s=1.3&since=3&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=4' + assert req.path == '/api/splitChanges?s=1.3&since=4&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Split kill req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=4' + assert req.path == '/api/splitChanges?s=1.3&since=4&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=5' + assert req.path == '/api/splitChanges?s=1.3&since=5&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Cleanup @@ -435,12 +447,14 @@ def test_start_without_occupancy(self): } split_changes = { - -1: { - 'since': -1, - 'till': 1, - 'splits': [make_simple_split('split1', 1, True, False, 'off', 'user', True)] + -1: {'ff': { + 's': -1, + 't': 1, + 'd': [make_simple_split('split1', 1, True, False, 'off', 'user', True)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} }, - 1: {'since': 1, 'till': 1, 'splits': []} + 1: {'ff': {'s': 1, 't': 1, 'd': []}, + 'rbs': {'t': -1, 's': -1, 'd': []}} } segment_changes = {} @@ -478,11 +492,13 @@ def test_start_without_occupancy(self): # After restoring occupancy, the sdk should switch to polling # and perform a syncAll that gets this change split_changes[1] = { - 'since': 1, - 'till': 2, - 'splits': [make_simple_split('split1', 2, True, False, 'off', 'user', False)] + 'ff': {'s': 1, + 't': 2, + 'd': [make_simple_split('split1', 2, True, False, 'off', 'user', False)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[2] = {'since': 2, 'till': 2, 'splits': []} + split_changes[2] = {'ff': {'s': 2, 't': 2, 'd': []}, + 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_occupancy('control_sec', 1)) time.sleep(2) @@ -516,43 +532,43 @@ def test_start_without_occupancy(self): # Initial splits fetch req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=-1' + assert req.path == '/api/splitChanges?s=1.3&since=-1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Auth req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/v2/auth?s=1.1' + assert req.path == '/api/v2/auth?s=1.3' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after streaming connected req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after push down req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after push restored req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Second iteration of previous syncAll req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Cleanup @@ -562,7 +578,7 @@ def test_start_without_occupancy(self): sse_server.publish(sse_server.GRACEFUL_REQUEST_END) sse_server.stop() split_backend.stop() - + def test_streaming_status_changes(self): """Test changes between streaming enabled, paused and disabled.""" auth_server_response = { @@ -579,12 +595,14 @@ def test_streaming_status_changes(self): } split_changes = { - -1: { - 'since': -1, - 'till': 1, - 'splits': [make_simple_split('split1', 1, True, False, 'off', 'user', True)] + -1: {'ff': { + 's': -1, + 't': 1, + 'd': [make_simple_split('split1', 1, True, False, 'off', 'user', True)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} }, - 1: {'since': 1, 'till': 1, 'splits': []} + 1: {'ff': {'s': 1, 't': 1, 'd': []}, + 'rbs': {'t': -1, 's': -1, 'd': []}} } segment_changes = {} @@ -623,11 +641,12 @@ def test_streaming_status_changes(self): # After dropping occupancy, the sdk should switch to polling # and perform a syncAll that gets this change split_changes[1] = { - 'since': 1, - 'till': 2, - 'splits': [make_simple_split('split1', 2, True, False, 'off', 'user', False)] + 'ff': {'s': 1, + 't': 2, + 'd': [make_simple_split('split1', 2, True, False, 'off', 'user', False)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[2] = {'since': 2, 'till': 2, 'splits': []} + split_changes[2] = {'ff': {'s': 2, 't': 2, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_control_event('STREAMING_PAUSED', 1)) time.sleep(2) @@ -638,11 +657,12 @@ def test_streaming_status_changes(self): # We restore occupancy, and it should be fetched by the # sync all after streaming is restored. split_changes[2] = { - 'since': 2, - 'till': 3, - 'splits': [make_simple_split('split1', 3, True, False, 'off', 'user', True)] + 'ff': {'s': 2, + 't': 3, + 'd': [make_simple_split('split1', 3, True, False, 'off', 'user', True)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[3] = {'since': 3, 'till': 3, 'splits': []} + split_changes[3] = {'ff': {'s': 3, 't': 3, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_control_event('STREAMING_ENABLED', 2)) time.sleep(2) @@ -651,22 +671,26 @@ def test_streaming_status_changes(self): # Now we make another change and send an event so it's propagated split_changes[3] = { - 'since': 3, - 'till': 4, - 'splits': [make_simple_split('split1', 4, True, False, 'off', 'user', False)] + 'ff': {'s': 3, + 't': 4, + 'd': [make_simple_split('split1', 4, True, False, 'off', 'user', False)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[4] = {'since': 4, 'till': 4, 'splits': []} + split_changes[4] = {'ff': {'s': 4, 't': 4, 'd': []}, + 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_split_change_event(4)) time.sleep(2) assert factory.client().get_treatment('maldo', 'split1') == 'off' assert not task.running() split_changes[4] = { - 'since': 4, - 'till': 5, - 'splits': [make_simple_split('split1', 5, True, False, 'off', 'user', True)] + 'ff': {'s': 4, + 't': 5, + 'd': [make_simple_split('split1', 5, True, False, 'off', 'user', True)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[5] = {'since': 5, 'till': 5, 'splits': []} + split_changes[5] = {'ff': {'s': 5, 't': 5, 'd': []}, + 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_control_event('STREAMING_DISABLED', 2)) time.sleep(2) assert factory.client().get_treatment('maldo', 'split1') == 'on' @@ -700,73 +724,73 @@ def test_streaming_status_changes(self): # Initial splits fetch req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=-1' + assert req.path == '/api/splitChanges?s=1.3&since=-1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Auth req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/v2/auth?s=1.1' + assert req.path == '/api/v2/auth?s=1.3' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after streaming connected req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll on push down req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after push is up req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=3' + assert req.path == '/api/splitChanges?s=1.3&since=3&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Fetch after notification req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=3' + assert req.path == '/api/splitChanges?s=1.3&since=3&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=4' + assert req.path == '/api/splitChanges?s=1.3&since=4&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after streaming disabled req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=4' + assert req.path == '/api/splitChanges?s=1.3&since=4&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=5' + assert req.path == '/api/splitChanges?s=1.3&since=5&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Cleanup @@ -793,15 +817,17 @@ def test_server_closes_connection(self): } split_changes = { - -1: { - 'since': -1, - 'till': 1, - 'splits': [make_simple_split('split1', 1, True, False, 'on', 'user', True)] + -1: {'ff': { + 's': -1, + 't': 1, + 'd': [make_simple_split('split1', 1, True, False, 'on', 'user', True)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} }, - 1: { - 'since': 1, - 'till': 1, - 'splits': [] + 1: {'ff': { + 's': 1, + 't': 1, + 'd': []}, + 'rbs': {'t': -1, 's': -1, 'd': []} } } @@ -836,12 +862,14 @@ def test_server_closes_connection(self): assert not task.running() time.sleep(1) - split_changes[1] = { - 'since': 1, - 'till': 2, - 'splits': [make_simple_split('split1', 2, True, False, 'off', 'user', False)] - } - split_changes[2] = {'since': 2, 'till': 2, 'splits': []} + split_changes[1] = {'ff': { + 's': 1, + 't': 2, + 'd': [make_simple_split('split1', 2, True, False, 'off', 'user', False)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} + } + split_changes[2] = {'ff': {'s': 2, 't': 2, 'd': []}, + 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_split_change_event(2)) time.sleep(1) assert factory.client().get_treatment('maldo', 'split1') == 'off' @@ -860,12 +888,14 @@ def test_server_closes_connection(self): time.sleep(2) assert not task.running() - split_changes[2] = { - 'since': 2, - 'till': 3, - 'splits': [make_simple_split('split1', 3, True, False, 'off', 'user', True)] - } - split_changes[3] = {'since': 3, 'till': 3, 'splits': []} + split_changes[2] = {'ff': { + 's': 2, + 't': 3, + 'd': [make_simple_split('split1', 3, True, False, 'off', 'user', True)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} + } + split_changes[3] = {'ff': {'s': 3, 't': 3, 'd': [], + 'rbs': {'t': -1, 's': -1, 'd': []}}} sse_server.publish(make_split_change_event(3)) time.sleep(1) assert factory.client().get_treatment('maldo', 'split1') == 'on' @@ -921,67 +951,67 @@ def test_server_closes_connection(self): # Initial splits fetch req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=-1' + assert req.path == '/api/splitChanges?s=1.3&since=-1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Auth req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/v2/auth?s=1.1' + assert req.path == '/api/v2/auth?s=1.3' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after streaming connected req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Fetch after first notification req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll on retryable error handling req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Auth after connection breaks req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/v2/auth?s=1.1' + assert req.path == '/api/v2/auth?s=1.3' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after streaming connected again req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Fetch after new notification req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=3' + assert req.path == '/api/splitChanges?s=1.3&since=3&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Cleanup @@ -1015,12 +1045,14 @@ def test_ably_errors_handling(self): } split_changes = { - -1: { - 'since': -1, - 'till': 1, - 'splits': [make_simple_split('split1', 1, True, False, 'off', 'user', True)] + -1: {'ff': { + 's': -1, + 't': 1, + 'd': [make_simple_split('split1', 1, True, False, 'off', 'user', True)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} }, - 1: {'since': 1, 'till': 1, 'splits': []} + 1: {'ff': {'s': 1, 't': 1, 'd': []}, + 'rbs': {'t': -1, 's': -1, 'd': []}} } segment_changes = {} @@ -1057,12 +1089,14 @@ def test_ably_errors_handling(self): # Make a change in the BE but don't send the event. # We'll send an ignorable error and check it has nothing happened - split_changes[1] = { - 'since': 1, - 'till': 2, - 'splits': [make_simple_split('split1', 2, True, False, 'off', 'user', False)] + split_changes[1] = {'ff': { + 's': 1, + 't': 2, + 'd': [make_simple_split('split1', 2, True, False, 'off', 'user', False)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[2] = {'since': 2, 'till': 2, 'splits': []} + split_changes[2] = {'ff': {'s': 2, 't': 2, 'd': []}, + 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_ably_error_event(60000, 600)) time.sleep(1) @@ -1083,12 +1117,14 @@ def test_ably_errors_handling(self): assert not task.running() # Assert streaming is working properly - split_changes[2] = { - 'since': 2, - 'till': 3, - 'splits': [make_simple_split('split1', 3, True, False, 'off', 'user', True)] - } - split_changes[3] = {'since': 3, 'till': 3, 'splits': []} + split_changes[2] = {'ff': { + 's': 2, + 't': 3, + 'd': [make_simple_split('split1', 3, True, False, 'off', 'user', True)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} + } + split_changes[3] = {'ff': {'s': 3, 't': 3, 'd': []}, + 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_split_change_event(3)) time.sleep(2) assert factory.client().get_treatment('maldo', 'split1') == 'on' @@ -1152,67 +1188,67 @@ def test_ably_errors_handling(self): # Initial splits fetch req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=-1' + assert req.path == '/api/splitChanges?s=1.3&since=-1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Auth req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/v2/auth?s=1.1' + assert req.path == '/api/v2/auth?s=1.3' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after streaming connected req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll retriable error req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Auth again req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/v2/auth?s=1.1' + assert req.path == '/api/v2/auth?s=1.3' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after push is up req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Fetch after notification req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=3' + assert req.path == '/api/splitChanges?s=1.3&since=3&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after non recoverable ably error req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=3' + assert req.path == '/api/splitChanges?s=1.3&since=3&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Cleanup @@ -1239,12 +1275,14 @@ def test_change_number(mocker): } split_changes = { - -1: { - 'since': -1, - 'till': 1, - 'splits': [make_simple_split('split1', 1, True, False, 'off', 'user', True)] + -1: {'ff': { + 's': -1, + 't': 1, + 'd': [make_simple_split('split1', 1, True, False, 'off', 'user', True)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} }, - 1: {'since': 1, 'till': 1, 'splits': []} + 1: {'ff': {'s': 1, 't': 1, 'd': []}, + 'rbs': {'t': -1, 's': -1, 'd': []}} } segment_changes = {} @@ -1312,15 +1350,17 @@ async def test_happiness(self): } split_changes = { - -1: { - 'since': -1, - 'till': 1, - 'splits': [make_simple_split('split1', 1, True, False, 'on', 'user', True)] + -1: {'ff': { + 's': -1, + 't': 1, + 'd': [make_simple_split('split1', 1, True, False, 'on', 'user', True)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} }, - 1: { - 'since': 1, - 'till': 1, - 'splits': [] + 1: {'ff': { + 's': 1, + 't': 1, + 'd': []}, + 'rbs': {'t': -1, 's': -1, 'd': []} } } @@ -1353,23 +1393,27 @@ async def test_happiness(self): await asyncio.sleep(1) assert(factory._telemetry_evaluation_producer._telemetry_storage._streaming_events._streaming_events[len(factory._telemetry_evaluation_producer._telemetry_storage._streaming_events._streaming_events)-1]._type == StreamingEventTypes.SYNC_MODE_UPDATE.value) assert(factory._telemetry_evaluation_producer._telemetry_storage._streaming_events._streaming_events[len(factory._telemetry_evaluation_producer._telemetry_storage._streaming_events._streaming_events)-1]._data == SSESyncMode.STREAMING.value) - split_changes[1] = { - 'since': 1, - 'till': 2, - 'splits': [make_simple_split('split1', 2, True, False, 'off', 'user', False)] - } - split_changes[2] = {'since': 2, 'till': 2, 'splits': []} + split_changes[1] = {'ff': { + 's': 1, + 't': 2, + 'd': [make_simple_split('split1', 2, True, False, 'off', 'user', False)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} + } + split_changes[2] = {'ff': {'s': 2, 't': 2, 'd': []}, + 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_split_change_event(2)) await asyncio.sleep(1) assert await factory.client().get_treatment('maldo', 'split1') == 'off' - split_changes[2] = { - 'since': 2, - 'till': 3, - 'splits': [make_split_with_segment('split2', 2, True, False, - 'off', 'user', 'off', 'segment1')] + split_changes[2] = {'ff': { + 's': 2, + 't': 3, + 'd': [make_split_with_segment('split2', 2, True, False, + 'off', 'user', 'off', 'segment1')]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[3] = {'since': 3, 'till': 3, 'splits': []} + split_changes[3] = {'ff': {'s': 3, 't': 3, 'd': []}, + 'rbs': {'t': -1, 's': -1, 'd': []}} segment_changes[('segment1', -1)] = { 'name': 'segment1', 'added': ['maldo'], @@ -1415,49 +1459,49 @@ async def test_happiness(self): # Initial splits fetch req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=-1' + assert req.path == '/api/splitChanges?s=1.3&since=-1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Auth req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/v2/auth?s=1.1' + assert req.path == '/api/v2/auth?s=1.3' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after streaming connected req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Fetch after first notification req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Fetch after second notification req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=3' + assert req.path == '/api/splitChanges?s=1.3&since=3&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Segment change notification @@ -1495,12 +1539,14 @@ async def test_occupancy_flicker(self): } split_changes = { - -1: { - 'since': -1, - 'till': 1, - 'splits': [make_simple_split('split1', 1, True, False, 'off', 'user', True)] + -1: {'ff': { + 's': -1, + 't': 1, + 'd': [make_simple_split('split1', 1, True, False, 'off', 'user', True)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} }, - 1: {'since': 1, 'till': 1, 'splits': []} + 1: {'ff': {'s': 1, 't': 1, 'd': []}, + 'rbs': {'t': -1, 's': -1, 'd': []}} } segment_changes = {} @@ -1538,13 +1584,13 @@ async def test_occupancy_flicker(self): # Make a change in the BE but don't send the event. # After dropping occupancy, the sdk should switch to polling # and perform a syncAll that gets this change - split_changes[1] = { - 'since': 1, - 'till': 2, - 'splits': [make_simple_split('split1', 2, True, False, 'off', 'user', False)] + split_changes[1] = {'ff': { + 's': 1, + 't': 2, + 'd': [make_simple_split('split1', 2, True, False, 'off', 'user', False)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[2] = {'since': 2, 'till': 2, 'splits': []} - + split_changes[2] = {'ff': {'s': 2, 't': 2, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_occupancy('control_pri', 0)) sse_server.publish(make_occupancy('control_sec', 0)) await asyncio.sleep(2) @@ -1554,36 +1600,38 @@ async def test_occupancy_flicker(self): # We make another chagne in the BE and don't send the event. # We restore occupancy, and it should be fetched by the # sync all after streaming is restored. - split_changes[2] = { - 'since': 2, - 'till': 3, - 'splits': [make_simple_split('split1', 3, True, False, 'off', 'user', True)] + split_changes[2] = {'ff': { + 's': 2, + 't': 3, + 'd': [make_simple_split('split1', 3, True, False, 'off', 'user', True)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[3] = {'since': 3, 'till': 3, 'splits': []} - + split_changes[3] = {'ff': {'s': 3, 't': 3, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_occupancy('control_pri', 1)) await asyncio.sleep(2) assert await factory.client().get_treatment('maldo', 'split1') == 'on' assert not task.running() # Now we make another change and send an event so it's propagated - split_changes[3] = { - 'since': 3, - 'till': 4, - 'splits': [make_simple_split('split1', 4, True, False, 'off', 'user', False)] + split_changes[3] = {'ff': { + 's': 3, + 't': 4, + 'd': [make_simple_split('split1', 4, True, False, 'off', 'user', False)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[4] = {'since': 4, 'till': 4, 'splits': []} + split_changes[4] = {'ff': {'s': 4, 't': 4, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_split_change_event(4)) await asyncio.sleep(2) assert await factory.client().get_treatment('maldo', 'split1') == 'off' # Kill the split - split_changes[4] = { - 'since': 4, - 'till': 5, - 'splits': [make_simple_split('split1', 5, True, True, 'frula', 'user', False)] + split_changes[4] = {'ff': { + 's': 4, + 't': 5, + 'd': [make_simple_split('split1', 5, True, True, 'frula', 'user', False)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[5] = {'since': 5, 'till': 5, 'splits': []} + split_changes[5] = {'ff': {'s': 5, 't': 5, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_split_kill_event('split1', 'frula', 5)) await asyncio.sleep(2) assert await factory.client().get_treatment('maldo', 'split1') == 'frula' @@ -1615,73 +1663,73 @@ async def test_occupancy_flicker(self): # Initial splits fetch req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=-1' + assert req.path == '/api/splitChanges?s=1.3&since=-1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Auth req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/v2/auth?s=1.1' + assert req.path == '/api/v2/auth?s=1.3' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after streaming connected req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Fetch after first notification req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Fetch after second notification req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=3' + assert req.path == '/api/splitChanges?s=1.3&since=3&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=3' + assert req.path == '/api/splitChanges?s=1.3&since=3&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=4' + assert req.path == '/api/splitChanges?s=1.3&since=4&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Split kill req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=4' + assert req.path == '/api/splitChanges?s=1.3&since=4&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=5' + assert req.path == '/api/splitChanges?s=1.3&since=5&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Cleanup @@ -1707,12 +1755,13 @@ async def test_start_without_occupancy(self): } split_changes = { - -1: { - 'since': -1, - 'till': 1, - 'splits': [make_simple_split('split1', 1, True, False, 'off', 'user', True)] + -1: {'ff': { + 's': -1, + 't': 1, + 'd': [make_simple_split('split1', 1, True, False, 'off', 'user', True)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} }, - 1: {'since': 1, 'till': 1, 'splits': []} + 1: {'ff': {'s': 1, 't': 1, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} } segment_changes = {} @@ -1752,12 +1801,13 @@ async def test_start_without_occupancy(self): # Make a change in the BE but don't send the event. # After restoring occupancy, the sdk should switch to polling # and perform a syncAll that gets this change - split_changes[1] = { - 'since': 1, - 'till': 2, - 'splits': [make_simple_split('split1', 2, True, False, 'off', 'user', False)] + split_changes[1] = {'ff': { + 's': 1, + 't': 2, + 'd': [make_simple_split('split1', 2, True, False, 'off', 'user', False)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[2] = {'since': 2, 'till': 2, 'splits': []} + split_changes[2] = {'ff': {'s': 2, 't': 2, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_occupancy('control_sec', 1)) await asyncio.sleep(2) @@ -1791,43 +1841,43 @@ async def test_start_without_occupancy(self): # Initial splits fetch req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=-1' + assert req.path == '/api/splitChanges?s=1.3&since=-1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Auth req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/v2/auth?s=1.1' + assert req.path == '/api/v2/auth?s=1.3' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after streaming connected req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after push down req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after push restored req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Second iteration of previous syncAll req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Cleanup @@ -1853,12 +1903,13 @@ async def test_streaming_status_changes(self): } split_changes = { - -1: { - 'since': -1, - 'till': 1, - 'splits': [make_simple_split('split1', 1, True, False, 'off', 'user', True)] + -1: {'ff': { + 's': -1, + 't': 1, + 'd': [make_simple_split('split1', 1, True, False, 'off', 'user', True)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} }, - 1: {'since': 1, 'till': 1, 'splits': []} + 1: {'ff': {'s': 1, 't': 1, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} } segment_changes = {} @@ -1899,12 +1950,13 @@ async def test_streaming_status_changes(self): # Make a change in the BE but don't send the event. # After dropping occupancy, the sdk should switch to polling # and perform a syncAll that gets this change - split_changes[1] = { - 'since': 1, - 'till': 2, - 'splits': [make_simple_split('split1', 2, True, False, 'off', 'user', False)] + split_changes[1] = {'ff': { + 's': 1, + 't': 2, + 'd': [make_simple_split('split1', 2, True, False, 'off', 'user', False)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[2] = {'since': 2, 'till': 2, 'splits': []} + split_changes[2] = {'ff': {'s': 2, 't': 2, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_control_event('STREAMING_PAUSED', 1)) await asyncio.sleep(4) @@ -1915,12 +1967,13 @@ async def test_streaming_status_changes(self): # We make another chagne in the BE and don't send the event. # We restore occupancy, and it should be fetched by the # sync all after streaming is restored. - split_changes[2] = { - 'since': 2, - 'till': 3, - 'splits': [make_simple_split('split1', 3, True, False, 'off', 'user', True)] + split_changes[2] = {'ff': { + 's': 2, + 't': 3, + 'd': [make_simple_split('split1', 3, True, False, 'off', 'user', True)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[3] = {'since': 3, 'till': 3, 'splits': []} + split_changes[3] = {'ff': {'s': 3, 't': 3, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_control_event('STREAMING_ENABLED', 2)) await asyncio.sleep(2) @@ -1929,24 +1982,26 @@ async def test_streaming_status_changes(self): assert not task.running() # Now we make another change and send an event so it's propagated - split_changes[3] = { - 'since': 3, - 'till': 4, - 'splits': [make_simple_split('split1', 4, True, False, 'off', 'user', False)] + split_changes[3] = {'ff': { + 's': 3, + 't': 4, + 'd': [make_simple_split('split1', 4, True, False, 'off', 'user', False)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[4] = {'since': 4, 'till': 4, 'splits': []} + split_changes[4] = {'ff': {'s': 4, 't': 4, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_split_change_event(4)) await asyncio.sleep(2) assert await factory.client().get_treatment('maldo', 'split1') == 'off' assert not task.running() - split_changes[4] = { - 'since': 4, - 'till': 5, - 'splits': [make_simple_split('split1', 5, True, False, 'off', 'user', True)] + split_changes[4] = {'ff': { + 's': 4, + 't': 5, + 'd': [make_simple_split('split1', 5, True, False, 'off', 'user', True)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[5] = {'since': 5, 'till': 5, 'splits': []} + split_changes[5] = {'ff': {'s': 5, 't': 5, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_control_event('STREAMING_DISABLED', 2)) await asyncio.sleep(2) @@ -1980,73 +2035,73 @@ async def test_streaming_status_changes(self): # Initial splits fetch req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=-1' + assert req.path == '/api/splitChanges?s=1.3&since=-1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Auth req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/v2/auth?s=1.1' + assert req.path == '/api/v2/auth?s=1.3' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after streaming connected req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll on push down req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after push is up req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=3' + assert req.path == '/api/splitChanges?s=1.3&since=3&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Fetch after notification req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=3' + assert req.path == '/api/splitChanges?s=1.3&since=3&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=4' + assert req.path == '/api/splitChanges?s=1.3&since=4&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after streaming disabled req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=4' + assert req.path == '/api/splitChanges?s=1.3&since=4&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=5' + assert req.path == '/api/splitChanges?s=1.3&since=5&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Cleanup @@ -2072,16 +2127,13 @@ async def test_server_closes_connection(self): } split_changes = { - -1: { - 'since': -1, - 'till': 1, - 'splits': [make_simple_split('split1', 1, True, False, 'on', 'user', True)] + -1: {'ff': { + 's': -1, + 't': 1, + 'd': [make_simple_split('split1', 1, True, False, 'on', 'user', True)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} }, - 1: { - 'since': 1, - 'till': 1, - 'splits': [] - } + 1: {'ff': {'s': 1, 't': 1, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} } segment_changes = {} @@ -2114,12 +2166,13 @@ async def test_server_closes_connection(self): assert not task.running() await asyncio.sleep(1) - split_changes[1] = { - 'since': 1, - 'till': 2, - 'splits': [make_simple_split('split1', 2, True, False, 'off', 'user', False)] + split_changes[1] = {'ff': { + 's': 1, + 't': 2, + 'd': [make_simple_split('split1', 2, True, False, 'off', 'user', False)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[2] = {'since': 2, 'till': 2, 'splits': []} + split_changes[2] = {'ff': {'s': 2, 't': 2, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_split_change_event(2)) await asyncio.sleep(1) assert await factory.client().get_treatment('maldo', 'split1') == 'off' @@ -2139,12 +2192,13 @@ async def test_server_closes_connection(self): await asyncio.sleep(2) assert not task.running() - split_changes[2] = { - 'since': 2, - 'till': 3, - 'splits': [make_simple_split('split1', 3, True, False, 'off', 'user', True)] + split_changes[2] = {'ff': { + 's': 2, + 't': 3, + 'd': [make_simple_split('split1', 3, True, False, 'off', 'user', True)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[3] = {'since': 3, 'till': 3, 'splits': []} + split_changes[3] = {'ff': {'s': 3, 't': 3, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_split_change_event(3)) await asyncio.sleep(1) @@ -2201,67 +2255,67 @@ async def test_server_closes_connection(self): # Initial splits fetch req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=-1' + assert req.path == '/api/splitChanges?s=1.3&since=-1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Auth req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/v2/auth?s=1.1' + assert req.path == '/api/v2/auth?s=1.3' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after streaming connected req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Fetch after first notification req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll on retryable error handling req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Auth after connection breaks req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/v2/auth?s=1.1' + assert req.path == '/api/v2/auth?s=1.3' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after streaming connected again req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Fetch after new notification req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=3' + assert req.path == '/api/splitChanges?s=1.3&since=3&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Cleanup @@ -2294,12 +2348,13 @@ async def test_ably_errors_handling(self): } split_changes = { - -1: { - 'since': -1, - 'till': 1, - 'splits': [make_simple_split('split1', 1, True, False, 'off', 'user', True)] + -1: {'ff': { + 's': -1, + 't': 1, + 'd': [make_simple_split('split1', 1, True, False, 'off', 'user', True)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} }, - 1: {'since': 1, 'till': 1, 'splits': []} + 1: {'ff': {'s': 1, 't': 1, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} } segment_changes = {} @@ -2338,12 +2393,13 @@ async def test_ably_errors_handling(self): # Make a change in the BE but don't send the event. # We'll send an ignorable error and check it has nothing happened - split_changes[1] = { - 'since': 1, - 'till': 2, - 'splits': [make_simple_split('split1', 2, True, False, 'off', 'user', False)] + split_changes[1] = {'ff': { + 's': 1, + 't': 2, + 'd': [make_simple_split('split1', 2, True, False, 'off', 'user', False)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[2] = {'since': 2, 'till': 2, 'splits': []} + split_changes[2] = {'ff': {'s': 2, 't': 2, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_ably_error_event(60000, 600)) await asyncio.sleep(1) @@ -2366,12 +2422,13 @@ async def test_ably_errors_handling(self): assert not task.running() # Assert streaming is working properly - split_changes[2] = { - 'since': 2, - 'till': 3, - 'splits': [make_simple_split('split1', 3, True, False, 'off', 'user', True)] + split_changes[2] = {'ff': { + 's': 2, + 't': 3, + 'd': [make_simple_split('split1', 3, True, False, 'off', 'user', True)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} } - split_changes[3] = {'since': 3, 'till': 3, 'splits': []} + split_changes[3] = {'ff': {'s': 3, 't': 3, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} sse_server.publish(make_split_change_event(3)) await asyncio.sleep(2) assert await factory.client().get_treatment('maldo', 'split1') == 'on' @@ -2434,67 +2491,67 @@ async def test_ably_errors_handling(self): # Initial splits fetch req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=-1' + assert req.path == '/api/splitChanges?s=1.3&since=-1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Auth req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/v2/auth?s=1.1' + assert req.path == '/api/v2/auth?s=1.3' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after streaming connected req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll retriable error req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=1' + assert req.path == '/api/splitChanges?s=1.3&since=1&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Auth again req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/v2/auth?s=1.1' + assert req.path == '/api/v2/auth?s=1.3' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after push is up req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Fetch after notification req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=2' + assert req.path == '/api/splitChanges?s=1.3&since=2&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Iteration until since == till req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=3' + assert req.path == '/api/splitChanges?s=1.3&since=3&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # SyncAll after non recoverable ably error req = split_backend_requests.get() assert req.method == 'GET' - assert req.path == '/api/splitChanges?s=1.1&since=3' + assert req.path == '/api/splitChanges?s=1.3&since=3&rbSince=-1' assert req.headers['authorization'] == 'Bearer some_apikey' # Cleanup @@ -2520,12 +2577,13 @@ async def test_change_number(mocker): } split_changes = { - -1: { - 'since': -1, - 'till': 1, - 'splits': [make_simple_split('split1', 1, True, False, 'off', 'user', True)] + -1: {'ff': { + 's': -1, + 't': 1, + 'd': [make_simple_split('split1', 1, True, False, 'off', 'user', True)]}, + 'rbs': {'t': -1, 's': -1, 'd': []} }, - 1: {'since': 1, 'till': 1, 'splits': []} + 1: {'ff': {'s': 1, 't': 1, 'd': []}, 'rbs': {'t': -1, 's': -1, 'd': []}} } segment_changes = {} diff --git a/tests/models/grammar/files/splits_prereq.json b/tests/models/grammar/files/splits_prereq.json new file mode 100644 index 00000000..5efa7fed --- /dev/null +++ b/tests/models/grammar/files/splits_prereq.json @@ -0,0 +1,293 @@ +{"ff": { + "d": [ + { + "trafficTypeName": "user", + "name": "test_prereq", + "prerequisites": [ + { "n": "feature_segment", "ts": ["off", "def_test"] }, + { "n": "rbs_flag", "ts": ["on"] } + ], + "trafficAllocation": 100, + "trafficAllocationSeed": 1582960494, + "seed": 1842944006, + "status": "ACTIVE", + "killed": false, + "defaultTreatment": "def_treatment", + "changeNumber": 1582741588594, + "algo": 2, + "configurations": {}, + "conditions": [ + { + "conditionType": "ROLLOUT", + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "keySelector": { + "trafficType": "user", + "attribute": null + }, + "matcherType": "ALL_KEYS", + "negate": false, + "userDefinedSegmentMatcherData": null, + "whitelistMatcherData": null, + "unaryNumericMatcherData": null, + "betweenMatcherData": null, + "booleanMatcherData": null, + "dependencyMatcherData": null, + "stringMatcherData": null + } + ] + }, + "partitions": [ + { + "treatment": "on", + "size": 100 + }, + { + "treatment": "off", + "size": 0 + } + ], + "label": "default rule" + } + ] + }, + { + "name":"feature_segment", + "trafficTypeId":"u", + "trafficTypeName":"User", + "trafficAllocation": 100, + "trafficAllocationSeed": 1582960494, + "seed":-1177551240, + "status":"ACTIVE", + "killed":false, + "defaultTreatment":"def_test", + "changeNumber": 1582741588594, + "algo": 2, + "configurations": {}, + "conditions":[ + { + "matcherGroup":{ + "combiner":"AND", + "matchers":[ + { + "matcherType":"IN_SEGMENT", + "negate":false, + "userDefinedSegmentMatcherData":{ + "segmentName":"segment-test" + }, + "whitelistMatcherData":null + } + ] + }, + "partitions":[ + { + "treatment":"on", + "size":100 + }, + { + "treatment":"off", + "size":0 + } + ], + "label": "default label" + } + ] + }, + { + "changeNumber": 10, + "trafficTypeName": "user", + "name": "rbs_flag", + "trafficAllocation": 100, + "trafficAllocationSeed": 1828377380, + "seed": -286617921, + "status": "ACTIVE", + "killed": false, + "defaultTreatment": "off", + "algo": 2, + "conditions": [ + { + "conditionType": "ROLLOUT", + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "keySelector": { + "trafficType": "user" + }, + "matcherType": "IN_RULE_BASED_SEGMENT", + "negate": false, + "userDefinedSegmentMatcherData": { + "segmentName": "sample_rule_based_segment" + } + } + ] + }, + "partitions": [ + { + "treatment": "on", + "size": 100 + }, + { + "treatment": "off", + "size": 0 + } + ], + "label": "in rule based segment sample_rule_based_segment" + }, + { + "conditionType": "ROLLOUT", + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "keySelector": { + "trafficType": "user" + }, + "matcherType": "ALL_KEYS", + "negate": false + } + ] + }, + "partitions": [ + { + "treatment": "on", + "size": 0 + }, + { + "treatment": "off", + "size": 100 + } + ], + "label": "default rule" + } + ], + "configurations": {}, + "sets": [], + "impressionsDisabled": false + }, + { + "trafficTypeName": "user", + "name": "prereq_chain", + "prerequisites": [ + { "n": "test_prereq", "ts": ["on"] } + ], + "trafficAllocation": 100, + "trafficAllocationSeed": -2092979940, + "seed": 105482719, + "status": "ACTIVE", + "killed": false, + "defaultTreatment": "on_default", + "changeNumber": 1585948850109, + "algo": 2, + "configurations": {}, + "conditions": [ + { + "conditionType": "WHITELIST", + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "keySelector": null, + "matcherType": "WHITELIST", + "negate": false, + "userDefinedSegmentMatcherData": null, + "whitelistMatcherData": { + "whitelist": [ + "bilal@split.io" + ] + }, + "unaryNumericMatcherData": null, + "betweenMatcherData": null, + "booleanMatcherData": null, + "dependencyMatcherData": null, + "stringMatcherData": null + } + ] + }, + "partitions": [ + { + "treatment": "on_whitelist", + "size": 100 + } + ], + "label": "whitelisted" + }, + { + "conditionType": "ROLLOUT", + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "keySelector": { + "trafficType": "user", + "attribute": null + }, + "matcherType": "ALL_KEYS", + "negate": false, + "userDefinedSegmentMatcherData": null, + "whitelistMatcherData": null, + "unaryNumericMatcherData": null, + "betweenMatcherData": null, + "booleanMatcherData": null, + "dependencyMatcherData": null, + "stringMatcherData": null + } + ] + }, + "partitions": [ + { + "treatment": "on", + "size": 100 + }, + { + "treatment": "off", + "size": 0 + }, + { + "treatment": "V1", + "size": 0 + } + ], + "label": "default rule" + } + ] + } + ], + "s": -1, + "t": 1585948850109 +}, "rbs":{"d": [ + { + "changeNumber": 5, + "name": "sample_rule_based_segment", + "status": "ACTIVE", + "trafficTypeName": "user", + "excluded":{ + "keys":["mauro@split.io","gaston@split.io"], + "segments":[] + }, + "conditions": [ + { + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "keySelector": { + "trafficType": "user", + "attribute": "email" + }, + "matcherType": "ENDS_WITH", + "negate": false, + "whitelistMatcherData": { + "whitelist": [ + "@split.io" + ] + } + } + ] + } + } + ] + }], "s": -1, "t": 1585948850109} +} diff --git a/tests/models/grammar/test_matchers.py b/tests/models/grammar/test_matchers.py index bf582917..71922431 100644 --- a/tests/models/grammar/test_matchers.py +++ b/tests/models/grammar/test_matchers.py @@ -11,7 +11,9 @@ from datetime import datetime from splitio.models.grammar import matchers +from splitio.models.grammar.matchers.prerequisites import PrerequisitesMatcher from splitio.models import splits +from splitio.models import rule_based_segments from splitio.models.grammar import condition from splitio.models.grammar.matchers.utils.utils import Semver from splitio.storage import SegmentStorage @@ -404,9 +406,9 @@ def test_matcher_behaviour(self, mocker): matcher = matchers.UserDefinedSegmentMatcher(self.raw) # Test that if the key if the storage wrapper finds the key in the segment, it matches. - assert matcher.evaluate('some_key', {}, {'evaluator': None, 'ec': EvaluationContext([],{'some_segment': True})}) is True + assert matcher.evaluate('some_key', {}, {'evaluator': None, 'ec': EvaluationContext([],{'some_segment': True}, {})}) is True # Test that if the key if the storage wrapper doesn't find the key in the segment, it fails. - assert matcher.evaluate('some_key', {}, {'evaluator': None, 'ec': EvaluationContext([], {'some_segment': False})}) is False + assert matcher.evaluate('some_key', {}, {'evaluator': None, 'ec': EvaluationContext([], {'some_segment': False}, {})}) is False def test_to_json(self): """Test that the object serializes to JSON properly.""" @@ -778,8 +780,8 @@ def test_matcher_behaviour(self, mocker): parsed = matchers.DependencyMatcher(cond_raw) evaluator = mocker.Mock(spec=Evaluator) - cond = condition.from_raw(splits_json["splitChange1_1"]["splits"][0]['conditions'][0]) - split = splits.from_raw(splits_json["splitChange1_1"]["splits"][0]) + cond = condition.from_raw(splits_json["splitChange1_1"]['ff']['d'][0]['conditions'][0]) + split = splits.from_raw(splits_json["splitChange1_1"]['ff']['d'][0]) evaluator.eval_with_context.return_value = {'treatment': 'on'} assert parsed.evaluate('SPLIT_2', {}, {'evaluator': evaluator, 'ec': [{'flags': [split], 'segment_memberships': {}}]}) is True @@ -1095,3 +1097,73 @@ def test_to_str(self): """Test that the object serializes to str properly.""" as_str = matchers.InListSemverMatcher(self.raw) assert str(as_str) == "in list semver ['2.1.8', '2.1.11']" + +class RuleBasedMatcherTests(MatcherTestsBase): + """Rule based segment matcher test cases.""" + + raw ={ + "keySelector": { + "trafficType": "user" + }, + "matcherType": "IN_RULE_BASED_SEGMENT", + "negate": False, + "userDefinedSegmentMatcherData": { + "segmentName": "sample_rule_based_segment" + } + } + + def test_from_raw(self, mocker): + """Test parsing from raw json/dict.""" + parsed = matchers.from_raw(self.raw) + assert isinstance(parsed, matchers.RuleBasedSegmentMatcher) + + def test_to_json(self): + """Test that the object serializes to JSON properly.""" + as_json = matchers.AllKeysMatcher(self.raw).to_json() + assert as_json['matcherType'] == 'IN_RULE_BASED_SEGMENT' + + def test_matcher_behaviour(self, mocker): + """Test if the matcher works properly.""" + rbs_segments = os.path.join(os.path.dirname(__file__), '../../engine/files', 'rule_base_segments3.json') + with open(rbs_segments, 'r') as flo: + data = json.loads(flo.read()) + + rbs = rule_based_segments.from_raw(data["rbs"]["d"][0]) + matcher = matchers.RuleBasedSegmentMatcher(self.raw) + ec ={'ec': EvaluationContext( + {}, + {"segment1": False}, + {"sample_rule_based_segment": rbs} + )} + assert matcher._match(None, context=ec) is False + assert matcher._match('bilal@split.io', context=ec) is False + assert matcher._match('bilal@split.io', {'email': 'bilal@split.io'}, context=ec) is True + +class PrerequisitesMatcherTests(MatcherTestsBase): + """tests for prerequisites matcher.""" + + def test_init(self, mocker): + """Test init.""" + split_load = os.path.join(os.path.dirname(__file__), 'files', 'splits_prereq.json') + with open(split_load, 'r') as flo: + data = json.loads(flo.read()) + + prereq = splits.from_raw_prerequisites(data['ff']['d'][0]['prerequisites']) + parsed = PrerequisitesMatcher(prereq) + assert parsed._prerequisites == prereq + + def test_matcher_behaviour(self, mocker): + """Test if the matcher works properly.""" + split_load = os.path.join(os.path.dirname(__file__), 'files', 'splits_prereq.json') + with open(split_load, 'r') as flo: + data = json.loads(flo.read()) + prereq = splits.from_raw_prerequisites(data['ff']['d'][3]['prerequisites']) + parsed = PrerequisitesMatcher(prereq) + evaluator = mocker.Mock(spec=Evaluator) + + + evaluator.eval_with_context.return_value = {'treatment': 'on'} + assert parsed.match('SPLIT_2', {}, {'evaluator': evaluator, 'ec': [{'flags': ['prereq_chain'], 'segment_memberships': {}}]}) is True + + evaluator.eval_with_context.return_value = {'treatment': 'off'} + assert parsed.match('SPLIT_2', {}, {'evaluator': evaluator, 'ec': [{'flags': ['prereq_chain'], 'segment_memberships': {}}]}) is False \ No newline at end of file diff --git a/tests/models/test_rule_based_segments.py b/tests/models/test_rule_based_segments.py new file mode 100644 index 00000000..98e35fe8 --- /dev/null +++ b/tests/models/test_rule_based_segments.py @@ -0,0 +1,103 @@ +"""Split model tests module.""" +import copy +from splitio.models import rule_based_segments +from splitio.models import splits +from splitio.models.grammar.condition import Condition +from splitio.models.grammar.matchers.rule_based_segment import RuleBasedSegmentMatcher + +class RuleBasedSegmentModelTests(object): + """Rule based segment model tests.""" + + raw = { + "changeNumber": 123, + "name": "sample_rule_based_segment", + "status": "ACTIVE", + "trafficTypeName": "user", + "excluded":{ + "keys":["mauro@split.io","gaston@split.io"], + "segments":[] + }, + "conditions": [ + { + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "keySelector": { + "trafficType": "user", + "attribute": "email" + }, + "matcherType": "ENDS_WITH", + "negate": False, + "whitelistMatcherData": { + "whitelist": [ + "@split.io" + ] + } + } + ] + } + } + ] + } + + def test_from_raw(self): + """Test split model parsing.""" + parsed = rule_based_segments.from_raw(self.raw) + assert isinstance(parsed, rule_based_segments.RuleBasedSegment) + assert parsed.change_number == 123 + assert parsed.name == 'sample_rule_based_segment' + assert parsed.status == splits.Status.ACTIVE + assert len(parsed.conditions) == 1 + assert parsed.excluded.get_excluded_keys() == ["mauro@split.io","gaston@split.io"] + assert parsed.excluded.get_excluded_segments() == [] + conditions = parsed.conditions[0].to_json() + assert conditions['matcherGroup']['matchers'][0] == { + 'betweenMatcherData': None, 'booleanMatcherData': None, 'dependencyMatcherData': None, + 'stringMatcherData': None, 'unaryNumericMatcherData': None, 'userDefinedSegmentMatcherData': None, + "keySelector": { + "attribute": "email" + }, + "matcherType": "ENDS_WITH", + "negate": False, + "whitelistMatcherData": { + "whitelist": [ + "@split.io" + ] + } + } + + def test_incorrect_matcher(self): + """Test incorrect matcher in split model parsing.""" + rbs = copy.deepcopy(self.raw) + rbs['conditions'][0]['matcherGroup']['matchers'][0]['matcherType'] = 'INVALID_MATCHER' + rbs = rule_based_segments.from_raw(rbs) + assert rbs.conditions[0].to_json() == splits._DEFAULT_CONDITIONS_TEMPLATE + + # using multiple conditions + rbs = copy.deepcopy(self.raw) + rbs['conditions'].append(rbs['conditions'][0]) + rbs['conditions'][0]['matcherGroup']['matchers'][0]['matcherType'] = 'INVALID_MATCHER' + parsed = rule_based_segments.from_raw(rbs) + assert parsed.conditions[0].to_json() == splits._DEFAULT_CONDITIONS_TEMPLATE + + def test_get_condition_segment_names(self): + rbs = copy.deepcopy(self.raw) + rbs['conditions'].append( + {"matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "matcherType": "IN_SEGMENT", + "negate": False, + "userDefinedSegmentMatcherData": { + "segmentName": "employees" + }, + "whitelistMatcherData": None + } + ] + }, + }) + rbs = rule_based_segments.from_raw(rbs) + + assert rbs.get_condition_segment_names() == {"employees"} \ No newline at end of file diff --git a/tests/models/test_splits.py b/tests/models/test_splits.py index 442a18d0..472ecde9 100644 --- a/tests/models/test_splits.py +++ b/tests/models/test_splits.py @@ -11,6 +11,10 @@ class SplitTests(object): 'changeNumber': 123, 'trafficTypeName': 'user', 'name': 'some_name', + 'prerequisites': [ + { 'n': 'flag1', 'ts': ['on','v1'] }, + { 'n': 'flag2', 'ts': ['off'] } + ], 'trafficAllocation': 100, 'trafficAllocationSeed': 123456, 'seed': 321654, @@ -83,14 +87,26 @@ def test_from_raw(self): assert parsed._configurations == {'on': '{"color": "blue", "size": 13}'} assert parsed.sets == {'set1', 'set2'} assert parsed.impressions_disabled == False - + assert len(parsed.prerequisites) == 2 + flag1 = False + flag2 = False + for prerequisite in parsed.prerequisites: + if prerequisite.feature_flag_name == 'flag1': + flag1 = True + assert prerequisite.treatments == ['on','v1'] + if prerequisite.feature_flag_name == 'flag2': + flag2 = True + assert prerequisite.treatments == ['off'] + assert flag1 + assert flag2 + def test_get_segment_names(self, mocker): """Test fetching segment names.""" cond1 = mocker.Mock(spec=Condition) cond2 = mocker.Mock(spec=Condition) cond1.get_segment_names.return_value = ['segment1', 'segment2'] cond2.get_segment_names.return_value = ['segment3', 'segment4'] - split1 = splits.Split( 'some_split', 123, False, 'off', 'user', 'ACTIVE', 123, [cond1, cond2]) + split1 = splits.Split( 'some_split', 123, False, 'off', 'user', 'ACTIVE', 123, [cond1, cond2], None) assert split1.get_segment_names() == ['segment%d' % i for i in range(1, 5)] def test_to_json(self): diff --git a/tests/push/test_parser.py b/tests/push/test_parser.py index 6f4b57ff..faffb3d0 100644 --- a/tests/push/test_parser.py +++ b/tests/push/test_parser.py @@ -66,7 +66,7 @@ def test_event_parsing(self): assert parsed1.change_number == 1591996685190 assert parsed1.previous_change_number == 12 assert parsed1.compression == 2 - assert parsed1.feature_flag_definition == 'eJzEUtFu2kAQ/BU0z4d0hw2Be0MFRVGJIx' + assert parsed1.object_definition == 'eJzEUtFu2kAQ/BU0z4d0hw2Be0MFRVGJIx' e1 = make_message( 'NDA5ODc2MTAyNg==_MzAyODY0NDkyOA==_splits', @@ -77,7 +77,7 @@ def test_event_parsing(self): assert parsed1.change_number == 1591996685190 assert parsed1.previous_change_number == None assert parsed1.compression == None - assert parsed1.feature_flag_definition == None + assert parsed1.object_definition == None e2 = make_message( 'NDA5ODc2MTAyNg==_MzAyODY0NDkyOA==_segments', diff --git a/tests/push/test_split_worker.py b/tests/push/test_split_worker.py index d792cada..0d3ac824 100644 --- a/tests/push/test_split_worker.py +++ b/tests/push/test_split_worker.py @@ -1,79 +1,127 @@ """Split Worker tests.""" import time import queue +import base64 import pytest from splitio.api import APIException from splitio.push.workers import SplitWorker, SplitWorkerAsync from splitio.models.notification import SplitChangeNotification from splitio.optional.loaders import asyncio -from splitio.push.parser import SplitChangeUpdate +from splitio.push.parser import SplitChangeUpdate, RBSChangeUpdate from splitio.engine.telemetry import TelemetryStorageProducer, TelemetryStorageProducerAsync from splitio.storage.inmemmory import InMemoryTelemetryStorage, InMemorySplitStorage, InMemorySegmentStorage, \ InMemoryTelemetryStorageAsync, InMemorySplitStorageAsync, InMemorySegmentStorageAsync change_number_received = None - - -def handler_sync(change_number): +rbs = { + "changeNumber": 5, + "name": "sample_rule_based_segment", + "status": "ACTIVE", + "trafficTypeName": "user", + "excluded":{ + "keys":["mauro@split.io","gaston@split.io"], + "segments":[] + }, + "conditions": [ + { + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "keySelector": { + "trafficType": "user", + "attribute": "email" + }, + "matcherType": "ENDS_WITH", + "negate": False, + "whitelistMatcherData": { + "whitelist": [ + "@split.io" + ] + } + } + ] + } + } + ] + } + +def handler_sync(change_number, rbs_change_number): global change_number_received + global rbs_change_number_received + change_number_received = change_number + rbs_change_number_received = rbs_change_number return -async def handler_async(change_number): +async def handler_async(change_number, rbs_change_number): global change_number_received + global rbs_change_number_received change_number_received = change_number + rbs_change_number_received = rbs_change_number return class SplitWorkerTests(object): - def test_on_error(self, mocker): - q = queue.Queue() - def handler_sync(change_number): - raise APIException('some') - - split_worker = SplitWorker(handler_sync, mocker.Mock(), q, mocker.Mock(), mocker.Mock(), mocker.Mock()) - split_worker.start() - assert split_worker.is_running() - - q.put(SplitChangeUpdate('some', 'SPLIT_UPDATE', 123456789, None, None, None)) - with pytest.raises(Exception): - split_worker._handler() - - assert split_worker.is_running() - assert split_worker._worker.is_alive() - split_worker.stop() - time.sleep(1) - assert not split_worker.is_running() - assert not split_worker._worker.is_alive() - def test_handler(self, mocker): q = queue.Queue() - split_worker = SplitWorker(handler_sync, mocker.Mock(), q, mocker.Mock(), mocker.Mock(), mocker.Mock()) + split_worker = SplitWorker(handler_sync, mocker.Mock(), q, mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock()) global change_number_received + global rbs_change_number_received assert not split_worker.is_running() split_worker.start() assert split_worker.is_running() - - # should call the handler - q.put(SplitChangeUpdate('some', 'SPLIT_UPDATE', 123456789, None, None, None)) - time.sleep(0.1) - assert change_number_received == 123456789 - + def get_change_number(): return 2345 split_worker._feature_flag_storage.get_change_number = get_change_number + def get_rbs_change_number(): + return 2345 + split_worker._rule_based_segment_storage.get_change_number = get_rbs_change_number + self._feature_flag_added = None self._feature_flag_deleted = None def update(feature_flag_add, feature_flag_delete, change_number): self._feature_flag_added = feature_flag_add - self._feature_flag_deleted = feature_flag_delete + self._feature_flag_deleted = feature_flag_delete split_worker._feature_flag_storage.update = update split_worker._feature_flag_storage.config_flag_sets_used = 0 + self._rbs_added = None + self._rbs_deleted = None + def update(rbs_add, rbs_delete, change_number): + self._rbs_added = rbs_add + self._rbs_deleted = rbs_delete + split_worker._rule_based_segment_storage.update = update + + # should not call the handler + rbs_change_number_received = 0 + rbs1 = str(rbs) + rbs1 = rbs1.replace("'", "\"") + rbs1 = rbs1.replace("False", "false") + encoded = base64.b64encode(bytes(rbs1, "utf-8")) + q.put(RBSChangeUpdate('some', 'RB_SEGMENT_UPDATE', 123456790, 2345, encoded, 0)) + time.sleep(0.1) + assert rbs_change_number_received == 0 + assert self._rbs_added[0].name == "sample_rule_based_segment" + + # should call the handler + q.put(SplitChangeUpdate('some', 'SPLIT_UPDATE', 123456789, None, None, None)) + time.sleep(0.1) + assert change_number_received == 123456789 + assert rbs_change_number_received == None + + # should call the handler + q.put(RBSChangeUpdate('some', 'RB_SEGMENT_UPDATE', 123456789, None, None, None)) + time.sleep(0.1) + assert rbs_change_number_received == 123456789 + assert change_number_received == None + + # should call the handler q.put(SplitChangeUpdate('some', 'SPLIT_UPDATE', 123456790, 12345, "{}", 1)) time.sleep(0.1) @@ -94,12 +142,32 @@ def update(feature_flag_add, feature_flag_delete, change_number): split_worker.stop() assert not split_worker.is_running() + def test_on_error(self, mocker): + q = queue.Queue() + def handler_sync(change_number): + raise APIException('some') + + split_worker = SplitWorker(handler_sync, mocker.Mock(), q, mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock()) + split_worker.start() + assert split_worker.is_running() + + q.put(SplitChangeUpdate('some', 'SPLIT_UPDATE', 123456789, None, None, None)) + with pytest.raises(Exception): + split_worker._handler() + + assert split_worker.is_running() + assert split_worker._worker.is_alive() + split_worker.stop() + time.sleep(1) + assert not split_worker.is_running() + assert not split_worker._worker.is_alive() + def test_compression(self, mocker): q = queue.Queue() telemetry_storage = InMemoryTelemetryStorage() telemetry_producer = TelemetryStorageProducer(telemetry_storage) telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() - split_worker = SplitWorker(handler_sync, mocker.Mock(), q, mocker.Mock(), mocker.Mock(), telemetry_runtime_producer) + split_worker = SplitWorker(handler_sync, mocker.Mock(), q, mocker.Mock(), mocker.Mock(), telemetry_runtime_producer, mocker.Mock()) global change_number_received split_worker.start() def get_change_number(): @@ -148,7 +216,7 @@ def update(feature_flag_add, feature_flag_delete, change_number): def test_edge_cases(self, mocker): q = queue.Queue() - split_worker = SplitWorker(handler_sync, mocker.Mock(), q, mocker.Mock(), mocker.Mock(), mocker.Mock()) + split_worker = SplitWorker(handler_sync, mocker.Mock(), q, mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock()) global change_number_received split_worker.start() @@ -201,7 +269,7 @@ def test_fetch_segment(self, mocker): def segment_handler_sync(segment_name, change_number): self.segment_name = segment_name return - split_worker = SplitWorker(handler_sync, segment_handler_sync, q, split_storage, segment_storage, mocker.Mock()) + split_worker = SplitWorker(handler_sync, segment_handler_sync, q, split_storage, segment_storage, mocker.Mock(), mocker.Mock()) split_worker.start() def get_change_number(): @@ -225,7 +293,7 @@ async def test_on_error(self, mocker): def handler_sync(change_number): raise APIException('some') - split_worker = SplitWorkerAsync(handler_async, mocker.Mock(), q, mocker.Mock(), mocker.Mock(), mocker.Mock()) + split_worker = SplitWorkerAsync(handler_async, mocker.Mock(), q, mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock()) split_worker.start() assert split_worker.is_running() @@ -253,7 +321,7 @@ def _worker_running(self): @pytest.mark.asyncio async def test_handler(self, mocker): q = asyncio.Queue() - split_worker = SplitWorkerAsync(handler_async, mocker.Mock(), q, mocker.Mock(), mocker.Mock(), mocker.Mock()) + split_worker = SplitWorkerAsync(handler_async, mocker.Mock(), q, mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock()) assert not split_worker.is_running() split_worker.start() @@ -261,7 +329,8 @@ async def test_handler(self, mocker): assert(self._worker_running()) global change_number_received - + global rbs_change_number_received + # should call the handler await q.put(SplitChangeUpdate('some', 'SPLIT_UPDATE', 123456789, None, None, None)) await asyncio.sleep(0.1) @@ -271,6 +340,10 @@ async def get_change_number(): return 2345 split_worker._feature_flag_storage.get_change_number = get_change_number + async def get_rbs_change_number(): + return 2345 + split_worker._rule_based_segment_storage.get_change_number = get_rbs_change_number + self.new_change_number = 0 self._feature_flag_added = None self._feature_flag_deleted = None @@ -289,6 +362,24 @@ async def record_update_from_sse(xx): pass split_worker._telemetry_runtime_producer.record_update_from_sse = record_update_from_sse + self._rbs_added = None + self._rbs_deleted = None + async def update_rbs(rbs_add, rbs_delete, change_number): + self._rbs_added = rbs_add + self._rbs_deleted = rbs_delete + split_worker._rule_based_segment_storage.update = update_rbs + + # should not call the handler + rbs_change_number_received = 0 + rbs1 = str(rbs) + rbs1 = rbs1.replace("'", "\"") + rbs1 = rbs1.replace("False", "false") + encoded = base64.b64encode(bytes(rbs1, "utf-8")) + await q.put(RBSChangeUpdate('some', 'RB_SEGMENT_UPDATE', 123456790, 2345, encoded, 0)) + await asyncio.sleep(0.1) + assert rbs_change_number_received == 0 + assert self._rbs_added[0].name == "sample_rule_based_segment" + # should call the handler await q.put(SplitChangeUpdate('some', 'SPLIT_UPDATE', 123456790, 12345, "{}", 1)) await asyncio.sleep(0.1) @@ -318,7 +409,7 @@ async def test_compression(self, mocker): telemetry_storage = await InMemoryTelemetryStorageAsync.create() telemetry_producer = TelemetryStorageProducerAsync(telemetry_storage) telemetry_runtime_producer = telemetry_producer.get_telemetry_runtime_producer() - split_worker = SplitWorkerAsync(handler_async, mocker.Mock(), q, mocker.Mock(), mocker.Mock(), telemetry_runtime_producer) + split_worker = SplitWorkerAsync(handler_async, mocker.Mock(), q, mocker.Mock(), mocker.Mock(), telemetry_runtime_producer, mocker.Mock()) global change_number_received split_worker.start() async def get_change_number(): @@ -343,6 +434,10 @@ async def update(feature_flag_add, feature_flag_delete, change_number): split_worker._feature_flag_storage.update = update split_worker._feature_flag_storage.config_flag_sets_used = 0 + async def contains(rbs): + return False + split_worker._rule_based_segment_storage.contains = contains + # compression 0 await q.put(SplitChangeUpdate('some', 'SPLIT_UPDATE', 123456790, 2345, 'eyJ0cmFmZmljVHlwZU5hbWUiOiJ1c2VyIiwiaWQiOiIzM2VhZmE1MC0xYTY1LTExZWQtOTBkZi1mYTMwZDk2OTA0NDUiLCJuYW1lIjoiYmlsYWxfc3BsaXQiLCJ0cmFmZmljQWxsb2NhdGlvbiI6MTAwLCJ0cmFmZmljQWxsb2NhdGlvblNlZWQiOi0xMzY0MTE5MjgyLCJzZWVkIjotNjA1OTM4ODQzLCJzdGF0dXMiOiJBQ1RJVkUiLCJraWxsZWQiOmZhbHNlLCJkZWZhdWx0VHJlYXRtZW50Ijoib2ZmIiwiY2hhbmdlTnVtYmVyIjoxNjg0MzQwOTA4NDc1LCJhbGdvIjoyLCJjb25maWd1cmF0aW9ucyI6e30sImNvbmRpdGlvbnMiOlt7ImNvbmRpdGlvblR5cGUiOiJST0xMT1VUIiwibWF0Y2hlckdyb3VwIjp7ImNvbWJpbmVyIjoiQU5EIiwibWF0Y2hlcnMiOlt7ImtleVNlbGVjdG9yIjp7InRyYWZmaWNUeXBlIjoidXNlciJ9LCJtYXRjaGVyVHlwZSI6IklOX1NFR01FTlQiLCJuZWdhdGUiOmZhbHNlLCJ1c2VyRGVmaW5lZFNlZ21lbnRNYXRjaGVyRGF0YSI6eyJzZWdtZW50TmFtZSI6ImJpbGFsX3NlZ21lbnQifX1dfSwicGFydGl0aW9ucyI6W3sidHJlYXRtZW50Ijoib24iLCJzaXplIjowfSx7InRyZWF0bWVudCI6Im9mZiIsInNpemUiOjEwMH1dLCJsYWJlbCI6ImluIHNlZ21lbnQgYmlsYWxfc2VnbWVudCJ9LHsiY29uZGl0aW9uVHlwZSI6IlJPTExPVVQiLCJtYXRjaGVyR3JvdXAiOnsiY29tYmluZXIiOiJBTkQiLCJtYXRjaGVycyI6W3sia2V5U2VsZWN0b3IiOnsidHJhZmZpY1R5cGUiOiJ1c2VyIn0sIm1hdGNoZXJUeXBlIjoiQUxMX0tFWVMiLCJuZWdhdGUiOmZhbHNlfV19LCJwYXJ0aXRpb25zIjpbeyJ0cmVhdG1lbnQiOiJvbiIsInNpemUiOjB9LHsidHJlYXRtZW50Ijoib2ZmIiwic2l6ZSI6MTAwfV0sImxhYmVsIjoiZGVmYXVsdCBydWxlIn1dfQ==', 0)) await asyncio.sleep(0.1) @@ -376,7 +471,7 @@ async def update(feature_flag_add, feature_flag_delete, change_number): @pytest.mark.asyncio async def test_edge_cases(self, mocker): q = asyncio.Queue() - split_worker = SplitWorkerAsync(handler_async, mocker.Mock(), q, mocker.Mock(), mocker.Mock(), mocker.Mock()) + split_worker = SplitWorkerAsync(handler_async, mocker.Mock(), q, mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock()) global change_number_received split_worker.start() @@ -434,7 +529,7 @@ async def test_fetch_segment(self, mocker): async def segment_handler_sync(segment_name, change_number): self.segment_name = segment_name return - split_worker = SplitWorkerAsync(handler_async, segment_handler_sync, q, split_storage, segment_storage, mocker.Mock()) + split_worker = SplitWorkerAsync(handler_async, segment_handler_sync, q, split_storage, segment_storage, mocker.Mock(), mocker.Mock()) split_worker.start() async def get_change_number(): diff --git a/tests/storage/test_inmemory_storage.py b/tests/storage/test_inmemory_storage.py index bf38ed57..9c5b6ed2 100644 --- a/tests/storage/test_inmemory_storage.py +++ b/tests/storage/test_inmemory_storage.py @@ -2,6 +2,7 @@ # pylint: disable=no-self-use import random import pytest +import copy from splitio.models.splits import Split from splitio.models.segments import Segment @@ -11,7 +12,9 @@ from splitio.engine.telemetry import TelemetryStorageProducer, TelemetryStorageProducerAsync from splitio.storage.inmemmory import InMemorySplitStorage, InMemorySegmentStorage, InMemorySegmentStorageAsync, InMemorySplitStorageAsync, \ InMemoryImpressionStorage, InMemoryEventStorage, InMemoryTelemetryStorage, InMemoryImpressionStorageAsync, InMemoryEventStorageAsync, \ - InMemoryTelemetryStorageAsync, FlagSets + InMemoryTelemetryStorageAsync, FlagSets, InMemoryRuleBasedSegmentStorage, InMemoryRuleBasedSegmentStorageAsync +from splitio.models.rule_based_segments import RuleBasedSegment +from splitio.models import rule_based_segments class FlagSetsFilterTests(object): """Flag sets filter storage tests.""" @@ -1807,3 +1810,120 @@ async def test_pop_latencies(self): assert(sync_latency == {'httpLatencies': {'split': [4] + [0] * 22, 'segment': [4] + [0] * 22, 'impression': [2] + [0] * 22, 'impressionCount': [2] + [0] * 22, 'event': [2] + [0] * 22, 'telemetry': [3] + [0] * 22, 'token': [3] + [0] * 22}}) + +class InMemoryRuleBasedSegmentStorageTests(object): + """In memory rule based segment storage test cases.""" + + def test_storing_retrieving_segments(self, mocker): + """Test storing and retrieving splits works.""" + rbs_storage = InMemoryRuleBasedSegmentStorage() + + segment1 = mocker.Mock(spec=RuleBasedSegment) + name_property = mocker.PropertyMock() + name_property.return_value = 'some_segment' + type(segment1).name = name_property + + segment2 = mocker.Mock() + name2_prop = mocker.PropertyMock() + name2_prop.return_value = 'segment2' + type(segment2).name = name2_prop + + rbs_storage.update([segment1, segment2], [], -1) + assert rbs_storage.get('some_segment') == segment1 + assert rbs_storage.get_segment_names() == ['some_segment', 'segment2'] + assert rbs_storage.get('nonexistant_segment') is None + + rbs_storage.update([], ['some_segment'], -1) + assert rbs_storage.get('some_segment') is None + + def test_store_get_changenumber(self): + """Test that storing and retrieving change numbers works.""" + storage = InMemoryRuleBasedSegmentStorage() + assert storage.get_change_number() == -1 + storage.update([], [], 5) + assert storage.get_change_number() == 5 + + def test_contains(self): + raw = { + "changeNumber": 123, + "name": "segment1", + "status": "ACTIVE", + "trafficTypeName": "user", + "excluded":{ + "keys":[], + "segments":[] + }, + "conditions": [] + } + segment1 = rule_based_segments.from_raw(raw) + raw2 = copy.deepcopy(raw) + raw2["name"] = "segment2" + segment2 = rule_based_segments.from_raw(raw2) + raw3 = copy.deepcopy(raw) + raw3["name"] = "segment3" + segment3 = rule_based_segments.from_raw(raw3) + storage = InMemoryRuleBasedSegmentStorage() + storage.update([segment1, segment2, segment3], [], -1) + assert storage.contains(["segment1"]) + assert storage.contains(["segment1", "segment3"]) + assert not storage.contains(["segment5"]) + +class InMemoryRuleBasedSegmentStorageAsyncTests(object): + """In memory rule based segment storage test cases.""" + + @pytest.mark.asyncio + async def test_storing_retrieving_segments(self, mocker): + """Test storing and retrieving splits works.""" + rbs_storage = InMemoryRuleBasedSegmentStorageAsync() + + segment1 = mocker.Mock(spec=RuleBasedSegment) + name_property = mocker.PropertyMock() + name_property.return_value = 'some_segment' + type(segment1).name = name_property + + segment2 = mocker.Mock() + name2_prop = mocker.PropertyMock() + name2_prop.return_value = 'segment2' + type(segment2).name = name2_prop + + await rbs_storage.update([segment1, segment2], [], -1) + assert await rbs_storage.get('some_segment') == segment1 + assert await rbs_storage.get_segment_names() == ['some_segment', 'segment2'] + assert await rbs_storage.get('nonexistant_segment') is None + + await rbs_storage.update([], ['some_segment'], -1) + assert await rbs_storage.get('some_segment') is None + + @pytest.mark.asyncio + async def test_store_get_changenumber(self): + """Test that storing and retrieving change numbers works.""" + storage = InMemoryRuleBasedSegmentStorageAsync() + assert await storage.get_change_number() == -1 + await storage.update([], [], 5) + assert await storage.get_change_number() == 5 + + @pytest.mark.asyncio + async def test_contains(self): + raw = { + "changeNumber": 123, + "name": "segment1", + "status": "ACTIVE", + "trafficTypeName": "user", + "excluded":{ + "keys":[], + "segments":[] + }, + "conditions": [] + } + segment1 = rule_based_segments.from_raw(raw) + raw2 = copy.deepcopy(raw) + raw2["name"] = "segment2" + segment2 = rule_based_segments.from_raw(raw2) + raw3 = copy.deepcopy(raw) + raw3["name"] = "segment3" + segment3 = rule_based_segments.from_raw(raw3) + storage = InMemoryRuleBasedSegmentStorageAsync() + await storage.update([segment1, segment2, segment3], [], -1) + assert await storage.contains(["segment1"]) + assert await storage.contains(["segment1", "segment3"]) + assert not await storage.contains(["segment5"]) diff --git a/tests/storage/test_pluggable.py b/tests/storage/test_pluggable.py index 439049e5..283eb8e3 100644 --- a/tests/storage/test_pluggable.py +++ b/tests/storage/test_pluggable.py @@ -1,20 +1,21 @@ """Pluggable storage test module.""" import json import threading +import copy import pytest from splitio.optional.loaders import asyncio from splitio.models.splits import Split -from splitio.models import splits, segments +from splitio.models import splits, segments, rule_based_segments from splitio.models.segments import Segment from splitio.models.impressions import Impression from splitio.models.events import Event, EventWrapper from splitio.storage.pluggable import PluggableSplitStorage, PluggableSegmentStorage, PluggableImpressionsStorage, PluggableEventsStorage, \ PluggableTelemetryStorage, PluggableEventsStorageAsync, PluggableSegmentStorageAsync, PluggableImpressionsStorageAsync,\ - PluggableSplitStorageAsync, PluggableTelemetryStorageAsync + PluggableSplitStorageAsync, PluggableTelemetryStorageAsync, PluggableRuleBasedSegmentsStorage, PluggableRuleBasedSegmentsStorageAsync from splitio.client.util import get_metadata, SdkMetadata from splitio.models.telemetry import MAX_TAGS, MethodExceptionsAndLatencies, OperationMode -from tests.integration import splits_json +from tests.integration import splits_json, rbsegments_json class StorageMockAdapter(object): def __init__(self): @@ -274,19 +275,19 @@ def test_get(self): for sprefix in [None, 'myprefix']: pluggable_split_storage = PluggableSplitStorage(self.mock_adapter, prefix=sprefix) - split1 = splits.from_raw(splits_json['splitChange1_2']['splits'][0]) - split_name = splits_json['splitChange1_2']['splits'][0]['name'] + split1 = splits.from_raw(splits_json['splitChange1_2']['ff']['d'][0]) + split_name = splits_json['splitChange1_2']['ff']['d'][0]['name'] self.mock_adapter.set(pluggable_split_storage._prefix.format(feature_flag_name=split_name), split1.to_json()) - assert(pluggable_split_storage.get(split_name).to_json() == splits.from_raw(splits_json['splitChange1_2']['splits'][0]).to_json()) + assert(pluggable_split_storage.get(split_name).to_json() == splits.from_raw(splits_json['splitChange1_2']['ff']['d'][0]).to_json()) assert(pluggable_split_storage.get('not_existing') == None) def test_fetch_many(self): self.mock_adapter._keys = {} for sprefix in [None, 'myprefix']: pluggable_split_storage = PluggableSplitStorage(self.mock_adapter, prefix=sprefix) - split1 = splits.from_raw(splits_json['splitChange1_2']['splits'][0]) - split2_temp = splits_json['splitChange1_2']['splits'][0].copy() + split1 = splits.from_raw(splits_json['splitChange1_2']['ff']['d'][0]) + split2_temp = splits_json['splitChange1_2']['ff']['d'][0].copy() split2_temp['name'] = 'another_split' split2 = splits.from_raw(split2_temp) @@ -325,8 +326,8 @@ def test_get_split_names(self): self.mock_adapter._keys = {} for sprefix in [None, 'myprefix']: pluggable_split_storage = PluggableSplitStorage(self.mock_adapter, prefix=sprefix) - split1 = splits.from_raw(splits_json['splitChange1_2']['splits'][0]) - split2_temp = splits_json['splitChange1_2']['splits'][0].copy() + split1 = splits.from_raw(splits_json['splitChange1_2']['ff']['d'][0]) + split2_temp = splits_json['splitChange1_2']['ff']['d'][0].copy() split2_temp['name'] = 'another_split' split2 = splits.from_raw(split2_temp) self.mock_adapter.set(pluggable_split_storage._prefix.format(feature_flag_name=split1.name), split1.to_json()) @@ -410,12 +411,12 @@ async def test_get(self): for sprefix in [None, 'myprefix']: pluggable_split_storage = PluggableSplitStorageAsync(self.mock_adapter, prefix=sprefix) - split1 = splits.from_raw(splits_json['splitChange1_2']['splits'][0]) - split_name = splits_json['splitChange1_2']['splits'][0]['name'] + split1 = splits.from_raw(splits_json['splitChange1_2']['ff']['d'][0]) + split_name = splits_json['splitChange1_2']['ff']['d'][0]['name'] await self.mock_adapter.set(pluggable_split_storage._prefix.format(feature_flag_name=split_name), split1.to_json()) split = await pluggable_split_storage.get(split_name) - assert(split.to_json() == splits.from_raw(splits_json['splitChange1_2']['splits'][0]).to_json()) + assert(split.to_json() == splits.from_raw(splits_json['splitChange1_2']['ff']['d'][0]).to_json()) assert(await pluggable_split_storage.get('not_existing') == None) @pytest.mark.asyncio @@ -423,8 +424,8 @@ async def test_fetch_many(self): self.mock_adapter._keys = {} for sprefix in [None, 'myprefix']: pluggable_split_storage = PluggableSplitStorageAsync(self.mock_adapter, prefix=sprefix) - split1 = splits.from_raw(splits_json['splitChange1_2']['splits'][0]) - split2_temp = splits_json['splitChange1_2']['splits'][0].copy() + split1 = splits.from_raw(splits_json['splitChange1_2']['ff']['d'][0]) + split2_temp = splits_json['splitChange1_2']['ff']['d'][0].copy() split2_temp['name'] = 'another_split' split2 = splits.from_raw(split2_temp) @@ -451,8 +452,8 @@ async def test_get_split_names(self): self.mock_adapter._keys = {} for sprefix in [None, 'myprefix']: pluggable_split_storage = PluggableSplitStorageAsync(self.mock_adapter, prefix=sprefix) - split1 = splits.from_raw(splits_json['splitChange1_2']['splits'][0]) - split2_temp = splits_json['splitChange1_2']['splits'][0].copy() + split1 = splits.from_raw(splits_json['splitChange1_2']['ff']['d'][0]) + split2_temp = splits_json['splitChange1_2']['ff']['d'][0].copy() split2_temp['name'] = 'another_split' split2 = splits.from_raw(split2_temp) await self.mock_adapter.set(pluggable_split_storage._prefix.format(feature_flag_name=split1.name), split1.to_json()) @@ -1372,3 +1373,124 @@ async def test_push_config_stats(self): await pluggable_telemetry_storage.record_active_and_redundant_factories(2, 1) await pluggable_telemetry_storage.push_config_stats() assert(self.mock_adapter._keys[pluggable_telemetry_storage._telemetry_config_key + "::" + pluggable_telemetry_storage._sdk_metadata] == '{"aF": 2, "rF": 1, "sT": "memory", "oM": 0, "t": []}') + +class PluggableRuleBasedSegmentStorageTests(object): + """In memory rule based segment storage test cases.""" + + def setup_method(self): + """Prepare storages with test data.""" + self.mock_adapter = StorageMockAdapter() + + def test_get(self): + self.mock_adapter._keys = {} + for sprefix in [None, 'myprefix']: + pluggable_rbs_storage = PluggableRuleBasedSegmentsStorage(self.mock_adapter, prefix=sprefix) + + rbs1 = rule_based_segments.from_raw(rbsegments_json[0]) + rbs_name = rbsegments_json[0]['name'] + + self.mock_adapter.set(pluggable_rbs_storage._prefix.format(segment_name=rbs_name), rbs1.to_json()) + assert(pluggable_rbs_storage.get(rbs_name).to_json() == rule_based_segments.from_raw(rbsegments_json[0]).to_json()) + assert(pluggable_rbs_storage.get('not_existing') == None) + + def test_get_change_number(self): + self.mock_adapter._keys = {} + for sprefix in [None, 'myprefix']: + pluggable_rbs_storage = PluggableRuleBasedSegmentsStorage(self.mock_adapter, prefix=sprefix) + if sprefix == 'myprefix': + prefix = 'myprefix.' + else: + prefix = '' + self.mock_adapter.set(prefix + "SPLITIO.rbsegments.till", 1234) + assert(pluggable_rbs_storage.get_change_number() == 1234) + + def test_get_segment_names(self): + self.mock_adapter._keys = {} + for sprefix in [None, 'myprefix']: + pluggable_rbs_storage = PluggableRuleBasedSegmentsStorage(self.mock_adapter, prefix=sprefix) + rbs1 = rule_based_segments.from_raw(rbsegments_json[0]) + rbs2_temp = copy.deepcopy(rbsegments_json[0]) + rbs2_temp['name'] = 'another_segment' + rbs2 = rule_based_segments.from_raw(rbs2_temp) + self.mock_adapter.set(pluggable_rbs_storage._prefix.format(segment_name=rbs1.name), rbs1.to_json()) + self.mock_adapter.set(pluggable_rbs_storage._prefix.format(segment_name=rbs2.name), rbs2.to_json()) + assert(pluggable_rbs_storage.get_segment_names() == [rbs1.name, rbs2.name]) + + def test_contains(self): + self.mock_adapter._keys = {} + for sprefix in [None, 'myprefix']: + pluggable_rbs_storage = PluggableRuleBasedSegmentsStorage(self.mock_adapter, prefix=sprefix) + rbs1 = rule_based_segments.from_raw(rbsegments_json[0]) + rbs2_temp = copy.deepcopy(rbsegments_json[0]) + rbs2_temp['name'] = 'another_segment' + rbs2 = rule_based_segments.from_raw(rbs2_temp) + self.mock_adapter.set(pluggable_rbs_storage._prefix.format(segment_name=rbs1.name), rbs1.to_json()) + self.mock_adapter.set(pluggable_rbs_storage._prefix.format(segment_name=rbs2.name), rbs2.to_json()) + + assert(pluggable_rbs_storage.contains([rbs1.name, rbs2.name])) + assert(pluggable_rbs_storage.contains([rbs2.name])) + assert(not pluggable_rbs_storage.contains(['none-exists', rbs2.name])) + assert(not pluggable_rbs_storage.contains(['none-exists', 'none-exists2'])) + +class PluggableRuleBasedSegmentStorageAsyncTests(object): + """In memory rule based segment storage test cases.""" + + def setup_method(self): + """Prepare storages with test data.""" + self.mock_adapter = StorageMockAdapterAsync() + + @pytest.mark.asyncio + async def test_get(self): + self.mock_adapter._keys = {} + for sprefix in [None, 'myprefix']: + pluggable_rbs_storage = PluggableRuleBasedSegmentsStorageAsync(self.mock_adapter, prefix=sprefix) + + rbs1 = rule_based_segments.from_raw(rbsegments_json[0]) + rbs_name = rbsegments_json[0]['name'] + + await self.mock_adapter.set(pluggable_rbs_storage._prefix.format(segment_name=rbs_name), rbs1.to_json()) + rbs = await pluggable_rbs_storage.get(rbs_name) + assert(rbs.to_json() == rule_based_segments.from_raw(rbsegments_json[0]).to_json()) + assert(await pluggable_rbs_storage.get('not_existing') == None) + + @pytest.mark.asyncio + async def test_get_change_number(self): + self.mock_adapter._keys = {} + for sprefix in [None, 'myprefix']: + pluggable_rbs_storage = PluggableRuleBasedSegmentsStorageAsync(self.mock_adapter, prefix=sprefix) + if sprefix == 'myprefix': + prefix = 'myprefix.' + else: + prefix = '' + await self.mock_adapter.set(prefix + "SPLITIO.rbsegments.till", 1234) + assert(await pluggable_rbs_storage.get_change_number() == 1234) + + @pytest.mark.asyncio + async def test_get_segment_names(self): + self.mock_adapter._keys = {} + for sprefix in [None, 'myprefix']: + pluggable_rbs_storage = PluggableRuleBasedSegmentsStorageAsync(self.mock_adapter, prefix=sprefix) + rbs1 = rule_based_segments.from_raw(rbsegments_json[0]) + rbs2_temp = copy.deepcopy(rbsegments_json[0]) + rbs2_temp['name'] = 'another_segment' + rbs2 = rule_based_segments.from_raw(rbs2_temp) + await self.mock_adapter.set(pluggable_rbs_storage._prefix.format(segment_name=rbs1.name), rbs1.to_json()) + await self.mock_adapter.set(pluggable_rbs_storage._prefix.format(segment_name=rbs2.name), rbs2.to_json()) + assert(await pluggable_rbs_storage.get_segment_names() == [rbs1.name, rbs2.name]) + + @pytest.mark.asyncio + async def test_contains(self): + self.mock_adapter._keys = {} + for sprefix in [None, 'myprefix']: + pluggable_rbs_storage = PluggableRuleBasedSegmentsStorageAsync(self.mock_adapter, prefix=sprefix) + rbs1 = rule_based_segments.from_raw(rbsegments_json[0]) + rbs2_temp = copy.deepcopy(rbsegments_json[0]) + rbs2_temp['name'] = 'another_segment' + rbs2 = rule_based_segments.from_raw(rbs2_temp) + await self.mock_adapter.set(pluggable_rbs_storage._prefix.format(segment_name=rbs1.name), rbs1.to_json()) + await self.mock_adapter.set(pluggable_rbs_storage._prefix.format(segment_name=rbs2.name), rbs2.to_json()) + + assert(await pluggable_rbs_storage.contains([rbs1.name, rbs2.name])) + assert(await pluggable_rbs_storage.contains([rbs2.name])) + assert(not await pluggable_rbs_storage.contains(['none-exists', rbs2.name])) + assert(not await pluggable_rbs_storage.contains(['none-exists', 'none-exists2'])) diff --git a/tests/storage/test_redis.py b/tests/storage/test_redis.py index cce9a43d..4537998c 100644 --- a/tests/storage/test_redis.py +++ b/tests/storage/test_redis.py @@ -12,7 +12,8 @@ from splitio.optional.loaders import asyncio from splitio.storage import FlagSetsFilter from splitio.storage.redis import RedisEventsStorage, RedisEventsStorageAsync, RedisImpressionsStorage, RedisImpressionsStorageAsync, \ - RedisSegmentStorage, RedisSegmentStorageAsync, RedisSplitStorage, RedisSplitStorageAsync, RedisTelemetryStorage, RedisTelemetryStorageAsync + RedisSegmentStorage, RedisSegmentStorageAsync, RedisSplitStorage, RedisSplitStorageAsync, RedisTelemetryStorage, RedisTelemetryStorageAsync, \ + RedisRuleBasedSegmentsStorage, RedisRuleBasedSegmentsStorageAsync from splitio.storage.adapters.redis import RedisAdapter, RedisAdapterException, build from redis.asyncio.client import Redis as aioredis from splitio.storage.adapters import redis @@ -1230,3 +1231,204 @@ async def expire(*args): await redis_telemetry.expire_keys('key', 12, 2, 2) assert(self.called) + +class RedisRuleBasedSegmentStorageTests(object): + """Redis rule based segment storage test cases.""" + + def test_get_segment(self, mocker): + """Test retrieving a rule based segment works.""" + adapter = mocker.Mock(spec=RedisAdapter) + adapter.get.return_value = '{"name": "some_segment"}' + from_raw = mocker.Mock() + mocker.patch('splitio.storage.redis.rule_based_segments.from_raw', new=from_raw) + + storage = RedisRuleBasedSegmentsStorage(adapter) + storage.get('some_segment') + + assert adapter.get.mock_calls == [mocker.call('SPLITIO.rbsegment.some_segment')] + assert from_raw.mock_calls == [mocker.call({"name": "some_segment"})] + + # Test that a missing split returns None and doesn't call from_raw + adapter.reset_mock() + from_raw.reset_mock() + adapter.get.return_value = None + result = storage.get('some_segment') + assert result is None + assert adapter.get.mock_calls == [mocker.call('SPLITIO.rbsegment.some_segment')] + assert not from_raw.mock_calls + + def test_get_changenumber(self, mocker): + """Test fetching changenumber.""" + adapter = mocker.Mock(spec=RedisAdapter) + storage = RedisRuleBasedSegmentsStorage(adapter) + adapter.get.return_value = '-1' + assert storage.get_change_number() == -1 + assert adapter.get.mock_calls == [mocker.call('SPLITIO.rbsegments.till')] + + def test_get_segment_names(self, mocker): + """Test getching rule based segment names.""" + adapter = mocker.Mock(spec=RedisAdapter) + storage = RedisRuleBasedSegmentsStorage(adapter) + adapter.keys.return_value = [ + 'SPLITIO.rbsegment.segment1', + 'SPLITIO.rbsegment.segment2', + 'SPLITIO.rbsegment.segment3' + ] + assert storage.get_segment_names() == ['segment1', 'segment2', 'segment3'] + + def test_contains(self, mocker): + """Test storage containing rule based segment names.""" + adapter = mocker.Mock(spec=RedisAdapter) + storage = RedisRuleBasedSegmentsStorage(adapter) + adapter.keys.return_value = [ + 'SPLITIO.rbsegment.segment1', + 'SPLITIO.rbsegment.segment2', + 'SPLITIO.rbsegment.segment3' + ] + assert storage.contains(['segment1', 'segment3']) + assert not storage.contains(['segment1', 'segment4']) + assert storage.contains(['segment1']) + assert not storage.contains(['segment4', 'segment5']) + + def test_fetch_many(self, mocker): + """Test retrieving a list of passed splits.""" + adapter = mocker.Mock(spec=RedisAdapter) + storage = RedisRuleBasedSegmentsStorage(adapter) + from_raw = mocker.Mock() + mocker.patch('splitio.storage.redis.rule_based_segments.from_raw', new=from_raw) + + adapter.mget.return_value = ['{"name": "rbs1"}', '{"name": "rbs2"}', None] + + result = storage.fetch_many(['rbs1', 'rbs2', 'rbs3']) + assert len(result) == 3 + + assert mocker.call({'name': 'rbs1'}) in from_raw.mock_calls + assert mocker.call({'name': 'rbs2'}) in from_raw.mock_calls + + assert result['rbs1'] is not None + assert result['rbs2'] is not None + assert 'rbs3' in result + +class RedisRuleBasedSegmentStorageAsyncTests(object): + """Redis rule based segment storage test cases.""" + + @pytest.mark.asyncio + async def test_get_segment(self, mocker): + """Test retrieving a rule based segment works.""" + redis_mock = await aioredis.from_url("redis://localhost") + adapter = redis.RedisAdapterAsync(redis_mock, 'some_prefix') + + self.redis_ret = None + self.name = None + async def get(sel, name): + self.name = name + self.redis_ret = '{"changeNumber": "12", "name": "some_segment", "status": "ACTIVE","trafficTypeName": "user","excluded":{"keys":[],"segments":[]},"conditions": []}' + return self.redis_ret + mocker.patch('splitio.storage.adapters.redis.RedisAdapterAsync.get', new=get) + + storage = RedisRuleBasedSegmentsStorageAsync(adapter) + await storage.get('some_segment') + + assert self.name == 'SPLITIO.rbsegment.some_segment' + assert self.redis_ret == '{"changeNumber": "12", "name": "some_segment", "status": "ACTIVE","trafficTypeName": "user","excluded":{"keys":[],"segments":[]},"conditions": []}' + + # Test that a missing split returns None and doesn't call from_raw + + self.name = None + async def get2(sel, name): + self.name = name + return None + mocker.patch('splitio.storage.adapters.redis.RedisAdapterAsync.get', new=get2) + + result = await storage.get('some_segment') + assert result is None + assert self.name == 'SPLITIO.rbsegment.some_segment' + + # Test that a missing split returns None and doesn't call from_raw + result = await storage.get('some_segment2') + assert result is None + + @pytest.mark.asyncio + async def test_get_changenumber(self, mocker): + """Test fetching changenumber.""" + redis_mock = await aioredis.from_url("redis://localhost") + adapter = redis.RedisAdapterAsync(redis_mock, 'some_prefix') + storage = RedisRuleBasedSegmentsStorageAsync(adapter) + + self.redis_ret = None + self.name = None + async def get(sel, name): + self.name = name + self.redis_ret = '-1' + return self.redis_ret + mocker.patch('splitio.storage.adapters.redis.RedisAdapterAsync.get', new=get) + + assert await storage.get_change_number() == -1 + assert self.name == 'SPLITIO.rbsegments.till' + + @pytest.mark.asyncio + async def test_get_segment_names(self, mocker): + """Test getching rule based segment names.""" + redis_mock = await aioredis.from_url("redis://localhost") + adapter = redis.RedisAdapterAsync(redis_mock, 'some_prefix') + storage = RedisRuleBasedSegmentsStorageAsync(adapter) + + self.key = None + self.keys_ret = None + async def keys(sel, key): + self.key = key + self.keys_ret = [ + 'SPLITIO.rbsegment.segment1', + 'SPLITIO.rbsegment.segment2', + 'SPLITIO.rbsegment.segment3' + ] + return self.keys_ret + mocker.patch('splitio.storage.adapters.redis.RedisAdapterAsync.keys', new=keys) + + assert await storage.get_segment_names() == ['segment1', 'segment2', 'segment3'] + + @pytest.mark.asyncio + async def test_contains(self, mocker): + """Test storage containing rule based segment names.""" + redis_mock = await aioredis.from_url("redis://localhost") + adapter = redis.RedisAdapterAsync(redis_mock, 'some_prefix') + storage = RedisRuleBasedSegmentsStorageAsync(adapter) + + self.key = None + self.keys_ret = None + async def keys(sel, key): + self.key = key + self.keys_ret = [ + 'SPLITIO.rbsegment.segment1', + 'SPLITIO.rbsegment.segment2', + 'SPLITIO.rbsegment.segment3' + ] + return self.keys_ret + mocker.patch('splitio.storage.adapters.redis.RedisAdapterAsync.keys', new=keys) + + assert await storage.contains(['segment1', 'segment3']) + assert not await storage.contains(['segment1', 'segment4']) + assert await storage.contains(['segment1']) + assert not await storage.contains(['segment4', 'segment5']) + + @pytest.mark.asyncio + async def test_fetch_many(self, mocker): + """Test retrieving a list of passed splits.""" + adapter = mocker.Mock(spec=RedisAdapter) + storage = RedisRuleBasedSegmentsStorageAsync(adapter) + from_raw = mocker.Mock() + mocker.patch('splitio.storage.redis.rule_based_segments.from_raw', new=from_raw) + async def mget(*_): + return ['{"name": "rbs1"}', '{"name": "rbs2"}', None] + adapter.mget = mget + + result = await storage.fetch_many(['rbs1', 'rbs2', 'rbs3']) + assert len(result) == 3 + + assert mocker.call({'name': 'rbs1'}) in from_raw.mock_calls + assert mocker.call({'name': 'rbs2'}) in from_raw.mock_calls + + assert result['rbs1'] is not None + assert result['rbs2'] is not None + assert 'rbs3' in result + diff --git a/tests/sync/test_manager.py b/tests/sync/test_manager.py index b99c63a8..47ac3f01 100644 --- a/tests/sync/test_manager.py +++ b/tests/sync/test_manager.py @@ -24,7 +24,7 @@ from splitio.sync.event import EventSynchronizer from splitio.sync.synchronizer import Synchronizer, SynchronizerAsync, SplitTasks, SplitSynchronizers, RedisSynchronizer, RedisSynchronizerAsync from splitio.sync.manager import Manager, ManagerAsync, RedisManager, RedisManagerAsync -from splitio.storage import SplitStorage +from splitio.storage import SplitStorage, RuleBasedSegmentsStorage from splitio.api import APIException from splitio.client.util import SdkMetadata @@ -38,6 +38,7 @@ def test_error(self, mocker): mocker.Mock(), mocker.Mock()) storage = mocker.Mock(spec=SplitStorage) + rb_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) api = mocker.Mock() def run(x): @@ -46,7 +47,7 @@ def run(x): api.fetch_splits.side_effect = run storage.get_change_number.return_value = -1 - split_sync = SplitSynchronizer(api, storage) + split_sync = SplitSynchronizer(api, storage, rb_storage) synchronizers = SplitSynchronizers(split_sync, mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock()) @@ -102,6 +103,7 @@ async def test_error(self, mocker): mocker.Mock(), mocker.Mock()) storage = mocker.Mock(spec=SplitStorage) + rb_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) api = mocker.Mock() async def run(x): @@ -112,7 +114,7 @@ async def get_change_number(): return -1 storage.get_change_number = get_change_number - split_sync = SplitSynchronizerAsync(api, storage) + split_sync = SplitSynchronizerAsync(api, storage, rb_storage) synchronizers = SplitSynchronizers(split_sync, mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock()) diff --git a/tests/sync/test_segments_synchronizer.py b/tests/sync/test_segments_synchronizer.py index 6e8f7f78..e88db2fa 100644 --- a/tests/sync/test_segments_synchronizer.py +++ b/tests/sync/test_segments_synchronizer.py @@ -5,10 +5,11 @@ from splitio.util.backoff import Backoff from splitio.api import APIException from splitio.api.commons import FetchOptions -from splitio.storage import SplitStorage, SegmentStorage +from splitio.storage import SplitStorage, SegmentStorage, RuleBasedSegmentsStorage from splitio.storage.inmemmory import InMemorySegmentStorage, InMemorySegmentStorageAsync, InMemorySplitStorage, InMemorySplitStorageAsync from splitio.sync.segment import SegmentSynchronizer, SegmentSynchronizerAsync, LocalSegmentSynchronizer, LocalSegmentSynchronizerAsync from splitio.models.segments import Segment +from splitio.models import rule_based_segments from splitio.optional.loaders import aiofiles, asyncio import pytest @@ -23,6 +24,8 @@ def test_synchronize_segments_error(self, mocker): storage = mocker.Mock(spec=SegmentStorage) storage.get_change_number.return_value = -1 + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + rbs_storage.get_segment_names.return_value = [] api = mocker.Mock() @@ -30,7 +33,7 @@ def run(x): raise APIException("something broke") api.fetch_segment.side_effect = run - segments_synchronizer = SegmentSynchronizer(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizer(api, split_storage, storage, rbs_storage) assert not segments_synchronizer.synchronize_segments() def test_synchronize_segments(self, mocker): @@ -38,6 +41,10 @@ def test_synchronize_segments(self, mocker): split_storage = mocker.Mock(spec=SplitStorage) split_storage.get_segment_names.return_value = ['segmentA', 'segmentB', 'segmentC'] + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + rbs_storage.get_segment_names.return_value = ['rbs'] + rbs_storage.get.return_value = rule_based_segments.from_raw({'name': 'rbs', 'conditions': [], 'trafficTypeName': 'user', 'changeNumber': 123, 'status': 'ACTIVE', 'excluded': {'keys': [], 'segments': [{'type': 'standard', 'name': 'segmentD'}]}}) + # Setup a mocked segment storage whose changenumber returns -1 on first fetch and # 123 afterwards. storage = mocker.Mock(spec=SegmentStorage) @@ -52,10 +59,14 @@ def change_number_mock(segment_name): if segment_name == 'segmentC' and change_number_mock._count_c == 0: change_number_mock._count_c = 1 return -1 + if segment_name == 'segmentD' and change_number_mock._count_d == 0: + change_number_mock._count_d = 1 + return -1 return 123 change_number_mock._count_a = 0 change_number_mock._count_b = 0 change_number_mock._count_c = 0 + change_number_mock._count_d = 0 storage.get_change_number.side_effect = change_number_mock # Setup a mocked segment api to return segments mentioned before. @@ -72,27 +83,35 @@ def fetch_segment_mock(segment_name, change_number, fetch_options): fetch_segment_mock._count_c = 1 return {'name': 'segmentC', 'added': ['key7', 'key8', 'key9'], 'removed': [], 'since': -1, 'till': 123} + if segment_name == 'segmentD' and fetch_segment_mock._count_d == 0: + fetch_segment_mock._count_d = 1 + return {'name': 'segmentD', 'added': ['key10'], 'removed': [], + 'since': -1, 'till': 123} return {'added': [], 'removed': [], 'since': 123, 'till': 123} fetch_segment_mock._count_a = 0 fetch_segment_mock._count_b = 0 fetch_segment_mock._count_c = 0 + fetch_segment_mock._count_d = 0 api = mocker.Mock() api.fetch_segment.side_effect = fetch_segment_mock - segments_synchronizer = SegmentSynchronizer(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizer(api, split_storage, storage, rbs_storage) assert segments_synchronizer.synchronize_segments() api_calls = [call for call in api.fetch_segment.mock_calls] - assert mocker.call('segmentA', -1, FetchOptions(True, None, None, None)) in api_calls - assert mocker.call('segmentB', -1, FetchOptions(True, None, None, None)) in api_calls - assert mocker.call('segmentC', -1, FetchOptions(True, None, None, None)) in api_calls - assert mocker.call('segmentA', 123, FetchOptions(True, None, None, None)) in api_calls - assert mocker.call('segmentB', 123, FetchOptions(True, None, None, None)) in api_calls - assert mocker.call('segmentC', 123, FetchOptions(True, None, None, None)) in api_calls + + assert mocker.call('segmentA', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert mocker.call('segmentB', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert mocker.call('segmentC', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert mocker.call('segmentD', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert mocker.call('segmentA', 123, FetchOptions(True, None, None, None, None)) in api_calls + assert mocker.call('segmentB', 123, FetchOptions(True, None, None, None, None)) in api_calls + assert mocker.call('segmentC', 123, FetchOptions(True, None, None, None, None)) in api_calls + assert mocker.call('segmentD', 123, FetchOptions(True, None, None, None, None)) in api_calls segment_put_calls = storage.put.mock_calls - segments_to_validate = set(['segmentA', 'segmentB', 'segmentC']) + segments_to_validate = set(['segmentA', 'segmentB', 'segmentC', 'segmentD']) for call in segment_put_calls: _, positional_args, _ = call segment = positional_args[0] @@ -104,6 +123,8 @@ def test_synchronize_segment(self, mocker): """Test particular segment update.""" split_storage = mocker.Mock(spec=SplitStorage) storage = mocker.Mock(spec=SegmentStorage) + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + rbs_storage.get_segment_names.return_value = [] def change_number_mock(segment_name): if change_number_mock._count_a == 0: @@ -124,12 +145,12 @@ def fetch_segment_mock(segment_name, change_number, fetch_options): api = mocker.Mock() api.fetch_segment.side_effect = fetch_segment_mock - segments_synchronizer = SegmentSynchronizer(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizer(api, split_storage, storage, rbs_storage) segments_synchronizer.synchronize_segment('segmentA') api_calls = [call for call in api.fetch_segment.mock_calls] - assert mocker.call('segmentA', -1, FetchOptions(True, None, None, None)) in api_calls - assert mocker.call('segmentA', 123, FetchOptions(True, None, None, None)) in api_calls + assert mocker.call('segmentA', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert mocker.call('segmentA', 123, FetchOptions(True, None, None, None, None)) in api_calls def test_synchronize_segment_cdn(self, mocker): """Test particular segment update cdn bypass.""" @@ -137,6 +158,8 @@ def test_synchronize_segment_cdn(self, mocker): split_storage = mocker.Mock(spec=SplitStorage) storage = mocker.Mock(spec=SegmentStorage) + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + rbs_storage.get_segment_names.return_value = [] def change_number_mock(segment_name): change_number_mock._count_a += 1 @@ -170,20 +193,20 @@ def fetch_segment_mock(segment_name, change_number, fetch_options): api = mocker.Mock() api.fetch_segment.side_effect = fetch_segment_mock - segments_synchronizer = SegmentSynchronizer(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizer(api, split_storage, storage, rbs_storage) segments_synchronizer.synchronize_segment('segmentA') - assert mocker.call('segmentA', -1, FetchOptions(True, None, None, None)) in api.fetch_segment.mock_calls - assert mocker.call('segmentA', 123, FetchOptions(True, None, None, None)) in api.fetch_segment.mock_calls + assert mocker.call('segmentA', -1, FetchOptions(True, None, None, None, None)) in api.fetch_segment.mock_calls + assert mocker.call('segmentA', 123, FetchOptions(True, None, None, None, None)) in api.fetch_segment.mock_calls segments_synchronizer._backoff = Backoff(1, 0.1) segments_synchronizer.synchronize_segment('segmentA', 12345) - assert mocker.call('segmentA', 12345, FetchOptions(True, 1234, None, None)) in api.fetch_segment.mock_calls + assert mocker.call('segmentA', 12345, FetchOptions(True, 1234, None, None, None)) in api.fetch_segment.mock_calls assert len(api.fetch_segment.mock_calls) == 8 # 2 ok + BACKOFF(2 since==till + 2 re-attempts) + CDN(2 since==till) def test_recreate(self, mocker): """Test recreate logic.""" - segments_synchronizer = SegmentSynchronizer(mocker.Mock(), mocker.Mock(), mocker.Mock()) + segments_synchronizer = SegmentSynchronizer(mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock()) current_pool = segments_synchronizer._worker_pool segments_synchronizer.recreate() assert segments_synchronizer._worker_pool != current_pool @@ -196,6 +219,11 @@ class SegmentsSynchronizerAsyncTests(object): async def test_synchronize_segments_error(self, mocker): """On error.""" split_storage = mocker.Mock(spec=SplitStorage) + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + + async def get_segment_names_rbs(): + return [] + rbs_storage.get_segment_names = get_segment_names_rbs async def get_segment_names(): return ['segmentA', 'segmentB', 'segmentC'] @@ -215,7 +243,7 @@ async def run(*args): raise APIException("something broke") api.fetch_segment = run - segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage, rbs_storage) assert not await segments_synchronizer.synchronize_segments() await segments_synchronizer.shutdown() @@ -227,6 +255,15 @@ async def get_segment_names(): return ['segmentA', 'segmentB', 'segmentC'] split_storage.get_segment_names = get_segment_names + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + async def get_segment_names_rbs(): + return ['rbs'] + rbs_storage.get_segment_names = get_segment_names_rbs + + async def get_rbs(segment_name): + return rule_based_segments.from_raw({'name': 'rbs', 'conditions': [], 'trafficTypeName': 'user', 'changeNumber': 123, 'status': 'ACTIVE', 'excluded': {'keys': [], 'segments': [{'type': 'standard', 'name': 'segmentD'}]}}) + rbs_storage.get = get_rbs + # Setup a mocked segment storage whose changenumber returns -1 on first fetch and # 123 afterwards. storage = mocker.Mock(spec=SegmentStorage) @@ -241,10 +278,14 @@ async def change_number_mock(segment_name): if segment_name == 'segmentC' and change_number_mock._count_c == 0: change_number_mock._count_c = 1 return -1 + if segment_name == 'segmentD' and change_number_mock._count_d == 0: + change_number_mock._count_d = 1 + return -1 return 123 change_number_mock._count_a = 0 change_number_mock._count_b = 0 change_number_mock._count_c = 0 + change_number_mock._count_d = 0 storage.get_change_number = change_number_mock self.segment_put = [] @@ -276,25 +317,36 @@ async def fetch_segment_mock(segment_name, change_number, fetch_options): fetch_segment_mock._count_c = 1 return {'name': 'segmentC', 'added': ['key7', 'key8', 'key9'], 'removed': [], 'since': -1, 'till': 123} + if segment_name == 'segmentD' and fetch_segment_mock._count_d == 0: + fetch_segment_mock._count_d = 1 + return {'name': 'segmentD', 'added': ['key10'], 'removed': [], + 'since': -1, 'till': 123} return {'added': [], 'removed': [], 'since': 123, 'till': 123} fetch_segment_mock._count_a = 0 fetch_segment_mock._count_b = 0 fetch_segment_mock._count_c = 0 + fetch_segment_mock._count_d = 0 api = mocker.Mock() api.fetch_segment = fetch_segment_mock - segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage, rbs_storage) assert await segments_synchronizer.synchronize_segments() - assert (self.segment[0], self.change[0], self.options[0]) == ('segmentA', -1, FetchOptions(True, None, None, None)) - assert (self.segment[1], self.change[1], self.options[1]) == ('segmentA', 123, FetchOptions(True, None, None, None)) - assert (self.segment[2], self.change[2], self.options[2]) == ('segmentB', -1, FetchOptions(True, None, None, None)) - assert (self.segment[3], self.change[3], self.options[3]) == ('segmentB', 123, FetchOptions(True, None, None, None)) - assert (self.segment[4], self.change[4], self.options[4]) == ('segmentC', -1, FetchOptions(True, None, None, None)) - assert (self.segment[5], self.change[5], self.options[5]) == ('segmentC', 123, FetchOptions(True, None, None, None)) - - segments_to_validate = set(['segmentA', 'segmentB', 'segmentC']) + api_calls = [] + for i in range(8): + api_calls.append((self.segment[i], self.change[i], self.options[i])) + + assert ('segmentD', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentD', 123, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentA', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentA', 123, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentB', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentB', 123, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentC', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentC', 123, FetchOptions(True, None, None, None, None)) in api_calls + + segments_to_validate = set(['segmentA', 'segmentB', 'segmentC', 'segmentD']) for segment in self.segment_put: assert isinstance(segment, Segment) assert segment.name in segments_to_validate @@ -307,6 +359,11 @@ async def test_synchronize_segment(self, mocker): """Test particular segment update.""" split_storage = mocker.Mock(spec=SplitStorage) storage = mocker.Mock(spec=SegmentStorage) + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + + async def get_segment_names_rbs(): + return [] + rbs_storage.get_segment_names = get_segment_names_rbs async def change_number_mock(segment_name): if change_number_mock._count_a == 0: @@ -340,11 +397,11 @@ async def fetch_segment_mock(segment_name, change_number, fetch_options): api = mocker.Mock() api.fetch_segment = fetch_segment_mock - segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage, rbs_storage) await segments_synchronizer.synchronize_segment('segmentA') - assert (self.segment[0], self.change[0], self.options[0]) == ('segmentA', -1, FetchOptions(True, None, None, None)) - assert (self.segment[1], self.change[1], self.options[1]) == ('segmentA', 123, FetchOptions(True, None, None, None)) + assert (self.segment[0], self.change[0], self.options[0]) == ('segmentA', -1, FetchOptions(True, None, None, None, None)) + assert (self.segment[1], self.change[1], self.options[1]) == ('segmentA', 123, FetchOptions(True, None, None, None, None)) await segments_synchronizer.shutdown() @@ -355,6 +412,11 @@ async def test_synchronize_segment_cdn(self, mocker): split_storage = mocker.Mock(spec=SplitStorage) storage = mocker.Mock(spec=SegmentStorage) + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + + async def get_segment_names_rbs(): + return [] + rbs_storage.get_segment_names = get_segment_names_rbs async def change_number_mock(segment_name): change_number_mock._count_a += 1 @@ -400,22 +462,22 @@ async def fetch_segment_mock(segment_name, change_number, fetch_options): api = mocker.Mock() api.fetch_segment = fetch_segment_mock - segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage, rbs_storage) await segments_synchronizer.synchronize_segment('segmentA') - assert (self.segment[0], self.change[0], self.options[0]) == ('segmentA', -1, FetchOptions(True, None, None, None)) - assert (self.segment[1], self.change[1], self.options[1]) == ('segmentA', 123, FetchOptions(True, None, None, None)) + assert (self.segment[0], self.change[0], self.options[0]) == ('segmentA', -1, FetchOptions(True, None, None, None, None)) + assert (self.segment[1], self.change[1], self.options[1]) == ('segmentA', 123, FetchOptions(True, None, None, None, None)) segments_synchronizer._backoff = Backoff(1, 0.1) await segments_synchronizer.synchronize_segment('segmentA', 12345) - assert (self.segment[7], self.change[7], self.options[7]) == ('segmentA', 12345, FetchOptions(True, 1234, None, None)) + assert (self.segment[7], self.change[7], self.options[7]) == ('segmentA', 12345, FetchOptions(True, 1234, None, None, None)) assert len(self.segment) == 8 # 2 ok + BACKOFF(2 since==till + 2 re-attempts) + CDN(2 since==till) await segments_synchronizer.shutdown() @pytest.mark.asyncio async def test_recreate(self, mocker): """Test recreate logic.""" - segments_synchronizer = SegmentSynchronizerAsync(mocker.Mock(), mocker.Mock(), mocker.Mock()) + segments_synchronizer = SegmentSynchronizerAsync(mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock()) current_pool = segments_synchronizer._worker_pool await segments_synchronizer.shutdown() segments_synchronizer.recreate() diff --git a/tests/sync/test_splits_synchronizer.py b/tests/sync/test_splits_synchronizer.py index b5aafd51..fd9ac585 100644 --- a/tests/sync/test_splits_synchronizer.py +++ b/tests/sync/test_splits_synchronizer.py @@ -8,13 +8,14 @@ from splitio.util.backoff import Backoff from splitio.api import APIException from splitio.api.commons import FetchOptions -from splitio.storage import SplitStorage -from splitio.storage.inmemmory import InMemorySplitStorage, InMemorySplitStorageAsync +from splitio.storage import SplitStorage, RuleBasedSegmentsStorage +from splitio.storage.inmemmory import InMemorySplitStorage, InMemorySplitStorageAsync, InMemoryRuleBasedSegmentStorage, InMemoryRuleBasedSegmentStorageAsync from splitio.storage import FlagSetsFilter from splitio.models.splits import Split +from splitio.models.rule_based_segments import RuleBasedSegment from splitio.sync.split import SplitSynchronizer, SplitSynchronizerAsync, LocalSplitSynchronizer, LocalSplitSynchronizerAsync, LocalhostMode from splitio.optional.loaders import aiofiles, asyncio -from tests.integration import splits_json +from tests.integration import splits_json, rbsegments_json splits_raw = [{ 'changeNumber': 123, @@ -52,42 +53,112 @@ 'sets': ['set1', 'set2'] }] -json_body = {'splits': [{ - 'changeNumber': 123, - 'trafficTypeName': 'user', - 'name': 'some_name', - 'trafficAllocation': 100, - 'trafficAllocationSeed': 123456, - 'seed': 321654, - 'status': 'ACTIVE', - 'killed': False, - 'defaultTreatment': 'off', - 'algo': 2, - 'conditions': [ - { - 'partitions': [ - {'treatment': 'on', 'size': 50}, - {'treatment': 'off', 'size': 50} - ], - 'contitionType': 'WHITELIST', - 'label': 'some_label', - 'matcherGroup': { - 'matchers': [ - { - 'matcherType': 'WHITELIST', - 'whitelistMatcherData': { - 'whitelist': ['k1', 'k2', 'k3'] - }, - 'negate': False, - } +json_body = { + "ff": { + "t":1675095324253, + "s":-1, + 'd': [{ + 'changeNumber': 123, + 'trafficTypeName': 'user', + 'name': 'some_name', + 'trafficAllocation': 100, + 'trafficAllocationSeed': 123456, + 'seed': 321654, + 'status': 'ACTIVE', + 'killed': False, + 'defaultTreatment': 'off', + 'algo': 2, + 'conditions': [ + { + 'partitions': [ + {'treatment': 'on', 'size': 50}, + {'treatment': 'off', 'size': 50} ], - 'combiner': 'AND' + 'contitionType': 'WHITELIST', + 'label': 'some_label', + 'matcherGroup': { + 'matchers': [ + { + 'matcherType': 'WHITELIST', + 'whitelistMatcherData': { + 'whitelist': ['k1', 'k2', 'k3'] + }, + 'negate': False, + } + ], + 'combiner': 'AND' + } + }, + { + "conditionType": "ROLLOUT", + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "keySelector": { + "trafficType": "user" + }, + "matcherType": "IN_RULE_BASED_SEGMENT", + "negate": False, + "userDefinedSegmentMatcherData": { + "segmentName": "sample_rule_based_segment" + } + } + ] + }, + "partitions": [ + { + "treatment": "on", + "size": 100 + }, + { + "treatment": "off", + "size": 0 + } + ], + "label": "in rule based segment sample_rule_based_segment" + }, + ], + 'sets': ['set1', 'set2']}] + }, + "rbs": { + "t": 1675095324253, + "s": -1, + "d": [ + { + "changeNumber": 5, + "name": "sample_rule_based_segment", + "status": "ACTIVE", + "trafficTypeName": "user", + "excluded":{ + "keys":["mauro@split.io","gaston@split.io"], + "segments":[] + }, + "conditions": [ + { + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "keySelector": { + "trafficType": "user", + "attribute": "email" + }, + "matcherType": "ENDS_WITH", + "negate": False, + "whitelistMatcherData": { + "whitelist": [ + "@split.io" + ] + } + } + ] } - } - ], - 'sets': ['set1', 'set2']}], - "till":1675095324253, - "since":-1, + } + ] + } + ] + } } class SplitsSynchronizerTests(object): @@ -98,13 +169,16 @@ class SplitsSynchronizerTests(object): def test_synchronize_splits_error(self, mocker): """Test that if fetching splits fails at some_point, the task will continue running.""" storage = mocker.Mock(spec=InMemorySplitStorage) + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorage) api = mocker.Mock() - def run(x, c): + def run(x, y, c): raise APIException("something broke") run._calls = 0 api.fetch_splits.side_effect = run storage.get_change_number.return_value = -1 + rbs_storage.get_change_number.return_value = -1 + class flag_set_filter(): def should_filter(): return False @@ -115,7 +189,7 @@ def intersect(sets): storage.flag_set_filter.flag_sets = {} storage.flag_set_filter.sorted_flag_sets = [] - split_synchronizer = SplitSynchronizer(api, storage) + split_synchronizer = SplitSynchronizer(api, storage, rbs_storage) with pytest.raises(APIException): split_synchronizer.synchronize_splits(1) @@ -123,21 +197,32 @@ def intersect(sets): def test_synchronize_splits(self, mocker): """Test split sync.""" storage = mocker.Mock(spec=InMemorySplitStorage) + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorage) def change_number_mock(): change_number_mock._calls += 1 if change_number_mock._calls == 1: return -1 return 123 + + def rbs_change_number_mock(): + rbs_change_number_mock._calls += 1 + if rbs_change_number_mock._calls == 1: + return -1 + return 123 + change_number_mock._calls = 0 + rbs_change_number_mock._calls = 0 storage.get_change_number.side_effect = change_number_mock - + rbs_storage.get_change_number.side_effect = rbs_change_number_mock + class flag_set_filter(): def should_filter(): return False def intersect(sets): return True + storage.flag_set_filter = flag_set_filter storage.flag_set_filter.flag_sets = {} storage.flag_set_filter.sorted_flag_sets = [] @@ -147,35 +232,46 @@ def get_changes(*args, **kwargs): get_changes.called += 1 if get_changes.called == 1: - return { - 'splits': self.splits, - 'since': -1, - 'till': 123 - } + return json_body else: return { - 'splits': [], - 'since': 123, - 'till': 123 + "ff": { + "t":123, + "s":123, + 'd': [] + }, + "rbs": { + "t": 5, + "s": 5, + "d": [] + } } + get_changes.called = 0 api.fetch_splits.side_effect = get_changes - split_synchronizer = SplitSynchronizer(api, storage) + split_synchronizer = SplitSynchronizer(api, storage, rbs_storage) split_synchronizer.synchronize_splits() - + assert api.fetch_splits.mock_calls[0][1][0] == -1 - assert api.fetch_splits.mock_calls[0][1][1].cache_control_headers == True + assert api.fetch_splits.mock_calls[0][1][2].cache_control_headers == True assert api.fetch_splits.mock_calls[1][1][0] == 123 - assert api.fetch_splits.mock_calls[1][1][1].cache_control_headers == True + assert api.fetch_splits.mock_calls[1][1][1] == 123 + assert api.fetch_splits.mock_calls[1][1][2].cache_control_headers == True inserted_split = storage.update.mock_calls[0][1][0][0] assert isinstance(inserted_split, Split) assert inserted_split.name == 'some_name' + inserted_rbs = rbs_storage.update.mock_calls[0][1][0][0] + assert isinstance(inserted_rbs, RuleBasedSegment) + assert inserted_rbs.name == 'sample_rule_based_segment' + def test_not_called_on_till(self, mocker): """Test that sync is not called when till is less than previous changenumber""" storage = mocker.Mock(spec=InMemorySplitStorage) + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorage) + class flag_set_filter(): def should_filter(): return False @@ -189,6 +285,7 @@ def intersect(sets): def change_number_mock(): return 2 storage.get_change_number.side_effect = change_number_mock + rbs_storage.get_change_number.side_effect = change_number_mock def get_changes(*args, **kwargs): get_changes.called += 1 @@ -199,7 +296,7 @@ def get_changes(*args, **kwargs): api = mocker.Mock() api.fetch_splits.side_effect = get_changes - split_synchronizer = SplitSynchronizer(api, storage) + split_synchronizer = SplitSynchronizer(api, storage, rbs_storage) split_synchronizer.synchronize_splits(1) assert get_changes.called == 0 @@ -209,6 +306,7 @@ def test_synchronize_splits_cdn(self, mocker): mocker.patch('splitio.sync.split._ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES', new=3) storage = mocker.Mock(spec=InMemorySplitStorage) + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorage) def change_number_mock(): change_number_mock._calls += 1 @@ -219,24 +317,48 @@ def change_number_mock(): elif change_number_mock._calls <= 7: return 1234 return 12345 # Return proper cn for CDN Bypass + + def rbs_change_number_mock(): + rbs_change_number_mock._calls += 1 + if rbs_change_number_mock._calls == 1: + return -1 + elif change_number_mock._calls >= 2 and change_number_mock._calls <= 3: + return 555 + elif change_number_mock._calls <= 9: + return 555 + return 666 # Return proper cn for CDN Bypass + change_number_mock._calls = 0 + rbs_change_number_mock._calls = 0 storage.get_change_number.side_effect = change_number_mock + rbs_storage.get_change_number.side_effect = rbs_change_number_mock api = mocker.Mock() - + rbs_1 = copy.deepcopy(json_body['rbs']['d']) def get_changes(*args, **kwargs): get_changes.called += 1 if get_changes.called == 1: - return { 'splits': self.splits, 'since': -1, 'till': 123 } + return { 'ff': { 'd': self.splits, 's': -1, 't': 123 }, + 'rbs': {"t": 555, "s": -1, "d": rbs_1}} elif get_changes.called == 2: - return { 'splits': [], 'since': 123, 'till': 123 } + return { 'ff': { 'd': [], 's': 123, 't': 123 }, + 'rbs': {"t": 555, "s": 555, "d": []}} elif get_changes.called == 3: - return { 'splits': [], 'since': 123, 'till': 1234 } + return { 'ff': { 'd': [], 's': 123, 't': 1234 }, + 'rbs': {"t": 555, "s": 555, "d": []}} elif get_changes.called >= 4 and get_changes.called <= 6: - return { 'splits': [], 'since': 1234, 'till': 1234 } + return { 'ff': { 'd': [], 's': 1234, 't': 1234 }, + 'rbs': {"t": 555, "s": 555, "d": []}} elif get_changes.called == 7: - return { 'splits': [], 'since': 1234, 'till': 12345 } - return { 'splits': [], 'since': 12345, 'till': 12345 } + return { 'ff': { 'd': [], 's': 1234, 't': 12345 }, + 'rbs': {"t": 555, "s": 555, "d": []}} + elif get_changes.called == 8: + return { 'ff': { 'd': [], 's': 12345, 't': 12345 }, + 'rbs': {"t": 555, "s": 555, "d": []}} + rbs_1[0]['excluded']['keys'] = ['bilal@split.io'] + return { 'ff': { 'd': [], 's': 12345, 't': 12345 }, + 'rbs': {"t": 666, "s": 666, "d": rbs_1}} + get_changes.called = 0 api.fetch_splits.side_effect = get_changes @@ -251,53 +373,65 @@ def intersect(sets): storage.flag_set_filter.flag_sets = {} storage.flag_set_filter.sorted_flag_sets = [] - split_synchronizer = SplitSynchronizer(api, storage) + split_synchronizer = SplitSynchronizer(api, storage, rbs_storage) split_synchronizer._backoff = Backoff(1, 1) split_synchronizer.synchronize_splits() assert api.fetch_splits.mock_calls[0][1][0] == -1 - assert api.fetch_splits.mock_calls[0][1][1].cache_control_headers == True + assert api.fetch_splits.mock_calls[0][1][2].cache_control_headers == True assert api.fetch_splits.mock_calls[1][1][0] == 123 - assert api.fetch_splits.mock_calls[1][1][1].cache_control_headers == True + assert api.fetch_splits.mock_calls[1][1][2].cache_control_headers == True split_synchronizer._backoff = Backoff(1, 0.1) split_synchronizer.synchronize_splits(12345) assert api.fetch_splits.mock_calls[3][1][0] == 1234 - assert api.fetch_splits.mock_calls[3][1][1].cache_control_headers == True + assert api.fetch_splits.mock_calls[3][1][2].cache_control_headers == True assert len(api.fetch_splits.mock_calls) == 8 # 2 ok + BACKOFF(2 since==till + 2 re-attempts) + CDN(2 since==till) inserted_split = storage.update.mock_calls[0][1][0][0] assert isinstance(inserted_split, Split) assert inserted_split.name == 'some_name' + inserted_rbs = rbs_storage.update.mock_calls[0][1][0][0] + assert inserted_rbs.excluded.get_excluded_keys() == ["mauro@split.io","gaston@split.io"] + split_synchronizer._backoff = Backoff(1, 0.1) + split_synchronizer.synchronize_splits(None, 666) + inserted_rbs = rbs_storage.update.mock_calls[8][1][0][0] + assert inserted_rbs.excluded.get_excluded_keys() == ['bilal@split.io'] + def test_sync_flag_sets_with_config_sets(self, mocker): """Test split sync with flag sets.""" storage = InMemorySplitStorage(['set1', 'set2']) - - split = self.splits[0].copy() + rbs_storage = InMemoryRuleBasedSegmentStorage() + + split = copy.deepcopy(self.splits[0]) split['name'] = 'second' splits1 = [self.splits[0].copy(), split] - splits2 = self.splits.copy() - splits3 = self.splits.copy() - splits4 = self.splits.copy() + splits2 = copy.deepcopy(self.splits) + splits3 = copy.deepcopy(self.splits) + splits4 = copy.deepcopy(self.splits) api = mocker.Mock() def get_changes(*args, **kwargs): get_changes.called += 1 if get_changes.called == 1: - return { 'splits': splits1, 'since': 123, 'till': 123 } + return { 'ff': { 'd': splits1, 's': 123, 't': 123 }, + 'rbs': {'t': 123, 's': 123, 'd': []}} elif get_changes.called == 2: splits2[0]['sets'] = ['set3'] - return { 'splits': splits2, 'since': 124, 'till': 124 } + return { 'ff': { 'd': splits2, 's': 124, 't': 124 }, + 'rbs': {'t': 124, 's': 124, 'd': []}} elif get_changes.called == 3: splits3[0]['sets'] = ['set1'] - return { 'splits': splits3, 'since': 12434, 'till': 12434 } + return { 'ff': { 'd': splits3, 's': 12434, 't': 12434 }, + 'rbs': {'t': 12434, 's': 12434, 'd': []}} splits4[0]['sets'] = ['set6'] splits4[0]['name'] = 'new_split' - return { 'splits': splits4, 'since': 12438, 'till': 12438 } + return { 'ff': { 'd': splits4, 's': 12438, 't': 12438 }, + 'rbs': {'t': 12438, 's': 12438, 'd': []}} get_changes.called = 0 api.fetch_splits.side_effect = get_changes - split_synchronizer = SplitSynchronizer(api, storage) + split_synchronizer = SplitSynchronizer(api, storage, rbs_storage) split_synchronizer._backoff = Backoff(1, 1) split_synchronizer.synchronize_splits() assert isinstance(storage.get('some_name'), Split) @@ -314,40 +448,44 @@ def get_changes(*args, **kwargs): def test_sync_flag_sets_without_config_sets(self, mocker): """Test split sync with flag sets.""" storage = InMemorySplitStorage() - - split = self.splits[0].copy() + rbs_storage = InMemoryRuleBasedSegmentStorage() + split = copy.deepcopy(self.splits[0]) split['name'] = 'second' splits1 = [self.splits[0].copy(), split] - splits2 = self.splits.copy() - splits3 = self.splits.copy() - splits4 = self.splits.copy() + splits2 = copy.deepcopy(self.splits) + splits3 = copy.deepcopy(self.splits) + splits4 = copy.deepcopy(self.splits) api = mocker.Mock() def get_changes(*args, **kwargs): get_changes.called += 1 if get_changes.called == 1: - return { 'splits': splits1, 'since': 123, 'till': 123 } + return { 'ff': { 'd': splits1, 's': 123, 't': 123 }, + 'rbs': {"t": 123, "s": 123, "d": []}} elif get_changes.called == 2: splits2[0]['sets'] = ['set3'] - return { 'splits': splits2, 'since': 124, 'till': 124 } + return { 'ff': { 'd': splits2, 's': 124, 't': 124 }, + 'rbs': {"t": 124, "s": 124, "d": []}} elif get_changes.called == 3: splits3[0]['sets'] = ['set1'] - return { 'splits': splits3, 'since': 12434, 'till': 12434 } + return { 'ff': { 'd': splits3, 's': 12434, 't': 12434 }, + 'rbs': {"t": 12434, "s": 12434, "d": []}} splits4[0]['sets'] = ['set6'] splits4[0]['name'] = 'third_split' - return { 'splits': splits4, 'since': 12438, 'till': 12438 } + return { 'ff': { 'd': splits4, 's': 12438, 't': 12438 }, + 'rbs': {"t": 12438, "s": 12438, "d": []}} get_changes.called = 0 api.fetch_splits.side_effect = get_changes - split_synchronizer = SplitSynchronizer(api, storage) + split_synchronizer = SplitSynchronizer(api, storage, rbs_storage) split_synchronizer._backoff = Backoff(1, 1) split_synchronizer.synchronize_splits() - assert isinstance(storage.get('new_split'), Split) + assert isinstance(storage.get('some_name'), Split) split_synchronizer.synchronize_splits(124) - assert isinstance(storage.get('new_split'), Split) + assert isinstance(storage.get('some_name'), Split) split_synchronizer.synchronize_splits(12434) - assert isinstance(storage.get('new_split'), Split) + assert isinstance(storage.get('some_name'), Split) split_synchronizer.synchronize_splits(12438) assert isinstance(storage.get('third_split'), Split) @@ -361,17 +499,19 @@ class SplitsSynchronizerAsyncTests(object): async def test_synchronize_splits_error(self, mocker): """Test that if fetching splits fails at some_point, the task will continue running.""" storage = mocker.Mock(spec=InMemorySplitStorageAsync) + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorageAsync) api = mocker.Mock() - async def run(x, c): + async def run(x, y, c): raise APIException("something broke") run._calls = 0 api.fetch_splits = run async def get_change_number(*args): return -1 - storage.get_change_number = get_change_number - + storage.get_change_number = get_change_number + rbs_storage.get_change_number = get_change_number + class flag_set_filter(): def should_filter(): return False @@ -382,7 +522,7 @@ def intersect(sets): storage.flag_set_filter.flag_sets = {} storage.flag_set_filter.sorted_flag_sets = [] - split_synchronizer = SplitSynchronizerAsync(api, storage) + split_synchronizer = SplitSynchronizerAsync(api, storage, rbs_storage) with pytest.raises(APIException): await split_synchronizer.synchronize_splits(1) @@ -391,15 +531,24 @@ def intersect(sets): async def test_synchronize_splits(self, mocker): """Test split sync.""" storage = mocker.Mock(spec=InMemorySplitStorageAsync) - + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorageAsync) + async def change_number_mock(): change_number_mock._calls += 1 if change_number_mock._calls == 1: return -1 return 123 + async def rbs_change_number_mock(): + rbs_change_number_mock._calls += 1 + if rbs_change_number_mock._calls == 1: + return -1 + return 123 + change_number_mock._calls = 0 + rbs_change_number_mock._calls = 0 storage.get_change_number = change_number_mock - + rbs_storage.get_change_number.side_effect = rbs_change_number_mock + class flag_set_filter(): def should_filter(): return False @@ -416,33 +565,53 @@ async def update(parsed_split, deleted, chanhe_number): self.parsed_split = parsed_split storage.update = update + self.parsed_rbs = None + async def update(parsed_rbs, deleted, chanhe_number): + if len(parsed_rbs) > 0: + self.parsed_rbs = parsed_rbs + rbs_storage.update = update + + self.clear = False + async def clear(): + self.clear = True + storage.clear = clear + + self.clear2 = False + async def clear(): + self.clear2 = True + rbs_storage.clear = clear + api = mocker.Mock() self.change_number_1 = None self.fetch_options_1 = None self.change_number_2 = None self.fetch_options_2 = None - async def get_changes(change_number, fetch_options): + async def get_changes(change_number, rbs_change_number, fetch_options): get_changes.called += 1 if get_changes.called == 1: self.change_number_1 = change_number self.fetch_options_1 = fetch_options - return { - 'splits': self.splits, - 'since': -1, - 'till': 123 - } + return json_body else: self.change_number_2 = change_number self.fetch_options_2 = fetch_options return { - 'splits': [], - 'since': 123, - 'till': 123 + "ff": { + "t":123, + "s":123, + 'd': [] + }, + "rbs": { + "t": 123, + "s": 123, + "d": [] + } } get_changes.called = 0 api.fetch_splits = get_changes + api.clear_storage.return_value = False - split_synchronizer = SplitSynchronizerAsync(api, storage) + split_synchronizer = SplitSynchronizerAsync(api, storage, rbs_storage) await split_synchronizer.synchronize_splits() assert (-1, FetchOptions(True)._cache_control_headers) == (self.change_number_1, self.fetch_options_1._cache_control_headers) @@ -451,10 +620,17 @@ async def get_changes(change_number, fetch_options): assert isinstance(inserted_split, Split) assert inserted_split.name == 'some_name' + inserted_rbs = self.parsed_rbs[0] + assert isinstance(inserted_rbs, RuleBasedSegment) + assert inserted_rbs.name == 'sample_rule_based_segment' + + @pytest.mark.asyncio async def test_not_called_on_till(self, mocker): """Test that sync is not called when till is less than previous changenumber""" storage = mocker.Mock(spec=InMemorySplitStorageAsync) + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorageAsync) + class flag_set_filter(): def should_filter(): return False @@ -468,7 +644,8 @@ def intersect(sets): async def change_number_mock(): return 2 storage.get_change_number = change_number_mock - + rbs_storage.get_change_number.side_effect = change_number_mock + async def get_changes(*args, **kwargs): get_changes.called += 1 return None @@ -476,7 +653,7 @@ async def get_changes(*args, **kwargs): api = mocker.Mock() api.fetch_splits = get_changes - split_synchronizer = SplitSynchronizerAsync(api, storage) + split_synchronizer = SplitSynchronizerAsync(api, storage, rbs_storage) await split_synchronizer.synchronize_splits(1) assert get_changes.called == 0 @@ -485,7 +662,7 @@ async def test_synchronize_splits_cdn(self, mocker): """Test split sync with bypassing cdn.""" mocker.patch('splitio.sync.split._ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES', new=3) storage = mocker.Mock(spec=InMemorySplitStorageAsync) - + rbs_storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorageAsync) async def change_number_mock(): change_number_mock._calls += 1 if change_number_mock._calls == 1: @@ -495,15 +672,33 @@ async def change_number_mock(): elif change_number_mock._calls <= 7: return 1234 return 12345 # Return proper cn for CDN Bypass + async def rbs_change_number_mock(): + rbs_change_number_mock._calls += 1 + if rbs_change_number_mock._calls == 1: + return -1 + elif change_number_mock._calls >= 2 and change_number_mock._calls <= 3: + return 555 + elif change_number_mock._calls <= 9: + return 555 + return 666 # Return proper cn for CDN Bypass + change_number_mock._calls = 0 + rbs_change_number_mock._calls = 0 storage.get_change_number = change_number_mock - + rbs_storage.get_change_number = rbs_change_number_mock + self.parsed_split = None async def update(parsed_split, deleted, change_number): if len(parsed_split) > 0: self.parsed_split = parsed_split storage.update = update + self.parsed_rbs = None + async def rbs_update(parsed, deleted, change_number): + if len(parsed) > 0: + self.parsed_rbs = parsed + rbs_storage.update = rbs_update + api = mocker.Mock() self.change_number_1 = None self.fetch_options_1 = None @@ -511,25 +706,38 @@ async def update(parsed_split, deleted, change_number): self.fetch_options_2 = None self.change_number_3 = None self.fetch_options_3 = None - async def get_changes(change_number, fetch_options): + rbs_1 = copy.deepcopy(json_body['rbs']['d']) + + async def get_changes(change_number, rbs_change_number, fetch_options): get_changes.called += 1 if get_changes.called == 1: self.change_number_1 = change_number self.fetch_options_1 = fetch_options - return { 'splits': self.splits, 'since': -1, 'till': 123 } + return { 'ff': { 'd': self.splits, 's': -1, 't': 123 }, + 'rbs': {"t": 555, "s": -1, "d": rbs_1}} elif get_changes.called == 2: self.change_number_2 = change_number self.fetch_options_2 = fetch_options - return { 'splits': [], 'since': 123, 'till': 123 } + return { 'ff': { 'd': [], 's': 123, 't': 123 }, + 'rbs': {"t": 555, "s": 555, "d": []}} elif get_changes.called == 3: - return { 'splits': [], 'since': 123, 'till': 1234 } + return { 'ff': { 'd': [], 's': 123, 't': 1234 }, + 'rbs': {"t": 555, "s": 555, "d": []}} elif get_changes.called >= 4 and get_changes.called <= 6: - return { 'splits': [], 'since': 1234, 'till': 1234 } + return { 'ff': { 'd': [], 's': 1234, 't': 1234 }, + 'rbs': {"t": 555, "s": 555, "d": []}} elif get_changes.called == 7: - return { 'splits': [], 'since': 1234, 'till': 12345 } - self.change_number_3 = change_number - self.fetch_options_3 = fetch_options - return { 'splits': [], 'since': 12345, 'till': 12345 } + return { 'ff': { 'd': [], 's': 1234, 't': 12345 }, + 'rbs': {"t": 555, "s": 555, "d": []}} + elif get_changes.called == 8: + self.change_number_3 = change_number + self.fetch_options_3 = fetch_options + return { 'ff': { 'd': [], 's': 12345, 't': 12345 }, + 'rbs': {"t": 555, "s": 555, "d": []}} + rbs_1[0]['excluded']['keys'] = ['bilal@split.io'] + return { 'ff': { 'd': [], 's': 12345, 't': 12345 }, + 'rbs': {"t": 666, "s": 666, "d": rbs_1}} + get_changes.called = 0 api.fetch_splits = get_changes @@ -544,7 +752,17 @@ def intersect(sets): storage.flag_set_filter.flag_sets = {} storage.flag_set_filter.sorted_flag_sets = [] - split_synchronizer = SplitSynchronizerAsync(api, storage) + self.clear = False + async def clear(): + self.clear = True + storage.clear = clear + + self.clear2 = False + async def clear(): + self.clear2 = True + rbs_storage.clear = clear + + split_synchronizer = SplitSynchronizerAsync(api, storage, rbs_storage) split_synchronizer._backoff = Backoff(1, 1) await split_synchronizer.synchronize_splits() @@ -559,12 +777,20 @@ def intersect(sets): inserted_split = self.parsed_split[0] assert isinstance(inserted_split, Split) assert inserted_split.name == 'some_name' + inserted_rbs = self.parsed_rbs[0] + assert inserted_rbs.excluded.get_excluded_keys() == ["mauro@split.io","gaston@split.io"] + split_synchronizer._backoff = Backoff(1, 0.1) + await split_synchronizer.synchronize_splits(None, 666) + inserted_rbs = self.parsed_rbs[0] + assert inserted_rbs.excluded.get_excluded_keys() == ['bilal@split.io'] + @pytest.mark.asyncio async def test_sync_flag_sets_with_config_sets(self, mocker): """Test split sync with flag sets.""" storage = InMemorySplitStorageAsync(['set1', 'set2']) - + rbs_storage = InMemoryRuleBasedSegmentStorageAsync() + split = self.splits[0].copy() split['name'] = 'second' splits1 = [self.splits[0].copy(), split] @@ -575,20 +801,25 @@ async def test_sync_flag_sets_with_config_sets(self, mocker): async def get_changes(*args, **kwargs): get_changes.called += 1 if get_changes.called == 1: - return { 'splits': splits1, 'since': 123, 'till': 123 } + return { 'ff': { 'd': splits1, 's': 123, 't': 123 }, + 'rbs': {'t': 123, 's': 123, 'd': []}} elif get_changes.called == 2: splits2[0]['sets'] = ['set3'] - return { 'splits': splits2, 'since': 124, 'till': 124 } + return { 'ff': { 'd': splits2, 's': 124, 't': 124 }, + 'rbs': {'t': 124, 's': 124, 'd': []}} elif get_changes.called == 3: splits3[0]['sets'] = ['set1'] - return { 'splits': splits3, 'since': 12434, 'till': 12434 } + return { 'ff': { 'd': splits3, 's': 12434, 't': 12434 }, + 'rbs': {'t': 12434, 's': 12434, 'd': []}} splits4[0]['sets'] = ['set6'] splits4[0]['name'] = 'new_split' - return { 'splits': splits4, 'since': 12438, 'till': 12438 } + return { 'ff': { 'd': splits4, 's': 12438, 't': 12438 }, + 'rbs': {'t': 12438, 's': 12438, 'd': []}} + get_changes.called = 0 api.fetch_splits = get_changes - split_synchronizer = SplitSynchronizerAsync(api, storage) + split_synchronizer = SplitSynchronizerAsync(api, storage, rbs_storage) split_synchronizer._backoff = Backoff(1, 1) await split_synchronizer.synchronize_splits() assert isinstance(await storage.get('some_name'), Split) @@ -606,7 +837,7 @@ async def get_changes(*args, **kwargs): async def test_sync_flag_sets_without_config_sets(self, mocker): """Test split sync with flag sets.""" storage = InMemorySplitStorageAsync() - + rbs_storage = InMemoryRuleBasedSegmentStorageAsync() split = self.splits[0].copy() split['name'] = 'second' splits1 = [self.splits[0].copy(), split] @@ -617,20 +848,24 @@ async def test_sync_flag_sets_without_config_sets(self, mocker): async def get_changes(*args, **kwargs): get_changes.called += 1 if get_changes.called == 1: - return { 'splits': splits1, 'since': 123, 'till': 123 } + return { 'ff': { 'd': splits1, 's': 123, 't': 123 }, + 'rbs': {"t": 123, "s": 123, "d": []}} elif get_changes.called == 2: splits2[0]['sets'] = ['set3'] - return { 'splits': splits2, 'since': 124, 'till': 124 } + return { 'ff': { 'd': splits2, 's': 124, 't': 124 }, + 'rbs': {"t": 124, "s": 124, "d": []}} elif get_changes.called == 3: splits3[0]['sets'] = ['set1'] - return { 'splits': splits3, 'since': 12434, 'till': 12434 } + return { 'ff': { 'd': splits3, 's': 12434, 't': 12434 }, + 'rbs': {"t": 12434, "s": 12434, "d": []}} splits4[0]['sets'] = ['set6'] splits4[0]['name'] = 'third_split' - return { 'splits': splits4, 'since': 12438, 'till': 12438 } + return { 'ff': { 'd': splits4, 's': 12438, 't': 12438 }, + 'rbs': {"t": 12438, "s": 12438, "d": []}} get_changes.called = 0 api.fetch_splits.side_effect = get_changes - split_synchronizer = SplitSynchronizerAsync(api, storage) + split_synchronizer = SplitSynchronizerAsync(api, storage, rbs_storage) split_synchronizer._backoff = Backoff(1, 1) await split_synchronizer.synchronize_splits() assert isinstance(await storage.get('new_split'), Split) @@ -647,12 +882,13 @@ async def get_changes(*args, **kwargs): class LocalSplitsSynchronizerTests(object): """Split synchronizer test cases.""" - splits = copy.deepcopy(splits_raw) + payload = copy.deepcopy(json_body) def test_synchronize_splits_error(self, mocker): """Test that if fetching splits fails at some_point, the task will continue running.""" storage = mocker.Mock(spec=SplitStorage) - split_synchronizer = LocalSplitSynchronizer("/incorrect_file", storage) + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + split_synchronizer = LocalSplitSynchronizer("/incorrect_file", storage, rbs_storage) with pytest.raises(Exception): split_synchronizer.synchronize_splits(1) @@ -660,74 +896,75 @@ def test_synchronize_splits_error(self, mocker): def test_synchronize_splits(self, mocker): """Test split sync.""" storage = InMemorySplitStorage() + rbs_storage = InMemoryRuleBasedSegmentStorage() - till = 123 def read_splits_from_json_file(*args, **kwargs): - return self.splits, till + return self.payload - split_synchronizer = LocalSplitSynchronizer("split.json", storage, LocalhostMode.JSON) + split_synchronizer = LocalSplitSynchronizer("split.json", storage, rbs_storage, LocalhostMode.JSON) split_synchronizer._read_feature_flags_from_json_file = read_splits_from_json_file split_synchronizer.synchronize_splits() - inserted_split = storage.get(self.splits[0]['name']) + inserted_split = storage.get(self.payload["ff"]["d"][0]['name']) assert isinstance(inserted_split, Split) assert inserted_split.name == 'some_name' # Should sync when changenumber is not changed - self.splits[0]['killed'] = True + self.payload["ff"]["d"][0]['killed'] = True split_synchronizer.synchronize_splits() - inserted_split = storage.get(self.splits[0]['name']) + inserted_split = storage.get(self.payload["ff"]["d"][0]['name']) assert inserted_split.killed # Should not sync when changenumber is less than stored - till = 122 - self.splits[0]['killed'] = False + self.payload["ff"]["t"] = 122 + self.payload["ff"]["d"][0]['killed'] = False split_synchronizer.synchronize_splits() - inserted_split = storage.get(self.splits[0]['name']) + inserted_split = storage.get(self.payload["ff"]["d"][0]['name']) assert inserted_split.killed # Should sync when changenumber is higher than stored - till = 124 + self.payload["ff"]["t"] = 1675095324999 split_synchronizer._current_json_sha = "-1" split_synchronizer.synchronize_splits() - inserted_split = storage.get(self.splits[0]['name']) + inserted_split = storage.get(self.payload["ff"]["d"][0]['name']) assert inserted_split.killed == False # Should sync when till is default (-1) - till = -1 + self.payload["ff"]["t"] = -1 split_synchronizer._current_json_sha = "-1" - self.splits[0]['killed'] = True + self.payload["ff"]["d"][0]['killed'] = True split_synchronizer.synchronize_splits() - inserted_split = storage.get(self.splits[0]['name']) + inserted_split = storage.get(self.payload["ff"]["d"][0]['name']) assert inserted_split.killed == True def test_sync_flag_sets_with_config_sets(self, mocker): """Test split sync with flag sets.""" storage = InMemorySplitStorage(['set1', 'set2']) - - split = self.splits[0].copy() + rbs_storage = InMemoryRuleBasedSegmentStorage() + + split = self.payload["ff"]["d"][0].copy() split['name'] = 'second' - splits1 = [self.splits[0].copy(), split] - splits2 = self.splits.copy() - splits3 = self.splits.copy() - splits4 = self.splits.copy() + splits1 = [self.payload["ff"]["d"][0].copy(), split] + splits2 = self.payload["ff"]["d"].copy() + splits3 = self.payload["ff"]["d"].copy() + splits4 = self.payload["ff"]["d"].copy() self.called = 0 def read_feature_flags_from_json_file(*args, **kwargs): self.called += 1 if self.called == 1: - return splits1, 123 + return {"ff": {"d": splits1, "t": 123, "s": -1}, "rbs": {"d": [], "t": -1, "s": -1}} elif self.called == 2: splits2[0]['sets'] = ['set3'] - return splits2, 124 + return {"ff": {"d": splits2, "t": 124, "s": -1}, "rbs": {"d": [], "t": -1, "s": -1}} elif self.called == 3: splits3[0]['sets'] = ['set1'] - return splits3, 12434 + return {"ff": {"d": splits3, "t": 12434, "s": -1}, "rbs": {"d": [], "t": -1, "s": -1}} splits4[0]['sets'] = ['set6'] splits4[0]['name'] = 'new_split' - return splits4, 12438 + return {"ff": {"d": splits4, "t": 12438, "s": -1}, "rbs": {"d": [], "t": -1, "s": -1}} - split_synchronizer = LocalSplitSynchronizer("split.json", storage, LocalhostMode.JSON) + split_synchronizer = LocalSplitSynchronizer("split.json", storage, rbs_storage, LocalhostMode.JSON) split_synchronizer._read_feature_flags_from_json_file = read_feature_flags_from_json_file split_synchronizer.synchronize_splits() @@ -745,30 +982,31 @@ def read_feature_flags_from_json_file(*args, **kwargs): def test_sync_flag_sets_without_config_sets(self, mocker): """Test split sync with flag sets.""" storage = InMemorySplitStorage() + rbs_storage = InMemoryRuleBasedSegmentStorage() - split = self.splits[0].copy() + split = self.payload["ff"]["d"][0].copy() split['name'] = 'second' - splits1 = [self.splits[0].copy(), split] - splits2 = self.splits.copy() - splits3 = self.splits.copy() - splits4 = self.splits.copy() + splits1 = [self.payload["ff"]["d"][0].copy(), split] + splits2 = self.payload["ff"]["d"].copy() + splits3 = self.payload["ff"]["d"].copy() + splits4 = self.payload["ff"]["d"].copy() self.called = 0 def read_feature_flags_from_json_file(*args, **kwargs): self.called += 1 if self.called == 1: - return splits1, 123 + return {"ff": {"d": splits1, "t": 123, "s": -1}, "rbs": {"d": [], "t": -1, "s": -1}} elif self.called == 2: splits2[0]['sets'] = ['set3'] - return splits2, 124 + return {"ff": {"d": splits2, "t": 124, "s": -1}, "rbs": {"d": [], "t": -1, "s": -1}} elif self.called == 3: splits3[0]['sets'] = ['set1'] - return splits3, 12434 + return {"ff": {"d": splits3, "t": 12434, "s": -1}, "rbs": {"d": [], "t": -1, "s": -1}} splits4[0]['sets'] = ['set6'] splits4[0]['name'] = 'third_split' - return splits4, 12438 + return {"ff": {"d": splits4, "t": 12438, "s": -1}, "rbs": {"d": [], "t": -1, "s": -1}} - split_synchronizer = LocalSplitSynchronizer("split.json", storage, LocalhostMode.JSON) + split_synchronizer = LocalSplitSynchronizer("split.json", storage, rbs_storage, LocalhostMode.JSON) split_synchronizer._read_feature_flags_from_json_file = read_feature_flags_from_json_file split_synchronizer.synchronize_splits() @@ -786,217 +1024,213 @@ def read_feature_flags_from_json_file(*args, **kwargs): def test_reading_json(self, mocker): """Test reading json file.""" f = open("./splits.json", "w") - json_body = {'splits': [{ - 'changeNumber': 123, - 'trafficTypeName': 'user', - 'name': 'some_name', - 'trafficAllocation': 100, - 'trafficAllocationSeed': 123456, - 'seed': 321654, - 'status': 'ACTIVE', - 'killed': False, - 'defaultTreatment': 'off', - 'algo': 2, - 'conditions': [ - { - 'partitions': [ - {'treatment': 'on', 'size': 50}, - {'treatment': 'off', 'size': 50} - ], - 'contitionType': 'WHITELIST', - 'label': 'some_label', - 'matcherGroup': { - 'matchers': [ - { - 'matcherType': 'WHITELIST', - 'whitelistMatcherData': { - 'whitelist': ['k1', 'k2', 'k3'] - }, - 'negate': False, - } - ], - 'combiner': 'AND' - } - } - ], - 'sets': ['set1'] - }], - "till":1675095324253, - "since":-1, - } - - f.write(json.dumps(json_body)) + f.write(json.dumps(self.payload)) f.close() storage = InMemorySplitStorage() - split_synchronizer = LocalSplitSynchronizer("./splits.json", storage, LocalhostMode.JSON) + rbs_storage = InMemoryRuleBasedSegmentStorage() + split_synchronizer = LocalSplitSynchronizer("./splits.json", storage, rbs_storage, LocalhostMode.JSON) split_synchronizer.synchronize_splits() - inserted_split = storage.get(json_body['splits'][0]['name']) + inserted_split = storage.get(self.payload['ff']['d'][0]['name']) assert isinstance(inserted_split, Split) - assert inserted_split.name == 'some_name' + assert inserted_split.name == self.payload['ff']['d'][0]['name'] + + inserted_rbs = rbs_storage.get(self.payload['rbs']['d'][0]['name']) + assert isinstance(inserted_rbs, RuleBasedSegment) + assert inserted_rbs.name == self.payload['rbs']['d'][0]['name'] os.remove("./splits.json") def test_json_elements_sanitization(self, mocker): """Test sanitization.""" - split_synchronizer = LocalSplitSynchronizer(mocker.Mock(), mocker.Mock(), mocker.Mock()) + split_synchronizer = LocalSplitSynchronizer(mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock()) # check no changes if all elements exist with valid values - parsed = {"splits": [], "since": -1, "till": -1} + parsed = {"ff": {"d": [], "s": -1, "t": -1}, "rbs": {"d": [], "s": -1, "t": -1}} assert (split_synchronizer._sanitize_json_elements(parsed) == parsed) # check set since to -1 when is None parsed2 = parsed.copy() - parsed2['since'] = None + parsed2['ff']['s'] = None assert (split_synchronizer._sanitize_json_elements(parsed2) == parsed) # check no changes if since > -1 parsed2 = parsed.copy() - parsed2['since'] = 12 + parsed2['ff']['s'] = 12 assert (split_synchronizer._sanitize_json_elements(parsed2) == parsed) # check set till to -1 when is None parsed2 = parsed.copy() - parsed2['till'] = None + parsed2['ff']['t'] = None assert (split_synchronizer._sanitize_json_elements(parsed2) == parsed) # check add since when missing - parsed2 = {"splits": [], "till": -1} + parsed2 = {"ff": {"d": [], "t": -1}, "rbs": {"d": [], "s": -1, "t": -1}} assert (split_synchronizer._sanitize_json_elements(parsed2) == parsed) # check add till when missing - parsed2 = {"splits": [], "since": -1} + parsed2 = {"ff": {"d": [], "s": -1}, "rbs": {"d": [], "s": -1, "t": -1}} assert (split_synchronizer._sanitize_json_elements(parsed2) == parsed) # check add splits when missing - parsed2 = {"since": -1, "till": -1} + parsed2 = {"ff": {"s": -1, "t": -1}, "rbs": {"d": [], "s": -1, "t": -1}} + assert (split_synchronizer._sanitize_json_elements(parsed2) == parsed) + + # check add since when missing + parsed2 = {"ff": {"d": [], "t": -1}, "rbs": {"d": [], "t": -1}} assert (split_synchronizer._sanitize_json_elements(parsed2) == parsed) - def test_split_elements_sanitization(self, mocker): + # check add till when missing + parsed2 = {"ff": {"d": [], "s": -1}, "rbs": {"d": [], "s": -1}} + assert (split_synchronizer._sanitize_json_elements(parsed2) == parsed) + + # check add splits when missing + parsed2 = {"ff": {"s": -1, "t": -1}, "rbs": {"s": -1, "t": -1}} + assert (split_synchronizer._sanitize_json_elements(parsed2) == parsed) + + def test_elements_sanitization(self, mocker): """Test sanitization.""" - split_synchronizer = LocalSplitSynchronizer(mocker.Mock(), mocker.Mock(), mocker.Mock()) + split_synchronizer = LocalSplitSynchronizer(mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock()) # No changes when split structure is good - assert (split_synchronizer._sanitize_feature_flag_elements(splits_json["splitChange1_1"]["splits"]) == splits_json["splitChange1_1"]["splits"]) + assert (split_synchronizer._sanitize_feature_flag_elements(splits_json["splitChange1_1"]['ff']['d']) == splits_json["splitChange1_1"]['ff']['d']) # test 'trafficTypeName' value None - split = splits_json["splitChange1_1"]["splits"].copy() + split = splits_json["splitChange1_1"]['ff']['d'].copy() split[0]['trafficTypeName'] = None - assert (split_synchronizer._sanitize_feature_flag_elements(split) == splits_json["splitChange1_1"]["splits"]) + assert (split_synchronizer._sanitize_feature_flag_elements(split) == splits_json["splitChange1_1"]['ff']['d']) # test 'trafficAllocation' value None - split = splits_json["splitChange1_1"]["splits"].copy() + split = splits_json["splitChange1_1"]['ff']['d'].copy() split[0]['trafficAllocation'] = None - assert (split_synchronizer._sanitize_feature_flag_elements(split) == splits_json["splitChange1_1"]["splits"]) + assert (split_synchronizer._sanitize_feature_flag_elements(split) == splits_json["splitChange1_1"]['ff']['d']) # test 'trafficAllocation' valid value should not change - split = splits_json["splitChange1_1"]["splits"].copy() + split = splits_json["splitChange1_1"]['ff']['d'].copy() split[0]['trafficAllocation'] = 50 assert (split_synchronizer._sanitize_feature_flag_elements(split) == split) # test 'trafficAllocation' invalid value should change - split = splits_json["splitChange1_1"]["splits"].copy() + split = splits_json["splitChange1_1"]['ff']['d'].copy() split[0]['trafficAllocation'] = 110 - assert (split_synchronizer._sanitize_feature_flag_elements(split) == splits_json["splitChange1_1"]["splits"]) + assert (split_synchronizer._sanitize_feature_flag_elements(split) == splits_json["splitChange1_1"]['ff']['d']) # test 'trafficAllocationSeed' is set to millisec epoch when None - split = splits_json["splitChange1_1"]["splits"].copy() + split = splits_json["splitChange1_1"]['ff']['d'].copy() split[0]['trafficAllocationSeed'] = None assert (split_synchronizer._sanitize_feature_flag_elements(split)[0]['trafficAllocationSeed'] > 0) # test 'trafficAllocationSeed' is set to millisec epoch when 0 - split = splits_json["splitChange1_1"]["splits"].copy() + split = splits_json["splitChange1_1"]['ff']['d'].copy() split[0]['trafficAllocationSeed'] = 0 assert (split_synchronizer._sanitize_feature_flag_elements(split)[0]['trafficAllocationSeed'] > 0) # test 'seed' is set to millisec epoch when None - split = splits_json["splitChange1_1"]["splits"].copy() + split = splits_json["splitChange1_1"]['ff']['d'].copy() split[0]['seed'] = None assert (split_synchronizer._sanitize_feature_flag_elements(split)[0]['seed'] > 0) # test 'seed' is set to millisec epoch when its 0 - split = splits_json["splitChange1_1"]["splits"].copy() + split = splits_json["splitChange1_1"]['ff']['d'].copy() split[0]['seed'] = 0 assert (split_synchronizer._sanitize_feature_flag_elements(split)[0]['seed'] > 0) # test 'status' is set to ACTIVE when None - split = splits_json["splitChange1_1"]["splits"].copy() + split = splits_json["splitChange1_1"]['ff']['d'].copy() split[0]['status'] = None - assert (split_synchronizer._sanitize_feature_flag_elements(split) == splits_json["splitChange1_1"]["splits"]) + assert (split_synchronizer._sanitize_feature_flag_elements(split) == splits_json["splitChange1_1"]['ff']['d']) # test 'status' is set to ACTIVE when incorrect - split = splits_json["splitChange1_1"]["splits"].copy() + split = splits_json["splitChange1_1"]['ff']['d'].copy() split[0]['status'] = 'ww' - assert (split_synchronizer._sanitize_feature_flag_elements(split) == splits_json["splitChange1_1"]["splits"]) + assert (split_synchronizer._sanitize_feature_flag_elements(split) == splits_json["splitChange1_1"]['ff']['d']) # test ''killed' is set to False when incorrect - split = splits_json["splitChange1_1"]["splits"].copy() + split = splits_json["splitChange1_1"]['ff']['d'].copy() split[0]['killed'] = None - assert (split_synchronizer._sanitize_feature_flag_elements(split) == splits_json["splitChange1_1"]["splits"]) + assert (split_synchronizer._sanitize_feature_flag_elements(split) == splits_json["splitChange1_1"]['ff']['d']) # test 'defaultTreatment' is set to on when None - split = splits_json["splitChange1_1"]["splits"].copy() + split = splits_json["splitChange1_1"]['ff']['d'].copy() split[0]['defaultTreatment'] = None assert (split_synchronizer._sanitize_feature_flag_elements(split)[0]['defaultTreatment'] == 'control') # test 'defaultTreatment' is set to on when its empty - split = splits_json["splitChange1_1"]["splits"].copy() + split = splits_json["splitChange1_1"]['ff']['d'].copy() split[0]['defaultTreatment'] = ' ' assert (split_synchronizer._sanitize_feature_flag_elements(split)[0]['defaultTreatment'] == 'control') # test 'changeNumber' is set to 0 when None - split = splits_json["splitChange1_1"]["splits"].copy() + split = splits_json["splitChange1_1"]['ff']['d'].copy() split[0]['changeNumber'] = None assert (split_synchronizer._sanitize_feature_flag_elements(split)[0]['changeNumber'] == 0) # test 'changeNumber' is set to 0 when invalid - split = splits_json["splitChange1_1"]["splits"].copy() + split = splits_json["splitChange1_1"]['ff']['d'].copy() split[0]['changeNumber'] = -33 assert (split_synchronizer._sanitize_feature_flag_elements(split)[0]['changeNumber'] == 0) # test 'algo' is set to 2 when None - split = splits_json["splitChange1_1"]["splits"].copy() + split = splits_json["splitChange1_1"]['ff']['d'].copy() split[0]['algo'] = None assert (split_synchronizer._sanitize_feature_flag_elements(split)[0]['algo'] == 2) # test 'algo' is set to 2 when higher than 2 - split = splits_json["splitChange1_1"]["splits"].copy() + split = splits_json["splitChange1_1"]['ff']['d'].copy() split[0]['algo'] = 3 assert (split_synchronizer._sanitize_feature_flag_elements(split)[0]['algo'] == 2) # test 'algo' is set to 2 when lower than 2 - split = splits_json["splitChange1_1"]["splits"].copy() + split = splits_json["splitChange1_1"]['ff']['d'].copy() split[0]['algo'] = 1 assert (split_synchronizer._sanitize_feature_flag_elements(split)[0]['algo'] == 2) - def test_split_condition_sanitization(self, mocker): + split = splits_json["splitChange1_1"]['ff']['d'].copy() + del split[0]['prerequisites'] + assert (split_synchronizer._sanitize_feature_flag_elements(split)[0]['prerequisites'] == []) + + # test 'status' is set to ACTIVE when None + rbs = copy.deepcopy(json_body["rbs"]["d"]) + rbs[0]['status'] = None + assert (split_synchronizer._sanitize_rb_segment_elements(rbs)[0]['status'] == 'ACTIVE') + + # test 'changeNumber' is set to 0 when invalid + rbs = copy.deepcopy(json_body["rbs"]["d"]) + rbs[0]['changeNumber'] = -2 + assert (split_synchronizer._sanitize_rb_segment_elements(rbs)[0]['changeNumber'] == 0) + + rbs = copy.deepcopy(json_body["rbs"]["d"]) + del rbs[0]['conditions'] + assert (len(split_synchronizer._sanitize_rb_segment_elements(rbs)[0]['conditions']) == 1) + + def test_condition_sanitization(self, mocker): """Test sanitization.""" split_synchronizer = LocalSplitSynchronizer(mocker.Mock(), mocker.Mock(), mocker.Mock()) # test missing all conditions with default rule set to 100% off - split = splits_json["splitChange1_1"]["splits"].copy() - target_split = splits_json["splitChange1_1"]["splits"].copy() + split = splits_json["splitChange1_1"]['ff']['d'].copy() + target_split = splits_json["splitChange1_1"]['ff']['d'].copy() target_split[0]["conditions"][0]['partitions'][0]['size'] = 0 target_split[0]["conditions"][0]['partitions'][1]['size'] = 100 del split[0]["conditions"] assert (split_synchronizer._sanitize_feature_flag_elements(split) == target_split) # test missing ALL_KEYS condition matcher with default rule set to 100% off - split = splits_json["splitChange1_1"]["splits"].copy() - target_split = splits_json["splitChange1_1"]["splits"].copy() + split = splits_json["splitChange1_1"]['ff']['d'].copy() + target_split = splits_json["splitChange1_1"]['ff']['d'].copy() split[0]["conditions"][0]["matcherGroup"]["matchers"][0]["matcherType"] = "IN_STR" target_split = split.copy() - target_split[0]["conditions"].append(splits_json["splitChange1_1"]["splits"][0]["conditions"][0]) + target_split[0]["conditions"].append(splits_json["splitChange1_1"]['ff']['d'][0]["conditions"][0]) target_split[0]["conditions"][1]['partitions'][0]['size'] = 0 target_split[0]["conditions"][1]['partitions'][1]['size'] = 100 assert (split_synchronizer._sanitize_feature_flag_elements(split) == target_split) # test missing ROLLOUT condition type with default rule set to 100% off - split = splits_json["splitChange1_1"]["splits"].copy() - target_split = splits_json["splitChange1_1"]["splits"].copy() + split = splits_json["splitChange1_1"]['ff']['d'].copy() + target_split = splits_json["splitChange1_1"]['ff']['d'].copy() split[0]["conditions"][0]["conditionType"] = "NOT" target_split = split.copy() - target_split[0]["conditions"].append(splits_json["splitChange1_1"]["splits"][0]["conditions"][0]) + target_split[0]["conditions"].append(splits_json["splitChange1_1"]['ff']['d'][0]["conditions"][0]) target_split[0]["conditions"][1]['partitions'][0]['size'] = 0 target_split[0]["conditions"][1]['partitions'][1]['size'] = 100 assert (split_synchronizer._sanitize_feature_flag_elements(split) == target_split) @@ -1004,13 +1238,14 @@ def test_split_condition_sanitization(self, mocker): class LocalSplitsSynchronizerAsyncTests(object): """Split synchronizer test cases.""" - splits = copy.deepcopy(splits_raw) + payload = copy.deepcopy(json_body) @pytest.mark.asyncio async def test_synchronize_splits_error(self, mocker): """Test that if fetching splits fails at some_point, the task will continue running.""" storage = mocker.Mock(spec=SplitStorage) - split_synchronizer = LocalSplitSynchronizerAsync("/incorrect_file", storage) + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + split_synchronizer = LocalSplitSynchronizerAsync("/incorrect_file", storage, rbs_storage) with pytest.raises(Exception): await split_synchronizer.synchronize_splits(1) @@ -1019,75 +1254,76 @@ async def test_synchronize_splits_error(self, mocker): async def test_synchronize_splits(self, mocker): """Test split sync.""" storage = InMemorySplitStorageAsync() + rbs_storage = InMemoryRuleBasedSegmentStorageAsync() - till = 123 async def read_splits_from_json_file(*args, **kwargs): - return self.splits, till + return self.payload - split_synchronizer = LocalSplitSynchronizerAsync("split.json", storage, LocalhostMode.JSON) + split_synchronizer = LocalSplitSynchronizerAsync("split.json", storage, rbs_storage, LocalhostMode.JSON) split_synchronizer._read_feature_flags_from_json_file = read_splits_from_json_file await split_synchronizer.synchronize_splits() - inserted_split = await storage.get(self.splits[0]['name']) + inserted_split = await storage.get(self.payload["ff"]["d"][0]['name']) assert isinstance(inserted_split, Split) assert inserted_split.name == 'some_name' # Should sync when changenumber is not changed - self.splits[0]['killed'] = True + self.payload["ff"]["d"][0]['killed'] = True await split_synchronizer.synchronize_splits() - inserted_split = await storage.get(self.splits[0]['name']) + inserted_split = await storage.get(self.payload["ff"]["d"][0]['name']) assert inserted_split.killed # Should not sync when changenumber is less than stored - till = 122 - self.splits[0]['killed'] = False + self.payload["ff"]["t"] = 122 + self.payload["ff"]["d"][0]['killed'] = False await split_synchronizer.synchronize_splits() - inserted_split = await storage.get(self.splits[0]['name']) + inserted_split = await storage.get(self.payload["ff"]["d"][0]['name']) assert inserted_split.killed # Should sync when changenumber is higher than stored - till = 124 + self.payload["ff"]["t"] = 1675095324999 split_synchronizer._current_json_sha = "-1" await split_synchronizer.synchronize_splits() - inserted_split = await storage.get(self.splits[0]['name']) + inserted_split = await storage.get(self.payload["ff"]["d"][0]['name']) assert inserted_split.killed == False # Should sync when till is default (-1) - till = -1 + self.payload["ff"]["t"] = -1 split_synchronizer._current_json_sha = "-1" - self.splits[0]['killed'] = True + self.payload["ff"]["d"][0]['killed'] = True await split_synchronizer.synchronize_splits() - inserted_split = await storage.get(self.splits[0]['name']) + inserted_split = await storage.get(self.payload["ff"]["d"][0]['name']) assert inserted_split.killed == True @pytest.mark.asyncio async def test_sync_flag_sets_with_config_sets(self, mocker): """Test split sync with flag sets.""" storage = InMemorySplitStorageAsync(['set1', 'set2']) - - split = self.splits[0].copy() + rbs_storage = InMemoryRuleBasedSegmentStorageAsync() + + split = self.payload["ff"]["d"][0].copy() split['name'] = 'second' - splits1 = [self.splits[0].copy(), split] - splits2 = self.splits.copy() - splits3 = self.splits.copy() - splits4 = self.splits.copy() + splits1 = [self.payload["ff"]["d"][0].copy(), split] + splits2 = self.payload["ff"]["d"].copy() + splits3 = self.payload["ff"]["d"].copy() + splits4 = self.payload["ff"]["d"].copy() self.called = 0 async def read_feature_flags_from_json_file(*args, **kwargs): self.called += 1 if self.called == 1: - return splits1, 123 + return {"ff": {"d": splits1, "t": 123, "s": -1}, "rbs": {"d": [], "t": -1, "s": -1}} elif self.called == 2: splits2[0]['sets'] = ['set3'] - return splits2, 124 + return {"ff": {"d": splits2, "t": 124, "s": -1}, "rbs": {"d": [], "t": -1, "s": -1}} elif self.called == 3: splits3[0]['sets'] = ['set1'] - return splits3, 12434 + return {"ff": {"d": splits3, "t": 12434, "s": -1}, "rbs": {"d": [], "t": -1, "s": -1}} splits4[0]['sets'] = ['set6'] splits4[0]['name'] = 'new_split' - return splits4, 12438 + return {"ff": {"d": splits4, "t": 12438, "s": -1}, "rbs": {"d": [], "t": -1, "s": -1}} - split_synchronizer = LocalSplitSynchronizerAsync("split.json", storage, LocalhostMode.JSON) + split_synchronizer = LocalSplitSynchronizerAsync("split.json", storage, rbs_storage, LocalhostMode.JSON) split_synchronizer._read_feature_flags_from_json_file = read_feature_flags_from_json_file await split_synchronizer.synchronize_splits() @@ -1106,30 +1342,30 @@ async def read_feature_flags_from_json_file(*args, **kwargs): async def test_sync_flag_sets_without_config_sets(self, mocker): """Test split sync with flag sets.""" storage = InMemorySplitStorageAsync() - - split = self.splits[0].copy() + rbs_storage = InMemoryRuleBasedSegmentStorageAsync() + + split = self.payload["ff"]["d"][0].copy() split['name'] = 'second' - splits1 = [self.splits[0].copy(), split] - splits2 = self.splits.copy() - splits3 = self.splits.copy() - splits4 = self.splits.copy() + splits1 = [self.payload["ff"]["d"][0].copy(), split] + splits2 = self.payload["ff"]["d"].copy() + splits3 = self.payload["ff"]["d"].copy() + splits4 = self.payload["ff"]["d"].copy() self.called = 0 async def read_feature_flags_from_json_file(*args, **kwargs): self.called += 1 if self.called == 1: - return splits1, 123 + return {"ff": {"d": splits1, "t": 123, "s": -1}, "rbs": {"d": [], "t": -1, "s": -1}} elif self.called == 2: - splits2[0]['sets'] = ['set3'] - return splits2, 124 + return {"ff": {"d": splits2, "t": 124, "s": -1}, "rbs": {"d": [], "t": -1, "s": -1}} elif self.called == 3: splits3[0]['sets'] = ['set1'] - return splits3, 12434 + return {"ff": {"d": splits3, "t": 12434, "s": -1}, "rbs": {"d": [], "t": -1, "s": -1}} splits4[0]['sets'] = ['set6'] splits4[0]['name'] = 'third_split' - return splits4, 12438 + return {"ff": {"d": splits4, "t": 12438, "s": -1}, "rbs": {"d": [], "t": -1, "s": -1}} - split_synchronizer = LocalSplitSynchronizerAsync("split.json", storage, LocalhostMode.JSON) + split_synchronizer = LocalSplitSynchronizerAsync("split.json", storage, rbs_storage, LocalhostMode.JSON) split_synchronizer._read_feature_flags_from_json_file = read_feature_flags_from_json_file await split_synchronizer.synchronize_splits() @@ -1148,13 +1384,18 @@ async def read_feature_flags_from_json_file(*args, **kwargs): async def test_reading_json(self, mocker): """Test reading json file.""" async with aiofiles.open("./splits.json", "w") as f: - await f.write(json.dumps(json_body)) + await f.write(json.dumps(self.payload)) storage = InMemorySplitStorageAsync() - split_synchronizer = LocalSplitSynchronizerAsync("./splits.json", storage, LocalhostMode.JSON) + rbs_storage = InMemoryRuleBasedSegmentStorageAsync() + split_synchronizer = LocalSplitSynchronizerAsync("./splits.json", storage, rbs_storage, LocalhostMode.JSON) await split_synchronizer.synchronize_splits() - inserted_split = await storage.get(json_body['splits'][0]['name']) + inserted_split = await storage.get(self.payload['ff']['d'][0]['name']) assert isinstance(inserted_split, Split) - assert inserted_split.name == 'some_name' + assert inserted_split.name == self.payload['ff']['d'][0]['name'] + + inserted_rbs = await rbs_storage.get(self.payload['rbs']['d'][0]['name']) + assert isinstance(inserted_rbs, RuleBasedSegment) + assert inserted_rbs.name == self.payload['rbs']['d'][0]['name'] os.remove("./splits.json") diff --git a/tests/sync/test_synchronizer.py b/tests/sync/test_synchronizer.py index 8e10d771..60ab7993 100644 --- a/tests/sync/test_synchronizer.py +++ b/tests/sync/test_synchronizer.py @@ -1,6 +1,4 @@ """Synchronizer tests.""" - -from turtle import clear import unittest.mock as mock import pytest @@ -14,11 +12,12 @@ from splitio.sync.segment import SegmentSynchronizer, SegmentSynchronizerAsync, LocalSegmentSynchronizer, LocalSegmentSynchronizerAsync from splitio.sync.impression import ImpressionSynchronizer, ImpressionSynchronizerAsync, ImpressionsCountSynchronizer, ImpressionsCountSynchronizerAsync from splitio.sync.event import EventSynchronizer, EventSynchronizerAsync -from splitio.storage import SegmentStorage, SplitStorage +from splitio.storage import SegmentStorage, SplitStorage, RuleBasedSegmentsStorage from splitio.api import APIException, APIUriException from splitio.models.splits import Split from splitio.models.segments import Segment -from splitio.storage.inmemmory import InMemorySegmentStorage, InMemorySplitStorage, InMemorySegmentStorageAsync, InMemorySplitStorageAsync +from splitio.storage.inmemmory import InMemorySegmentStorage, InMemorySplitStorage, InMemorySegmentStorageAsync, InMemorySplitStorageAsync, \ + InMemoryRuleBasedSegmentStorage, InMemoryRuleBasedSegmentStorageAsync splits = [{ 'changeNumber': 123, @@ -63,11 +62,11 @@ def intersect(sets): storage.flag_set_filter.flag_sets = {} storage.flag_set_filter.sorted_flag_sets = [] - def run(x, c): + def run(x, y, c): raise APIException("something broke") api.fetch_splits.side_effect = run - split_sync = SplitSynchronizer(api, storage) + split_sync = SplitSynchronizer(api, storage, mocker.Mock()) split_synchronizers = SplitSynchronizers(split_sync, mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock()) sychronizer = Synchronizer(split_synchronizers, mocker.Mock(spec=SplitTasks)) @@ -89,11 +88,11 @@ def intersect(sets): storage.flag_set_filter.flag_sets = {} storage.flag_set_filter.sorted_flag_sets = [] - def run(x, c): + def run(x, y, c): raise APIException("something broke", 414) api.fetch_splits.side_effect = run - split_sync = SplitSynchronizer(api, storage) + split_sync = SplitSynchronizer(api, storage, mocker.Mock()) split_synchronizers = SplitSynchronizers(split_sync, mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock()) synchronizer = Synchronizer(split_synchronizers, mocker.Mock(spec=SplitTasks)) @@ -107,14 +106,16 @@ def test_sync_all_failed_segments(self, mocker): storage = mocker.Mock() split_storage = mocker.Mock(spec=SplitStorage) split_storage.get_segment_names.return_value = ['segmentA'] + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + rbs_storage.get_segment_names.return_value = [] split_sync = mocker.Mock(spec=SplitSynchronizer) split_sync.synchronize_splits.return_value = None - def run(x, y): + def run(x, y, c): raise APIException("something broke") api.fetch_segment.side_effect = run - segment_sync = SegmentSynchronizer(api, split_storage, storage) + segment_sync = SegmentSynchronizer(api, split_storage, storage, rbs_storage) split_synchronizers = SplitSynchronizers(split_sync, segment_sync, mocker.Mock(), mocker.Mock(), mocker.Mock()) sychronizer = Synchronizer(split_synchronizers, mocker.Mock(spec=SplitTasks)) @@ -124,15 +125,16 @@ def run(x, y): def test_synchronize_splits(self, mocker): split_storage = InMemorySplitStorage() + rbs_storage = InMemoryRuleBasedSegmentStorage() split_api = mocker.Mock() - split_api.fetch_splits.return_value = {'splits': splits, 'since': 123, - 'till': 123} - split_sync = SplitSynchronizer(split_api, split_storage) + split_api.fetch_splits.return_value = {'ff': {'d': splits, 's': 123, + 't': 123}, 'rbs': {'d': [], 's': -1, 't': -1}} + split_sync = SplitSynchronizer(split_api, split_storage, rbs_storage) segment_storage = InMemorySegmentStorage() segment_api = mocker.Mock() segment_api.fetch_segment.return_value = {'name': 'segmentA', 'added': ['key1', 'key2', 'key3'], 'removed': [], 'since': 123, 'till': 123} - segment_sync = SegmentSynchronizer(segment_api, split_storage, segment_storage) + segment_sync = SegmentSynchronizer(segment_api, split_storage, segment_storage, rbs_storage) split_synchronizers = SplitSynchronizers(split_sync, segment_sync, mocker.Mock(), mocker.Mock(), mocker.Mock()) synchronizer = Synchronizer(split_synchronizers, mocker.Mock(spec=SplitTasks)) @@ -150,10 +152,12 @@ def test_synchronize_splits(self, mocker): def test_synchronize_splits_calling_segment_sync_once(self, mocker): split_storage = InMemorySplitStorage() + rbs_storage = InMemoryRuleBasedSegmentStorage() split_api = mocker.Mock() - split_api.fetch_splits.return_value = {'splits': splits, 'since': 123, - 'till': 123} - split_sync = SplitSynchronizer(split_api, split_storage) + split_api.fetch_splits.return_value = {'ff': {'d': splits, 's': 123, + 't': 123}, 'rbs': {'d': [], 's': -1, 't': -1}} + + split_sync = SplitSynchronizer(split_api, split_storage, rbs_storage) counts = {'segments': 0} def sync_segments(*_): @@ -173,6 +177,8 @@ def sync_segments(*_): def test_sync_all(self, mocker): split_storage = mocker.Mock(spec=SplitStorage) + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + rbs_storage.get_segment_names.return_value = [] split_storage.get_change_number.return_value = 123 split_storage.get_segment_names.return_value = ['segmentA'] class flag_set_filter(): @@ -185,21 +191,28 @@ def intersect(sets): split_storage.flag_set_filter.sorted_flag_sets = [] split_api = mocker.Mock() - split_api.fetch_splits.return_value = {'splits': splits, 'since': 123, - 'till': 123} - split_sync = SplitSynchronizer(split_api, split_storage) + split_api.fetch_splits.return_value = {'ff': {'d': splits, 's': 123, + 't': 123}, 'rbs': {'d': [], 's': -1, 't': -1}} + split_sync = SplitSynchronizer(split_api, split_storage, rbs_storage) segment_storage = mocker.Mock(spec=SegmentStorage) segment_storage.get_change_number.return_value = 123 segment_api = mocker.Mock() segment_api.fetch_segment.return_value = {'name': 'segmentA', 'added': ['key1', 'key2', 'key3'], 'removed': [], 'since': 123, 'till': 123} - segment_sync = SegmentSynchronizer(segment_api, split_storage, segment_storage) + segment_sync = SegmentSynchronizer(segment_api, split_storage, segment_storage, rbs_storage) split_synchronizers = SplitSynchronizers(split_sync, segment_sync, mocker.Mock(), mocker.Mock(), mocker.Mock()) synchronizer = Synchronizer(split_synchronizers, mocker.Mock(spec=SplitTasks)) +# pytest.set_trace() + self.clear = False + def clear(): + self.clear = True + split_storage.clear = clear + rbs_storage.clear = clear + synchronizer.sync_all() inserted_split = split_storage.update.mock_calls[0][1][0][0] @@ -391,6 +404,7 @@ class SynchronizerAsyncTests(object): async def test_sync_all_failed_splits(self, mocker): api = mocker.Mock() storage = mocker.Mock() + rbs_storage = mocker.Mock() class flag_set_filter(): def should_filter(): return False @@ -400,15 +414,16 @@ def intersect(sets): storage.flag_set_filter.flag_sets = {} storage.flag_set_filter.sorted_flag_sets = [] - async def run(x, c): + async def run(x, y, c): raise APIException("something broke") api.fetch_splits = run async def get_change_number(): return 1234 storage.get_change_number = get_change_number + rbs_storage.get_change_number = get_change_number - split_sync = SplitSynchronizerAsync(api, storage) + split_sync = SplitSynchronizerAsync(api, storage, rbs_storage) split_synchronizers = SplitSynchronizers(split_sync, mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock()) sychronizer = SynchronizerAsync(split_synchronizers, mocker.Mock(spec=SplitTasks)) @@ -422,6 +437,7 @@ async def get_change_number(): async def test_sync_all_failed_splits_with_flagsets(self, mocker): api = mocker.Mock() storage = mocker.Mock() + rbs_storage = mocker.Mock() class flag_set_filter(): def should_filter(): return False @@ -434,12 +450,13 @@ def intersect(sets): async def get_change_number(): pass storage.get_change_number = get_change_number - - async def run(x, c): + rbs_storage.get_change_number = get_change_number + + async def run(x, y, c): raise APIException("something broke", 414) api.fetch_splits = run - split_sync = SplitSynchronizerAsync(api, storage) + split_sync = SplitSynchronizerAsync(api, storage, rbs_storage) split_synchronizers = SplitSynchronizers(split_sync, mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock()) synchronizer = SynchronizerAsync(split_synchronizers, mocker.Mock(spec=SplitTasks)) @@ -455,11 +472,11 @@ async def test_sync_all_failed_segments(self, mocker): api = mocker.Mock() storage = mocker.Mock() split_storage = mocker.Mock(spec=SplitStorage) - split_storage.get_segment_names.return_value = ['segmentA'] + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) split_sync = mocker.Mock(spec=SplitSynchronizer) split_sync.synchronize_splits.return_value = None - async def run(x, y): + async def run(x, y, c): raise APIException("something broke") api.fetch_segment = run @@ -467,7 +484,11 @@ async def get_segment_names(): return ['seg'] split_storage.get_segment_names = get_segment_names - segment_sync = SegmentSynchronizerAsync(api, split_storage, storage) + async def get_segment_names_rbs(): + return [] + rbs_storage.get_segment_names = get_segment_names_rbs + + segment_sync = SegmentSynchronizerAsync(api, split_storage, storage, rbs_storage) split_synchronizers = SplitSynchronizers(split_sync, segment_sync, mocker.Mock(), mocker.Mock(), mocker.Mock()) sychronizer = SynchronizerAsync(split_synchronizers, mocker.Mock(spec=SplitTasks)) @@ -479,14 +500,16 @@ async def get_segment_names(): @pytest.mark.asyncio async def test_synchronize_splits(self, mocker): split_storage = InMemorySplitStorageAsync() + rbs_storage = InMemoryRuleBasedSegmentStorageAsync() split_api = mocker.Mock() - async def fetch_splits(change, options): - return {'splits': splits, 'since': 123, - 'till': 123} + async def fetch_splits(change, rb, options): + return {'ff': {'d': splits, 's': 123, + 't': 123}, 'rbs': {'d': [], 's': -1, 't': -1}} + split_api.fetch_splits = fetch_splits - split_sync = SplitSynchronizerAsync(split_api, split_storage) + split_sync = SplitSynchronizerAsync(split_api, split_storage, rbs_storage) segment_storage = InMemorySegmentStorageAsync() segment_api = mocker.Mock() @@ -499,7 +522,7 @@ async def fetch_segment(segment_name, change, options): 'key3'], 'removed': [], 'since': 123, 'till': 123} segment_api.fetch_segment = fetch_segment - segment_sync = SegmentSynchronizerAsync(segment_api, split_storage, segment_storage) + segment_sync = SegmentSynchronizerAsync(segment_api, split_storage, segment_storage, rbs_storage) split_synchronizers = SplitSynchronizers(split_sync, segment_sync, mocker.Mock(), mocker.Mock(), mocker.Mock()) synchronizer = SynchronizerAsync(split_synchronizers, mocker.Mock(spec=SplitTasks)) @@ -520,17 +543,18 @@ async def fetch_segment(segment_name, change, options): @pytest.mark.asyncio async def test_synchronize_splits_calling_segment_sync_once(self, mocker): split_storage = InMemorySplitStorageAsync() + rbs_storage = InMemoryRuleBasedSegmentStorageAsync() async def get_change_number(): return 123 split_storage.get_change_number = get_change_number split_api = mocker.Mock() - async def fetch_splits(change, options): - return {'splits': splits, 'since': 123, - 'till': 123} + async def fetch_splits(change, rb, options): + return {'ff': {'d': splits, 's': 123, + 't': 123}, 'rbs': {'d': [], 's': -1, 't': -1}} split_api.fetch_splits = fetch_splits - split_sync = SplitSynchronizerAsync(split_api, split_storage) + split_sync = SplitSynchronizerAsync(split_api, split_storage, rbs_storage) counts = {'segments': 0} segment_sync = mocker.Mock() @@ -554,6 +578,7 @@ async def segment_exist_in_storage(segment): @pytest.mark.asyncio async def test_sync_all(self, mocker): split_storage = InMemorySplitStorageAsync() + rbs_storage = InMemoryRuleBasedSegmentStorageAsync() async def get_change_number(): return 123 split_storage.get_change_number = get_change_number @@ -578,11 +603,12 @@ def intersect(sets): split_storage.flag_set_filter.sorted_flag_sets = [] split_api = mocker.Mock() - async def fetch_splits(change, options): - return {'splits': splits, 'since': 123, 'till': 123} + async def fetch_splits(change, rb, options): + return {'ff': {'d': splits, 's': 123, + 't': 123}, 'rbs': {'d': [], 's': -1, 't': -1}} split_api.fetch_splits = fetch_splits - split_sync = SplitSynchronizerAsync(split_api, split_storage) + split_sync = SplitSynchronizerAsync(split_api, split_storage, rbs_storage) segment_storage = InMemorySegmentStorageAsync() async def get_change_number(segment): return 123 @@ -601,7 +627,7 @@ async def fetch_segment(segment_name, change, options): 'removed': [], 'since': 123, 'till': 123} segment_api.fetch_segment = fetch_segment - segment_sync = SegmentSynchronizerAsync(segment_api, split_storage, segment_storage) + segment_sync = SegmentSynchronizerAsync(segment_api, split_storage, segment_storage, rbs_storage) split_synchronizers = SplitSynchronizers(split_sync, segment_sync, mocker.Mock(), mocker.Mock(), mocker.Mock()) synchronizer = SynchronizerAsync(split_synchronizers, mocker.Mock(spec=SplitTasks)) @@ -673,7 +699,6 @@ def test_start_periodic_data_recording(self, mocker): assert len(impression_count_task.start.mock_calls) == 1 assert len(event_task.start.mock_calls) == 1 - class RedisSynchronizerTests(object): def test_start_periodic_data_recording(self, mocker): impression_count_task = mocker.Mock(spec=ImpressionsCountSyncTask) @@ -746,7 +771,6 @@ def stop_mock(event): assert len(unique_keys_task.stop.mock_calls) == 1 assert len(clear_filter_task.stop.mock_calls) == 1 - class RedisSynchronizerAsyncTests(object): @pytest.mark.asyncio async def test_start_periodic_data_recording(self, mocker): diff --git a/tests/sync/test_telemetry.py b/tests/sync/test_telemetry.py index c3aaac52..898216f8 100644 --- a/tests/sync/test_telemetry.py +++ b/tests/sync/test_telemetry.py @@ -169,7 +169,7 @@ def record_stats(*args, **kwargs): "spC": 1, "seC": 1, "skC": 0, - "ufs": {"sp": 3}, + "ufs": {"rbs": 0, "sp": 3}, "t": ['tag1'] }) @@ -294,6 +294,6 @@ async def record_stats(*args, **kwargs): "spC": 1, "seC": 1, "skC": 0, - "ufs": {"sp": 3}, + "ufs": {"rbs": 0, "sp": 3}, "t": ['tag1'] }) diff --git a/tests/tasks/test_segment_sync.py b/tests/tasks/test_segment_sync.py index 930d3f86..cc701e52 100644 --- a/tests/tasks/test_segment_sync.py +++ b/tests/tasks/test_segment_sync.py @@ -6,7 +6,7 @@ from splitio.api.commons import FetchOptions from splitio.tasks import segment_sync -from splitio.storage import SegmentStorage, SplitStorage +from splitio.storage import SegmentStorage, SplitStorage, RuleBasedSegmentsStorage from splitio.models.splits import Split from splitio.models.segments import Segment from splitio.models.grammar.condition import Condition @@ -21,6 +21,8 @@ def test_normal_operation(self, mocker): """Test the normal operation flow.""" split_storage = mocker.Mock(spec=SplitStorage) split_storage.get_segment_names.return_value = ['segmentA', 'segmentB', 'segmentC'] + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + rbs_storage.get_segment_names.return_value = [] # Setup a mocked segment storage whose changenumber returns -1 on first fetch and # 123 afterwards. @@ -62,10 +64,10 @@ def fetch_segment_mock(segment_name, change_number, fetch_options): fetch_segment_mock._count_c = 0 api = mocker.Mock() - fetch_options = FetchOptions(True, None, None, None) + fetch_options = FetchOptions(True, None, None, None, None) api.fetch_segment.side_effect = fetch_segment_mock - segments_synchronizer = SegmentSynchronizer(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizer(api, split_storage, storage, rbs_storage) task = segment_sync.SegmentSynchronizationTask(segments_synchronizer.synchronize_segments, 0.5) task.start() @@ -99,6 +101,8 @@ def test_that_errors_dont_stop_task(self, mocker): """Test that if fetching segments fails at some_point, the task will continue running.""" split_storage = mocker.Mock(spec=SplitStorage) split_storage.get_segment_names.return_value = ['segmentA', 'segmentB', 'segmentC'] + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + rbs_storage.get_segment_names.return_value = [] # Setup a mocked segment storage whose changenumber returns -1 on first fetch and # 123 afterwards. @@ -139,10 +143,10 @@ def fetch_segment_mock(segment_name, change_number, fetch_options): fetch_segment_mock._count_c = 0 api = mocker.Mock() - fetch_options = FetchOptions(True, None, None, None) + fetch_options = FetchOptions(True, None, None, None, None) api.fetch_segment.side_effect = fetch_segment_mock - segments_synchronizer = SegmentSynchronizer(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizer(api, split_storage, storage, rbs_storage) task = segment_sync.SegmentSynchronizationTask(segments_synchronizer.synchronize_segments, 0.5) task.start() @@ -183,6 +187,11 @@ async def get_segment_names(): return ['segmentA', 'segmentB', 'segmentC'] split_storage.get_segment_names = get_segment_names + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + async def get_segment_names_rbs(): + return [] + rbs_storage.get_segment_names = get_segment_names_rbs + # Setup a mocked segment storage whose changenumber returns -1 on first fetch and # 123 afterwards. storage = mocker.Mock(spec=SegmentStorage) @@ -238,10 +247,10 @@ async def fetch_segment_mock(segment_name, change_number, fetch_options): fetch_segment_mock._count_c = 0 api = mocker.Mock() - fetch_options = FetchOptions(True, None, None, None) + fetch_options = FetchOptions(True, None, None, None, None) api.fetch_segment = fetch_segment_mock - segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage, rbs_storage) task = segment_sync.SegmentSynchronizationTaskAsync(segments_synchronizer.synchronize_segments, 0.5) task.start() @@ -251,12 +260,16 @@ async def fetch_segment_mock(segment_name, change_number, fetch_options): await task.stop() assert not task.is_running() - assert (self.segment_name[0], self.change_number[0], self.fetch_options[0]) == ('segmentA', -1, fetch_options) - assert (self.segment_name[1], self.change_number[1], self.fetch_options[1]) == ('segmentA', 123, fetch_options) - assert (self.segment_name[2], self.change_number[2], self.fetch_options[2]) == ('segmentB', -1, fetch_options) - assert (self.segment_name[3], self.change_number[3], self.fetch_options[3]) == ('segmentB', 123, fetch_options) - assert (self.segment_name[4], self.change_number[4], self.fetch_options[4]) == ('segmentC', -1, fetch_options) - assert (self.segment_name[5], self.change_number[5], self.fetch_options[5]) == ('segmentC', 123, fetch_options) + api_calls = [] + for i in range(6): + api_calls.append((self.segment_name[i], self.change_number[i], self.fetch_options[i])) + + assert ('segmentA', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentA', 123, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentB', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentB', 123, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentC', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentC', 123, FetchOptions(True, None, None, None, None)) in api_calls segments_to_validate = set(['segmentA', 'segmentB', 'segmentC']) for segment in self.segments: @@ -272,6 +285,11 @@ async def get_segment_names(): return ['segmentA', 'segmentB', 'segmentC'] split_storage.get_segment_names = get_segment_names + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + async def get_segment_names_rbs(): + return [] + rbs_storage.get_segment_names = get_segment_names_rbs + # Setup a mocked segment storage whose changenumber returns -1 on first fetch and # 123 afterwards. storage = mocker.Mock(spec=SegmentStorage) @@ -326,10 +344,10 @@ async def fetch_segment_mock(segment_name, change_number, fetch_options): fetch_segment_mock._count_c = 0 api = mocker.Mock() - fetch_options = FetchOptions(True, None, None, None) + fetch_options = FetchOptions(True, None, None, None, None) api.fetch_segment = fetch_segment_mock - segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage, rbs_storage) task = segment_sync.SegmentSynchronizationTaskAsync(segments_synchronizer.synchronize_segments, 0.5) task.start() @@ -338,12 +356,16 @@ async def fetch_segment_mock(segment_name, change_number, fetch_options): await task.stop() assert not task.is_running() - - assert (self.segment_name[0], self.change_number[0], self.fetch_options[0]) == ('segmentA', -1, fetch_options) - assert (self.segment_name[1], self.change_number[1], self.fetch_options[1]) == ('segmentA', 123, fetch_options) - assert (self.segment_name[2], self.change_number[2], self.fetch_options[2]) == ('segmentB', -1, fetch_options) - assert (self.segment_name[3], self.change_number[3], self.fetch_options[3]) == ('segmentC', -1, fetch_options) - assert (self.segment_name[4], self.change_number[4], self.fetch_options[4]) == ('segmentC', 123, fetch_options) + + api_calls = [] + for i in range(5): + api_calls.append((self.segment_name[i], self.change_number[i], self.fetch_options[i])) + + assert ('segmentA', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentA', 123, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentB', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentC', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentC', 123, FetchOptions(True, None, None, None, None)) in api_calls segments_to_validate = set(['segmentA', 'segmentB', 'segmentC']) for segment in self.segments: diff --git a/tests/tasks/test_split_sync.py b/tests/tasks/test_split_sync.py index 9e9267e5..c9a0c692 100644 --- a/tests/tasks/test_split_sync.py +++ b/tests/tasks/test_split_sync.py @@ -6,7 +6,7 @@ from splitio.api import APIException from splitio.api.commons import FetchOptions from splitio.tasks import split_sync -from splitio.storage import SplitStorage +from splitio.storage import SplitStorage, RuleBasedSegmentsStorage from splitio.models.splits import Split from splitio.sync.split import SplitSynchronizer, SplitSynchronizerAsync from splitio.optional.loaders import asyncio @@ -53,6 +53,7 @@ class SplitSynchronizationTests(object): def test_normal_operation(self, mocker): """Test the normal operation flow.""" storage = mocker.Mock(spec=SplitStorage) + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) def change_number_mock(): change_number_mock._calls += 1 @@ -72,28 +73,31 @@ def intersect(sets): storage.flag_set_filter.flag_sets = {} storage.flag_set_filter.sorted_flag_sets = [] + self.clear = False + def clear(): + self.clear = True + storage.clear = clear + rbs_storage.clear = clear + api = mocker.Mock() def get_changes(*args, **kwargs): get_changes.called += 1 if get_changes.called == 1: - return { - 'splits': splits, - 'since': -1, - 'till': 123 + return {'ff': { + 'd': splits, + 's': -1, + 't': 123}, 'rbs': {'d': [], 't': -1, 's': -1} } else: - return { - 'splits': [], - 'since': 123, - 'till': 123 - } + return {'ff': {'d': [],'s': 123, 't': 123}, + 'rbs': {'d': [], 't': -1, 's': -1}} get_changes.called = 0 fetch_options = FetchOptions(True) api.fetch_splits.side_effect = get_changes - split_synchronizer = SplitSynchronizer(api, storage) + split_synchronizer = SplitSynchronizer(api, storage, rbs_storage) task = split_sync.SplitSynchronizationTask(split_synchronizer.synchronize_splits, 0.5) task.start() time.sleep(0.7) @@ -103,9 +107,9 @@ def get_changes(*args, **kwargs): stop_event.wait() assert not task.is_running() assert api.fetch_splits.mock_calls[0][1][0] == -1 - assert api.fetch_splits.mock_calls[0][1][1].cache_control_headers == True + assert api.fetch_splits.mock_calls[0][1][2].cache_control_headers == True assert api.fetch_splits.mock_calls[1][1][0] == 123 - assert api.fetch_splits.mock_calls[1][1][1].cache_control_headers == True + assert api.fetch_splits.mock_calls[1][1][2].cache_control_headers == True inserted_split = storage.update.mock_calls[0][1][0][0] assert isinstance(inserted_split, Split) @@ -114,20 +118,23 @@ def get_changes(*args, **kwargs): def test_that_errors_dont_stop_task(self, mocker): """Test that if fetching splits fails at some_point, the task will continue running.""" storage = mocker.Mock(spec=SplitStorage) + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) api = mocker.Mock() def run(x): run._calls += 1 if run._calls == 1: - return {'splits': [], 'since': -1, 'till': -1} + return {'ff': {'d': [],'s': -1, 't': -1}, + 'rbs': {'d': [], 't': -1, 's': -1}} if run._calls == 2: - return {'splits': [], 'since': -1, 'till': -1} + return {'ff': {'d': [],'s': -1, 't': -1}, + 'rbs': {'d': [], 't': -1, 's': -1}} raise APIException("something broke") run._calls = 0 api.fetch_splits.side_effect = run storage.get_change_number.return_value = -1 - split_synchronizer = SplitSynchronizer(api, storage) + split_synchronizer = SplitSynchronizer(api, storage, rbs_storage) task = split_sync.SplitSynchronizationTask(split_synchronizer.synchronize_splits, 0.5) task.start() time.sleep(0.1) @@ -144,6 +151,7 @@ class SplitSynchronizationAsyncTests(object): async def test_normal_operation(self, mocker): """Test the normal operation flow.""" storage = mocker.Mock(spec=SplitStorage) + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) async def change_number_mock(): change_number_mock._calls += 1 @@ -152,6 +160,9 @@ async def change_number_mock(): return 123 change_number_mock._calls = 0 storage.get_change_number = change_number_mock + async def rb_change_number_mock(): + return -1 + rbs_storage.get_change_number = rb_change_number_mock class flag_set_filter(): def should_filter(): @@ -168,25 +179,25 @@ async def set_change_number(*_): change_number_mock._calls = 0 storage.set_change_number = set_change_number + self.clear = False + async def clear(): + self.clear = True + storage.clear = clear + rbs_storage.clear = clear + api = mocker.Mock() self.change_number = [] self.fetch_options = [] - async def get_changes(change_number, fetch_options): + async def get_changes(change_number, rb_change_number, fetch_options): self.change_number.append(change_number) self.fetch_options.append(fetch_options) get_changes.called += 1 if get_changes.called == 1: - return { - 'splits': splits, - 'since': -1, - 'till': 123 - } + return {'ff': {'d': splits,'s': -1, 't': 123}, + 'rbs': {'d': [], 't': -1, 's': -1}} else: - return { - 'splits': [], - 'since': 123, - 'till': 123 - } + return {'ff': {'d': [],'s': 123, 't': 123}, + 'rbs': {'d': [], 't': -1, 's': -1}} api.fetch_splits = get_changes get_changes.called = 0 self.inserted_split = None @@ -194,12 +205,15 @@ async def update(split, deleted, change_number): if len(split) > 0: self.inserted_split = split storage.update = update - + async def rbs_update(split, deleted, change_number): + pass + rbs_storage.update = rbs_update + fetch_options = FetchOptions(True) - split_synchronizer = SplitSynchronizerAsync(api, storage) + split_synchronizer = SplitSynchronizerAsync(api, storage, rbs_storage) task = split_sync.SplitSynchronizationTaskAsync(split_synchronizer.synchronize_splits, 0.5) task.start() - await asyncio.sleep(1) + await asyncio.sleep(2) assert task.is_running() await task.stop() assert not task.is_running() @@ -212,14 +226,17 @@ async def update(split, deleted, change_number): async def test_that_errors_dont_stop_task(self, mocker): """Test that if fetching splits fails at some_point, the task will continue running.""" storage = mocker.Mock(spec=SplitStorage) + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) api = mocker.Mock() async def run(x): run._calls += 1 if run._calls == 1: - return {'splits': [], 'since': -1, 'till': -1} + return {'ff': {'d': [],'s': -1, 't': -1}, + 'rbs': {'d': [], 't': -1, 's': -1}} if run._calls == 2: - return {'splits': [], 'since': -1, 'till': -1} + return {'ff': {'d': [],'s': -1, 't': -1}, + 'rbs': {'d': [], 't': -1, 's': -1}} raise APIException("something broke") run._calls = 0 api.fetch_splits = run @@ -228,7 +245,7 @@ async def get_change_number(): return -1 storage.get_change_number = get_change_number - split_synchronizer = SplitSynchronizerAsync(api, storage) + split_synchronizer = SplitSynchronizerAsync(api, storage, rbs_storage) task = split_sync.SplitSynchronizationTaskAsync(split_synchronizer.synchronize_splits, 0.5) task.start() await asyncio.sleep(0.1) diff --git a/tests/util/test_storage_helper.py b/tests/util/test_storage_helper.py index 7608306d..5804a6fa 100644 --- a/tests/util/test_storage_helper.py +++ b/tests/util/test_storage_helper.py @@ -1,14 +1,44 @@ """Storage Helper tests.""" import pytest -from splitio.util.storage_helper import update_feature_flag_storage, get_valid_flag_sets, combine_valid_flag_sets -from splitio.storage.inmemmory import InMemorySplitStorage -from splitio.models import splits +from splitio.util.storage_helper import update_feature_flag_storage, get_valid_flag_sets, combine_valid_flag_sets, \ + update_rule_based_segment_storage, update_rule_based_segment_storage_async, update_feature_flag_storage_async, \ + get_standard_segment_names_in_rbs_storage_async, get_standard_segment_names_in_rbs_storage +from splitio.storage.inmemmory import InMemorySplitStorage, InMemoryRuleBasedSegmentStorage, InMemoryRuleBasedSegmentStorageAsync, \ + InMemorySplitStorageAsync +from splitio.models import splits, rule_based_segments from splitio.storage import FlagSetsFilter from tests.sync.test_splits_synchronizer import splits_raw as split_sample class StorageHelperTests(object): + rbs = rule_based_segments.from_raw({ + "changeNumber": 123, + "name": "sample_rule_based_segment", + "status": "ACTIVE", + "trafficTypeName": "user", + "excluded":{ + "keys":["mauro@split.io","gaston@split.io"], + "segments":[{"name":"excluded_segment", "type": "standard"}] + }, + "conditions": [ + {"matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "matcherType": "IN_SEGMENT", + "negate": False, + "userDefinedSegmentMatcherData": { + "segmentName": "employees" + }, + "whitelistMatcherData": None + } + ] + }, + } + ] + }) + def test_update_feature_flag_storage(self, mocker): storage = mocker.Mock(spec=InMemorySplitStorage) split = splits.from_raw(split_sample[0]) @@ -34,10 +64,16 @@ def intersect(sets): storage.flag_set_filter = flag_set_filter storage.flag_set_filter.flag_sets = {} - update_feature_flag_storage(storage, [split], 123) + self.clear = 0 + def clear(): + self.clear += 1 + storage.clear = clear + + update_feature_flag_storage(storage, [split], 123, True) assert self.added[0] == split assert self.deleted == [] assert self.change_number == 123 + assert self.clear == 1 class flag_set_filter2(): def should_filter(): @@ -47,9 +83,11 @@ def intersect(sets): storage.flag_set_filter = flag_set_filter2 storage.flag_set_filter.flag_sets = set({'set1', 'set2'}) + self.clear = 0 update_feature_flag_storage(storage, [split], 123) assert self.added == [] assert self.deleted[0] == split.name + assert self.clear == 0 class flag_set_filter3(): def should_filter(): @@ -126,4 +164,173 @@ def test_combine_valid_flag_sets(self): assert combine_valid_flag_sets(results_set) == {'set2', 'set3'} results_set = ['set1', {'set2', 'set3'}] - assert combine_valid_flag_sets(results_set) == {'set2', 'set3'} \ No newline at end of file + assert combine_valid_flag_sets(results_set) == {'set2', 'set3'} + + def test_update_rule_base_segment_storage(self, mocker): + storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorage) + self.added = [] + self.deleted = [] + self.change_number = 0 + def update(to_add, to_delete, change_number): + self.added = to_add + self.deleted = to_delete + self.change_number = change_number + storage.update = update + + self.clear = 0 + def clear(): + self.clear += 1 + storage.clear = clear + + segments = update_rule_based_segment_storage(storage, [self.rbs], 123) + assert self.added[0] == self.rbs + assert self.deleted == [] + assert self.change_number == 123 + assert segments == {'excluded_segment', 'employees'} + assert self.clear == 0 + + segments = update_rule_based_segment_storage(storage, [self.rbs], 123, True) + assert self.clear == 1 + + def test_get_standard_segment_in_rbs_storage(self, mocker): + storage = InMemoryRuleBasedSegmentStorage() + segments = update_rule_based_segment_storage(storage, [self.rbs], 123) + assert get_standard_segment_names_in_rbs_storage(storage) == {'excluded_segment', 'employees'} + + @pytest.mark.asyncio + async def test_get_standard_segment_in_rbs_storage(self, mocker): + storage = InMemoryRuleBasedSegmentStorageAsync() + segments = await update_rule_based_segment_storage_async(storage, [self.rbs], 123) + assert await get_standard_segment_names_in_rbs_storage_async(storage) == {'excluded_segment', 'employees'} + + @pytest.mark.asyncio + async def test_update_rule_base_segment_storage_async(self, mocker): + storage = mocker.Mock(spec=InMemoryRuleBasedSegmentStorageAsync) + self.added = [] + self.deleted = [] + self.change_number = 0 + async def update(to_add, to_delete, change_number): + self.added = to_add + self.deleted = to_delete + self.change_number = change_number + storage.update = update + + self.clear = 0 + async def clear(): + self.clear += 1 + storage.clear = clear + + segments = await update_rule_based_segment_storage_async(storage, [self.rbs], 123) + assert self.added[0] == self.rbs + assert self.deleted == [] + assert self.change_number == 123 + assert segments == {'excluded_segment', 'employees'} + + segments = await update_rule_based_segment_storage_async(storage, [self.rbs], 123, True) + assert self.clear == 1 + + @pytest.mark.asyncio + async def test_update_feature_flag_storage_async(self, mocker): + storage = mocker.Mock(spec=InMemorySplitStorageAsync) + split = splits.from_raw(split_sample[0]) + + self.added = [] + self.deleted = [] + self.change_number = 0 + async def get(flag_name): + return None + storage.get = get + + async def update(to_add, to_delete, change_number): + self.added = to_add + self.deleted = to_delete + self.change_number = change_number + storage.update = update + + async def is_flag_set_exist(flag_set): + return False + storage.is_flag_set_exist = is_flag_set_exist + + class flag_set_filter(): + def should_filter(): + return False + def intersect(sets): + return True + storage.flag_set_filter = flag_set_filter + storage.flag_set_filter.flag_sets = {} + + self.clear = 0 + async def clear(): + self.clear += 1 + storage.clear = clear + + await update_feature_flag_storage_async(storage, [split], 123, True) + assert self.added[0] == split + assert self.deleted == [] + assert self.change_number == 123 + assert self.clear == 1 + + class flag_set_filter2(): + def should_filter(): + return True + def intersect(sets): + return False + storage.flag_set_filter = flag_set_filter2 + storage.flag_set_filter.flag_sets = set({'set1', 'set2'}) + + async def get(flag_name): + return split + storage.get = get + + self.clear = 0 + await update_feature_flag_storage_async(storage, [split], 123) + assert self.added == [] + assert self.deleted[0] == split.name + assert self.clear == 0 + + class flag_set_filter3(): + def should_filter(): + return True + def intersect(sets): + return True + storage.flag_set_filter = flag_set_filter3 + storage.flag_set_filter.flag_sets = set({'set1', 'set2'}) + + async def is_flag_set_exist2(flag_set): + return True + storage.is_flag_set_exist = is_flag_set_exist2 + await update_feature_flag_storage_async(storage, [split], 123) + assert self.added[0] == split + assert self.deleted == [] + + split_json = split_sample[0] + split_json['conditions'].append({ + "matcherGroup": { + "combiner": "AND", + "matchers": [ + { + "matcherType": "IN_SEGMENT", + "negate": False, + "userDefinedSegmentMatcherData": { + "segmentName": "segment1" + }, + "whitelistMatcherData": None + } + ] + }, + "partitions": [ + { + "treatment": "on", + "size": 30 + }, + { + "treatment": "off", + "size": 70 + } + ] + } + ) + + split = splits.from_raw(split_json) + storage.config_flag_sets_used = 0 + assert await update_feature_flag_storage_async(storage, [split], 123) == {'segment1'} \ No newline at end of file