From 04b916c8aedfe76937740a08f27021c8fd4a370b Mon Sep 17 00:00:00 2001 From: Young Date: Tue, 16 Feb 2021 15:07:14 +0000 Subject: [PATCH 01/61] safe yaml loader --- qlib/__init__.py | 2 +- qlib/utils/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qlib/__init__.py b/qlib/__init__.py index 596202061e..20c2cab4a3 100644 --- a/qlib/__init__.py +++ b/qlib/__init__.py @@ -147,7 +147,7 @@ def init_from_yaml_conf(conf_path, **kwargs): """ with open(conf_path) as f: - config = yaml.load(f, Loader=yaml.FullLoader) + config = yaml.load(f, Loader=yaml.SafeLoader) config.update(kwargs) default_conf = config.pop("default_conf", "client") init(default_conf, **config) diff --git a/qlib/utils/__init__.py b/qlib/utils/__init__.py index 799ab377a7..f5e981c24d 100644 --- a/qlib/utils/__init__.py +++ b/qlib/utils/__init__.py @@ -128,7 +128,7 @@ def parse_config(config): # Check whether config is file if os.path.exists(config): with open(config, "r") as f: - return yaml.load(f) + return yaml.load(f, Loader=yaml.SafeLoader) # Check whether the str can be parsed try: return yaml.load(config) From 83237ba4ed8244aa6eb89109b4443e5ca6881dda Mon Sep 17 00:00:00 2001 From: Young Date: Wed, 17 Feb 2021 05:17:18 +0000 Subject: [PATCH 02/61] yml afe load --- qlib/__init__.py | 2 +- qlib/contrib/online/manager.py | 2 +- qlib/contrib/online/utils.py | 2 +- qlib/contrib/tuner/config.py | 2 +- qlib/utils/__init__.py | 4 ++-- qlib/workflow/cli.py | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/qlib/__init__.py b/qlib/__init__.py index 20c2cab4a3..99035e5014 100644 --- a/qlib/__init__.py +++ b/qlib/__init__.py @@ -147,7 +147,7 @@ def init_from_yaml_conf(conf_path, **kwargs): """ with open(conf_path) as f: - config = yaml.load(f, Loader=yaml.SafeLoader) + config = yaml.safe_load(f) config.update(kwargs) default_conf = config.pop("default_conf", "client") init(default_conf, **config) diff --git a/qlib/contrib/online/manager.py b/qlib/contrib/online/manager.py index cf850b9dac..b0be7f0768 100644 --- a/qlib/contrib/online/manager.py +++ b/qlib/contrib/online/manager.py @@ -110,7 +110,7 @@ def add_user(self, user_id, config_file, add_date): raise ValueError("User data for {} already exists".format(user_id)) with config_file.open("r") as fp: - config = yaml.load(fp) + config = yaml.safe_load(fp) # load model model = init_instance_by_config(config["model"]) diff --git a/qlib/contrib/online/utils.py b/qlib/contrib/online/utils.py index 611af63e4a..71a6a91ec2 100644 --- a/qlib/contrib/online/utils.py +++ b/qlib/contrib/online/utils.py @@ -88,7 +88,7 @@ def prepare(um, today, user_id, exchange_config=None): dates.append(get_next_trading_date(dates[-1], future=True)) if exchange_config: with pathlib.Path(exchange_config).open("r") as fp: - exchange_paras = yaml.load(fp) + exchange_paras = yaml.safe_load(fp) else: exchange_paras = {} trade_exchange = Exchange(trade_dates=dates, **exchange_paras) diff --git a/qlib/contrib/tuner/config.py b/qlib/contrib/tuner/config.py index f23d1b8740..247fa6a4fa 100644 --- a/qlib/contrib/tuner/config.py +++ b/qlib/contrib/tuner/config.py @@ -14,7 +14,7 @@ def __init__(self, config_path): self.config_path = config_path with open(config_path) as fp: - config = yaml.load(fp) + config = yaml.safe_load(fp) self.config = copy.deepcopy(config) self.pipeline_ex_config = PipelineExperimentConfig(config.get("experiment", dict()), self) diff --git a/qlib/utils/__init__.py b/qlib/utils/__init__.py index f5e981c24d..be7969b658 100644 --- a/qlib/utils/__init__.py +++ b/qlib/utils/__init__.py @@ -128,10 +128,10 @@ def parse_config(config): # Check whether config is file if os.path.exists(config): with open(config, "r") as f: - return yaml.load(f, Loader=yaml.SafeLoader) + return yaml.safe_load(f) # Check whether the str can be parsed try: - return yaml.load(config) + return yaml.safe_load(config) except BaseException: raise ValueError("cannot parse config!") diff --git a/qlib/workflow/cli.py b/qlib/workflow/cli.py index f7455797bd..6eba962779 100644 --- a/qlib/workflow/cli.py +++ b/qlib/workflow/cli.py @@ -44,7 +44,7 @@ def sys_config(config, config_path): # worflow handler function def workflow(config_path, experiment_name="workflow", uri_folder="mlruns"): with open(config_path) as fp: - config = yaml.load(fp, Loader=yaml.SafeLoader) + config = yaml.safe_load(fp) # config the `sys` section sys_config(config, config_path) From 1e5cf1c17400accafc7453e4b09779aac751520f Mon Sep 17 00:00:00 2001 From: Young Date: Fri, 26 Feb 2021 09:14:40 +0000 Subject: [PATCH 03/61] init version of online serving and rolling --- qlib/model/ens/__init__.py | 0 qlib/workflow/task/__init__.py | 13 ++ qlib/workflow/task/collect.py | 52 ++++++ qlib/workflow/task/gen.py | 133 +++++++++++++++ qlib/workflow/task/manage.py | 290 +++++++++++++++++++++++++++++++++ qlib/workflow/task/utils.py | 134 +++++++++++++++ setup.py | 1 + 7 files changed, 623 insertions(+) create mode 100644 qlib/model/ens/__init__.py create mode 100644 qlib/workflow/task/__init__.py create mode 100644 qlib/workflow/task/collect.py create mode 100644 qlib/workflow/task/gen.py create mode 100644 qlib/workflow/task/manage.py create mode 100644 qlib/workflow/task/utils.py diff --git a/qlib/model/ens/__init__.py b/qlib/model/ens/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/qlib/workflow/task/__init__.py b/qlib/workflow/task/__init__.py new file mode 100644 index 0000000000..cc338cca4d --- /dev/null +++ b/qlib/workflow/task/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +""" +Task related workflow is implemented in this folder + +A typical task workflow + +| Step | Description | +|-----------------------+------------------------------------------------| +| TaskGen | Generating tasks. | +| TaskManager(optional) | Manage generated tasks | +| run task | retrive tasks from TaskManager and run tasks. | +""" diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py new file mode 100644 index 0000000000..13b5869de5 --- /dev/null +++ b/qlib/workflow/task/collect.py @@ -0,0 +1,52 @@ +from qlib.workflow import R +import pandas as pd +from typing import Union +from tqdm.auto import tqdm + + +class RollingEnsemble: + ''' + Rolling Models Ensemble based on (R)ecord + + This shares nothing with Ensemble + ''' + # TODO: 这边还可以加加速 + def __init__(self, get_key_func, flt_func=None): + self.get_key_func = get_key_func + self.flt_func = flt_func + + def __call__(self, exp_name) -> Union[pd.Series, dict]: + # TODO; + # Should we split the scripts into several sub functions? + exp = R.get_exp(experiment_name=exp_name) + + # filter records + recs = exp.list_recorders() + + recs_flt = {} + for rid, rec in tqdm(recs.items(), desc="Loading data"): + # rec = exp.get_recorder(recorder_id=rid) + params = rec.load_object("param") + if rec.status == rec.STATUS_FI: + if self.flt_func is None or self.flt_func(params): + rec.params = params + recs_flt[rid] = rec + + # group + recs_group = {} + for _, rec in recs_flt.items(): + params = rec.params + group_key = self.get_key_func(params) + recs_group.setdefault(group_key, []).append(rec) + + # reduce group + reduce_group = {} + for k, rec_l in recs_group.items(): + pred_l = [] + for rec in rec_l: + pred_l.append(rec.load_object('pred.pkl').iloc[:, 0]) + pred = pd.concat(pred_l).sort_index() + reduce_group[k] = pred + + return reduce_group + diff --git a/qlib/workflow/task/gen.py b/qlib/workflow/task/gen.py new file mode 100644 index 0000000000..66529f3a5a --- /dev/null +++ b/qlib/workflow/task/gen.py @@ -0,0 +1,133 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +''' +this is a task generator +''' +import abc +import copy +import typing +from .utils import TimeAdjuster + + +class TaskGen(metaclass=abc.ABCMeta): + @abc.abstractmethod + def __call__(self, *args, **kwargs) -> typing.List[dict]: + """ + generate + + Parameters + ---------- + args, kwargs: + The info for generating tasks + Example 1): + input: a specific task template + output: rolling version of the tasks + Example 2): + input: a specific task template + output: a set of tasks with different losses + + Returns + ------- + typing.List[dict]: + A list of tasks + """ + pass + + +class RollingGen(TaskGen): + + ROLL_EX = TimeAdjuster.SHIFT_EX + ROLL_SD = TimeAdjuster.SHIFT_SD + + def __init__(self, step: int = 40, rtype: str = ROLL_EX): + """ + Generate tasks for rolling + + Parameters + ---------- + step : int + step to rolling + rtype : str + rolling type (expanding, rolling) + """ + self.step = step + self.rtype = rtype + self.ta = TimeAdjuster(future=True) # 为了保证test最后的日期不是None, 所以这边要改一改 + + self.test_key = 'test' + self.train_key = 'train' + + def __call__(self, task: dict): + """ + Converting the task into a rolling task + + Parameters + ---------- + task : dict + A dict describing a task. For example. + + DEFAULT_TASK = { + "model": { + "class": "LGBModel", + "module_path": "qlib.contrib.model.gbdt", + }, + "dataset": { + "class": "DatasetH", + "module_path": "qlib.data.dataset", + "kwargs": { + "handler": { + "class": "Alpha158", + "module_path": "qlib.contrib.data.handler", + "kwargs": data_handler_config, + }, + "segments": { + "train": ("2008-01-01", "2014-12-31"), + "valid": ("2015-01-01", "2016-12-20"), # Please avoid leaking the future test data into validation + "test": ("2017-01-01", "2020-08-01"), + }, + }, + }, + # You shoud record the data in specific sequence + # "record": ['SignalRecord', 'SigAnaRecord', 'PortAnaRecord'], + } + """ + res = [] + + prev_seg = None + test_end = None + while True: + t = copy.deepcopy(task) + + # calculate segments + if prev_seg is None: + # First rolling + # 1) prepare the end porint + segments = copy.deepcopy(self.ta.align_seg(t['dataset']['kwargs']['segments'])) + test_end = self.ta.max() if segments[self.test_key][1] is None else segments[self.test_key][1] + # 2) and the init test segments + test_start_idx = self.ta.align_idx(segments[self.test_key][0]) + segments[self.test_key] = (self.ta.get(test_start_idx), self.ta.get(test_start_idx + self.step - 1)) + else: + segments = {} + try: + for k, seg in prev_seg.items(): + # 决定怎么shift + if k == self.train_key and self.rtype == self.ROLL_EX: + rtype = self.ta.SHIFT_EX + else: + rtype = self.ta.SHIFT_SD + # 整段数据做shift + segments[k] = self.ta.shift(seg, step=self.step, rtype=rtype) + if segments[self.test_key][0] > test_end: + break + except KeyError: + # We reach the end of tasks + # No more rolling + break + + t['dataset']['kwargs']['segments'] = copy.deepcopy(segments) + prev_seg = segments + res.append(t) + return res + + diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py new file mode 100644 index 0000000000..6407279f06 --- /dev/null +++ b/qlib/workflow/task/manage.py @@ -0,0 +1,290 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +""" +A task consists of 2 parts +- tasks description: the desc will define the task +- tasks status: the status of the task +- tasks result information : A user can get the task with the task description and task result. + +""" +from bson.binary import Binary +import pickle +from pymongo.errors import InvalidDocument +from fire import Fire +from bson.objectid import ObjectId +from contextlib import contextmanager +from loguru import logger +from tqdm.cli import tqdm +import time +import concurrent +import pymongo +from qlib.config import C + + +class TaskManager: + """TaskManager + here is the what will a task looks like + { + 'def': pickle serialized task definition. using pickle will make it easier + 'filter': json-like data. This is for filtering the tasks. + 'status': 'waiting' | 'running' | 'done' + 'res': pickle serialized task result, + } + + The tasks manager assume that you will only update the tasks you fetched. + The mongo fetch one and update will make it date updating secure. + + Usage Examples from the CLI. + python -m blocks.tasks.__init__ task_stat --task_pool meta_task_rule + + + NOTE: + - 假设： 存储在db里面的都是encode过的， 拿出来的都是decode过的 + """ + STATUS_WAITING = 'waiting' + STATUS_RUNNING = 'running' + STATUS_DONE = 'done' + STATUS_PART_DONE = 'part_done' + + ENCODE_FIELDS_PREFIX = ['def', 'res'] + + def __init__(self, task_pool=None): + self.mdb = get_mongodb() + self.task_pool = task_pool + + def list(self): + return self.mdb.list_collection_names() + + def _encode_task(self, task): + for prefix in self.ENCODE_FIELDS_PREFIX: + for k in list(task.keys()): + if k.startswith(prefix): + task[k] = Binary(pickle.dumps(task[k])) + return task + + def _decode_task(self, task): + for prefix in self.ENCODE_FIELDS_PREFIX: + for k in list(task.keys()): + if k.startswith(prefix): + task[k] = pickle.loads(task[k]) + return task + + def _get_task_pool(self, task_pool=None): + if task_pool is None: + task_pool = self.task_pool + if task_pool is None: + raise ValueError('You must specify a task pool.') + if isinstance(task_pool, str): + return getattr(self.mdb, task_pool) + return task_pool + + def _dict_to_str(self, flt): + return {k: str(v) for k, v in flt.items()} + + def replace_task(self, task, new_task, task_pool=None): + # 这里的假设是从接口拿出来的都是decode过的，在接口内部的都是 encode过的 + new_task = self._encode_task(new_task) + task_pool = self._get_task_pool(task_pool) + query = {'_id': ObjectId(task['_id'])} + try: + task_pool.replace_one(query, new_task) + except InvalidDocument: + task['filter'] = self._dict_to_str(task['filter']) + task_pool.replace_one(query, new_task) + + def insert_task(self, task, task_pool=None): + task_pool = self._get_task_pool(task_pool) + try: + task_pool.insert_one(task) + except InvalidDocument: + task['filter'] = self._dict_to_str(task['filter']) + task_pool.insert_one(task) + + def insert_task_def(self, task_def, task_pool=None): + task_pool = self._get_task_pool(task_pool) + task = self._encode_task({ + 'def': task_def, + 'filter': task_def, # FIXME: catch the raised error + 'status': self.STATUS_WAITING, + }) + self.insert_task(task, task_pool) + + def create_task(self, task_def_l, task_pool=None, dry_run=False, print_nt=False): + task_pool = self._get_task_pool(task_pool) + new_tasks = [] + for t in task_def_l: + try: + r = task_pool.find_one({'filter': t}) + except InvalidDocument: + r = task_pool.find_one({'filter': self._dict_to_str(t)}) + if r is None: + new_tasks.append(t) + print("Total Tasks, New Tasks:", len(task_def_l), len(new_tasks)) + + if print_nt: # print new task + for t in new_tasks: + print(t) + + if dry_run: + return + + for t in new_tasks: + self.insert_task_def(t, task_pool) + + def fetch_task(self, query={}, task_pool=None): + task_pool = self._get_task_pool(task_pool) + query = query.copy() + if '_id' in query: + query['_id'] = ObjectId(query['_id']) + query.update({'status': self.STATUS_WAITING}) + task = task_pool.find_one_and_update(query, {'$set': { + 'status': self.STATUS_RUNNING + }}, + sort=[('priority', pymongo.DESCENDING)]) + # 这里我的 priority 必须是 高数优先级更高，因为 null会被在 ASCENDING时被排在最前面 + if task is None: + return None + task['status'] = self.STATUS_RUNNING + return self._decode_task(task) + + @contextmanager + def safe_fetch_task(self, query={}, task_pool=None): + task = self.fetch_task(query=query, task_pool=task_pool) + try: + yield task + except Exception: + if task is not None: + logger.info('Returning task before raising error') + self.return_task(task) + logger.info('Task returned') + raise + + def task_fetcher_iter(self, query={}, task_pool=None): + while True: + with self.safe_fetch_task(query=query, task_pool=task_pool) as task: + if task is None: + break + yield task + + def query(self, query={}, decode=True, task_pool=None): + """query + This function may raise exception `pymongo.errors.CursorNotFound: cursor id not found` if it takes too long to iterate the generator + + :param query: + :param decode: + :param task_pool: + """ + query = query.copy() + if '_id' in query: + query['_id'] = ObjectId(query['_id']) + task_pool = self._get_task_pool(task_pool) + for t in task_pool.find(query): + yield self._decode_task(t) + + def commit_task_res(self, task, res, status=None, task_pool=None): + task_pool = self._get_task_pool(task_pool) + # A workaround to use the class attribute. + if status is None: + status = TaskManager.STATUS_DONE + task_pool.update_one({"_id": task['_id']}, {'$set': {'status': status, 'res': Binary(pickle.dumps(res))}}) + + def return_task(self, task, status=None, task_pool=None): + task_pool = self._get_task_pool(task_pool) + if status is None: + status = TaskManager.STATUS_WAITING + update_dict = {'$set': {'status': status}} + task_pool.update_one({"_id": task['_id']}, update_dict) + + def remove(self, query={}, task_pool=None): + query = query.copy() + task_pool = self._get_task_pool(task_pool) + if '_id' in query: + query['_id'] = ObjectId(query['_id']) + task_pool.delete_many(query) + + def task_stat(self, query={}, task_pool=None): + query = query.copy() + if '_id' in query: + query['_id'] = ObjectId(query['_id']) + tasks = self.query(task_pool=task_pool, query=query, decode=False) + status_stat = {} + for t in tasks: + status_stat[t['status']] = status_stat.get(t['status'], 0) + 1 + return status_stat + + def reset_waiting(self, query={}, task_pool=None): + query = query.copy() + # default query + if 'status' not in query: + query['status'] = self.STATUS_RUNNING + return self.reset_status(query=query, status=self.STATUS_WAITING, task_pool=task_pool) + + def reset_status(self, query, status, task_pool=None): + query = query.copy() + task_pool = self._get_task_pool(task_pool) + if '_id' in query: + query['_id'] = ObjectId(query['_id']) + print(task_pool.update_many(query, {"$set": {"status": status}})) + + def _get_undone_n(self, task_stat): + return task_stat.get(self.STATUS_WAITING, 0) + task_stat.get(self.STATUS_RUNNING, 0) + + def _get_total(self, task_stat): + return sum(task_stat.values()) + + def wait(self, query={}, task_pool=None): + task_stat = self.task_stat(query, task_pool) + total = self._get_total(task_stat) + last_undone_n = self._get_undone_n(task_stat) + with tqdm(total=total, initial=total - last_undone_n) as pbar: + while True: + time.sleep(10) + undone_n = self._get_undone_n(self.task_stat(query, task_pool)) + pbar.update(last_undone_n - undone_n) + last_undone_n = undone_n + if undone_n == 0: + break + + def __str__(self): + return f"TaskManager({self.task_pool})" + + +def run_task(task_func, task_pool, force_release=False, *args, **kwargs): + """run_task. + While task pool is not empty, use task_func to fetch and run tasks in task_pool + + Parameters + ---------- + task_func : def (task_def, *args, **kwargs) -> + the function to run the task + task_pool : + The name of the task pool + force_release : + will the program force to release the resource + args : + args + kwargs : + kwargs + """ + tm = TaskManager(task_pool) + + ever_run = False + + while True: + with tm.safe_fetch_task() as task: + if task is None: + break + logger.info(task['def']) + if force_release: + with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: + res = executor.submit(task_func, task['def'], *args, **kwargs).result() + else: + res = task_func(task['def'], *args, **kwargs) + tm.commit_task_res(task, res) + ever_run = True + + return ever_run + + +if __name__ == '__main__': + Fire(TaskManager) diff --git a/qlib/workflow/task/utils.py b/qlib/workflow/task/utils.py new file mode 100644 index 0000000000..3d8fe89964 --- /dev/null +++ b/qlib/workflow/task/utils.py @@ -0,0 +1,134 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +import bisect +import pandas as pd +from qlib.data import D +from qlib.config import C +from qlib.log import get_module_logger +from pymongo import MongoClient + + +def get_mongodb(): + try: + cfg = C['mongo'] + except KeyError: + get_module_logger("task").error("Please configure `C['mongo']` before using TaskManager") + raise + + client = MongoClient(cfg['task_url']) + return client.get_database(name=cfg['task_db_name']) + + +class TimeAdjuster: + '''找到合适的日期，然后adjust date''' + def __init__(self, future=False): + self.cals = D.calendar(future=future) + + def get(self, idx: int): + """ + Get datetime by index + + Parameters + ---------- + idx : int + index of the calendar + """ + if idx >= len(self.cals): + return None + return self.cals[idx] + + def max(self): + """ + Return return the max calendar date + """ + return max(self.cals) + + def align_idx(self, time_point, tp_type="start"): + time_point = pd.Timestamp(time_point) + if tp_type == 'start': + idx = bisect.bisect_left(self.cals, time_point) + elif tp_type == 'end': + idx = bisect.bisect_right(self.cals, time_point) - 1 + else: + raise NotImplementedError(f"This type of input is not supported") + return idx + + def align_time(self, time_point, tp_type="start"): + """ + Align a timepoint to calendar weekdays + + Parameters + ---------- + time_point : + Time point + tp_type : str + time point type (`"start"`, `"end"`) + """ + return self.cals[self.align_idx(time_point, tp_type=tp_type)] + + def align_seg(self, segment): + if isinstance(segment, dict): + return {k: self.align_seg(seg) for k, seg in segment.items()} + elif isinstance(segment, tuple): + return self.align_time(segment[0], tp_type="start"), self.align_time(segment[1], tp_type="end") + else: + raise NotImplementedError(f"This type of input is not supported") + + def truncate(self, segment, test_start, days: int): + """ + truncate the segment based on the test_start date + + Parameters + ---------- + segment : + time segment + days : int + The trading days to be truncated + 大部分情况是因为这个时间段的数据(一般是特征)会用到 `days` 天的数据 + """ + test_idx = self.align_idx(test_start) + if isinstance(segment, tuple): + new_seg = [] + for time_point in segment: + tp_idx = min(self.align_idx(time_point), test_idx - days) + assert (tp_idx > 0) + new_seg.append(self.get(tp_idx)) + return tuple(new_seg) + else: + raise NotImplementedError(f"This type of input is not supported") + + SHIFT_SD = "sliding" + SHIFT_EX = "expanding" + + def shift(self, seg, step: int, rtype=SHIFT_SD): + """ + shift the datatiem of segment + + Parameters + ---------- + seg : + datetime segment + step : int + rolling step + rtype : str + rolling type ("sliding" or "expanding") + + Raises + ------ + KeyError: + shift will raise error if the index(both start and end) is out of self.cal + """ + if isinstance(seg, tuple): + start_idx, end_idx = self.align_idx(seg[0], tp_type="start"), self.align_idx(seg[1], tp_type="end") + if rtype == self.SHIFT_SD: + start_idx += step + end_idx += step + elif rtype == self.SHIFT_EX: + end_idx += step + else: + raise NotImplementedError(f"This type of input is not supported") + if start_idx > len(self.cals): + raise KeyError("The segment is out of valid calendar") + return self.get(start_idx), self.get(end_idx) + else: + raise NotImplementedError(f"This type of input is not supported") diff --git a/setup.py b/setup.py index f759945fd5..c83d092a3b 100644 --- a/setup.py +++ b/setup.py @@ -55,6 +55,7 @@ "tornado", "joblib>=0.17.0", "ruamel.yaml>=0.16.12", + "pymongo==3.7.2", # For task management ] # Numpy include From 24024d51c7762dae44313628ff357821a65283d9 Mon Sep 17 00:00:00 2001 From: Young Date: Sat, 27 Feb 2021 09:44:01 +0000 Subject: [PATCH 04/61] qlib auto init basedon project & black format --- qlib/__init__.py | 75 +++++++++++++++++++++++++++- qlib/config.py | 17 +++++++ qlib/workflow/task/collect.py | 8 +-- qlib/workflow/task/gen.py | 16 +++--- qlib/workflow/task/manage.py | 93 ++++++++++++++++++----------------- qlib/workflow/task/utils.py | 15 +++--- 6 files changed, 158 insertions(+), 66 deletions(-) diff --git a/qlib/__init__.py b/qlib/__init__.py index 99035e5014..816e5a5852 100644 --- a/qlib/__init__.py +++ b/qlib/__init__.py @@ -3,6 +3,7 @@ __version__ = "0.6.3.99" +__version__bak = __version__ # This version is backup for QlibConfig.reset_qlib_version import os @@ -10,12 +11,13 @@ import logging import platform import subprocess +from pathlib import Path +from .log import get_module_logger # init qlib def init(default_conf="client", **kwargs): from .config import C - from .log import get_module_logger from .data.cache import H H.clear() @@ -48,7 +50,6 @@ def init(default_conf="client", **kwargs): def _mount_nfs_uri(C): - from .log import get_module_logger LOG = get_module_logger("mount nfs", level=logging.INFO) @@ -151,3 +152,73 @@ def init_from_yaml_conf(conf_path, **kwargs): config.update(kwargs) default_conf = config.pop("default_conf", "client") init(default_conf, **config) + + +def get_project_path(config_name="config.yaml") -> Path: + """ + If users are building a project follow the following pattern. + - Qlib is a sub folder in project path + - There is a file named `config.yaml` in qlib. + + For example: + If your project file system stucuture follows such a pattern + + / + - config.yaml + - ...some folders... + - qlib/ + + This folder will return + + NOTE: link is not supported here. + + + This method is often used when + - user want to use a relative config path instead of hard-coding qlib config path in code + + Raises + ------ + FileNotFoundError: + If project path is not found + """ + cur_path = Path(__file__).absolute().resolve() + while True: + if (cur_path / config_name).exists(): + return cur_path + if cur_path == cur_path.parent: + raise FileNotFoundError("We can't find the project path") + cur_path = cur_path.parent + + +def auto_init(**kwargs): + """ + This function will init qlib automatically with following priority + - Find the project configuration and init qlib + - The parsing process will be affected by the `conf_type` of the configuration file + - Init qlib with default config + """ + + try: + pp = get_project_path() + except FileNotFoundError: + init(**kwargs) + else: + + conf_pp = pp / "config.yaml" + with conf_pp.open() as f: + conf = yaml.safe_load(f) + + conf_type = conf.get("conf_type", "origin") + if conf_type == "origin": + # The type of config is just like original qlib config + init_from_yaml_conf(conf_pp, **kwargs) + elif conf_type == "ref": + # This config type will be more convenient in following scenario + # - There is a shared configure file and you don't want to edit it inplace. + # - The shared configure may be updated later and you don't want to copy it. + # - You have some customized config. + qlib_conf_path = conf["qlib_cfg"] + qlib_conf_update = conf.get("qlib_cfg_update") + init_from_yaml_conf(qlib_conf_path, **qlib_conf_update, **kwargs) + logger = get_module_logger("Initialization") + logger.info(f"Auto load project config: {conf_pp}") diff --git a/qlib/config.py b/qlib/config.py index 52b05568d5..b245cc1df5 100644 --- a/qlib/config.py +++ b/qlib/config.py @@ -33,6 +33,9 @@ def __getattr__(self, attr): raise AttributeError(f"No such {attr} in self._config") + def get(self, key, default=None): + return self.__dict__["_config"].get(key, default) + def __setitem__(self, key, value): self.__dict__["_config"][key] = value @@ -310,8 +313,22 @@ def register(self): # clean up experiment when python program ends experiment_exit_handler() + # Supporting user reset qlib version (useful when user want to connect to qlib server with old version) + self.reset_qlib_version() + self._registered = True + def reset_qlib_version(self): + import qlib + + reset_version = self.get("qlib_reset_version", None) + if reset_version is not None: + qlib.__version__ = reset_version + else: + qlib.__version__ = getattr(qlib, "__version__bak") + # Due to a bug? that converting __version__ to _QlibConfig__version__bak + # Using __version__bak instead of __version__ + @property def registered(self): return self._registered diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index 13b5869de5..7cdca30fae 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -5,11 +5,12 @@ class RollingEnsemble: - ''' + """ Rolling Models Ensemble based on (R)ecord This shares nothing with Ensemble - ''' + """ + # TODO: 这边还可以加加速 def __init__(self, get_key_func, flt_func=None): self.get_key_func = get_key_func @@ -44,9 +45,8 @@ def __call__(self, exp_name) -> Union[pd.Series, dict]: for k, rec_l in recs_group.items(): pred_l = [] for rec in rec_l: - pred_l.append(rec.load_object('pred.pkl').iloc[:, 0]) + pred_l.append(rec.load_object("pred.pkl").iloc[:, 0]) pred = pd.concat(pred_l).sort_index() reduce_group[k] = pred return reduce_group - diff --git a/qlib/workflow/task/gen.py b/qlib/workflow/task/gen.py index 66529f3a5a..9b031435ed 100644 --- a/qlib/workflow/task/gen.py +++ b/qlib/workflow/task/gen.py @@ -1,8 +1,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -''' +""" this is a task generator -''' +""" import abc import copy import typing @@ -54,8 +54,8 @@ def __init__(self, step: int = 40, rtype: str = ROLL_EX): self.rtype = rtype self.ta = TimeAdjuster(future=True) # 为了保证test最后的日期不是None, 所以这边要改一改 - self.test_key = 'test' - self.train_key = 'train' + self.test_key = "test" + self.train_key = "train" def __call__(self, task: dict): """ @@ -102,7 +102,7 @@ def __call__(self, task: dict): if prev_seg is None: # First rolling # 1) prepare the end porint - segments = copy.deepcopy(self.ta.align_seg(t['dataset']['kwargs']['segments'])) + segments = copy.deepcopy(self.ta.align_seg(t["dataset"]["kwargs"]["segments"])) test_end = self.ta.max() if segments[self.test_key][1] is None else segments[self.test_key][1] # 2) and the init test segments test_start_idx = self.ta.align_idx(segments[self.test_key][0]) @@ -120,14 +120,12 @@ def __call__(self, task: dict): segments[k] = self.ta.shift(seg, step=self.step, rtype=rtype) if segments[self.test_key][0] > test_end: break - except KeyError: + except KeyError: # We reach the end of tasks # No more rolling break - t['dataset']['kwargs']['segments'] = copy.deepcopy(segments) + t["dataset"]["kwargs"]["segments"] = copy.deepcopy(segments) prev_seg = segments res.append(t) return res - - diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 6407279f06..3bcac83600 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -19,6 +19,8 @@ import concurrent import pymongo from qlib.config import C +from .utils import get_mongodb +from qlib import auto_init class TaskManager: @@ -41,12 +43,13 @@ class TaskManager: NOTE: - 假设： 存储在db里面的都是encode过的， 拿出来的都是decode过的 """ - STATUS_WAITING = 'waiting' - STATUS_RUNNING = 'running' - STATUS_DONE = 'done' - STATUS_PART_DONE = 'part_done' - ENCODE_FIELDS_PREFIX = ['def', 'res'] + STATUS_WAITING = "waiting" + STATUS_RUNNING = "running" + STATUS_DONE = "done" + STATUS_PART_DONE = "part_done" + + ENCODE_FIELDS_PREFIX = ["def", "res"] def __init__(self, task_pool=None): self.mdb = get_mongodb() @@ -73,7 +76,7 @@ def _get_task_pool(self, task_pool=None): if task_pool is None: task_pool = self.task_pool if task_pool is None: - raise ValueError('You must specify a task pool.') + raise ValueError("You must specify a task pool.") if isinstance(task_pool, str): return getattr(self.mdb, task_pool) return task_pool @@ -85,11 +88,11 @@ def replace_task(self, task, new_task, task_pool=None): # 这里的假设是从接口拿出来的都是decode过的，在接口内部的都是 encode过的 new_task = self._encode_task(new_task) task_pool = self._get_task_pool(task_pool) - query = {'_id': ObjectId(task['_id'])} + query = {"_id": ObjectId(task["_id"])} try: task_pool.replace_one(query, new_task) except InvalidDocument: - task['filter'] = self._dict_to_str(task['filter']) + task["filter"] = self._dict_to_str(task["filter"]) task_pool.replace_one(query, new_task) def insert_task(self, task, task_pool=None): @@ -97,16 +100,18 @@ def insert_task(self, task, task_pool=None): try: task_pool.insert_one(task) except InvalidDocument: - task['filter'] = self._dict_to_str(task['filter']) + task["filter"] = self._dict_to_str(task["filter"]) task_pool.insert_one(task) def insert_task_def(self, task_def, task_pool=None): task_pool = self._get_task_pool(task_pool) - task = self._encode_task({ - 'def': task_def, - 'filter': task_def, # FIXME: catch the raised error - 'status': self.STATUS_WAITING, - }) + task = self._encode_task( + { + "def": task_def, + "filter": task_def, # FIXME: catch the raised error + "status": self.STATUS_WAITING, + } + ) self.insert_task(task, task_pool) def create_task(self, task_def_l, task_pool=None, dry_run=False, print_nt=False): @@ -114,9 +119,9 @@ def create_task(self, task_def_l, task_pool=None, dry_run=False, print_nt=False) new_tasks = [] for t in task_def_l: try: - r = task_pool.find_one({'filter': t}) + r = task_pool.find_one({"filter": t}) except InvalidDocument: - r = task_pool.find_one({'filter': self._dict_to_str(t)}) + r = task_pool.find_one({"filter": self._dict_to_str(t)}) if r is None: new_tasks.append(t) print("Total Tasks, New Tasks:", len(task_def_l), len(new_tasks)) @@ -134,17 +139,16 @@ def create_task(self, task_def_l, task_pool=None, dry_run=False, print_nt=False) def fetch_task(self, query={}, task_pool=None): task_pool = self._get_task_pool(task_pool) query = query.copy() - if '_id' in query: - query['_id'] = ObjectId(query['_id']) - query.update({'status': self.STATUS_WAITING}) - task = task_pool.find_one_and_update(query, {'$set': { - 'status': self.STATUS_RUNNING - }}, - sort=[('priority', pymongo.DESCENDING)]) + if "_id" in query: + query["_id"] = ObjectId(query["_id"]) + query.update({"status": self.STATUS_WAITING}) + task = task_pool.find_one_and_update( + query, {"$set": {"status": self.STATUS_RUNNING}}, sort=[("priority", pymongo.DESCENDING)] + ) # 这里我的 priority 必须是 高数优先级更高，因为 null会被在 ASCENDING时被排在最前面 if task is None: return None - task['status'] = self.STATUS_RUNNING + task["status"] = self.STATUS_RUNNING return self._decode_task(task) @contextmanager @@ -154,9 +158,9 @@ def safe_fetch_task(self, query={}, task_pool=None): yield task except Exception: if task is not None: - logger.info('Returning task before raising error') + logger.info("Returning task before raising error") self.return_task(task) - logger.info('Task returned') + logger.info("Task returned") raise def task_fetcher_iter(self, query={}, task_pool=None): @@ -175,8 +179,8 @@ def query(self, query={}, decode=True, task_pool=None): :param task_pool: """ query = query.copy() - if '_id' in query: - query['_id'] = ObjectId(query['_id']) + if "_id" in query: + query["_id"] = ObjectId(query["_id"]) task_pool = self._get_task_pool(task_pool) for t in task_pool.find(query): yield self._decode_task(t) @@ -186,44 +190,44 @@ def commit_task_res(self, task, res, status=None, task_pool=None): # A workaround to use the class attribute. if status is None: status = TaskManager.STATUS_DONE - task_pool.update_one({"_id": task['_id']}, {'$set': {'status': status, 'res': Binary(pickle.dumps(res))}}) + task_pool.update_one({"_id": task["_id"]}, {"$set": {"status": status, "res": Binary(pickle.dumps(res))}}) def return_task(self, task, status=None, task_pool=None): task_pool = self._get_task_pool(task_pool) if status is None: status = TaskManager.STATUS_WAITING - update_dict = {'$set': {'status': status}} - task_pool.update_one({"_id": task['_id']}, update_dict) + update_dict = {"$set": {"status": status}} + task_pool.update_one({"_id": task["_id"]}, update_dict) def remove(self, query={}, task_pool=None): query = query.copy() task_pool = self._get_task_pool(task_pool) - if '_id' in query: - query['_id'] = ObjectId(query['_id']) + if "_id" in query: + query["_id"] = ObjectId(query["_id"]) task_pool.delete_many(query) def task_stat(self, query={}, task_pool=None): query = query.copy() - if '_id' in query: - query['_id'] = ObjectId(query['_id']) + if "_id" in query: + query["_id"] = ObjectId(query["_id"]) tasks = self.query(task_pool=task_pool, query=query, decode=False) status_stat = {} for t in tasks: - status_stat[t['status']] = status_stat.get(t['status'], 0) + 1 + status_stat[t["status"]] = status_stat.get(t["status"], 0) + 1 return status_stat def reset_waiting(self, query={}, task_pool=None): query = query.copy() # default query - if 'status' not in query: - query['status'] = self.STATUS_RUNNING + if "status" not in query: + query["status"] = self.STATUS_RUNNING return self.reset_status(query=query, status=self.STATUS_WAITING, task_pool=task_pool) def reset_status(self, query, status, task_pool=None): query = query.copy() task_pool = self._get_task_pool(task_pool) - if '_id' in query: - query['_id'] = ObjectId(query['_id']) + if "_id" in query: + query["_id"] = ObjectId(query["_id"]) print(task_pool.update_many(query, {"$set": {"status": status}})) def _get_undone_n(self, task_stat): @@ -274,17 +278,18 @@ def run_task(task_func, task_pool, force_release=False, *args, **kwargs): with tm.safe_fetch_task() as task: if task is None: break - logger.info(task['def']) + logger.info(task["def"]) if force_release: with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: - res = executor.submit(task_func, task['def'], *args, **kwargs).result() + res = executor.submit(task_func, task["def"], *args, **kwargs).result() else: - res = task_func(task['def'], *args, **kwargs) + res = task_func(task["def"], *args, **kwargs) tm.commit_task_res(task, res) ever_run = True return ever_run -if __name__ == '__main__': +if __name__ == "__main__": + auto_init() Fire(TaskManager) diff --git a/qlib/workflow/task/utils.py b/qlib/workflow/task/utils.py index 3d8fe89964..d6089ff66d 100644 --- a/qlib/workflow/task/utils.py +++ b/qlib/workflow/task/utils.py @@ -10,17 +10,18 @@ def get_mongodb(): try: - cfg = C['mongo'] + cfg = C["mongo"] except KeyError: get_module_logger("task").error("Please configure `C['mongo']` before using TaskManager") raise - client = MongoClient(cfg['task_url']) - return client.get_database(name=cfg['task_db_name']) + client = MongoClient(cfg["task_url"]) + return client.get_database(name=cfg["task_db_name"]) class TimeAdjuster: - '''找到合适的日期，然后adjust date''' + """找到合适的日期，然后adjust date""" + def __init__(self, future=False): self.cals = D.calendar(future=future) @@ -45,9 +46,9 @@ def max(self): def align_idx(self, time_point, tp_type="start"): time_point = pd.Timestamp(time_point) - if tp_type == 'start': + if tp_type == "start": idx = bisect.bisect_left(self.cals, time_point) - elif tp_type == 'end': + elif tp_type == "end": idx = bisect.bisect_right(self.cals, time_point) - 1 else: raise NotImplementedError(f"This type of input is not supported") @@ -91,7 +92,7 @@ def truncate(self, segment, test_start, days: int): new_seg = [] for time_point in segment: tp_idx = min(self.align_idx(time_point), test_idx - days) - assert (tp_idx > 0) + assert tp_idx > 0 new_seg.append(self.get(tp_idx)) return tuple(new_seg) else: From b84156fde8bab93c220c8df25fa973f75bd2ccb0 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Wed, 3 Mar 2021 11:25:37 +0800 Subject: [PATCH 05/61] Consider more situations about task_config. Save the "param" file which is collect.py need. --- qlib/model/trainer.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index f0bc0b780a..71cf9061f8 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -27,16 +27,22 @@ def task_train(task_config: dict, experiment_name): model.fit(dataset) recorder = R.get_recorder() R.save_objects(**{"params.pkl": model}) + R.save_objects(param=task_config) # keep the original format and datatype # generate records: prediction, backtest, and analysis - for record in task_config["record"]: + records = task_config.get('record', []) + if isinstance(records, dict): # prevent only one dict + records = [records] + for record in records: if record["class"] == SignalRecord.__name__: srconf = {"model": model, "dataset": dataset, "recorder": recorder} + record.setdefault("kwargs", {}) record["kwargs"].update(srconf) sr = init_instance_by_config(record) sr.generate() else: rconf = {"recorder": recorder} + record.setdefault("kwargs", {}) record["kwargs"].update(rconf) ar = init_instance_by_config(record) ar.generate() From 05cf0e1edcdbe0b696b7e8c1cde538e3a5168dfa Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Wed, 3 Mar 2021 15:42:39 +0800 Subject: [PATCH 06/61] add task_generator method and update some hint --- qlib/workflow/task/gen.py | 66 ++++++++++++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 7 deletions(-) diff --git a/qlib/workflow/task/gen.py b/qlib/workflow/task/gen.py index 9b031435ed..efbfe94a6c 100644 --- a/qlib/workflow/task/gen.py +++ b/qlib/workflow/task/gen.py @@ -9,11 +9,64 @@ from .utils import TimeAdjuster +def task_generator(*args, **kwargs) -> list: + """ + Accept the dict of task config and the TaskGen to generate different tasks. + There is no limit to the number and position of input. + The key of input will add to task config. + + for example: + There are 3 task_config(a,b,c) and 2 TaskGen(A,B). A will double the task_config and B will triple. + task_generator(a=a, b=b, c=c, A=A, B=B) will finally generate 18 task_config. + + Parameters + ---------- + args : dict or TaskGen + kwargs : dict or TaskGen + + Returns + ------- + gen_task_list : list + a list of task config after generating + """ + tasks_list = [] + gen_list = [] + + tmp_id = 1 + for task in args: + if isinstance(task, dict): + task["task_key"] = tmp_id + tmp_id += 1 + tasks_list.append(task) + elif isinstance(task, TaskGen): + gen_list.append(task) + else: + raise NotImplementedError(f"{type(task)} is not supported in task_generator") + + for key, task in kwargs.items(): + if isinstance(task, dict): + task["task_key"] = key + tasks_list.append(task) + elif isinstance(task, TaskGen): + gen_list.append(task) + else: + raise NotImplementedError(f"{type(task)} is not supported in task_generator") + + # generate gen_task_list + gen_task_list = [] + for gen in gen_list: + new_task_list = [] + for task in tasks_list: + new_task_list.extend(gen(task)) + gen_task_list = new_task_list + return gen_task_list + + class TaskGen(metaclass=abc.ABCMeta): @abc.abstractmethod def __call__(self, *args, **kwargs) -> typing.List[dict]: """ - generate + the base class for generate different tasks Parameters ---------- @@ -35,9 +88,8 @@ def __call__(self, *args, **kwargs) -> typing.List[dict]: class RollingGen(TaskGen): - - ROLL_EX = TimeAdjuster.SHIFT_EX - ROLL_SD = TimeAdjuster.SHIFT_SD + ROLL_EX = TimeAdjuster.SHIFT_EX # fixed start date, expanding end date + ROLL_SD = TimeAdjuster.SHIFT_SD # fixed window size, slide it from start date def __init__(self, step: int = 40, rtype: str = ROLL_EX): """ @@ -48,7 +100,7 @@ def __init__(self, step: int = 40, rtype: str = ROLL_EX): step : int step to rolling rtype : str - rolling type (expanding, rolling) + rolling type (expanding, sliding) """ self.step = step self.rtype = rtype @@ -111,12 +163,12 @@ def __call__(self, task: dict): segments = {} try: for k, seg in prev_seg.items(): - # 决定怎么shift + # decide how to shift if k == self.train_key and self.rtype == self.ROLL_EX: rtype = self.ta.SHIFT_EX else: rtype = self.ta.SHIFT_SD - # 整段数据做shift + # shift the segments data segments[k] = self.ta.shift(seg, step=self.step, rtype=rtype) if segments[self.test_key][0] > test_end: break From fd2c1ba1ed1c3919b6ddd418f0b3f82239f0baf5 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Wed, 3 Mar 2021 16:36:15 +0800 Subject: [PATCH 07/61] Update some hint --- qlib/workflow/task/collect.py | 9 ++++----- qlib/workflow/task/manage.py | 8 ++------ qlib/workflow/task/utils.py | 29 +++++++++++++++++++++-------- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index 7cdca30fae..9a67d8e06f 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -4,17 +4,17 @@ from tqdm.auto import tqdm -class RollingEnsemble: +class RollingCollector: """ Rolling Models Ensemble based on (R)ecord This shares nothing with Ensemble """ - # TODO: 这边还可以加加速 + # TODO: speed up this class def __init__(self, get_key_func, flt_func=None): - self.get_key_func = get_key_func - self.flt_func = flt_func + self.get_key_func = get_key_func # user need to implement this method to get the key of a task based on task config + self.flt_func = flt_func # determine whether a task can be retained based on task config def __call__(self, exp_name) -> Union[pd.Series, dict]: # TODO; @@ -26,7 +26,6 @@ def __call__(self, exp_name) -> Union[pd.Series, dict]: recs_flt = {} for rid, rec in tqdm(recs.items(), desc="Loading data"): - # rec = exp.get_recorder(recorder_id=rid) params = rec.load_object("param") if rec.status == rec.STATUS_FI: if self.flt_func is None or self.flt_func(params): diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 3bcac83600..1a4c341de2 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -36,12 +36,8 @@ class TaskManager: The tasks manager assume that you will only update the tasks you fetched. The mongo fetch one and update will make it date updating secure. - Usage Examples from the CLI. - python -m blocks.tasks.__init__ task_stat --task_pool meta_task_rule - - NOTE: - - 假设： 存储在db里面的都是encode过的， 拿出来的都是decode过的 + - assumption: the data in MongoDB was encoded and the data out of MongoDB was decoded """ STATUS_WAITING = "waiting" @@ -85,7 +81,7 @@ def _dict_to_str(self, flt): return {k: str(v) for k, v in flt.items()} def replace_task(self, task, new_task, task_pool=None): - # 这里的假设是从接口拿出来的都是decode过的，在接口内部的都是 encode过的 + # assume that the data out of interface was decoded and the data in interface was encoded new_task = self._encode_task(new_task) task_pool = self._get_task_pool(task_pool) query = {"_id": ObjectId(task["_id"])} diff --git a/qlib/workflow/task/utils.py b/qlib/workflow/task/utils.py index d6089ff66d..719359d5b5 100644 --- a/qlib/workflow/task/utils.py +++ b/qlib/workflow/task/utils.py @@ -6,9 +6,19 @@ from qlib.config import C from qlib.log import get_module_logger from pymongo import MongoClient - +from typing import Union def get_mongodb(): + """ + + get database in MongoDB, which means you need to declare the address and the name of database. + for example: + C["mongo"] = { + "task_url" : "mongodb://localhost:27017/", + "task_db_name" : "rolling_db" + } + + """ try: cfg = C["mongo"] except KeyError: @@ -20,7 +30,9 @@ def get_mongodb(): class TimeAdjuster: - """找到合适的日期，然后adjust date""" + """ + find appropriate date and adjust date. + """ def __init__(self, future=False): self.cals = D.calendar(future=future) @@ -40,7 +52,7 @@ def get(self, idx: int): def max(self): """ - Return return the max calendar date + Return the max calendar date """ return max(self.cals) @@ -56,7 +68,7 @@ def align_idx(self, time_point, tp_type="start"): def align_time(self, time_point, tp_type="start"): """ - Align a timepoint to calendar weekdays + Align a timepoint to calendar weekdays Parameters ---------- @@ -67,7 +79,7 @@ def align_time(self, time_point, tp_type="start"): """ return self.cals[self.align_idx(time_point, tp_type=tp_type)] - def align_seg(self, segment): + def align_seg(self, segment: Union[dict, tuple]): if isinstance(segment, dict): return {k: self.align_seg(seg) for k, seg in segment.items()} elif isinstance(segment, tuple): @@ -75,14 +87,15 @@ def align_seg(self, segment): else: raise NotImplementedError(f"This type of input is not supported") - def truncate(self, segment, test_start, days: int): + def truncate(self, segment: tuple, test_start, days: int): """ truncate the segment based on the test_start date Parameters ---------- - segment : + segment : tuple time segment + test_start days : int The trading days to be truncated 大部分情况是因为这个时间段的数据(一般是特征)会用到 `days` 天的数据 @@ -101,7 +114,7 @@ def truncate(self, segment, test_start, days: int): SHIFT_SD = "sliding" SHIFT_EX = "expanding" - def shift(self, seg, step: int, rtype=SHIFT_SD): + def shift(self, seg: tuple, step: int, rtype=SHIFT_SD): """ shift the datatiem of segment From 2882929c5d91e2f655265036fd26ca6c50cbdd6a Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Wed, 3 Mar 2021 16:58:05 +0800 Subject: [PATCH 08/61] Add an example about workflow using RollingGen. --- examples/workflow_task_rolling.ipynb | 177 +++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 examples/workflow_task_rolling.ipynb diff --git a/examples/workflow_task_rolling.ipynb b/examples/workflow_task_rolling.ipynb new file mode 100644 index 0000000000..c2d399be0d --- /dev/null +++ b/examples/workflow_task_rolling.ipynb @@ -0,0 +1,177 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import qlib\n", + "from qlib.config import REG_CN\n", + "from qlib.workflow.task.gen import RollingGen, task_generator\n", + "from qlib.workflow.task.manage import TaskManager\n", + "from qlib.config import C\n", + "\n", + "data_handler_config = {\n", + " \"start_time\": \"2008-01-01\",\n", + " \"end_time\": \"2020-08-01\",\n", + " \"fit_start_time\": \"2008-01-01\",\n", + " \"fit_end_time\": \"2014-12-31\",\n", + " \"instruments\": 'csi100',\n", + "}\n", + "\n", + "dataset_config = {\n", + " \"class\": \"DatasetH\",\n", + " \"module_path\": \"qlib.data.dataset\",\n", + " \"kwargs\": {\n", + " \"handler\": {\n", + " \"class\": \"Alpha158\",\n", + " \"module_path\": \"qlib.contrib.data.handler\",\n", + " \"kwargs\": data_handler_config,\n", + " },\n", + " \"segments\": {\n", + " \"train\": (\"2008-01-01\", \"2014-12-31\"),\n", + " \"valid\": (\"2015-01-01\", \"2016-12-31\"),\n", + " \"test\": (\"2017-01-01\", \"2020-08-01\"),\n", + " },\n", + " },\n", + " }\n", + "\n", + "record_config = [\n", + " {\n", + " \"class\": \"SignalRecord\",\n", + " \"module_path\": \"qlib.workflow.record_temp\",\n", + " },\n", + " {\n", + " \"class\": \"SigAnaRecord\",\n", + " \"module_path\": \"qlib.workflow.record_temp\",\n", + " }\n", + "]\n", + "\n", + "# use lgb\n", + "task_lgb_config = {\n", + " \"model\": {\n", + " \"class\": \"LGBModel\",\n", + " \"module_path\": \"qlib.contrib.model.gbdt\",\n", + " },\n", + " \"dataset\": dataset_config,\n", + " \"record\": record_config,\n", + "}\n", + "\n", + "# use xgboost\n", + "task_xgboost_config = {\n", + " \"model\": {\n", + " \"class\": \"XGBModel\",\n", + " \"module_path\": \"qlib.contrib.model.xgboost\",\n", + " },\n", + " \"dataset\": dataset_config,\n", + " \"record\": record_config,\n", + "}\n", + "provider_uri = r\"../qlib-main/qlib_data/cn_data\"\n", + "#provider_uri = \"~/.qlib/qlib_data/cn_data\" # target_dir\n", + "qlib.init(provider_uri=provider_uri, region=REG_CN)\n", + "\n", + "C[\"mongo\"] = {\n", + " \"task_url\" : \"mongodb://localhost:27017/\", # maybe you need to change it to your url\n", + " \"task_db_name\" : \"rolling_db\"\n", + "}\n", + "\n", + "exp_name = 'rolling_exp' # experiment name, will be used as the experiment in MLflow\n", + "task_pool = 'rolling_task' # task pool name, will be used as the document in MongoDB" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "tasks = task_generator(\n", + " task_xgboost_config, # default task name\n", + " RollingGen(step=550,rtype=RollingGen.ROLL_SD), # generate different date segment\n", + " task_lgb=task_lgb_config # use \"task_lgb\" as the task name\n", + ")\n", + "# Uncomment next two lines to see the generated tasks\n", + "# from pprint import pprint\n", + "# pprint(tasks)\n", + "tm = TaskManager(task_pool=task_pool)\n", + "tm.create_task(tasks) # all tasks will be saved to MongoDB" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "from qlib.workflow.task.manage import run_task\n", + "from qlib.workflow.task.collect import RollingCollector\n", + "from qlib.model.trainer import task_train\n", + "\n", + "run_task(task_train, task_pool, experiment_name=exp_name) # all tasks will be trained using \"task_train\" method" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "def get_task_key(task_config):\n", + " task_key = task_config[\"task_key\"]\n", + " rolling_end_timestamp = task_config[\"dataset\"][\"kwargs\"][\"segments\"][\"test\"][1]\n", + " rolling_end_datatime = rolling_end_timestamp.to_pydatetime()\n", + " return task_key, rolling_end_datatime.strftime('%Y-%m-%d')\n", + "\n", + "def my_filter(task_config):\n", + " # only choose the results of \"task_lgb\" and test in 2019 from all tasks\n", + " task_key, rolling_end = get_task_key(task_config)\n", + " if task_key==\"task_lgb\" and rolling_end.startswith('2019'):\n", + " return True\n", + " return False\n", + "\n", + "collector = RollingCollector(get_task_key, my_filter)\n", + "pred_rolling = collector(exp_name) # name tasks by \"get_task_key\" and filter tasks by \"my_filter\"\n", + "pred_rolling" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file From a244f87f95e70ca2f97a687be10e3e3f606517a0 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Mon, 8 Mar 2021 13:25:11 +0800 Subject: [PATCH 09/61] modified the comments --- qlib/workflow/task/gen.py | 47 +++++++++------ qlib/workflow/task/manage.py | 114 +++++++++++++++++++++++++++++------ qlib/workflow/task/utils.py | 47 +++++++++++++-- 3 files changed, 168 insertions(+), 40 deletions(-) diff --git a/qlib/workflow/task/gen.py b/qlib/workflow/task/gen.py index efbfe94a6c..60fc5c2216 100644 --- a/qlib/workflow/task/gen.py +++ b/qlib/workflow/task/gen.py @@ -17,7 +17,7 @@ def task_generator(*args, **kwargs) -> list: for example: There are 3 task_config(a,b,c) and 2 TaskGen(A,B). A will double the task_config and B will triple. - task_generator(a=a, b=b, c=c, A=A, B=B) will finally generate 18 task_config. + task_generator(a_key=a, b_key=b, c_key=c, A, B) will finally generate 3*2*3 = 18 task_config. Parameters ---------- @@ -57,27 +57,37 @@ def task_generator(*args, **kwargs) -> list: for gen in gen_list: new_task_list = [] for task in tasks_list: - new_task_list.extend(gen(task)) + new_task_list.extend(gen.generate(task)) gen_task_list = new_task_list return gen_task_list class TaskGen(metaclass=abc.ABCMeta): + """ + the base class for generate different tasks + + Example 1: + + input: a specific task template and rolling steps + + output: rolling version of the tasks + + Example 2: + + input: a specific task template and losses list + + output: a set of tasks with different losses + + """ @abc.abstractmethod - def __call__(self, *args, **kwargs) -> typing.List[dict]: + def generate(self, task: dict) -> typing.List[dict]: """ - the base class for generate different tasks + generate different tasks based on a task template Parameters ---------- - args, kwargs: - The info for generating tasks - Example 1): - input: a specific task template - output: rolling version of the tasks - Example 2): - input: a specific task template - output: a set of tasks with different losses + task: dict + a task template Returns ------- @@ -89,7 +99,7 @@ def __call__(self, *args, **kwargs) -> typing.List[dict]: class RollingGen(TaskGen): ROLL_EX = TimeAdjuster.SHIFT_EX # fixed start date, expanding end date - ROLL_SD = TimeAdjuster.SHIFT_SD # fixed window size, slide it from start date + ROLL_SD = TimeAdjuster.SHIFT_SD # fixed segments size, slide it from start date def __init__(self, step: int = 40, rtype: str = ROLL_EX): """ @@ -104,12 +114,13 @@ def __init__(self, step: int = 40, rtype: str = ROLL_EX): """ self.step = step self.rtype = rtype - self.ta = TimeAdjuster(future=True) # 为了保证test最后的日期不是None, 所以这边要改一改 + # TODO: Ask pengrong to update future date in dataset + self.ta = TimeAdjuster(future=True) self.test_key = "test" self.train_key = "train" - def __call__(self, task: dict): + def generate(self, task: dict): """ Converting the task into a rolling task @@ -153,9 +164,9 @@ def __call__(self, task: dict): # calculate segments if prev_seg is None: # First rolling - # 1) prepare the end porint + # 1) prepare the end point segments = copy.deepcopy(self.ta.align_seg(t["dataset"]["kwargs"]["segments"])) - test_end = self.ta.max() if segments[self.test_key][1] is None else segments[self.test_key][1] + test_end = self.ta.last_date() if segments[self.test_key][1] is None else segments[self.test_key][1] # 2) and the init test segments test_start_idx = self.ta.align_idx(segments[self.test_key][0]) segments[self.test_key] = (self.ta.get(test_start_idx), self.ta.get(test_start_idx + self.step - 1)) @@ -164,6 +175,7 @@ def __call__(self, task: dict): try: for k, seg in prev_seg.items(): # decide how to shift + # expanding only for train data, the segments size of test data and valid data won't change if k == self.train_key and self.rtype == self.ROLL_EX: rtype = self.ta.SHIFT_EX else: @@ -177,6 +189,7 @@ def __call__(self, task: dict): # No more rolling break + # update segments of this task t["dataset"]["kwargs"]["segments"] = copy.deepcopy(segments) prev_seg = segments res.append(t) diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 1a4c341de2..ae4aee147d 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """ -A task consists of 2 parts +A task consists of 3 parts - tasks description: the desc will define the task - tasks status: the status of the task - tasks result information : A user can get the task with the task description and task result. @@ -26,18 +26,22 @@ class TaskManager: """TaskManager here is the what will a task looks like - { - 'def': pickle serialized task definition. using pickle will make it easier - 'filter': json-like data. This is for filtering the tasks. - 'status': 'waiting' | 'running' | 'done' - 'res': pickle serialized task result, - } + + .. code-block:: python + + { + 'def': pickle serialized task definition. using pickle will make it easier + 'filter': json-like data. This is for filtering the tasks. + 'status': 'waiting' | 'running' | 'done' + 'res': pickle serialized task result, + } The tasks manager assume that you will only update the tasks you fetched. The mongo fetch one and update will make it date updating secure. - NOTE: - - assumption: the data in MongoDB was encoded and the data out of MongoDB was decoded + .. note:: + + assumption: the data in MongoDB was encoded and the data out of MongoDB was decoded """ STATUS_WAITING = "waiting" @@ -48,6 +52,14 @@ class TaskManager: ENCODE_FIELDS_PREFIX = ["def", "res"] def __init__(self, task_pool=None): + """ + init Task Manager, remember to make the statement of MongoDB url and database name firstly. + + Parameters + ---------- + task_pool: str + the name of Collection in MongoDB + """ self.mdb = get_mongodb() self.task_pool = task_pool @@ -100,6 +112,19 @@ def insert_task(self, task, task_pool=None): task_pool.insert_one(task) def insert_task_def(self, task_def, task_pool=None): + """ + insert a task to task_pool + + Parameters + ---------- + task_def: dict + task_pool: str + the name of Collection in MongoDB + + Returns + ------- + + """ task_pool = self._get_task_pool(task_pool) task = self._encode_task( { @@ -111,6 +136,23 @@ def insert_task_def(self, task_def, task_pool=None): self.insert_task(task, task_pool) def create_task(self, task_def_l, task_pool=None, dry_run=False, print_nt=False): + """ + if the tasks in task_def_l is new, then insert new tasks into the task_pool + + Parameters + ---------- + task_def_l: list + a list of task + task_pool: str + the name of task_pool (collection name of MongoDB) + dry_run: bool + if insert those new tasks to task pool + print_nt: bool + if print new task + Returns + ------- + + """ task_pool = self._get_task_pool(task_pool) new_tasks = [] for t in task_def_l: @@ -141,7 +183,7 @@ def fetch_task(self, query={}, task_pool=None): task = task_pool.find_one_and_update( query, {"$set": {"status": self.STATUS_RUNNING}}, sort=[("priority", pymongo.DESCENDING)] ) - # 这里我的 priority 必须是 高数优先级更高，因为 null会被在 ASCENDING时被排在最前面 + # null will be at the top after sorting when using ASCENDING, so the larger the number higher, the higher the priority if task is None: return None task["status"] = self.STATUS_RUNNING @@ -149,6 +191,20 @@ def fetch_task(self, query={}, task_pool=None): @contextmanager def safe_fetch_task(self, query={}, task_pool=None): + """ + fetch task from task_pool using query with contextmanager + + Parameters + ---------- + query: dict + the dict of query + task_pool: str + the name of Collection in MongoDB + + Returns + ------- + + """ task = self.fetch_task(query=query, task_pool=task_pool) try: yield task @@ -167,12 +223,20 @@ def task_fetcher_iter(self, query={}, task_pool=None): yield task def query(self, query={}, decode=True, task_pool=None): - """query + """ This function may raise exception `pymongo.errors.CursorNotFound: cursor id not found` if it takes too long to iterate the generator - :param query: - :param decode: - :param task_pool: + Parameters + ---------- + query: dict + the dict of query + decode: bool + task_pool: str + the name of Collection in MongoDB + + Returns + ------- + """ query = query.copy() if "_id" in query: @@ -196,6 +260,20 @@ def return_task(self, task, status=None, task_pool=None): task_pool.update_one({"_id": task["_id"]}, update_dict) def remove(self, query={}, task_pool=None): + """ + remove the task using query + + Parameters + ---------- + query: dict + the dict of query + task_pool: str + the name of Collection in MongoDB + + Returns + ------- + + """ query = query.copy() task_pool = self._get_task_pool(task_pool) if "_id" in query: @@ -250,15 +328,15 @@ def __str__(self): def run_task(task_func, task_pool, force_release=False, *args, **kwargs): - """run_task. - While task pool is not empty, use task_func to fetch and run tasks in task_pool + """ + While task pool is not empty (has WAITING tasks), use task_func to fetch and run tasks in task_pool Parameters ---------- task_func : def (task_def, *args, **kwargs) -> the function to run the task - task_pool : - The name of the task pool + task_pool : str + the name of the task pool (Collection in MongoDB) force_release : will the program force to release the resource args : diff --git a/qlib/workflow/task/utils.py b/qlib/workflow/task/utils.py index 719359d5b5..5e94f55ae4 100644 --- a/qlib/workflow/task/utils.py +++ b/qlib/workflow/task/utils.py @@ -52,11 +52,30 @@ def get(self, idx: int): def max(self): """ - Return the max calendar date + (Deprecated) + Return the max calendar datetime """ return max(self.cals) + def last_date(self) -> pd.Timestamp: + """ + Return the last datetime in the calendar + """ + return self.cals[-1] + def align_idx(self, time_point, tp_type="start"): + """ + align the index of time_point in the calendar + + Parameters + ---------- + time_point + tp_type : str + + Returns + ------- + index : int + """ time_point = pd.Timestamp(time_point) if tp_type == "start": idx = bisect.bisect_left(self.cals, time_point) @@ -68,11 +87,11 @@ def align_idx(self, time_point, tp_type="start"): def align_time(self, time_point, tp_type="start"): """ - Align a timepoint to calendar weekdays + Align time_point to trade date of calendar Parameters ---------- - time_point : + time_point Time point tp_type : str time point type (`"start"`, `"end"`) @@ -80,6 +99,24 @@ def align_time(self, time_point, tp_type="start"): return self.cals[self.align_idx(time_point, tp_type=tp_type)] def align_seg(self, segment: Union[dict, tuple]): + """ + align the given date to trade date + + for example: + input: {'train': ('2008-01-01', '2014-12-31'), 'valid': ('2015-01-01', '2016-12-31'), 'test': ('2017-01-01', '2020-08-01')} + + output: {'train': (Timestamp('2008-01-02 00:00:00'), Timestamp('2014-12-31 00:00:00')), + 'valid': (Timestamp('2015-01-05 00:00:00'), Timestamp('2016-12-30 00:00:00')), + 'test': (Timestamp('2017-01-03 00:00:00'), Timestamp('2020-07-31 00:00:00'))} + + Parameters + ---------- + segment + + Returns + ------- + the start and end trade date (pd.Timestamp) between the given start and end date. + """ if isinstance(segment, dict): return {k: self.align_seg(seg) for k, seg in segment.items()} elif isinstance(segment, tuple): @@ -98,7 +135,7 @@ def truncate(self, segment: tuple, test_start, days: int): test_start days : int The trading days to be truncated - 大部分情况是因为这个时间段的数据(一般是特征)会用到 `days` 天的数据 + the data in this segment may need 'days' data """ test_idx = self.align_idx(test_start) if isinstance(segment, tuple): @@ -116,7 +153,7 @@ def truncate(self, segment: tuple, test_start, days: int): def shift(self, seg: tuple, step: int, rtype=SHIFT_SD): """ - shift the datatiem of segment + shift the datatime of segment Parameters ---------- From def132e1407bc97585efa2d261feefd8386c34f6 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Mon, 8 Mar 2021 16:10:16 +0800 Subject: [PATCH 10/61] modified format and added TaskCollector --- qlib/model/trainer.py | 6 ++-- qlib/workflow/task/collect.py | 58 ++++++++++++++++++++++++++++++++++- qlib/workflow/task/gen.py | 1 + qlib/workflow/task/utils.py | 1 + 4 files changed, 63 insertions(+), 3 deletions(-) diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index 71cf9061f8..91061636d4 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -6,7 +6,7 @@ from qlib.workflow.record_temp import SignalRecord -def task_train(task_config: dict, experiment_name): +def task_train(task_config: dict, experiment_name: str): """ task based training @@ -14,6 +14,8 @@ def task_train(task_config: dict, experiment_name): ---------- task_config : dict A dict describes a task setting. + experiment_name: str + The name of experiment """ # model initiaiton @@ -30,7 +32,7 @@ def task_train(task_config: dict, experiment_name): R.save_objects(param=task_config) # keep the original format and datatype # generate records: prediction, backtest, and analysis - records = task_config.get('record', []) + records = task_config.get("record", []) if isinstance(records, dict): # prevent only one dict records = [records] for record in records: diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index 9a67d8e06f..4562a1cec7 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -4,6 +4,62 @@ from tqdm.auto import tqdm +class TaskCollector: + """ + Collect the record results of the finished tasks with key and filter + """ + + @staticmethod + def collect( + experiment_name: str, + get_key_func, + filter_func=None, + ): + """ + + Parameters + ---------- + experiment_name : str + get_key_func : function(task: dict) -> Union[Number, str, tuple] + get the key of a task when collect it + filter_func : function(task: dict) -> bool + to judge a task will be collected or not + + Returns + ------- + + """ + exp = R.get_exp(experiment_name=experiment_name) + # filter records + recs = exp.list_recorders() + + recs_flt = {} + for rid, rec in tqdm(recs.items(), desc="Loading data"): + params = rec.load_object("param") + if rec.status == rec.STATUS_FI: + if filter_func is None or filter_func(params): + rec.params = params + recs_flt[rid] = rec + + # group + recs_group = {} + for _, rec in recs_flt.items(): + params = rec.params + group_key = get_key_func(params) + recs_group.setdefault(group_key, []).append(rec) + + # reduce group + reduce_group = {} + for k, rec_l in recs_group.items(): + pred_l = [] + for rec in rec_l: + pred_l.append(rec.load_object("pred.pkl").iloc[:, 0]) + pred = pd.concat(pred_l).sort_index() + reduce_group[k] = pred + + return reduce_group + + class RollingCollector: """ Rolling Models Ensemble based on (R)ecord @@ -13,7 +69,7 @@ class RollingCollector: # TODO: speed up this class def __init__(self, get_key_func, flt_func=None): - self.get_key_func = get_key_func # user need to implement this method to get the key of a task based on task config + self.get_key_func = get_key_func # get the key of a task based on task config self.flt_func = flt_func # determine whether a task can be retained based on task config def __call__(self, exp_name) -> Union[pd.Series, dict]: diff --git a/qlib/workflow/task/gen.py b/qlib/workflow/task/gen.py index 60fc5c2216..b1c2e0ce2c 100644 --- a/qlib/workflow/task/gen.py +++ b/qlib/workflow/task/gen.py @@ -79,6 +79,7 @@ class TaskGen(metaclass=abc.ABCMeta): output: a set of tasks with different losses """ + @abc.abstractmethod def generate(self, task: dict) -> typing.List[dict]: """ diff --git a/qlib/workflow/task/utils.py b/qlib/workflow/task/utils.py index 5e94f55ae4..63563e2f6d 100644 --- a/qlib/workflow/task/utils.py +++ b/qlib/workflow/task/utils.py @@ -8,6 +8,7 @@ from pymongo import MongoClient from typing import Union + def get_mongodb(): """ From 83dbdfb45e5f429ed26186466fd8b0e56108a2ce Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Tue, 9 Mar 2021 17:22:36 +0800 Subject: [PATCH 11/61] finished document and example --- docs/advanced/task_managment.rst | 67 +++ .../taskmanager/task_manager_rolling.ipynb | 445 ++++++++++++++++++ examples/taskmanager/task_manager_rolling.py | 108 +++++ examples/workflow_task_rolling.ipynb | 177 ------- 4 files changed, 620 insertions(+), 177 deletions(-) create mode 100644 docs/advanced/task_managment.rst create mode 100644 examples/taskmanager/task_manager_rolling.ipynb create mode 100644 examples/taskmanager/task_manager_rolling.py delete mode 100644 examples/workflow_task_rolling.ipynb diff --git a/docs/advanced/task_managment.rst b/docs/advanced/task_managment.rst new file mode 100644 index 0000000000..78ac62410e --- /dev/null +++ b/docs/advanced/task_managment.rst @@ -0,0 +1,67 @@ +.. _task_managment: + +================================= +Task Management +================================= +.. currentmodule:: qlib + + +Introduction +============= + +The `Workflow <../component/introduction.html>`_ part introduce how to run research workflow in a loosely-coupled way. But it can only execute one ``task`` when you use ``qrun``. To automatically generate and execute different tasks, Task Management module provide a whole process including `Task Generating`_, `Task Storing`_, `Task Running`_ and `Task Collecting`_. +With this module, users can run their ``task`` automatically at different periods, in different losses or even by different models. + +An example of the entire process is shown `here <>`_. + +Task Generating +=============== +A ``task`` consists of `Model`, `Dataset`, `Record` or anything added by users. +The specific task template can be viewed in +`Task Section <../component/workflow.html#task-section>`_. +Even though the task template is fixed, Users can use ``TaskGen`` to generate different ``task`` by task template. + +Here is the base class of TaskGen: + +.. autoclass:: qlib.workflow.task.gen.TaskGen + :members: + +``Qlib`` provider a class `RollingGen`_ to generate a list of ``task`` of dataset in different date segments. +This allows users to verify the effect of data from different periods on the model in one experiment. + +Task Storing +=============== +In order to achieve higher efficiency and the possibility of cluster operation, ``Task Manager`` will store all tasks in `MongoDB `_. +Users **MUST** finished the configuration of `MongoDB `_ when using this module. + +Users need to provide the url and database of ``task`` storing like this. + + .. code-block:: python + + from qlib.config import C + C["mongo"] = { + "task_url" : "mongodb://localhost:27017/", # maybe you need to change it to your url + "task_db_name" : "rolling_db" # you can custom database name + } + +The CRUD methods of ``task`` can be found in TaskManager. More methods can be seen in the `Github`_. + +.. autoclass:: qlib.workflow.task.manage.TaskManager + :members: + +Task Running +=============== +After generating and storing those ``task``, it's time to run the ``task`` in the *WAITING* status. +``qlib`` provide a method to run those ``task`` in task pool, however users can also customize how tasks are executed. +An easy way to get the ``task_func`` is using ``qlib.model.trainer.task_train`` directly. +It will run the whole workflow defined by ``task``, which includes *Model*, *Dataset*, *Record*. + +.. autofunction:: qlib.workflow.task.manage.run_task + +Task Collecting +=============== +To see the results of ``task`` after running, ``Qlib`` provide a task collector to collect the tasks by filter condition (optional). +The collector will return a dict of filtered key (users defined by task config) and value (predict scores from ``pred.pkl``). + +.. autoclass:: qlib.workflow.task.collect.TaskCollector + :members: \ No newline at end of file diff --git a/examples/taskmanager/task_manager_rolling.ipynb b/examples/taskmanager/task_manager_rolling.ipynb new file mode 100644 index 0000000000..43ae5b1d10 --- /dev/null +++ b/examples/taskmanager/task_manager_rolling.ipynb @@ -0,0 +1,445 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "import mlflow\n", + "mlflow.end_run()" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "collapsed": true + }, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "[8348:MainThread](2021-03-09 14:55:48,543) INFO - qlib.Initialization - [config.py:279] - default_conf: client.\n", + "[8348:MainThread](2021-03-09 14:55:50,592) WARNING - qlib.Initialization - [config.py:295] - redis connection failed(host=127.0.0.1 port=6379), cache will not be used!\n", + "[8348:MainThread](2021-03-09 14:55:50,597) INFO - qlib.Initialization - [__init__.py:48] - qlib successfully initialized based on client settings.\n", + "[8348:MainThread](2021-03-09 14:55:50,601) INFO - qlib.Initialization - [__init__.py:49] - data_path=C:\\Users\\lzh222333\\.qlib\\qlib_data\\cn_data\n" + ] + } + ], + "source": [ + "import qlib\n", + "from qlib.config import REG_CN\n", + "from qlib.workflow.task.gen import RollingGen, task_generator\n", + "from qlib.workflow.task.manage import TaskManager\n", + "from qlib.config import C\n", + "\n", + "data_handler_template = {\n", + " \"start_time\": \"2008-01-01\",\n", + " \"end_time\": \"2020-08-01\",\n", + " \"fit_start_time\": \"2008-01-01\",\n", + " \"fit_end_time\": \"2014-12-31\",\n", + " \"instruments\": 'csi100',\n", + "}\n", + "\n", + "dataset_template = {\n", + " \"class\": \"DatasetH\",\n", + " \"module_path\": \"qlib.data.dataset\",\n", + " \"kwargs\": {\n", + " \"handler\": {\n", + " \"class\": \"Alpha158\",\n", + " \"module_path\": \"qlib.contrib.data.handler\",\n", + " \"kwargs\": data_handler_template,\n", + " },\n", + " \"segments\": {\n", + " \"train\": (\"2008-01-01\", \"2014-12-31\"),\n", + " \"valid\": (\"2015-01-01\", \"2016-12-31\"),\n", + " \"test\": (\"2017-01-01\", \"2020-08-01\"),\n", + " },\n", + " },\n", + " }\n", + "\n", + "record_template = [\n", + " {\n", + " \"class\": \"SignalRecord\",\n", + " \"module_path\": \"qlib.workflow.record_temp\",\n", + " },\n", + " {\n", + " \"class\": \"SigAnaRecord\",\n", + " \"module_path\": \"qlib.workflow.record_temp\",\n", + " }\n", + "]\n", + "\n", + "# use lgb\n", + "lgb_task_template = {\n", + " \"model\": {\n", + " \"class\": \"LGBModel\",\n", + " \"module_path\": \"qlib.contrib.model.gbdt\",\n", + " },\n", + " \"dataset\": dataset_template,\n", + " \"record\": record_template,\n", + "}\n", + "\n", + "# use xgboost\n", + "xgboost_task_template = {\n", + " \"model\": {\n", + " \"class\": \"XGBModel\",\n", + " \"module_path\": \"qlib.contrib.model.xgboost\",\n", + " },\n", + " \"dataset\": dataset_template,\n", + " \"record\": record_template,\n", + "}\n", + "\n", + "provider_uri = \"~/.qlib/qlib_data/cn_data\" # target_dir\n", + "qlib.init(provider_uri=provider_uri, region=REG_CN)\n", + "\n", + "C[\"mongo\"] = {\n", + " \"task_url\" : \"mongodb://localhost:27017/\", # maybe you need to change it to your url\n", + " \"task_db_name\" : \"rolling_db3\"\n", + "}\n", + "\n", + "exp_name = 'rolling_exp3' # experiment name, will be used as the experiment in MLflow\n", + "task_pool = 'rolling_task3' # task pool name, will be used as the document in MongoDB" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[{'dataset': {'class': 'DatasetH',\n", + " 'kwargs': {'handler': {'class': 'Alpha158',\n", + " 'kwargs': {'end_time': '2020-08-01',\n", + " 'fit_end_time': '2014-12-31',\n", + " 'fit_start_time': '2008-01-01',\n", + " 'instruments': 'csi100',\n", + " 'start_time': '2008-01-01'},\n", + " 'module_path': 'qlib.contrib.data.handler'},\n", + " 'segments': {'test': (Timestamp('2017-01-03 00:00:00'),\n", + " Timestamp('2019-04-08 00:00:00')),\n", + " 'train': (Timestamp('2008-01-02 00:00:00'),\n", + " Timestamp('2014-12-31 00:00:00')),\n", + " 'valid': (Timestamp('2015-01-05 00:00:00'),\n", + " Timestamp('2016-12-30 00:00:00'))}},\n", + " 'module_path': 'qlib.data.dataset'},\n", + " 'model': {'class': 'XGBModel', 'module_path': 'qlib.contrib.model.xgboost'},\n", + " 'record': [{'class': 'SignalRecord',\n", + " 'module_path': 'qlib.workflow.record_temp'},\n", + " {'class': 'SigAnaRecord',\n", + " 'module_path': 'qlib.workflow.record_temp'}],\n", + " 'task_key': 1},\n", + " {'dataset': {'class': 'DatasetH',\n", + " 'kwargs': {'handler': {'class': 'Alpha158',\n", + " 'kwargs': {'end_time': '2020-08-01',\n", + " 'fit_end_time': '2014-12-31',\n", + " 'fit_start_time': '2008-01-01',\n", + " 'instruments': 'csi100',\n", + " 'start_time': '2008-01-01'},\n", + " 'module_path': 'qlib.contrib.data.handler'},\n", + " 'segments': {'test': (Timestamp('2019-04-09 00:00:00'),\n", + " Timestamp('2021-07-12 00:00:00')),\n", + " 'train': (Timestamp('2010-04-23 00:00:00'),\n", + " Timestamp('2017-05-24 00:00:00')),\n", + " 'valid': (Timestamp('2017-05-25 00:00:00'),\n", + " Timestamp('2019-04-08 00:00:00'))}},\n", + " 'module_path': 'qlib.data.dataset'},\n", + " 'model': {'class': 'XGBModel', 'module_path': 'qlib.contrib.model.xgboost'},\n", + " 'record': [{'class': 'SignalRecord',\n", + " 'module_path': 'qlib.workflow.record_temp'},\n", + " {'class': 'SigAnaRecord',\n", + " 'module_path': 'qlib.workflow.record_temp'}],\n", + " 'task_key': 1},\n", + " {'dataset': {'class': 'DatasetH',\n", + " 'kwargs': {'handler': {'class': 'Alpha158',\n", + " 'kwargs': {'end_time': '2020-08-01',\n", + " 'fit_end_time': '2014-12-31',\n", + " 'fit_start_time': '2008-01-01',\n", + " 'instruments': 'csi100',\n", + " 'start_time': '2008-01-01'},\n", + " 'module_path': 'qlib.contrib.data.handler'},\n", + " 'segments': {'test': (Timestamp('2017-01-03 00:00:00'),\n", + " Timestamp('2019-04-08 00:00:00')),\n", + " 'train': (Timestamp('2008-01-02 00:00:00'),\n", + " Timestamp('2014-12-31 00:00:00')),\n", + " 'valid': (Timestamp('2015-01-05 00:00:00'),\n", + " Timestamp('2016-12-30 00:00:00'))}},\n", + " 'module_path': 'qlib.data.dataset'},\n", + " 'model': {'class': 'LGBModel', 'module_path': 'qlib.contrib.model.gbdt'},\n", + " 'record': [{'class': 'SignalRecord',\n", + " 'module_path': 'qlib.workflow.record_temp'},\n", + " {'class': 'SigAnaRecord',\n", + " 'module_path': 'qlib.workflow.record_temp'}],\n", + " 'task_key': 'task_lgb'},\n", + " {'dataset': {'class': 'DatasetH',\n", + " 'kwargs': {'handler': {'class': 'Alpha158',\n", + " 'kwargs': {'end_time': '2020-08-01',\n", + " 'fit_end_time': '2014-12-31',\n", + " 'fit_start_time': '2008-01-01',\n", + " 'instruments': 'csi100',\n", + " 'start_time': '2008-01-01'},\n", + " 'module_path': 'qlib.contrib.data.handler'},\n", + " 'segments': {'test': (Timestamp('2019-04-09 00:00:00'),\n", + " Timestamp('2021-07-12 00:00:00')),\n", + " 'train': (Timestamp('2010-04-23 00:00:00'),\n", + " Timestamp('2017-05-24 00:00:00')),\n", + " 'valid': (Timestamp('2017-05-25 00:00:00'),\n", + " Timestamp('2019-04-08 00:00:00'))}},\n", + " 'module_path': 'qlib.data.dataset'},\n", + " 'model': {'class': 'LGBModel', 'module_path': 'qlib.contrib.model.gbdt'},\n", + " 'record': [{'class': 'SignalRecord',\n", + " 'module_path': 'qlib.workflow.record_temp'},\n", + " {'class': 'SigAnaRecord',\n", + " 'module_path': 'qlib.workflow.record_temp'}],\n", + " 'task_key': 'task_lgb'}]\n", + "Total Tasks, New Tasks: 4 0\n" + ] + } + ], + "source": [ + "tasks = task_generator(\n", + " xgboost_task_template, # default task name\n", + " RollingGen(step=550,rtype=RollingGen.ROLL_SD), # generate different date segment\n", + " task_lgb=lgb_task_template # use \"task_lgb\" as the task name\n", + ")\n", + "# Uncomment next two lines to see the generated tasks\n", + "from pprint import pprint\n", + "pprint(tasks)\n", + "tm = TaskManager(task_pool=task_pool)\n", + "tm.create_task(tasks) # all tasks will be saved to MongoDB" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 26, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "2021-03-09 14:55:51.600 | INFO | qlib.workflow.task.manage:run_task:355 - {'model': {'class': 'XGBModel', 'module_path': 'qlib.contrib.model.xgboost'}, 'dataset': {'class': 'DatasetH', 'module_path': 'qlib.data.dataset', 'kwargs': {'handler': {'class': 'Alpha158', 'module_path': 'qlib.contrib.data.handler', 'kwargs': {'start_time': '2008-01-01', 'end_time': '2020-08-01', 'fit_start_time': '2008-01-01', 'fit_end_time': '2014-12-31', 'instruments': 'csi100'}}, 'segments': {'train': (Timestamp('2008-01-02 00:00:00'), Timestamp('2014-12-31 00:00:00')), 'valid': (Timestamp('2015-01-05 00:00:00'), Timestamp('2016-12-30 00:00:00')), 'test': (Timestamp('2017-01-03 00:00:00'), Timestamp('2019-04-08 00:00:00'))}}}, 'record': [{'class': 'SignalRecord', 'module_path': 'qlib.workflow.record_temp'}, {'class': 'SigAnaRecord', 'module_path': 'qlib.workflow.record_temp'}], 'task_key': 1}\n", + "[8348:MainThread](2021-03-09 14:56:46,051) INFO - qlib.timer - [log.py:81] - Time cost: 54.448s | Loading data Done\n", + "[8348:MainThread](2021-03-09 14:56:46,440) INFO - qlib.timer - [log.py:81] - Time cost: 0.322s | DropnaLabel Done\n", + "[8348:MainThread](2021-03-09 14:56:52,461) INFO - qlib.timer - [log.py:81] - Time cost: 6.019s | CSZScoreNorm Done\n", + "[8348:MainThread](2021-03-09 14:56:52,464) INFO - qlib.timer - [log.py:81] - Time cost: 6.411s | fit & process data Done\n", + "[8348:MainThread](2021-03-09 14:56:52,468) INFO - qlib.timer - [log.py:81] - Time cost: 60.865s | Init data Done\n", + "[8348:MainThread](2021-03-09 14:56:52,471) INFO - qlib.workflow - [expm.py:245] - No tracking URI is provided. Use the default tracking URI.\n", + "[8348:MainThread](2021-03-09 14:56:52,500) INFO - qlib.workflow - [exp.py:181] - Experiment 2 starts running ...\n", + "[8348:MainThread](2021-03-09 14:56:52,567) INFO - qlib.workflow - [recorder.py:233] - Recorder dd6bceb6d319493686ab6565633c0b5a starts running under Experiment 2 ...\n", + "[0]\ttrain-rmse:1.05165\tvalid-rmse:1.05565\n", + "[20]\ttrain-rmse:0.97071\tvalid-rmse:1.00077\n", + "[40]\ttrain-rmse:0.95124\tvalid-rmse:1.00609\n", + "[59]\ttrain-rmse:0.93833\tvalid-rmse:1.00945\n", + "[8348:MainThread](2021-03-09 14:59:37,266) INFO - qlib.workflow - [record_temp.py:126] - Signal record 'pred.pkl' has been saved as the artifact of the Experiment 2\n", + "'The following are prediction results of the XGBModel model.'\n", + " score\n", + "datetime instrument \n", + "2017-01-03 SH600000 -0.103259\n", + " SH600010 -0.084365\n", + " SH600015 -0.107433\n", + " SH600016 -0.064723\n", + " SH600018 -0.038639\n", + "{'IC': 0.05347474869798698,\n", + " 'ICIR': 0.29781294430945265,\n", + " 'Rank IC': 0.0484064337863249,\n", + " 'Rank ICIR': 0.36035393716962033}\n", + "2021-03-09 14:59:38.633 | INFO | qlib.workflow.task.manage:run_task:355 - {'model': {'class': 'XGBModel', 'module_path': 'qlib.contrib.model.xgboost'}, 'dataset': {'class': 'DatasetH', 'module_path': 'qlib.data.dataset', 'kwargs': {'handler': {'class': 'Alpha158', 'module_path': 'qlib.contrib.data.handler', 'kwargs': {'start_time': '2008-01-01', 'end_time': '2020-08-01', 'fit_start_time': '2008-01-01', 'fit_end_time': '2014-12-31', 'instruments': 'csi100'}}, 'segments': {'train': (Timestamp('2010-04-23 00:00:00'), Timestamp('2017-05-24 00:00:00')), 'valid': (Timestamp('2017-05-25 00:00:00'), Timestamp('2019-04-08 00:00:00')), 'test': (Timestamp('2019-04-09 00:00:00'), Timestamp('2021-07-12 00:00:00'))}}}, 'record': [{'class': 'SignalRecord', 'module_path': 'qlib.workflow.record_temp'}, {'class': 'SigAnaRecord', 'module_path': 'qlib.workflow.record_temp'}], 'task_key': 1}\n", + "[8348:MainThread](2021-03-09 15:00:36,591) INFO - qlib.timer - [log.py:81] - Time cost: 57.954s | Loading data Done\n", + "[8348:MainThread](2021-03-09 15:00:36,997) INFO - qlib.timer - [log.py:81] - Time cost: 0.338s | DropnaLabel Done\n", + "[8348:MainThread](2021-03-09 15:00:43,728) INFO - qlib.timer - [log.py:81] - Time cost: 6.728s | CSZScoreNorm Done\n", + "[8348:MainThread](2021-03-09 15:00:43,731) INFO - qlib.timer - [log.py:81] - Time cost: 7.137s | fit & process data Done\n", + "[8348:MainThread](2021-03-09 15:00:43,734) INFO - qlib.timer - [log.py:81] - Time cost: 65.097s | Init data Done\n", + "[8348:MainThread](2021-03-09 15:00:43,740) INFO - qlib.workflow - [expm.py:245] - No tracking URI is provided. Use the default tracking URI.\n", + "[8348:MainThread](2021-03-09 15:00:43,768) INFO - qlib.workflow - [exp.py:181] - Experiment 2 starts running ...\n", + "[8348:MainThread](2021-03-09 15:00:43,851) INFO - qlib.workflow - [recorder.py:233] - Recorder de2f892b569c436ba642a23e99f4f2b0 starts running under Experiment 2 ...\n", + "[0]\ttrain-rmse:1.05178\tvalid-rmse:1.05345\n", + "[20]\ttrain-rmse:0.96764\tvalid-rmse:0.99546\n", + "[40]\ttrain-rmse:0.94957\tvalid-rmse:0.99798\n", + "[57]\ttrain-rmse:0.93592\tvalid-rmse:1.00030\n", + "[8348:MainThread](2021-03-09 15:03:12,764) INFO - qlib.workflow - [record_temp.py:126] - Signal record 'pred.pkl' has been saved as the artifact of the Experiment 2\n", + "'The following are prediction results of the XGBModel model.'\n", + " score\n", + "datetime instrument \n", + "2019-04-09 SH600000 0.006996\n", + " SH600009 -0.102482\n", + " SH600010 0.016398\n", + " SH600011 0.004459\n", + " SH600015 -0.128315\n", + "{'IC': 0.013224093132176661,\n", + " 'ICIR': 0.08254897170570956,\n", + " 'Rank IC': 0.02472594591723197,\n", + " 'Rank ICIR': 0.16330982475433398}\n", + "2021-03-09 15:03:13.593 | INFO | qlib.workflow.task.manage:run_task:355 - {'model': {'class': 'LGBModel', 'module_path': 'qlib.contrib.model.gbdt'}, 'dataset': {'class': 'DatasetH', 'module_path': 'qlib.data.dataset', 'kwargs': {'handler': {'class': 'Alpha158', 'module_path': 'qlib.contrib.data.handler', 'kwargs': {'start_time': '2008-01-01', 'end_time': '2020-08-01', 'fit_start_time': '2008-01-01', 'fit_end_time': '2014-12-31', 'instruments': 'csi100'}}, 'segments': {'train': (Timestamp('2008-01-02 00:00:00'), Timestamp('2014-12-31 00:00:00')), 'valid': (Timestamp('2015-01-05 00:00:00'), Timestamp('2016-12-30 00:00:00')), 'test': (Timestamp('2017-01-03 00:00:00'), Timestamp('2019-04-08 00:00:00'))}}}, 'record': [{'class': 'SignalRecord', 'module_path': 'qlib.workflow.record_temp'}, {'class': 'SigAnaRecord', 'module_path': 'qlib.workflow.record_temp'}], 'task_key': 'task_lgb'}\n", + "[8348:MainThread](2021-03-09 15:04:06,545) INFO - qlib.timer - [log.py:81] - Time cost: 52.814s | Loading data Done\n", + "[8348:MainThread](2021-03-09 15:04:06,919) INFO - qlib.timer - [log.py:81] - Time cost: 0.312s | DropnaLabel Done\n", + "[8348:MainThread](2021-03-09 15:04:12,850) INFO - qlib.timer - [log.py:81] - Time cost: 5.928s | CSZScoreNorm Done\n", + "[8348:MainThread](2021-03-09 15:04:12,853) INFO - qlib.timer - [log.py:81] - Time cost: 6.305s | fit & process data Done\n", + "[8348:MainThread](2021-03-09 15:04:12,856) INFO - qlib.timer - [log.py:81] - Time cost: 59.125s | Init data Done\n", + "[8348:MainThread](2021-03-09 15:04:12,859) INFO - qlib.workflow - [expm.py:245] - No tracking URI is provided. Use the default tracking URI.\n", + "[8348:MainThread](2021-03-09 15:04:12,888) INFO - qlib.workflow - [exp.py:181] - Experiment 2 starts running ...\n", + "[8348:MainThread](2021-03-09 15:04:12,958) INFO - qlib.workflow - [recorder.py:233] - Recorder 15df799127a74656829978c1b9352e60 starts running under Experiment 2 ...\n", + "Training until validation scores don't improve for 50 rounds\n", + "[20]\ttrain's l2: 0.970491\tvalid's l2: 0.987723\n", + "[40]\ttrain's l2: 0.957984\tvalid's l2: 0.990056\n", + "[60]\ttrain's l2: 0.947201\tvalid's l2: 0.991459\n", + "Early stopping, best iteration is:\n", + "[18]\ttrain's l2: 0.971834\tvalid's l2: 0.987481\n", + "[8348:MainThread](2021-03-09 15:04:19,847) INFO - qlib.workflow - [record_temp.py:126] - Signal record 'pred.pkl' has been saved as the artifact of the Experiment 2\n", + "'The following are prediction results of the LGBModel model.'\n", + " score\n", + "datetime instrument \n", + "2017-01-03 SH600000 -0.013089\n", + " SH600010 -0.006642\n", + " SH600015 -0.035137\n", + " SH600016 -0.034634\n", + " SH600018 -0.029493\n", + "{'IC': 0.05704431372255674,\n", + " 'ICIR': 0.28879437007622133,\n", + " 'Rank IC': 0.05181220321608411,\n", + " 'Rank ICIR': 0.3233833799543165}\n", + "2021-03-09 15:04:21.111 | INFO | qlib.workflow.task.manage:run_task:355 - {'model': {'class': 'LGBModel', 'module_path': 'qlib.contrib.model.gbdt'}, 'dataset': {'class': 'DatasetH', 'module_path': 'qlib.data.dataset', 'kwargs': {'handler': {'class': 'Alpha158', 'module_path': 'qlib.contrib.data.handler', 'kwargs': {'start_time': '2008-01-01', 'end_time': '2020-08-01', 'fit_start_time': '2008-01-01', 'fit_end_time': '2014-12-31', 'instruments': 'csi100'}}, 'segments': {'train': (Timestamp('2010-04-23 00:00:00'), Timestamp('2017-05-24 00:00:00')), 'valid': (Timestamp('2017-05-25 00:00:00'), Timestamp('2019-04-08 00:00:00')), 'test': (Timestamp('2019-04-09 00:00:00'), Timestamp('2021-07-12 00:00:00'))}}}, 'record': [{'class': 'SignalRecord', 'module_path': 'qlib.workflow.record_temp'}, {'class': 'SigAnaRecord', 'module_path': 'qlib.workflow.record_temp'}], 'task_key': 'task_lgb'}\n", + "[8348:MainThread](2021-03-09 15:05:16,072) INFO - qlib.timer - [log.py:81] - Time cost: 54.958s | Loading data Done\n", + "[8348:MainThread](2021-03-09 15:05:16,466) INFO - qlib.timer - [log.py:81] - Time cost: 0.334s | DropnaLabel Done\n", + "[8348:MainThread](2021-03-09 15:05:22,281) INFO - qlib.timer - [log.py:81] - Time cost: 5.812s | CSZScoreNorm Done\n", + "[8348:MainThread](2021-03-09 15:05:22,283) INFO - qlib.timer - [log.py:81] - Time cost: 6.209s | fit & process data Done\n", + "[8348:MainThread](2021-03-09 15:05:22,286) INFO - qlib.timer - [log.py:81] - Time cost: 61.172s | Init data Done\n", + "[8348:MainThread](2021-03-09 15:05:22,291) INFO - qlib.workflow - [expm.py:245] - No tracking URI is provided. Use the default tracking URI.\n", + "[8348:MainThread](2021-03-09 15:05:22,317) INFO - qlib.workflow - [exp.py:181] - Experiment 2 starts running ...\n", + "[8348:MainThread](2021-03-09 15:05:22,386) INFO - qlib.workflow - [recorder.py:233] - Recorder 0c814539f55842b9b6310843fc5ec708 starts running under Experiment 2 ...\n", + "Training until validation scores don't improve for 50 rounds\n", + "[20]\ttrain's l2: 0.969033\tvalid's l2: 0.98571\n", + "[40]\ttrain's l2: 0.955399\tvalid's l2: 0.986164\n", + "[60]\ttrain's l2: 0.943514\tvalid's l2: 0.986301\n", + "Early stopping, best iteration is:\n", + "[26]\ttrain's l2: 0.964587\tvalid's l2: 0.985356\n", + "[8348:MainThread](2021-03-09 15:05:29,546) INFO - qlib.workflow - [record_temp.py:126] - Signal record 'pred.pkl' has been saved as the artifact of the Experiment 2\n", + "'The following are prediction results of the LGBModel model.'\n", + " score\n", + "datetime instrument \n", + "2019-04-09 SH600000 0.029586\n", + " SH600009 0.004306\n", + " SH600010 -0.004411\n", + " SH600011 0.002707\n", + " SH600015 -0.029124\n", + "{'IC': 0.020784811232504984,\n", + " 'ICIR': 0.11590182186569555,\n", + " 'Rank IC': 0.028925697036767055,\n", + " 'Rank ICIR': 0.16388058980901396}\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "True" + ] + }, + "metadata": {}, + "execution_count": 26 + } + ], + "source": [ + "from qlib.workflow.task.manage import run_task\n", + "from qlib.workflow.task.collect import TaskCollector\n", + "from qlib.model.trainer import task_train\n", + "\n", + "run_task(task_train, task_pool, experiment_name=exp_name) # all tasks will be trained using \"task_train\" method" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 27, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "Loading data: 100%|██████████| 4/4 [00:00<00:00, 37.38it/s]\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "{('task_lgb', '2019-04-08'): datetime instrument\n", + " 2017-01-03 SH600000 -0.013089\n", + " SH600010 -0.006642\n", + " SH600015 -0.035137\n", + " SH600016 -0.034634\n", + " SH600018 -0.029493\n", + " ... \n", + " 2019-04-08 SZ002415 0.049199\n", + " SZ002450 -0.013450\n", + " SZ002594 0.022395\n", + " SZ002736 0.091433\n", + " SZ300059 -0.016237\n", + " Name: score, Length: 55000, dtype: float64}" + ] + }, + "metadata": {}, + "execution_count": 27 + } + ], + "source": [ + "def get_task_key(task):\n", + " task_key = task[\"task_key\"]\n", + " rolling_end_timestamp = task[\"dataset\"][\"kwargs\"][\"segments\"][\"test\"][1]\n", + " #rolling_end_datatime = rolling_end_timestamp.to_pydatetime()\n", + " return task_key, rolling_end_timestamp.strftime('%Y-%m-%d')\n", + "\n", + "def my_filter(task):\n", + " # only choose the results of \"task_lgb\" and test segment end in 2019 from all tasks\n", + " task_key, rolling_end = get_task_key(task)\n", + " if task_key==\"task_lgb\" and rolling_end.startswith('2019'):\n", + " return True\n", + " return False\n", + "\n", + "# name tasks by \"get_task_key\" and filter tasks by \"my_filter\"\n", + "pred_rolling = TaskCollector.collect(exp_name, get_task_key, my_filter) \n", + "pred_rolling" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "3.6.5-final" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/examples/taskmanager/task_manager_rolling.py b/examples/taskmanager/task_manager_rolling.py new file mode 100644 index 0000000000..7519bc4be0 --- /dev/null +++ b/examples/taskmanager/task_manager_rolling.py @@ -0,0 +1,108 @@ +import qlib +from qlib.config import REG_CN +from qlib.workflow.task.gen import RollingGen, task_generator +from qlib.workflow.task.manage import TaskManager +from qlib.config import C + +data_handler_config = { + "start_time": "2008-01-01", + "end_time": "2020-08-01", + "fit_start_time": "2008-01-01", + "fit_end_time": "2014-12-31", + "instruments": 'csi100', +} + +dataset_config = { + "class": "DatasetH", + "module_path": "qlib.data.dataset", + "kwargs": { + "handler": { + "class": "Alpha158", + "module_path": "qlib.contrib.data.handler", + "kwargs": data_handler_config, + }, + "segments": { + "train": ("2008-01-01", "2014-12-31"), + "valid": ("2015-01-01", "2016-12-31"), + "test": ("2017-01-01", "2020-08-01"), + }, + }, + } + +record_config = [ + { + "class": "SignalRecord", + "module_path": "qlib.workflow.record_temp", + }, + { + "class": "SigAnaRecord", + "module_path": "qlib.workflow.record_temp", + } +] + +# use lgb +task_lgb_config = { + "model": { + "class": "LGBModel", + "module_path": "qlib.contrib.model.gbdt", + }, + "dataset": dataset_config, + "record": record_config, +} + +# use xgboost +task_xgboost_config = { + "model": { + "class": "XGBModel", + "module_path": "qlib.contrib.model.xgboost", + }, + "dataset": dataset_config, + "record": record_config, +} + +provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir +qlib.init(provider_uri=provider_uri, region=REG_CN) + +C["mongo"] = { + "task_url" : "mongodb://localhost:27017/", # maybe you need to change it to your url + "task_db_name" : "rolling_db" +} + +exp_name = 'rolling_exp' # experiment name, will be used as the experiment in MLflow +task_pool = 'rolling_task' # task pool name, will be used as the document in MongoDB + +tasks = task_generator( + task_xgboost_config, # default task name + RollingGen(step=550,rtype=RollingGen.ROLL_SD), # generate different date segment + task_lgb=task_lgb_config # use "task_lgb" as the task name +) + +# Uncomment next two lines to see the generated tasks +# from pprint import pprint +# pprint(tasks) + +tm = TaskManager(task_pool=task_pool) +tm.create_task(tasks) # all tasks will be saved to MongoDB + +from qlib.workflow.task.manage import run_task +from qlib.workflow.task.collect import RollingCollector +from qlib.model.trainer import task_train + +run_task(task_train, task_pool, experiment_name=exp_name) # all tasks will be trained using "task_train" method + +def get_task_key(task_config): + task_key = task_config["task_key"] + rolling_end_timestamp = task_config["dataset"]["kwargs"]["segments"]["test"][1] + #rolling_end_datatime = rolling_end_timestamp.to_pydatetime() + return task_key, rolling_end_timestamp.strftime('%Y-%m-%d') + +def my_filter(task_config): + # only choose the results of "task_lgb" and test in 2019 from all tasks + task_key, rolling_end = get_task_key(task_config) + if task_key=="task_lgb" and rolling_end.startswith('2019'): + return True + return False + +collector = RollingCollector(get_task_key, my_filter) +pred_rolling = collector(exp_name) # name tasks by "get_task_key" and filter tasks by "my_filter" +print(pred_rolling) \ No newline at end of file diff --git a/examples/workflow_task_rolling.ipynb b/examples/workflow_task_rolling.ipynb deleted file mode 100644 index c2d399be0d..0000000000 --- a/examples/workflow_task_rolling.ipynb +++ /dev/null @@ -1,177 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "import qlib\n", - "from qlib.config import REG_CN\n", - "from qlib.workflow.task.gen import RollingGen, task_generator\n", - "from qlib.workflow.task.manage import TaskManager\n", - "from qlib.config import C\n", - "\n", - "data_handler_config = {\n", - " \"start_time\": \"2008-01-01\",\n", - " \"end_time\": \"2020-08-01\",\n", - " \"fit_start_time\": \"2008-01-01\",\n", - " \"fit_end_time\": \"2014-12-31\",\n", - " \"instruments\": 'csi100',\n", - "}\n", - "\n", - "dataset_config = {\n", - " \"class\": \"DatasetH\",\n", - " \"module_path\": \"qlib.data.dataset\",\n", - " \"kwargs\": {\n", - " \"handler\": {\n", - " \"class\": \"Alpha158\",\n", - " \"module_path\": \"qlib.contrib.data.handler\",\n", - " \"kwargs\": data_handler_config,\n", - " },\n", - " \"segments\": {\n", - " \"train\": (\"2008-01-01\", \"2014-12-31\"),\n", - " \"valid\": (\"2015-01-01\", \"2016-12-31\"),\n", - " \"test\": (\"2017-01-01\", \"2020-08-01\"),\n", - " },\n", - " },\n", - " }\n", - "\n", - "record_config = [\n", - " {\n", - " \"class\": \"SignalRecord\",\n", - " \"module_path\": \"qlib.workflow.record_temp\",\n", - " },\n", - " {\n", - " \"class\": \"SigAnaRecord\",\n", - " \"module_path\": \"qlib.workflow.record_temp\",\n", - " }\n", - "]\n", - "\n", - "# use lgb\n", - "task_lgb_config = {\n", - " \"model\": {\n", - " \"class\": \"LGBModel\",\n", - " \"module_path\": \"qlib.contrib.model.gbdt\",\n", - " },\n", - " \"dataset\": dataset_config,\n", - " \"record\": record_config,\n", - "}\n", - "\n", - "# use xgboost\n", - "task_xgboost_config = {\n", - " \"model\": {\n", - " \"class\": \"XGBModel\",\n", - " \"module_path\": \"qlib.contrib.model.xgboost\",\n", - " },\n", - " \"dataset\": dataset_config,\n", - " \"record\": record_config,\n", - "}\n", - "provider_uri = r\"../qlib-main/qlib_data/cn_data\"\n", - "#provider_uri = \"~/.qlib/qlib_data/cn_data\" # target_dir\n", - "qlib.init(provider_uri=provider_uri, region=REG_CN)\n", - "\n", - "C[\"mongo\"] = {\n", - " \"task_url\" : \"mongodb://localhost:27017/\", # maybe you need to change it to your url\n", - " \"task_db_name\" : \"rolling_db\"\n", - "}\n", - "\n", - "exp_name = 'rolling_exp' # experiment name, will be used as the experiment in MLflow\n", - "task_pool = 'rolling_task' # task pool name, will be used as the document in MongoDB" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "tasks = task_generator(\n", - " task_xgboost_config, # default task name\n", - " RollingGen(step=550,rtype=RollingGen.ROLL_SD), # generate different date segment\n", - " task_lgb=task_lgb_config # use \"task_lgb\" as the task name\n", - ")\n", - "# Uncomment next two lines to see the generated tasks\n", - "# from pprint import pprint\n", - "# pprint(tasks)\n", - "tm = TaskManager(task_pool=task_pool)\n", - "tm.create_task(tasks) # all tasks will be saved to MongoDB" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "from qlib.workflow.task.manage import run_task\n", - "from qlib.workflow.task.collect import RollingCollector\n", - "from qlib.model.trainer import task_train\n", - "\n", - "run_task(task_train, task_pool, experiment_name=exp_name) # all tasks will be trained using \"task_train\" method" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "def get_task_key(task_config):\n", - " task_key = task_config[\"task_key\"]\n", - " rolling_end_timestamp = task_config[\"dataset\"][\"kwargs\"][\"segments\"][\"test\"][1]\n", - " rolling_end_datatime = rolling_end_timestamp.to_pydatetime()\n", - " return task_key, rolling_end_datatime.strftime('%Y-%m-%d')\n", - "\n", - "def my_filter(task_config):\n", - " # only choose the results of \"task_lgb\" and test in 2019 from all tasks\n", - " task_key, rolling_end = get_task_key(task_config)\n", - " if task_key==\"task_lgb\" and rolling_end.startswith('2019'):\n", - " return True\n", - " return False\n", - "\n", - "collector = RollingCollector(get_task_key, my_filter)\n", - "pred_rolling = collector(exp_name) # name tasks by \"get_task_key\" and filter tasks by \"my_filter\"\n", - "pred_rolling" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file From e2f58274ba91f1ef43e8bc87b6cc04e67416137b Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Wed, 10 Mar 2021 10:58:49 +0000 Subject: [PATCH 12/61] update task manager --- .../taskmanager/task_manager_rolling.ipynb | 297 +----------------- examples/taskmanager/task_manager_rolling.py | 9 +- qlib/model/trainer.py | 10 +- qlib/workflow/task/collect.py | 15 +- qlib/workflow/task/gen.py | 2 +- 5 files changed, 34 insertions(+), 299 deletions(-) diff --git a/examples/taskmanager/task_manager_rolling.ipynb b/examples/taskmanager/task_manager_rolling.ipynb index 43ae5b1d10..e8ec8d4a75 100644 --- a/examples/taskmanager/task_manager_rolling.ipynb +++ b/examples/taskmanager/task_manager_rolling.ipynb @@ -2,32 +2,11 @@ "cells": [ { "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [], - "source": [ - "import mlflow\n", - "mlflow.end_run()" - ] - }, - { - "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": { "collapsed": true }, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "[8348:MainThread](2021-03-09 14:55:48,543) INFO - qlib.Initialization - [config.py:279] - default_conf: client.\n", - "[8348:MainThread](2021-03-09 14:55:50,592) WARNING - qlib.Initialization - [config.py:295] - redis connection failed(host=127.0.0.1 port=6379), cache will not be used!\n", - "[8348:MainThread](2021-03-09 14:55:50,597) INFO - qlib.Initialization - [__init__.py:48] - qlib successfully initialized based on client settings.\n", - "[8348:MainThread](2021-03-09 14:55:50,601) INFO - qlib.Initialization - [__init__.py:49] - data_path=C:\\Users\\lzh222333\\.qlib\\qlib_data\\cn_data\n" - ] - } - ], + "outputs": [], "source": [ "import qlib\n", "from qlib.config import REG_CN\n", @@ -96,109 +75,17 @@ "\n", "C[\"mongo\"] = {\n", " \"task_url\" : \"mongodb://localhost:27017/\", # maybe you need to change it to your url\n", - " \"task_db_name\" : \"rolling_db3\"\n", + " \"task_db_name\" : \"rolling_db\"\n", "}\n", "\n", - "exp_name = 'rolling_exp3' # experiment name, will be used as the experiment in MLflow\n", - "task_pool = 'rolling_task3' # task pool name, will be used as the document in MongoDB" + "exp_name = 'rolling_exp' # experiment name, will be used as the experiment in MLflow\n", + "task_pool = 'rolling_task' # task pool name, will be used as the document in MongoDB" ] }, { "cell_type": "code", - "execution_count": 25, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "[{'dataset': {'class': 'DatasetH',\n", - " 'kwargs': {'handler': {'class': 'Alpha158',\n", - " 'kwargs': {'end_time': '2020-08-01',\n", - " 'fit_end_time': '2014-12-31',\n", - " 'fit_start_time': '2008-01-01',\n", - " 'instruments': 'csi100',\n", - " 'start_time': '2008-01-01'},\n", - " 'module_path': 'qlib.contrib.data.handler'},\n", - " 'segments': {'test': (Timestamp('2017-01-03 00:00:00'),\n", - " Timestamp('2019-04-08 00:00:00')),\n", - " 'train': (Timestamp('2008-01-02 00:00:00'),\n", - " Timestamp('2014-12-31 00:00:00')),\n", - " 'valid': (Timestamp('2015-01-05 00:00:00'),\n", - " Timestamp('2016-12-30 00:00:00'))}},\n", - " 'module_path': 'qlib.data.dataset'},\n", - " 'model': {'class': 'XGBModel', 'module_path': 'qlib.contrib.model.xgboost'},\n", - " 'record': [{'class': 'SignalRecord',\n", - " 'module_path': 'qlib.workflow.record_temp'},\n", - " {'class': 'SigAnaRecord',\n", - " 'module_path': 'qlib.workflow.record_temp'}],\n", - " 'task_key': 1},\n", - " {'dataset': {'class': 'DatasetH',\n", - " 'kwargs': {'handler': {'class': 'Alpha158',\n", - " 'kwargs': {'end_time': '2020-08-01',\n", - " 'fit_end_time': '2014-12-31',\n", - " 'fit_start_time': '2008-01-01',\n", - " 'instruments': 'csi100',\n", - " 'start_time': '2008-01-01'},\n", - " 'module_path': 'qlib.contrib.data.handler'},\n", - " 'segments': {'test': (Timestamp('2019-04-09 00:00:00'),\n", - " Timestamp('2021-07-12 00:00:00')),\n", - " 'train': (Timestamp('2010-04-23 00:00:00'),\n", - " Timestamp('2017-05-24 00:00:00')),\n", - " 'valid': (Timestamp('2017-05-25 00:00:00'),\n", - " Timestamp('2019-04-08 00:00:00'))}},\n", - " 'module_path': 'qlib.data.dataset'},\n", - " 'model': {'class': 'XGBModel', 'module_path': 'qlib.contrib.model.xgboost'},\n", - " 'record': [{'class': 'SignalRecord',\n", - " 'module_path': 'qlib.workflow.record_temp'},\n", - " {'class': 'SigAnaRecord',\n", - " 'module_path': 'qlib.workflow.record_temp'}],\n", - " 'task_key': 1},\n", - " {'dataset': {'class': 'DatasetH',\n", - " 'kwargs': {'handler': {'class': 'Alpha158',\n", - " 'kwargs': {'end_time': '2020-08-01',\n", - " 'fit_end_time': '2014-12-31',\n", - " 'fit_start_time': '2008-01-01',\n", - " 'instruments': 'csi100',\n", - " 'start_time': '2008-01-01'},\n", - " 'module_path': 'qlib.contrib.data.handler'},\n", - " 'segments': {'test': (Timestamp('2017-01-03 00:00:00'),\n", - " Timestamp('2019-04-08 00:00:00')),\n", - " 'train': (Timestamp('2008-01-02 00:00:00'),\n", - " Timestamp('2014-12-31 00:00:00')),\n", - " 'valid': (Timestamp('2015-01-05 00:00:00'),\n", - " Timestamp('2016-12-30 00:00:00'))}},\n", - " 'module_path': 'qlib.data.dataset'},\n", - " 'model': {'class': 'LGBModel', 'module_path': 'qlib.contrib.model.gbdt'},\n", - " 'record': [{'class': 'SignalRecord',\n", - " 'module_path': 'qlib.workflow.record_temp'},\n", - " {'class': 'SigAnaRecord',\n", - " 'module_path': 'qlib.workflow.record_temp'}],\n", - " 'task_key': 'task_lgb'},\n", - " {'dataset': {'class': 'DatasetH',\n", - " 'kwargs': {'handler': {'class': 'Alpha158',\n", - " 'kwargs': {'end_time': '2020-08-01',\n", - " 'fit_end_time': '2014-12-31',\n", - " 'fit_start_time': '2008-01-01',\n", - " 'instruments': 'csi100',\n", - " 'start_time': '2008-01-01'},\n", - " 'module_path': 'qlib.contrib.data.handler'},\n", - " 'segments': {'test': (Timestamp('2019-04-09 00:00:00'),\n", - " Timestamp('2021-07-12 00:00:00')),\n", - " 'train': (Timestamp('2010-04-23 00:00:00'),\n", - " Timestamp('2017-05-24 00:00:00')),\n", - " 'valid': (Timestamp('2017-05-25 00:00:00'),\n", - " Timestamp('2019-04-08 00:00:00'))}},\n", - " 'module_path': 'qlib.data.dataset'},\n", - " 'model': {'class': 'LGBModel', 'module_path': 'qlib.contrib.model.gbdt'},\n", - " 'record': [{'class': 'SignalRecord',\n", - " 'module_path': 'qlib.workflow.record_temp'},\n", - " {'class': 'SigAnaRecord',\n", - " 'module_path': 'qlib.workflow.record_temp'}],\n", - " 'task_key': 'task_lgb'}]\n", - "Total Tasks, New Tasks: 4 0\n" - ] - } - ], + "execution_count": null, + "outputs": [], "source": [ "tasks = task_generator(\n", " xgboost_task_template, # default task name\n", @@ -206,8 +93,8 @@ " task_lgb=lgb_task_template # use \"task_lgb\" as the task name\n", ")\n", "# Uncomment next two lines to see the generated tasks\n", - "from pprint import pprint\n", - "pprint(tasks)\n", + "# from pprint import pprint\n", + "# pprint(tasks)\n", "tm = TaskManager(task_pool=task_pool)\n", "tm.create_task(tasks) # all tasks will be saved to MongoDB" ], @@ -220,133 +107,8 @@ }, { "cell_type": "code", - "execution_count": 26, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "2021-03-09 14:55:51.600 | INFO | qlib.workflow.task.manage:run_task:355 - {'model': {'class': 'XGBModel', 'module_path': 'qlib.contrib.model.xgboost'}, 'dataset': {'class': 'DatasetH', 'module_path': 'qlib.data.dataset', 'kwargs': {'handler': {'class': 'Alpha158', 'module_path': 'qlib.contrib.data.handler', 'kwargs': {'start_time': '2008-01-01', 'end_time': '2020-08-01', 'fit_start_time': '2008-01-01', 'fit_end_time': '2014-12-31', 'instruments': 'csi100'}}, 'segments': {'train': (Timestamp('2008-01-02 00:00:00'), Timestamp('2014-12-31 00:00:00')), 'valid': (Timestamp('2015-01-05 00:00:00'), Timestamp('2016-12-30 00:00:00')), 'test': (Timestamp('2017-01-03 00:00:00'), Timestamp('2019-04-08 00:00:00'))}}}, 'record': [{'class': 'SignalRecord', 'module_path': 'qlib.workflow.record_temp'}, {'class': 'SigAnaRecord', 'module_path': 'qlib.workflow.record_temp'}], 'task_key': 1}\n", - "[8348:MainThread](2021-03-09 14:56:46,051) INFO - qlib.timer - [log.py:81] - Time cost: 54.448s | Loading data Done\n", - "[8348:MainThread](2021-03-09 14:56:46,440) INFO - qlib.timer - [log.py:81] - Time cost: 0.322s | DropnaLabel Done\n", - "[8348:MainThread](2021-03-09 14:56:52,461) INFO - qlib.timer - [log.py:81] - Time cost: 6.019s | CSZScoreNorm Done\n", - "[8348:MainThread](2021-03-09 14:56:52,464) INFO - qlib.timer - [log.py:81] - Time cost: 6.411s | fit & process data Done\n", - "[8348:MainThread](2021-03-09 14:56:52,468) INFO - qlib.timer - [log.py:81] - Time cost: 60.865s | Init data Done\n", - "[8348:MainThread](2021-03-09 14:56:52,471) INFO - qlib.workflow - [expm.py:245] - No tracking URI is provided. Use the default tracking URI.\n", - "[8348:MainThread](2021-03-09 14:56:52,500) INFO - qlib.workflow - [exp.py:181] - Experiment 2 starts running ...\n", - "[8348:MainThread](2021-03-09 14:56:52,567) INFO - qlib.workflow - [recorder.py:233] - Recorder dd6bceb6d319493686ab6565633c0b5a starts running under Experiment 2 ...\n", - "[0]\ttrain-rmse:1.05165\tvalid-rmse:1.05565\n", - "[20]\ttrain-rmse:0.97071\tvalid-rmse:1.00077\n", - "[40]\ttrain-rmse:0.95124\tvalid-rmse:1.00609\n", - "[59]\ttrain-rmse:0.93833\tvalid-rmse:1.00945\n", - "[8348:MainThread](2021-03-09 14:59:37,266) INFO - qlib.workflow - [record_temp.py:126] - Signal record 'pred.pkl' has been saved as the artifact of the Experiment 2\n", - "'The following are prediction results of the XGBModel model.'\n", - " score\n", - "datetime instrument \n", - "2017-01-03 SH600000 -0.103259\n", - " SH600010 -0.084365\n", - " SH600015 -0.107433\n", - " SH600016 -0.064723\n", - " SH600018 -0.038639\n", - "{'IC': 0.05347474869798698,\n", - " 'ICIR': 0.29781294430945265,\n", - " 'Rank IC': 0.0484064337863249,\n", - " 'Rank ICIR': 0.36035393716962033}\n", - "2021-03-09 14:59:38.633 | INFO | qlib.workflow.task.manage:run_task:355 - {'model': {'class': 'XGBModel', 'module_path': 'qlib.contrib.model.xgboost'}, 'dataset': {'class': 'DatasetH', 'module_path': 'qlib.data.dataset', 'kwargs': {'handler': {'class': 'Alpha158', 'module_path': 'qlib.contrib.data.handler', 'kwargs': {'start_time': '2008-01-01', 'end_time': '2020-08-01', 'fit_start_time': '2008-01-01', 'fit_end_time': '2014-12-31', 'instruments': 'csi100'}}, 'segments': {'train': (Timestamp('2010-04-23 00:00:00'), Timestamp('2017-05-24 00:00:00')), 'valid': (Timestamp('2017-05-25 00:00:00'), Timestamp('2019-04-08 00:00:00')), 'test': (Timestamp('2019-04-09 00:00:00'), Timestamp('2021-07-12 00:00:00'))}}}, 'record': [{'class': 'SignalRecord', 'module_path': 'qlib.workflow.record_temp'}, {'class': 'SigAnaRecord', 'module_path': 'qlib.workflow.record_temp'}], 'task_key': 1}\n", - "[8348:MainThread](2021-03-09 15:00:36,591) INFO - qlib.timer - [log.py:81] - Time cost: 57.954s | Loading data Done\n", - "[8348:MainThread](2021-03-09 15:00:36,997) INFO - qlib.timer - [log.py:81] - Time cost: 0.338s | DropnaLabel Done\n", - "[8348:MainThread](2021-03-09 15:00:43,728) INFO - qlib.timer - [log.py:81] - Time cost: 6.728s | CSZScoreNorm Done\n", - "[8348:MainThread](2021-03-09 15:00:43,731) INFO - qlib.timer - [log.py:81] - Time cost: 7.137s | fit & process data Done\n", - "[8348:MainThread](2021-03-09 15:00:43,734) INFO - qlib.timer - [log.py:81] - Time cost: 65.097s | Init data Done\n", - "[8348:MainThread](2021-03-09 15:00:43,740) INFO - qlib.workflow - [expm.py:245] - No tracking URI is provided. Use the default tracking URI.\n", - "[8348:MainThread](2021-03-09 15:00:43,768) INFO - qlib.workflow - [exp.py:181] - Experiment 2 starts running ...\n", - "[8348:MainThread](2021-03-09 15:00:43,851) INFO - qlib.workflow - [recorder.py:233] - Recorder de2f892b569c436ba642a23e99f4f2b0 starts running under Experiment 2 ...\n", - "[0]\ttrain-rmse:1.05178\tvalid-rmse:1.05345\n", - "[20]\ttrain-rmse:0.96764\tvalid-rmse:0.99546\n", - "[40]\ttrain-rmse:0.94957\tvalid-rmse:0.99798\n", - "[57]\ttrain-rmse:0.93592\tvalid-rmse:1.00030\n", - "[8348:MainThread](2021-03-09 15:03:12,764) INFO - qlib.workflow - [record_temp.py:126] - Signal record 'pred.pkl' has been saved as the artifact of the Experiment 2\n", - "'The following are prediction results of the XGBModel model.'\n", - " score\n", - "datetime instrument \n", - "2019-04-09 SH600000 0.006996\n", - " SH600009 -0.102482\n", - " SH600010 0.016398\n", - " SH600011 0.004459\n", - " SH600015 -0.128315\n", - "{'IC': 0.013224093132176661,\n", - " 'ICIR': 0.08254897170570956,\n", - " 'Rank IC': 0.02472594591723197,\n", - " 'Rank ICIR': 0.16330982475433398}\n", - "2021-03-09 15:03:13.593 | INFO | qlib.workflow.task.manage:run_task:355 - {'model': {'class': 'LGBModel', 'module_path': 'qlib.contrib.model.gbdt'}, 'dataset': {'class': 'DatasetH', 'module_path': 'qlib.data.dataset', 'kwargs': {'handler': {'class': 'Alpha158', 'module_path': 'qlib.contrib.data.handler', 'kwargs': {'start_time': '2008-01-01', 'end_time': '2020-08-01', 'fit_start_time': '2008-01-01', 'fit_end_time': '2014-12-31', 'instruments': 'csi100'}}, 'segments': {'train': (Timestamp('2008-01-02 00:00:00'), Timestamp('2014-12-31 00:00:00')), 'valid': (Timestamp('2015-01-05 00:00:00'), Timestamp('2016-12-30 00:00:00')), 'test': (Timestamp('2017-01-03 00:00:00'), Timestamp('2019-04-08 00:00:00'))}}}, 'record': [{'class': 'SignalRecord', 'module_path': 'qlib.workflow.record_temp'}, {'class': 'SigAnaRecord', 'module_path': 'qlib.workflow.record_temp'}], 'task_key': 'task_lgb'}\n", - "[8348:MainThread](2021-03-09 15:04:06,545) INFO - qlib.timer - [log.py:81] - Time cost: 52.814s | Loading data Done\n", - "[8348:MainThread](2021-03-09 15:04:06,919) INFO - qlib.timer - [log.py:81] - Time cost: 0.312s | DropnaLabel Done\n", - "[8348:MainThread](2021-03-09 15:04:12,850) INFO - qlib.timer - [log.py:81] - Time cost: 5.928s | CSZScoreNorm Done\n", - "[8348:MainThread](2021-03-09 15:04:12,853) INFO - qlib.timer - [log.py:81] - Time cost: 6.305s | fit & process data Done\n", - "[8348:MainThread](2021-03-09 15:04:12,856) INFO - qlib.timer - [log.py:81] - Time cost: 59.125s | Init data Done\n", - "[8348:MainThread](2021-03-09 15:04:12,859) INFO - qlib.workflow - [expm.py:245] - No tracking URI is provided. Use the default tracking URI.\n", - "[8348:MainThread](2021-03-09 15:04:12,888) INFO - qlib.workflow - [exp.py:181] - Experiment 2 starts running ...\n", - "[8348:MainThread](2021-03-09 15:04:12,958) INFO - qlib.workflow - [recorder.py:233] - Recorder 15df799127a74656829978c1b9352e60 starts running under Experiment 2 ...\n", - "Training until validation scores don't improve for 50 rounds\n", - "[20]\ttrain's l2: 0.970491\tvalid's l2: 0.987723\n", - "[40]\ttrain's l2: 0.957984\tvalid's l2: 0.990056\n", - "[60]\ttrain's l2: 0.947201\tvalid's l2: 0.991459\n", - "Early stopping, best iteration is:\n", - "[18]\ttrain's l2: 0.971834\tvalid's l2: 0.987481\n", - "[8348:MainThread](2021-03-09 15:04:19,847) INFO - qlib.workflow - [record_temp.py:126] - Signal record 'pred.pkl' has been saved as the artifact of the Experiment 2\n", - "'The following are prediction results of the LGBModel model.'\n", - " score\n", - "datetime instrument \n", - "2017-01-03 SH600000 -0.013089\n", - " SH600010 -0.006642\n", - " SH600015 -0.035137\n", - " SH600016 -0.034634\n", - " SH600018 -0.029493\n", - "{'IC': 0.05704431372255674,\n", - " 'ICIR': 0.28879437007622133,\n", - " 'Rank IC': 0.05181220321608411,\n", - " 'Rank ICIR': 0.3233833799543165}\n", - "2021-03-09 15:04:21.111 | INFO | qlib.workflow.task.manage:run_task:355 - {'model': {'class': 'LGBModel', 'module_path': 'qlib.contrib.model.gbdt'}, 'dataset': {'class': 'DatasetH', 'module_path': 'qlib.data.dataset', 'kwargs': {'handler': {'class': 'Alpha158', 'module_path': 'qlib.contrib.data.handler', 'kwargs': {'start_time': '2008-01-01', 'end_time': '2020-08-01', 'fit_start_time': '2008-01-01', 'fit_end_time': '2014-12-31', 'instruments': 'csi100'}}, 'segments': {'train': (Timestamp('2010-04-23 00:00:00'), Timestamp('2017-05-24 00:00:00')), 'valid': (Timestamp('2017-05-25 00:00:00'), Timestamp('2019-04-08 00:00:00')), 'test': (Timestamp('2019-04-09 00:00:00'), Timestamp('2021-07-12 00:00:00'))}}}, 'record': [{'class': 'SignalRecord', 'module_path': 'qlib.workflow.record_temp'}, {'class': 'SigAnaRecord', 'module_path': 'qlib.workflow.record_temp'}], 'task_key': 'task_lgb'}\n", - "[8348:MainThread](2021-03-09 15:05:16,072) INFO - qlib.timer - [log.py:81] - Time cost: 54.958s | Loading data Done\n", - "[8348:MainThread](2021-03-09 15:05:16,466) INFO - qlib.timer - [log.py:81] - Time cost: 0.334s | DropnaLabel Done\n", - "[8348:MainThread](2021-03-09 15:05:22,281) INFO - qlib.timer - [log.py:81] - Time cost: 5.812s | CSZScoreNorm Done\n", - "[8348:MainThread](2021-03-09 15:05:22,283) INFO - qlib.timer - [log.py:81] - Time cost: 6.209s | fit & process data Done\n", - "[8348:MainThread](2021-03-09 15:05:22,286) INFO - qlib.timer - [log.py:81] - Time cost: 61.172s | Init data Done\n", - "[8348:MainThread](2021-03-09 15:05:22,291) INFO - qlib.workflow - [expm.py:245] - No tracking URI is provided. Use the default tracking URI.\n", - "[8348:MainThread](2021-03-09 15:05:22,317) INFO - qlib.workflow - [exp.py:181] - Experiment 2 starts running ...\n", - "[8348:MainThread](2021-03-09 15:05:22,386) INFO - qlib.workflow - [recorder.py:233] - Recorder 0c814539f55842b9b6310843fc5ec708 starts running under Experiment 2 ...\n", - "Training until validation scores don't improve for 50 rounds\n", - "[20]\ttrain's l2: 0.969033\tvalid's l2: 0.98571\n", - "[40]\ttrain's l2: 0.955399\tvalid's l2: 0.986164\n", - "[60]\ttrain's l2: 0.943514\tvalid's l2: 0.986301\n", - "Early stopping, best iteration is:\n", - "[26]\ttrain's l2: 0.964587\tvalid's l2: 0.985356\n", - "[8348:MainThread](2021-03-09 15:05:29,546) INFO - qlib.workflow - [record_temp.py:126] - Signal record 'pred.pkl' has been saved as the artifact of the Experiment 2\n", - "'The following are prediction results of the LGBModel model.'\n", - " score\n", - "datetime instrument \n", - "2019-04-09 SH600000 0.029586\n", - " SH600009 0.004306\n", - " SH600010 -0.004411\n", - " SH600011 0.002707\n", - " SH600015 -0.029124\n", - "{'IC': 0.020784811232504984,\n", - " 'ICIR': 0.11590182186569555,\n", - " 'Rank IC': 0.028925697036767055,\n", - " 'Rank ICIR': 0.16388058980901396}\n" - ] - }, - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "True" - ] - }, - "metadata": {}, - "execution_count": 26 - } - ], + "execution_count": null, + "outputs": [], "source": [ "from qlib.workflow.task.manage import run_task\n", "from qlib.workflow.task.collect import TaskCollector\n", @@ -363,43 +125,12 @@ }, { "cell_type": "code", - "execution_count": 27, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "Loading data: 100%|██████████| 4/4 [00:00<00:00, 37.38it/s]\n" - ] - }, - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "{('task_lgb', '2019-04-08'): datetime instrument\n", - " 2017-01-03 SH600000 -0.013089\n", - " SH600010 -0.006642\n", - " SH600015 -0.035137\n", - " SH600016 -0.034634\n", - " SH600018 -0.029493\n", - " ... \n", - " 2019-04-08 SZ002415 0.049199\n", - " SZ002450 -0.013450\n", - " SZ002594 0.022395\n", - " SZ002736 0.091433\n", - " SZ300059 -0.016237\n", - " Name: score, Length: 55000, dtype: float64}" - ] - }, - "metadata": {}, - "execution_count": 27 - } - ], + "execution_count": null, + "outputs": [], "source": [ "def get_task_key(task):\n", " task_key = task[\"task_key\"]\n", " rolling_end_timestamp = task[\"dataset\"][\"kwargs\"][\"segments\"][\"test\"][1]\n", - " #rolling_end_datatime = rolling_end_timestamp.to_pydatetime()\n", " return task_key, rolling_end_timestamp.strftime('%Y-%m-%d')\n", "\n", "def my_filter(task):\n", @@ -410,7 +141,7 @@ " return False\n", "\n", "# name tasks by \"get_task_key\" and filter tasks by \"my_filter\"\n", - "pred_rolling = TaskCollector.collect(exp_name, get_task_key, my_filter) \n", + "pred_rolling = TaskCollector.collect_predictions(exp_name, get_task_key, my_filter) \n", "pred_rolling" ], "metadata": { diff --git a/examples/taskmanager/task_manager_rolling.py b/examples/taskmanager/task_manager_rolling.py index 7519bc4be0..db5d1817fb 100644 --- a/examples/taskmanager/task_manager_rolling.py +++ b/examples/taskmanager/task_manager_rolling.py @@ -85,7 +85,7 @@ tm.create_task(tasks) # all tasks will be saved to MongoDB from qlib.workflow.task.manage import run_task -from qlib.workflow.task.collect import RollingCollector +from qlib.workflow.task.collect import TaskCollector from qlib.model.trainer import task_train run_task(task_train, task_pool, experiment_name=exp_name) # all tasks will be trained using "task_train" method @@ -93,7 +93,6 @@ def get_task_key(task_config): task_key = task_config["task_key"] rolling_end_timestamp = task_config["dataset"]["kwargs"]["segments"]["test"][1] - #rolling_end_datatime = rolling_end_timestamp.to_pydatetime() return task_key, rolling_end_timestamp.strftime('%Y-%m-%d') def my_filter(task_config): @@ -103,6 +102,6 @@ def my_filter(task_config): return True return False -collector = RollingCollector(get_task_key, my_filter) -pred_rolling = collector(exp_name) # name tasks by "get_task_key" and filter tasks by "my_filter" -print(pred_rolling) \ No newline at end of file +# name tasks by "get_task_key" and filter tasks by "my_filter" +pred_rolling = TaskCollector.collect_predictions(exp_name, get_task_key, my_filter) +pred_rolling \ No newline at end of file diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index 91061636d4..82d770b960 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -6,7 +6,7 @@ from qlib.workflow.record_temp import SignalRecord -def task_train(task_config: dict, experiment_name: str): +def task_train(task_config: dict, experiment_name: str) -> str: """ task based training @@ -16,6 +16,11 @@ def task_train(task_config: dict, experiment_name: str): A dict describes a task setting. experiment_name: str The name of experiment + + Returns + ---------- + rid : str + The id of the recorder of this task """ # model initiaiton @@ -29,7 +34,7 @@ def task_train(task_config: dict, experiment_name: str): model.fit(dataset) recorder = R.get_recorder() R.save_objects(**{"params.pkl": model}) - R.save_objects(param=task_config) # keep the original format and datatype + R.save_objects(**{"task.pkl": task_config}) # keep the original format and datatype # generate records: prediction, backtest, and analysis records = task_config.get("record", []) @@ -48,3 +53,4 @@ def task_train(task_config: dict, experiment_name: str): record["kwargs"].update(rconf) ar = init_instance_by_config(record) ar.generate() + return record.info["id"] diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index 4562a1cec7..8341895617 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -1,7 +1,7 @@ from qlib.workflow import R import pandas as pd from typing import Union -from tqdm.auto import tqdm +from qlib import get_module_logger class TaskCollector: @@ -10,10 +10,8 @@ class TaskCollector: """ @staticmethod - def collect( - experiment_name: str, - get_key_func, - filter_func=None, + def collect_predictions( + experiment_name: str, get_key_func, filter_func=None, ): """ @@ -34,8 +32,8 @@ def collect( recs = exp.list_recorders() recs_flt = {} - for rid, rec in tqdm(recs.items(), desc="Loading data"): - params = rec.load_object("param") + for rid, rec in recs.items(): + params = rec.load_object("task.pkl") if rec.status == rec.STATUS_FI: if filter_func is None or filter_func(params): rec.params = params @@ -57,6 +55,7 @@ def collect( pred = pd.concat(pred_l).sort_index() reduce_group[k] = pred + get_module_logger("TaskCollector").info(f"Collect {len(reduce_group)} predictions in {experiment_name}") return reduce_group @@ -82,7 +81,7 @@ def __call__(self, exp_name) -> Union[pd.Series, dict]: recs_flt = {} for rid, rec in tqdm(recs.items(), desc="Loading data"): - params = rec.load_object("param") + params = rec.load_object("task.pkl") if rec.status == rec.STATUS_FI: if self.flt_func is None or self.flt_func(params): rec.params = params diff --git a/qlib/workflow/task/gen.py b/qlib/workflow/task/gen.py index b1c2e0ce2c..19793c485b 100644 --- a/qlib/workflow/task/gen.py +++ b/qlib/workflow/task/gen.py @@ -168,7 +168,7 @@ def generate(self, task: dict): # 1) prepare the end point segments = copy.deepcopy(self.ta.align_seg(t["dataset"]["kwargs"]["segments"])) test_end = self.ta.last_date() if segments[self.test_key][1] is None else segments[self.test_key][1] - # 2) and the init test segments + # 2) and init test segments test_start_idx = self.ta.align_idx(segments[self.test_key][0]) segments[self.test_key] = (self.ta.get(test_start_idx), self.ta.get(test_start_idx + self.step - 1)) else: From 2ca2071d959a007625b2879a57e84efb1507ac59 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Wed, 10 Mar 2021 17:06:08 +0000 Subject: [PATCH 13/61] format code --- examples/taskmanager/task_manager_rolling.py | 75 ++++++++------------ qlib/workflow/task/collect.py | 4 +- 2 files changed, 34 insertions(+), 45 deletions(-) diff --git a/examples/taskmanager/task_manager_rolling.py b/examples/taskmanager/task_manager_rolling.py index db5d1817fb..36ec819620 100644 --- a/examples/taskmanager/task_manager_rolling.py +++ b/examples/taskmanager/task_manager_rolling.py @@ -9,53 +9,37 @@ "end_time": "2020-08-01", "fit_start_time": "2008-01-01", "fit_end_time": "2014-12-31", - "instruments": 'csi100', + "instruments": "csi100", } dataset_config = { - "class": "DatasetH", - "module_path": "qlib.data.dataset", - "kwargs": { - "handler": { - "class": "Alpha158", - "module_path": "qlib.contrib.data.handler", - "kwargs": data_handler_config, - }, - "segments": { - "train": ("2008-01-01", "2014-12-31"), - "valid": ("2015-01-01", "2016-12-31"), - "test": ("2017-01-01", "2020-08-01"), - }, + "class": "DatasetH", + "module_path": "qlib.data.dataset", + "kwargs": { + "handler": {"class": "Alpha158", "module_path": "qlib.contrib.data.handler", "kwargs": data_handler_config,}, + "segments": { + "train": ("2008-01-01", "2014-12-31"), + "valid": ("2015-01-01", "2016-12-31"), + "test": ("2017-01-01", "2020-08-01"), }, - } + }, +} record_config = [ - { - "class": "SignalRecord", - "module_path": "qlib.workflow.record_temp", - }, - { - "class": "SigAnaRecord", - "module_path": "qlib.workflow.record_temp", - } + {"class": "SignalRecord", "module_path": "qlib.workflow.record_temp",}, + {"class": "SigAnaRecord", "module_path": "qlib.workflow.record_temp",}, ] # use lgb task_lgb_config = { - "model": { - "class": "LGBModel", - "module_path": "qlib.contrib.model.gbdt", - }, + "model": {"class": "LGBModel", "module_path": "qlib.contrib.model.gbdt",}, "dataset": dataset_config, "record": record_config, } # use xgboost task_xgboost_config = { - "model": { - "class": "XGBModel", - "module_path": "qlib.contrib.model.xgboost", - }, + "model": {"class": "XGBModel", "module_path": "qlib.contrib.model.xgboost",}, "dataset": dataset_config, "record": record_config, } @@ -64,17 +48,17 @@ qlib.init(provider_uri=provider_uri, region=REG_CN) C["mongo"] = { - "task_url" : "mongodb://localhost:27017/", # maybe you need to change it to your url - "task_db_name" : "rolling_db" + "task_url": "mongodb://localhost:27017/", # maybe you need to change it to your url + "task_db_name": "rolling_db", } -exp_name = 'rolling_exp' # experiment name, will be used as the experiment in MLflow -task_pool = 'rolling_task' # task pool name, will be used as the document in MongoDB +exp_name = "rolling_exp" # experiment name, will be used as the experiment in MLflow +task_pool = "rolling_task" # task pool name, will be used as the document in MongoDB tasks = task_generator( - task_xgboost_config, # default task name - RollingGen(step=550,rtype=RollingGen.ROLL_SD), # generate different date segment - task_lgb=task_lgb_config # use "task_lgb" as the task name + task_xgboost_config, # default task name + RollingGen(step=550, rtype=RollingGen.ROLL_SD), # generate different date segment + task_lgb=task_lgb_config, # use "task_lgb" as the task name ) # Uncomment next two lines to see the generated tasks @@ -82,26 +66,29 @@ # pprint(tasks) tm = TaskManager(task_pool=task_pool) -tm.create_task(tasks) # all tasks will be saved to MongoDB +tm.create_task(tasks) # all tasks will be saved to MongoDB from qlib.workflow.task.manage import run_task from qlib.workflow.task.collect import TaskCollector from qlib.model.trainer import task_train -run_task(task_train, task_pool, experiment_name=exp_name) # all tasks will be trained using "task_train" method +run_task(task_train, task_pool, experiment_name=exp_name) # all tasks will be trained using "task_train" method + def get_task_key(task_config): task_key = task_config["task_key"] rolling_end_timestamp = task_config["dataset"]["kwargs"]["segments"]["test"][1] - return task_key, rolling_end_timestamp.strftime('%Y-%m-%d') + return task_key, rolling_end_timestamp.strftime("%Y-%m-%d") + def my_filter(task_config): # only choose the results of "task_lgb" and test in 2019 from all tasks task_key, rolling_end = get_task_key(task_config) - if task_key=="task_lgb" and rolling_end.startswith('2019'): + if task_key == "task_lgb" and rolling_end.startswith("2019"): return True return False + # name tasks by "get_task_key" and filter tasks by "my_filter" -pred_rolling = TaskCollector.collect_predictions(exp_name, get_task_key, my_filter) -pred_rolling \ No newline at end of file +pred_rolling = TaskCollector.collect_predictions(exp_name, get_task_key, my_filter) +pred_rolling diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index 8341895617..ccd6ce169a 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -11,7 +11,9 @@ class TaskCollector: @staticmethod def collect_predictions( - experiment_name: str, get_key_func, filter_func=None, + experiment_name: str, + get_key_func, + filter_func=None, ): """ From 48f0fc147f1799b2eff4848318ebf0f222320865 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Thu, 11 Mar 2021 03:00:30 +0000 Subject: [PATCH 14/61] first version of online serving --- examples/taskmanager/update_online_pred.py | 77 +++++++++++ qlib/model/trainer.py | 2 +- qlib/workflow/task/collect.py | 4 +- qlib/workflow/task/update.py | 154 +++++++++++++++++++++ 4 files changed, 234 insertions(+), 3 deletions(-) create mode 100644 examples/taskmanager/update_online_pred.py create mode 100644 qlib/workflow/task/update.py diff --git a/examples/taskmanager/update_online_pred.py b/examples/taskmanager/update_online_pred.py new file mode 100644 index 0000000000..4dbd22b855 --- /dev/null +++ b/examples/taskmanager/update_online_pred.py @@ -0,0 +1,77 @@ +import qlib +from qlib.model.trainer import task_train +from qlib.workflow.task.update import ModelUpdater +from qlib.config import REG_CN +import fire + +data_handler_config = { + "start_time": "2008-01-01", + "end_time": "2020-08-01", + "fit_start_time": "2008-01-01", + "fit_end_time": "2014-12-31", + "instruments": "csi100", + } + +task = { + "model": { + "class": "LGBModel", + "module_path": "qlib.contrib.model.gbdt", + "kwargs": { + "loss": "mse", + "colsample_bytree": 0.8879, + "learning_rate": 0.0421, + "subsample": 0.8789, + "lambda_l1": 205.6999, + "lambda_l2": 580.9768, + "max_depth": 8, + "num_leaves": 210, + "num_threads": 20, + }, + }, + "dataset": { + "class": "DatasetH", + "module_path": "qlib.data.dataset", + "kwargs": { + "handler": { + "class": "Alpha158", + "module_path": "qlib.contrib.data.handler", + "kwargs": data_handler_config, + }, + "segments": { + "train": ("2008-01-01", "2014-12-31"), + "valid": ("2015-01-01", "2016-12-31"), + "test": ("2017-01-01", "2020-08-01"), + }, + }, + }, + "record": {"class": "SignalRecord", "module_path": "qlib.workflow.record_temp",}, +} + +provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir + +def first_train(experiment_name="online_svr"): + + + qlib.init(provider_uri=provider_uri, region=REG_CN) + model_updater = ModelUpdater(experiment_name) + + rid = task_train(task_config=task, experiment_name=experiment_name) + model_updater.reset_online_model(rid) + +def update_online_pred(experiment_name="online_svr"): + + qlib.init(provider_uri=provider_uri, region=REG_CN) + model_updater = ModelUpdater(experiment_name) + + print("Here are the online models waiting for update:") + for rid, rec in model_updater.list_online_model().items(): + print(rid) + + model_updater.update_online_pred() + +if __name__ == '__main__': + fire.Fire() + # to train a model and set it to online model, use the command below + # python update_online_pred.py first_train + # to update online predictions once a day, use the command below + # python update_online_pred.py update_online_pred diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index 82d770b960..5e62a141cd 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -53,4 +53,4 @@ def task_train(task_config: dict, experiment_name: str) -> str: record["kwargs"].update(rconf) ar = init_instance_by_config(record) ar.generate() - return record.info["id"] + return recorder.info["id"] diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index ccd6ce169a..059871ab1a 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -11,8 +11,8 @@ class TaskCollector: @staticmethod def collect_predictions( - experiment_name: str, - get_key_func, + experiment_name: str, + get_key_func, filter_func=None, ): """ diff --git a/qlib/workflow/task/update.py b/qlib/workflow/task/update.py new file mode 100644 index 0000000000..f9d03efbcc --- /dev/null +++ b/qlib/workflow/task/update.py @@ -0,0 +1,154 @@ +from typing import Union +from qlib.workflow import R +from tqdm.auto import tqdm +from qlib.data import D +import pandas as pd +from qlib.utils import init_instance_by_config +from qlib import get_module_logger +from qlib.workflow import R + + +class ModelUpdater: + """ + The model updater to re-train model or update predictions + """ + + ONLINE_TAG = "online_model" + ONLINE_TAG_TRUE = "True" + ONLINE_TAG_FALSE = "False" + + def __init__(self, experiment_name: str) -> None: + """ModelUpdater needs experiment name to find the records + + Parameters + ---------- + experiment_name : str + experiment name string + """ + self.exp_name = experiment_name + self.exp = R.get_exp(experiment_name=experiment_name) + self.logger = get_module_logger("ModelUpdater") + + def set_online_model(self, rid: str): + """online model will be identified at the tags of the record + + Parameters + ---------- + rid : str + the id of a record + """ + rec = self.exp.get_recorder(recorder_id=rid) + rec.set_tags(**{self.ONLINE_TAG: self.ONLINE_TAG_TRUE}) + + def cancel_online_model(self, rid: str): + rec = self.exp.get_recorder(recorder_id=rid) + rec.set_tags(**{self.ONLINE_TAG: self.ONLINE_TAG_FALSE}) + + def cancel_all_online_model(self): + recs = self.exp.list_recorders() + for rid, rec in recs.items(): + self.cancel_online_model(rid) + + def reset_online_model(self, rids: Union[str, list]): + """cancel all online model and reset the given model to online model + + Parameters + ---------- + rids : Union[str, list] + the name of a record or the list of the name of records + """ + self.cancel_all_online_model() + if isinstance(rids, str): + rids = [rids] + for rid in rids: + self.set_online_model(rid) + + def update_pred(self, rid: str): + """update predictions to the latest day in Calendar based on rid + + Parameters + ---------- + rid : str + the id of the record + """ + rec = self.exp.get_recorder(recorder_id=rid) + old_pred = rec.load_object("pred.pkl") + last_end = old_pred.index.get_level_values("datetime").max() + task_config = rec.load_object("task.pkl") + + # updated to the latest trading day + cal = D.calendar(start_time=last_end + pd.Timedelta(days=1), end_time=None) + + if len(cal) == 0: + self.logger.info(f"All prediction in {rid} of {self.exp_name} are latest. No need to update.") + return + + start_time, end_time = cal[0], cal[-1] + task_config["dataset"]["kwargs"]["segments"]["test"] = (start_time, end_time) + task_config["dataset"]["kwargs"]["handler"]["kwargs"]["end_time"] = end_time + + dataset = init_instance_by_config(task_config["dataset"]) + + model = rec.load_object("params.pkl") + new_pred = model.predict(dataset) + + cb_pred = pd.concat([old_pred, new_pred.to_frame("score")], axis=0) + cb_pred = cb_pred.sort_index() + + rec.save_objects(**{"pred.pkl": cb_pred}) + + self.logger.info(f"Finish updating new {new_pred.shape[0]} predictions in {rid} of {self.exp_name}.") + + def update_all_pred(self, filter_func=None): + """update all predictions in this experiment after filter. + + An example of filter function: + + .. code-block:: python + + def record_filter(record): + task_config = record.load_object("task.pkl") + if task_config["model"]["class"]=="LGBModel": + return True + return False + + Parameters + ---------- + filter_func : function, optional + the filter function to decide whether this record will be updated, by default None + + Returns + ---------- + cnt: int + the count of updated record + + """ + cnt = 0 + recs = self.exp.list_recorders() + for rid, rec in recs.items(): + if rec.status == rec.STATUS_FI: + if filter_func != None and filter_func(rec) == False: + # records that should be filtered out + continue + self.update_pred(rid) + cnt += 1 + return cnt + + def online_filter(self, record): + tags = record.list_tags() + if tags[self.ONLINE_TAG] == self.ONLINE_TAG_TRUE: + return True + return False + + def update_online_pred(self): + """update all online model predictions to the latest day in Calendar.""" + cnt = self.update_all_pred(self.online_filter) + self.logger.info(f"Finish updating {cnt} online model predictions of {self.exp_name}.") + + def list_online_model(self): + recs = self.exp.list_recorders() + online_rec = {} + for rid, rec in recs.items(): + if self.online_filter(rec): + online_rec[rid] = rec + return online_rec From 0df88c07f64549ab27aa924a7001e5a8e7beecc7 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Thu, 11 Mar 2021 16:25:46 +0000 Subject: [PATCH 15/61] bug fixed and update collect.py --- ...task_managment.rst => task_management.rst} | 0 qlib/workflow/task/collect.py | 95 +++++++++++++++---- qlib/workflow/task/manage.py | 4 - qlib/workflow/task/update.py | 11 ++- 4 files changed, 88 insertions(+), 22 deletions(-) rename docs/advanced/{task_managment.rst => task_management.rst} (100%) diff --git a/docs/advanced/task_managment.rst b/docs/advanced/task_management.rst similarity index 100% rename from docs/advanced/task_managment.rst rename to docs/advanced/task_management.rst diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index 059871ab1a..2e4746f594 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -1,6 +1,7 @@ from qlib.workflow import R import pandas as pd from typing import Union +from typing import Callable from qlib import get_module_logger @@ -9,9 +10,63 @@ class TaskCollector: Collect the record results of the finished tasks with key and filter """ - @staticmethod + def __init__(self, experiment_name: str) -> None: + self.exp_name = experiment_name + self.exp = R.get_exp(experiment_name=experiment_name) + self.logger = get_module_logger("TaskCollector") + + def list_recorders(self, rec_filter_func=None, task_filter_func=None, only_finished=True, only_have_task=False): + """ + Return a dict of {rid:recorder} by recorder filter and task filter. It is not necessary to use those filter. + If you don't train with "task_train", then there is no "task.pkl" which includes the task config. + If there is a "task.pkl", then it will become rec.task which can be get simply. + + Parameters + ---------- + rec_filter_func : Callable[[MLflowRecorder], bool], optional + judge whether you need this recorder, by default None + task_filter_func : Callable[[dict], bool], optional + judge whether you need this task, by default None + only_finished : bool, optional + whether always use finished recorder, by default True + only_have_task : bool, optional + whether it is necessary to get the task config + + Returns + ------- + dict + a dict of {rid:recorder} + + Raises + ------ + OSError + if you use a task filter, but there is no "task.pkl" which includes the task config + """ + recs = self.exp.list_recorders() + # return all recorders if the filter is None and you don't need task + if rec_filter_func==None and task_filter_func==None and only_have_task==False: + return recs + recs_flt = {} + for rid, rec in recs.items(): + if (only_finished and rec.status == rec.STATUS_FI) or only_finished==False: + if rec_filter_func is None or rec_filter_func(rec): + task = None + try: + task = rec.load_object("task.pkl") + except OSError: + if task_filter_func is not None: + raise OSError('Can not find "task.pkl" in your records, have you train with "task_train" method in qlib.model.trainer?') + if task is None and only_have_task: + continue + + if task_filter_func is None or task_filter_func(task): + rec.task = task + recs_flt[rid] = rec + + return recs_flt + def collect_predictions( - experiment_name: str, + self, get_key_func, filter_func=None, ): @@ -27,24 +82,15 @@ def collect_predictions( Returns ------- - + dict + the dict of predictions """ - exp = R.get_exp(experiment_name=experiment_name) - # filter records - recs = exp.list_recorders() - - recs_flt = {} - for rid, rec in recs.items(): - params = rec.load_object("task.pkl") - if rec.status == rec.STATUS_FI: - if filter_func is None or filter_func(params): - rec.params = params - recs_flt[rid] = rec + recs_flt = self.list_recorders(task_filter_func=filter_func) # group recs_group = {} for _, rec in recs_flt.items(): - params = rec.params + params = rec.task group_key = get_key_func(params) recs_group.setdefault(group_key, []).append(rec) @@ -57,9 +103,26 @@ def collect_predictions( pred = pd.concat(pred_l).sort_index() reduce_group[k] = pred - get_module_logger("TaskCollector").info(f"Collect {len(reduce_group)} predictions in {experiment_name}") + self.logger.info(f"Collect {len(reduce_group)} predictions in {self.exp_name}") return reduce_group + def collect_latest_records( + self, + filter_func=None, + ): + recs_flt = self.list_recorders(task_filter_func=filter_func,only_have_task=True) + + max_test = max(rec.task['dataset']['kwargs']['segments']['test'] for rec in recs_flt.values()) + + latest_record = {} + for rid, rec in recs_flt.items(): + if rec.task['dataset']['kwargs']['segments']['test'] == max_test: + latest_record[rid] = rec + + self.logger.info(f"Collect {len(latest_record)} latest records in {self.exp_name}") + return latest_record + + class RollingCollector: """ diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index ae4aee147d..f27d025947 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -363,7 +363,3 @@ def run_task(task_func, task_pool, force_release=False, *args, **kwargs): return ever_run - -if __name__ == "__main__": - auto_init() - Fire(TaskManager) diff --git a/qlib/workflow/task/update.py b/qlib/workflow/task/update.py index f9d03efbcc..5127a87da8 100644 --- a/qlib/workflow/task/update.py +++ b/qlib/workflow/task/update.py @@ -6,7 +6,7 @@ from qlib.utils import init_instance_by_config from qlib import get_module_logger from qlib.workflow import R - +from qlib.model.trainer import task_train class ModelUpdater: """ @@ -136,7 +136,7 @@ def record_filter(record): def online_filter(self, record): tags = record.list_tags() - if tags[self.ONLINE_TAG] == self.ONLINE_TAG_TRUE: + if tags.get(self.ONLINE_TAG, self.ONLINE_TAG_FALSE) == self.ONLINE_TAG_TRUE: return True return False @@ -146,6 +146,13 @@ def update_online_pred(self): self.logger.info(f"Finish updating {cnt} online model predictions of {self.exp_name}.") def list_online_model(self): + """list the record of online model + + Returns + ------- + dict + {rid : record of the online model} + """ recs = self.exp.list_recorders() online_rec = {} for rid, rec in recs.items(): From 44a7dc004d03d4e4a84e3e912c83a803711ea9b5 Mon Sep 17 00:00:00 2001 From: Young Date: Fri, 12 Mar 2021 07:50:17 +0000 Subject: [PATCH 16/61] update docs and fix duplicated pred bug --- qlib/data/dataset/handler.py | 7 +++++++ qlib/workflow/task/collect.py | 7 ++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/qlib/data/dataset/handler.py b/qlib/data/dataset/handler.py index 2889c4465e..25d02fdf62 100644 --- a/qlib/data/dataset/handler.py +++ b/qlib/data/dataset/handler.py @@ -50,6 +50,9 @@ class DataHandler(Serializable): SH600004 13.313329 11800983.0 13.313329 13.317701 0.183632 0.0042 SH600005 37.796539 12231662.0 38.258602 37.919757 0.970325 0.0289 + + Tips for improving the performance of datahandler + - Fetching data with `col_set=CS_RAW` will return the raw data and may avoid pandas from copying the data when calling `loc` """ def __init__( @@ -257,6 +260,10 @@ def get_range_iterator( class DataHandlerLP(DataHandler): """ DataHandler with **(L)earnable (P)rocessor** + + Tips to improving the performance of data handler + - To reduce the memory cost + - `drop_raw=True`: this will modify the data inplace on raw data; """ # data key diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index 7cdca30fae..6c4e45c72c 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -46,7 +46,12 @@ def __call__(self, exp_name) -> Union[pd.Series, dict]: pred_l = [] for rec in rec_l: pred_l.append(rec.load_object("pred.pkl").iloc[:, 0]) - pred = pd.concat(pred_l).sort_index() + # Make sure the pred are sorted according to the rolling start time + pred_l.sort(key=lambda pred: pred.index.get_level_values("datetime").min()) + pred = pd.concat(pred_l) + # If there are duplicated predition, we use the latest perdiction + pred = pred[~pred.index.duplicated(keep="last")] + pred = pred.sort_index() reduce_group[k] = pred return reduce_group From 6d8aa215d62866cd0934ac61f8d01a5981ee4fae Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Fri, 12 Mar 2021 08:04:08 +0000 Subject: [PATCH 17/61] the second version of online serving --- qlib/model/trainer.py | 2 +- qlib/workflow/task/collect.py | 90 ++++++++------------------------ qlib/workflow/task/manage.py | 11 ++-- qlib/workflow/task/update.py | 97 ++++++++++++++++------------------- qlib/workflow/task/utils.py | 4 +- 5 files changed, 76 insertions(+), 128 deletions(-) diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index 5e62a141cd..e901bc2521 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -34,7 +34,7 @@ def task_train(task_config: dict, experiment_name: str) -> str: model.fit(dataset) recorder = R.get_recorder() R.save_objects(**{"params.pkl": model}) - R.save_objects(**{"task.pkl": task_config}) # keep the original format and datatype + R.save_objects(**{"task": task_config}) # keep the original format and datatype # generate records: prediction, backtest, and analysis records = task_config.get("record", []) diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index 2e4746f594..c022e6e760 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -2,6 +2,7 @@ import pandas as pd from typing import Union from typing import Callable + from qlib import get_module_logger @@ -17,13 +18,13 @@ def __init__(self, experiment_name: str) -> None: def list_recorders(self, rec_filter_func=None, task_filter_func=None, only_finished=True, only_have_task=False): """ - Return a dict of {rid:recorder} by recorder filter and task filter. It is not necessary to use those filter. - If you don't train with "task_train", then there is no "task.pkl" which includes the task config. - If there is a "task.pkl", then it will become rec.task which can be get simply. + Return a dict of {rid:Recorder} by recorder filter and task filter. It is not necessary to use those filter. + If you don't train with "task_train", then there is no "task" which includes the task config. + If there is a "task", then it will become rec.task which can be get simply. Parameters ---------- - rec_filter_func : Callable[[MLflowRecorder], bool], optional + rec_filter_func : Callable[[Recorder], bool], optional judge whether you need this recorder, by default None task_filter_func : Callable[[dict], bool], optional judge whether you need this task, by default None @@ -35,30 +36,27 @@ def list_recorders(self, rec_filter_func=None, task_filter_func=None, only_finis Returns ------- dict - a dict of {rid:recorder} + a dict of {rid:Recorder} Raises ------ OSError - if you use a task filter, but there is no "task.pkl" which includes the task config + if you use a task filter, but there is no "task" which includes the task config """ recs = self.exp.list_recorders() - # return all recorders if the filter is None and you don't need task - if rec_filter_func==None and task_filter_func==None and only_have_task==False: - return recs recs_flt = {} + if task_filter_func is not None: + only_have_task = True for rid, rec in recs.items(): if (only_finished and rec.status == rec.STATUS_FI) or only_finished==False: if rec_filter_func is None or rec_filter_func(rec): task = None try: - task = rec.load_object("task.pkl") + task = rec.load_object("task") except OSError: - if task_filter_func is not None: - raise OSError('Can not find "task.pkl" in your records, have you train with "task_train" method in qlib.model.trainer?') + pass if task is None and only_have_task: continue - if task_filter_func is None or task_filter_func(task): rec.task = task recs_flt[rid] = rec @@ -68,7 +66,7 @@ def list_recorders(self, rec_filter_func=None, task_filter_func=None, only_finis def collect_predictions( self, get_key_func, - filter_func=None, + task_filter_func=None, ): """ @@ -85,7 +83,7 @@ def collect_predictions( dict the dict of predictions """ - recs_flt = self.list_recorders(task_filter_func=filter_func) + recs_flt = self.list_recorders(task_filter_func=task_filter_func,only_have_task=True) # group recs_group = {} @@ -108,11 +106,14 @@ def collect_predictions( def collect_latest_records( self, - filter_func=None, + task_filter_func=None, ): - recs_flt = self.list_recorders(task_filter_func=filter_func,only_have_task=True) - - max_test = max(rec.task['dataset']['kwargs']['segments']['test'] for rec in recs_flt.values()) + recs_flt = self.list_recorders(task_filter_func=task_filter_func,only_have_task=True) + + if len(recs_flt) == 0: + self.logger.warning("Can not collect any recorders...") + return None, None + max_test = max(rec.task['dataset']['kwargs']['segments']['test'] for rec in recs_flt.values()) latest_record = {} for rid, rec in recs_flt.items(): @@ -120,52 +121,5 @@ def collect_latest_records( latest_record[rid] = rec self.logger.info(f"Collect {len(latest_record)} latest records in {self.exp_name}") - return latest_record - - - -class RollingCollector: - """ - Rolling Models Ensemble based on (R)ecord - - This shares nothing with Ensemble - """ - - # TODO: speed up this class - def __init__(self, get_key_func, flt_func=None): - self.get_key_func = get_key_func # get the key of a task based on task config - self.flt_func = flt_func # determine whether a task can be retained based on task config - - def __call__(self, exp_name) -> Union[pd.Series, dict]: - # TODO; - # Should we split the scripts into several sub functions? - exp = R.get_exp(experiment_name=exp_name) - - # filter records - recs = exp.list_recorders() - - recs_flt = {} - for rid, rec in tqdm(recs.items(), desc="Loading data"): - params = rec.load_object("task.pkl") - if rec.status == rec.STATUS_FI: - if self.flt_func is None or self.flt_func(params): - rec.params = params - recs_flt[rid] = rec - - # group - recs_group = {} - for _, rec in recs_flt.items(): - params = rec.params - group_key = self.get_key_func(params) - recs_group.setdefault(group_key, []).append(rec) - - # reduce group - reduce_group = {} - for k, rec_l in recs_group.items(): - pred_l = [] - for rec in rec_l: - pred_l.append(rec.load_object("pred.pkl").iloc[:, 0]) - pred = pd.concat(pred_l).sort_index() - reduce_group[k] = pred - - return reduce_group + return latest_record, max_test + \ No newline at end of file diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index f27d025947..22b5430cc7 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -10,10 +10,8 @@ from bson.binary import Binary import pickle from pymongo.errors import InvalidDocument -from fire import Fire from bson.objectid import ObjectId from contextlib import contextmanager -from loguru import logger from tqdm.cli import tqdm import time import concurrent @@ -21,7 +19,7 @@ from qlib.config import C from .utils import get_mongodb from qlib import auto_init - +from qlib import get_module_logger class TaskManager: """TaskManager @@ -62,6 +60,7 @@ def __init__(self, task_pool=None): """ self.mdb = get_mongodb() self.task_pool = task_pool + self.logger = get_module_logger("TaskManager") def list(self): return self.mdb.list_collection_names() @@ -210,9 +209,9 @@ def safe_fetch_task(self, query={}, task_pool=None): yield task except Exception: if task is not None: - logger.info("Returning task before raising error") + self.logger.info("Returning task before raising error") self.return_task(task) - logger.info("Task returned") + self.logger.info("Task returned") raise def task_fetcher_iter(self, query={}, task_pool=None): @@ -352,7 +351,7 @@ def run_task(task_func, task_pool, force_release=False, *args, **kwargs): with tm.safe_fetch_task() as task: if task is None: break - logger.info(task["def"]) + get_module_logger("run_task").info(task["def"]) if force_release: with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: res = executor.submit(task_func, task["def"], *args, **kwargs).result() diff --git a/qlib/workflow/task/update.py b/qlib/workflow/task/update.py index 5127a87da8..9f1cc0a299 100644 --- a/qlib/workflow/task/update.py +++ b/qlib/workflow/task/update.py @@ -1,4 +1,4 @@ -from typing import Union +from typing import Union,List from qlib.workflow import R from tqdm.auto import tqdm from qlib.data import D @@ -7,8 +7,10 @@ from qlib import get_module_logger from qlib.workflow import R from qlib.model.trainer import task_train +from qlib.workflow.recorder import Recorder +from qlib.workflow.task.collect import TaskCollector -class ModelUpdater: +class ModelUpdater(TaskCollector): """ The model updater to re-train model or update predictions """ @@ -29,58 +31,59 @@ def __init__(self, experiment_name: str) -> None: self.exp = R.get_exp(experiment_name=experiment_name) self.logger = get_module_logger("ModelUpdater") - def set_online_model(self, rid: str): + def set_online_model(self, recorder: Union[str,Recorder]): """online model will be identified at the tags of the record Parameters ---------- - rid : str - the id of a record + recorder: Union[str,Recorder] + the id of a Recorder or the Recorder instance """ - rec = self.exp.get_recorder(recorder_id=rid) - rec.set_tags(**{self.ONLINE_TAG: self.ONLINE_TAG_TRUE}) + if isinstance(recorder,str): + recorder = self.exp.get_recorder(recorder_id=recorder) + recorder.set_tags(**{ModelUpdater.ONLINE_TAG: ModelUpdater.ONLINE_TAG_TRUE}) - def cancel_online_model(self, rid: str): - rec = self.exp.get_recorder(recorder_id=rid) - rec.set_tags(**{self.ONLINE_TAG: self.ONLINE_TAG_FALSE}) + def cancel_online_model(self, recorder: Union[str,Recorder]): + if isinstance(recorder,str): + recorder = self.exp.get_recorder(recorder_id=recorder) + recorder.set_tags(**{ModelUpdater.ONLINE_TAG: ModelUpdater.ONLINE_TAG_FALSE}) def cancel_all_online_model(self): recs = self.exp.list_recorders() for rid, rec in recs.items(): - self.cancel_online_model(rid) + self.cancel_online_model(rec) - def reset_online_model(self, rids: Union[str, list]): + def reset_online_model(self, recorders: List[Union[str,Recorder]]): """cancel all online model and reset the given model to online model Parameters ---------- - rids : Union[str, list] - the name of a record or the list of the name of records + recorders: List[Union[str,Recorder]] + the list of the id of a Recorder or the Recorder instance """ self.cancel_all_online_model() - if isinstance(rids, str): - rids = [rids] - for rid in rids: - self.set_online_model(rid) + for rec_or_rid in recorders: + self.set_online_model(rec_or_rid) - def update_pred(self, rid: str): + def update_pred(self, recorder: Union[str,Recorder]): """update predictions to the latest day in Calendar based on rid Parameters ---------- - rid : str - the id of the record + recorder: Union[str,Recorder] + the id of a Recorder or the Recorder instance """ - rec = self.exp.get_recorder(recorder_id=rid) - old_pred = rec.load_object("pred.pkl") + if isinstance(recorder,str): + recorder = self.exp.get_recorder(recorder_id=recorder) + old_pred = recorder.load_object("pred.pkl") last_end = old_pred.index.get_level_values("datetime").max() - task_config = rec.load_object("task.pkl") + task_config = recorder.load_object("task") # recorder.task # updated to the latest trading day cal = D.calendar(start_time=last_end + pd.Timedelta(days=1), end_time=None) if len(cal) == 0: - self.logger.info(f"All prediction in {rid} of {self.exp_name} are latest. No need to update.") + self.logger.info(f"The prediction in {recorder.info['id']} of {self.exp_name} are latest. No need to update.") return start_time, end_time = cal[0], cal[-1] @@ -89,32 +92,32 @@ def update_pred(self, rid: str): dataset = init_instance_by_config(task_config["dataset"]) - model = rec.load_object("params.pkl") + model = recorder.load_object("params.pkl") new_pred = model.predict(dataset) cb_pred = pd.concat([old_pred, new_pred.to_frame("score")], axis=0) cb_pred = cb_pred.sort_index() - rec.save_objects(**{"pred.pkl": cb_pred}) + recorder.save_objects(**{"pred.pkl": cb_pred}) - self.logger.info(f"Finish updating new {new_pred.shape[0]} predictions in {rid} of {self.exp_name}.") + self.logger.info(f"Finish updating new {new_pred.shape[0]} predictions in {recorder.info['id']} of {self.exp_name}.") - def update_all_pred(self, filter_func=None): + def update_all_pred(self, rec_filter_func=None): """update all predictions in this experiment after filter. An example of filter function: .. code-block:: python - def record_filter(record): - task_config = record.load_object("task.pkl") + def rec_filter_func(recorder): + task_config = recorder.load_object("task") if task_config["model"]["class"]=="LGBModel": return True return False Parameters ---------- - filter_func : function, optional + rec_filter_func : Callable[[Recorder], bool], optional the filter function to decide whether this record will be updated, by default None Returns @@ -123,20 +126,14 @@ def record_filter(record): the count of updated record """ - cnt = 0 - recs = self.exp.list_recorders() + recs = self.list_recorders(rec_filter_func=rec_filter_func,only_have_task=True) for rid, rec in recs.items(): - if rec.status == rec.STATUS_FI: - if filter_func != None and filter_func(rec) == False: - # records that should be filtered out - continue - self.update_pred(rid) - cnt += 1 - return cnt - - def online_filter(self, record): - tags = record.list_tags() - if tags.get(self.ONLINE_TAG, self.ONLINE_TAG_FALSE) == self.ONLINE_TAG_TRUE: + self.update_pred(rec) + return len(recs) + + def online_filter(self, recorder): + tags = recorder.list_tags() + if tags.get(ModelUpdater.ONLINE_TAG, ModelUpdater.ONLINE_TAG_FALSE) == ModelUpdater.ONLINE_TAG_TRUE: return True return False @@ -151,11 +148,7 @@ def list_online_model(self): Returns ------- dict - {rid : record of the online model} + {rid : recorder of the online model} """ - recs = self.exp.list_recorders() - online_rec = {} - for rid, rec in recs.items(): - if self.online_filter(rec): - online_rec[rid] = rec - return online_rec + + return self.list_recorders(rec_filter_func=self.online_filter) diff --git a/qlib/workflow/task/utils.py b/qlib/workflow/task/utils.py index 63563e2f6d..9445d3c680 100644 --- a/qlib/workflow/task/utils.py +++ b/qlib/workflow/task/utils.py @@ -50,7 +50,6 @@ def get(self, idx: int): if idx >= len(self.cals): return None return self.cals[idx] - def max(self): """ (Deprecated) @@ -86,6 +85,9 @@ def align_idx(self, time_point, tp_type="start"): raise NotImplementedError(f"This type of input is not supported") return idx + def cal_interval(self, time_point_A, time_point_B): + return self.align_idx(time_point_A) - self.align_idx(time_point_B) + def align_time(self, time_point, tp_type="start"): """ Align time_point to trade date of calendar From 9d84d389ab9026207a30bd3122925f67d4a276e5 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Fri, 12 Mar 2021 08:24:21 +0000 Subject: [PATCH 18/61] format code and add example --- .../taskmanager/task_manager_rolling.ipynb | 176 ------------- examples/taskmanager/task_manager_rolling.py | 26 +- .../task_manager_rolling_with_updating.py | 244 ++++++++++++++++++ examples/taskmanager/update_online_pred.py | 27 +- qlib/workflow/task/collect.py | 19 +- qlib/workflow/task/manage.py | 2 +- qlib/workflow/task/update.py | 31 ++- qlib/workflow/task/utils.py | 1 + 8 files changed, 310 insertions(+), 216 deletions(-) delete mode 100644 examples/taskmanager/task_manager_rolling.ipynb create mode 100644 examples/taskmanager/task_manager_rolling_with_updating.py diff --git a/examples/taskmanager/task_manager_rolling.ipynb b/examples/taskmanager/task_manager_rolling.ipynb deleted file mode 100644 index e8ec8d4a75..0000000000 --- a/examples/taskmanager/task_manager_rolling.ipynb +++ /dev/null @@ -1,176 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "import qlib\n", - "from qlib.config import REG_CN\n", - "from qlib.workflow.task.gen import RollingGen, task_generator\n", - "from qlib.workflow.task.manage import TaskManager\n", - "from qlib.config import C\n", - "\n", - "data_handler_template = {\n", - " \"start_time\": \"2008-01-01\",\n", - " \"end_time\": \"2020-08-01\",\n", - " \"fit_start_time\": \"2008-01-01\",\n", - " \"fit_end_time\": \"2014-12-31\",\n", - " \"instruments\": 'csi100',\n", - "}\n", - "\n", - "dataset_template = {\n", - " \"class\": \"DatasetH\",\n", - " \"module_path\": \"qlib.data.dataset\",\n", - " \"kwargs\": {\n", - " \"handler\": {\n", - " \"class\": \"Alpha158\",\n", - " \"module_path\": \"qlib.contrib.data.handler\",\n", - " \"kwargs\": data_handler_template,\n", - " },\n", - " \"segments\": {\n", - " \"train\": (\"2008-01-01\", \"2014-12-31\"),\n", - " \"valid\": (\"2015-01-01\", \"2016-12-31\"),\n", - " \"test\": (\"2017-01-01\", \"2020-08-01\"),\n", - " },\n", - " },\n", - " }\n", - "\n", - "record_template = [\n", - " {\n", - " \"class\": \"SignalRecord\",\n", - " \"module_path\": \"qlib.workflow.record_temp\",\n", - " },\n", - " {\n", - " \"class\": \"SigAnaRecord\",\n", - " \"module_path\": \"qlib.workflow.record_temp\",\n", - " }\n", - "]\n", - "\n", - "# use lgb\n", - "lgb_task_template = {\n", - " \"model\": {\n", - " \"class\": \"LGBModel\",\n", - " \"module_path\": \"qlib.contrib.model.gbdt\",\n", - " },\n", - " \"dataset\": dataset_template,\n", - " \"record\": record_template,\n", - "}\n", - "\n", - "# use xgboost\n", - "xgboost_task_template = {\n", - " \"model\": {\n", - " \"class\": \"XGBModel\",\n", - " \"module_path\": \"qlib.contrib.model.xgboost\",\n", - " },\n", - " \"dataset\": dataset_template,\n", - " \"record\": record_template,\n", - "}\n", - "\n", - "provider_uri = \"~/.qlib/qlib_data/cn_data\" # target_dir\n", - "qlib.init(provider_uri=provider_uri, region=REG_CN)\n", - "\n", - "C[\"mongo\"] = {\n", - " \"task_url\" : \"mongodb://localhost:27017/\", # maybe you need to change it to your url\n", - " \"task_db_name\" : \"rolling_db\"\n", - "}\n", - "\n", - "exp_name = 'rolling_exp' # experiment name, will be used as the experiment in MLflow\n", - "task_pool = 'rolling_task' # task pool name, will be used as the document in MongoDB" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "tasks = task_generator(\n", - " xgboost_task_template, # default task name\n", - " RollingGen(step=550,rtype=RollingGen.ROLL_SD), # generate different date segment\n", - " task_lgb=lgb_task_template # use \"task_lgb\" as the task name\n", - ")\n", - "# Uncomment next two lines to see the generated tasks\n", - "# from pprint import pprint\n", - "# pprint(tasks)\n", - "tm = TaskManager(task_pool=task_pool)\n", - "tm.create_task(tasks) # all tasks will be saved to MongoDB" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "from qlib.workflow.task.manage import run_task\n", - "from qlib.workflow.task.collect import TaskCollector\n", - "from qlib.model.trainer import task_train\n", - "\n", - "run_task(task_train, task_pool, experiment_name=exp_name) # all tasks will be trained using \"task_train\" method" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "def get_task_key(task):\n", - " task_key = task[\"task_key\"]\n", - " rolling_end_timestamp = task[\"dataset\"][\"kwargs\"][\"segments\"][\"test\"][1]\n", - " return task_key, rolling_end_timestamp.strftime('%Y-%m-%d')\n", - "\n", - "def my_filter(task):\n", - " # only choose the results of \"task_lgb\" and test segment end in 2019 from all tasks\n", - " task_key, rolling_end = get_task_key(task)\n", - " if task_key==\"task_lgb\" and rolling_end.startswith('2019'):\n", - " return True\n", - " return False\n", - "\n", - "# name tasks by \"get_task_key\" and filter tasks by \"my_filter\"\n", - "pred_rolling = TaskCollector.collect_predictions(exp_name, get_task_key, my_filter) \n", - "pred_rolling" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "3.6.5-final" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/examples/taskmanager/task_manager_rolling.py b/examples/taskmanager/task_manager_rolling.py index 36ec819620..9223ed8189 100644 --- a/examples/taskmanager/task_manager_rolling.py +++ b/examples/taskmanager/task_manager_rolling.py @@ -16,7 +16,11 @@ "class": "DatasetH", "module_path": "qlib.data.dataset", "kwargs": { - "handler": {"class": "Alpha158", "module_path": "qlib.contrib.data.handler", "kwargs": data_handler_config,}, + "handler": { + "class": "Alpha158", + "module_path": "qlib.contrib.data.handler", + "kwargs": data_handler_config, + }, "segments": { "train": ("2008-01-01", "2014-12-31"), "valid": ("2015-01-01", "2016-12-31"), @@ -26,20 +30,32 @@ } record_config = [ - {"class": "SignalRecord", "module_path": "qlib.workflow.record_temp",}, - {"class": "SigAnaRecord", "module_path": "qlib.workflow.record_temp",}, + { + "class": "SignalRecord", + "module_path": "qlib.workflow.record_temp", + }, + { + "class": "SigAnaRecord", + "module_path": "qlib.workflow.record_temp", + }, ] # use lgb task_lgb_config = { - "model": {"class": "LGBModel", "module_path": "qlib.contrib.model.gbdt",}, + "model": { + "class": "LGBModel", + "module_path": "qlib.contrib.model.gbdt", + }, "dataset": dataset_config, "record": record_config, } # use xgboost task_xgboost_config = { - "model": {"class": "XGBModel", "module_path": "qlib.contrib.model.xgboost",}, + "model": { + "class": "XGBModel", + "module_path": "qlib.contrib.model.xgboost", + }, "dataset": dataset_config, "record": record_config, } diff --git a/examples/taskmanager/task_manager_rolling_with_updating.py b/examples/taskmanager/task_manager_rolling_with_updating.py new file mode 100644 index 0000000000..c69b558bc1 --- /dev/null +++ b/examples/taskmanager/task_manager_rolling_with_updating.py @@ -0,0 +1,244 @@ +import qlib +import fire +import mlflow +from qlib.config import C +from qlib.workflow import R +from qlib.config import REG_CN +from qlib.model.trainer import task_train +from qlib.workflow.task.manage import run_task +from qlib.workflow.task.manage import TaskManager +from qlib.workflow.task.utils import TimeAdjuster +from qlib.workflow.task.update import ModelUpdater +from qlib.workflow.task.collect import TaskCollector +from qlib.workflow.task.gen import RollingGen, task_generator + + +data_handler_config = { + "start_time": "2013-01-01", + "end_time": "2020-09-25", + "fit_start_time": "2013-01-01", + "fit_end_time": "2014-12-31", + "instruments": "csi100", +} + +dataset_config = { + "class": "DatasetH", + "module_path": "qlib.data.dataset", + "kwargs": { + "handler": { + "class": "Alpha158", + "module_path": "qlib.contrib.data.handler", + "kwargs": data_handler_config, + }, + "segments": { + "train": ("2013-01-01", "2014-12-31"), + "valid": ("2015-01-01", "2015-12-31"), + "test": ("2016-01-01", "2017-01-01"), + }, + }, +} + +record_config = [ + { + "class": "SignalRecord", + "module_path": "qlib.workflow.record_temp", + }, + { + "class": "SigAnaRecord", + "module_path": "qlib.workflow.record_temp", + }, +] + +# use lgb model +task_lgb_config = { + "model": { + "class": "LGBModel", + "module_path": "qlib.contrib.model.gbdt", + }, + "dataset": dataset_config, + "record": record_config, +} + +# use xgboost model +task_xgboost_config = { + "model": { + "class": "XGBModel", + "module_path": "qlib.contrib.model.xgboost", + }, + "dataset": dataset_config, + "record": record_config, +} + +# This part corresponds to "Task Generating" in the document +def task_generating(**kwargs): + print("========================================= task_generating =========================================") + + rolling_generator = RollingGen(step=rolling_step, rtype=RollingGen.ROLL_EX) + + tasks = task_generator(rolling_generator, **kwargs) + + # See the generated tasks in a easy way + from pprint import pprint + + pprint(tasks) + + return tasks + + +# This part corresponds to "Task Storing" in the document +def task_storing(tasks): + print("========================================= task_storing =========================================") + tm = TaskManager(task_pool=task_pool) + tm.create_task(tasks) # all tasks will be saved to MongoDB + + +# This part corresponds to "Task Running" in the document +def task_running(): + print("========================================= task_running =========================================") + run_task(task_train, task_pool, experiment_name=exp_name) # all tasks will be trained using "task_train" method + + +# This part corresponds to "Task Collecting" in the document +def task_collecting(): + print("========================================= task_collecting =========================================") + + def get_task_key(task_config): + task_key = task_config["task_key"] + rolling_end_timestamp = task_config["dataset"]["kwargs"]["segments"]["test"][1] + if rolling_end_timestamp == None: + rolling_end_timestamp = TimeAdjuster().last_date() + return task_key, rolling_end_timestamp.strftime("%Y-%m-%d") + + def lgb_filter(task_config): + # only choose the results of "task_lgb" + task_key, rolling_end = get_task_key(task_config) + if task_key == "task_lgb": + return True + return False + + task_collector = TaskCollector(exp_name) + pred_rolling = task_collector.collect_predictions( + get_task_key, lgb_filter + ) # name tasks by "get_task_key" and filter tasks by "my_filter" + print(pred_rolling) + + +# Reset all things to the first status, be careful to save important data +def reset(force_end=False): + print("========================================= reset =========================================") + TaskManager(task_pool=task_pool).remove() + + exp = R.get_exp(experiment_name=exp_name) + recs = TaskCollector(exp_name).list_recorders(only_finished=True) + + for rid in recs: + exp.delete_recorder(rid) + + try: + if force_end: + mlflow.end_run() + except Exception: + pass + + +def set_online_model_to_latest(): + print( + "========================================= set_online_model_to_latest =========================================" + ) + model_updater = ModelUpdater(experiment_name=exp_name) + latest_records, latest_test = model_updater.collect_latest_records() + model_updater.reset_online_model(latest_records.values()) + + +# Run this firstly to see the workflow in Task Management +def first_run(): + print("========================================= first_run =========================================") + reset(force_end=True) + + # use "task_lgb" and "task_xgboost" as the task name + tasks = task_generating(**{"task_xgboost": task_xgboost_config, "task_lgb": task_lgb_config}) + task_storing(tasks) + task_running() + task_collecting() + set_online_model_to_latest() + + +# Update the predictions of online model +def update_predictions(): + print("========================================= update_predictions =========================================") + model_updater = ModelUpdater(experiment_name=exp_name) + model_updater.update_online_pred() + + +# Update the models using the latest date and set them to online model +def update_model(): + print("========================================= update_model =========================================") + # get the latest recorders + model_updater = ModelUpdater(experiment_name=exp_name) + latest_records, latest_test = model_updater.collect_latest_records() + # date adjustment based on trade day of Calendar in Qlib + time_adjuster = TimeAdjuster() + calendar_latest = time_adjuster.last_date() + print("The latest date is ", calendar_latest) + if time_adjuster.cal_interval(calendar_latest, latest_test[0]) > rolling_step: + print("Need update models!") + tasks = {} + for rid, rec in latest_records.items(): + old_task = rec.task + test_begin = old_task["dataset"]["kwargs"]["segments"]["test"][0] + # modify the test segment to generate new tasks + old_task["dataset"]["kwargs"]["segments"]["test"] = (test_begin, calendar_latest) + tasks[old_task["task_key"]] = old_task + + # retrain the latest model + new_tasks = task_generating(**tasks) + task_storing(new_tasks) + task_running() + task_collecting() + latest_records, _ = model_updater.collect_latest_records() + + # set the latest model to online model + model_updater.reset_online_model(latest_records.values()) + + +# Run whole workflow completely +def whole_workflow(): + print("========================================= whole_workflow =========================================") + # run this at the first time + first_run() + # run this every day + update_predictions() + # run this every "rolling_steps" day + update_model() + + +if __name__ == "__main__": + ####### to train the first version's models, use the command below + # python task_manager_rolling_with_updating.py first_run + + ####### to update the models using the latest date and set them to online model, use the command below + # python task_manager_rolling_with_updating.py update_model + + ####### to update the predictions to the latest date, use the command below + # python task_manager_rolling_with_updating.py update_predictions + + ####### to run whole workflow completely, use the command below + # python task_manager_rolling_with_updating.py whole_workflow + + #################### you need to finish the configurations below ######################### + + provider_uri = "~/.qlib/qlib_data/cn_data" # data_dir + qlib.init(provider_uri=provider_uri, region=REG_CN) + + C["mongo"] = { + "task_url": "mongodb://localhost:27017/", # your MongoDB url + "task_db_name": "rolling_db", # database name + } + + exp_name = "rolling_exp" # experiment name, will be used as the experiment in MLflow + task_pool = "rolling_task" # task pool name, will be used as the document in MongoDB + rolling_step = 550 + + ########################################################################################## + + fire.Fire() diff --git a/examples/taskmanager/update_online_pred.py b/examples/taskmanager/update_online_pred.py index 4dbd22b855..ba4beb8cf0 100644 --- a/examples/taskmanager/update_online_pred.py +++ b/examples/taskmanager/update_online_pred.py @@ -5,12 +5,12 @@ import fire data_handler_config = { - "start_time": "2008-01-01", - "end_time": "2020-08-01", - "fit_start_time": "2008-01-01", - "fit_end_time": "2014-12-31", - "instruments": "csi100", - } + "start_time": "2008-01-01", + "end_time": "2020-08-01", + "fit_start_time": "2008-01-01", + "fit_end_time": "2014-12-31", + "instruments": "csi100", +} task = { "model": { @@ -44,22 +44,26 @@ }, }, }, - "record": {"class": "SignalRecord", "module_path": "qlib.workflow.record_temp",}, + "record": { + "class": "SignalRecord", + "module_path": "qlib.workflow.record_temp", + }, } provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir + def first_train(experiment_name="online_svr"): - qlib.init(provider_uri=provider_uri, region=REG_CN) model_updater = ModelUpdater(experiment_name) rid = task_train(task_config=task, experiment_name=experiment_name) model_updater.reset_online_model(rid) + def update_online_pred(experiment_name="online_svr"): - + qlib.init(provider_uri=provider_uri, region=REG_CN) model_updater = ModelUpdater(experiment_name) @@ -68,8 +72,9 @@ def update_online_pred(experiment_name="online_svr"): print(rid) model_updater.update_online_pred() - -if __name__ == '__main__': + + +if __name__ == "__main__": fire.Fire() # to train a model and set it to online model, use the command below # python update_online_pred.py first_train diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index c022e6e760..45e51da360 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -18,7 +18,7 @@ def __init__(self, experiment_name: str) -> None: def list_recorders(self, rec_filter_func=None, task_filter_func=None, only_finished=True, only_have_task=False): """ - Return a dict of {rid:Recorder} by recorder filter and task filter. It is not necessary to use those filter. + Return a dict of {rid:Recorder} by recorder filter and task filter. It is not necessary to use those filter. If you don't train with "task_train", then there is no "task" which includes the task config. If there is a "task", then it will become rec.task which can be get simply. @@ -48,7 +48,7 @@ def list_recorders(self, rec_filter_func=None, task_filter_func=None, only_finis if task_filter_func is not None: only_have_task = True for rid, rec in recs.items(): - if (only_finished and rec.status == rec.STATUS_FI) or only_finished==False: + if (only_finished and rec.status == rec.STATUS_FI) or only_finished == False: if rec_filter_func is None or rec_filter_func(rec): task = None try: @@ -60,7 +60,7 @@ def list_recorders(self, rec_filter_func=None, task_filter_func=None, only_finis if task_filter_func is None or task_filter_func(task): rec.task = task recs_flt[rid] = rec - + return recs_flt def collect_predictions( @@ -83,7 +83,7 @@ def collect_predictions( dict the dict of predictions """ - recs_flt = self.list_recorders(task_filter_func=task_filter_func,only_have_task=True) + recs_flt = self.list_recorders(task_filter_func=task_filter_func, only_have_task=True) # group recs_group = {} @@ -108,18 +108,17 @@ def collect_latest_records( self, task_filter_func=None, ): - recs_flt = self.list_recorders(task_filter_func=task_filter_func,only_have_task=True) - + recs_flt = self.list_recorders(task_filter_func=task_filter_func, only_have_task=True) + if len(recs_flt) == 0: self.logger.warning("Can not collect any recorders...") return None, None - max_test = max(rec.task['dataset']['kwargs']['segments']['test'] for rec in recs_flt.values()) + max_test = max(rec.task["dataset"]["kwargs"]["segments"]["test"] for rec in recs_flt.values()) latest_record = {} for rid, rec in recs_flt.items(): - if rec.task['dataset']['kwargs']['segments']['test'] == max_test: + if rec.task["dataset"]["kwargs"]["segments"]["test"] == max_test: latest_record[rid] = rec - + self.logger.info(f"Collect {len(latest_record)} latest records in {self.exp_name}") return latest_record, max_test - \ No newline at end of file diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 22b5430cc7..a5741e3ed1 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -21,6 +21,7 @@ from qlib import auto_init from qlib import get_module_logger + class TaskManager: """TaskManager here is the what will a task looks like @@ -361,4 +362,3 @@ def run_task(task_func, task_pool, force_release=False, *args, **kwargs): ever_run = True return ever_run - diff --git a/qlib/workflow/task/update.py b/qlib/workflow/task/update.py index 9f1cc0a299..bafb6561b4 100644 --- a/qlib/workflow/task/update.py +++ b/qlib/workflow/task/update.py @@ -1,4 +1,4 @@ -from typing import Union,List +from typing import Union, List from qlib.workflow import R from tqdm.auto import tqdm from qlib.data import D @@ -10,6 +10,7 @@ from qlib.workflow.recorder import Recorder from qlib.workflow.task.collect import TaskCollector + class ModelUpdater(TaskCollector): """ The model updater to re-train model or update predictions @@ -31,7 +32,7 @@ def __init__(self, experiment_name: str) -> None: self.exp = R.get_exp(experiment_name=experiment_name) self.logger = get_module_logger("ModelUpdater") - def set_online_model(self, recorder: Union[str,Recorder]): + def set_online_model(self, recorder: Union[str, Recorder]): """online model will be identified at the tags of the record Parameters @@ -39,12 +40,12 @@ def set_online_model(self, recorder: Union[str,Recorder]): recorder: Union[str,Recorder] the id of a Recorder or the Recorder instance """ - if isinstance(recorder,str): + if isinstance(recorder, str): recorder = self.exp.get_recorder(recorder_id=recorder) recorder.set_tags(**{ModelUpdater.ONLINE_TAG: ModelUpdater.ONLINE_TAG_TRUE}) - def cancel_online_model(self, recorder: Union[str,Recorder]): - if isinstance(recorder,str): + def cancel_online_model(self, recorder: Union[str, Recorder]): + if isinstance(recorder, str): recorder = self.exp.get_recorder(recorder_id=recorder) recorder.set_tags(**{ModelUpdater.ONLINE_TAG: ModelUpdater.ONLINE_TAG_FALSE}) @@ -53,7 +54,7 @@ def cancel_all_online_model(self): for rid, rec in recs.items(): self.cancel_online_model(rec) - def reset_online_model(self, recorders: List[Union[str,Recorder]]): + def reset_online_model(self, recorders: List[Union[str, Recorder]]): """cancel all online model and reset the given model to online model Parameters @@ -65,7 +66,7 @@ def reset_online_model(self, recorders: List[Union[str,Recorder]]): for rec_or_rid in recorders: self.set_online_model(rec_or_rid) - def update_pred(self, recorder: Union[str,Recorder]): + def update_pred(self, recorder: Union[str, Recorder]): """update predictions to the latest day in Calendar based on rid Parameters @@ -73,17 +74,19 @@ def update_pred(self, recorder: Union[str,Recorder]): recorder: Union[str,Recorder] the id of a Recorder or the Recorder instance """ - if isinstance(recorder,str): + if isinstance(recorder, str): recorder = self.exp.get_recorder(recorder_id=recorder) old_pred = recorder.load_object("pred.pkl") last_end = old_pred.index.get_level_values("datetime").max() - task_config = recorder.load_object("task") # recorder.task + task_config = recorder.load_object("task") # recorder.task # updated to the latest trading day cal = D.calendar(start_time=last_end + pd.Timedelta(days=1), end_time=None) if len(cal) == 0: - self.logger.info(f"The prediction in {recorder.info['id']} of {self.exp_name} are latest. No need to update.") + self.logger.info( + f"The prediction in {recorder.info['id']} of {self.exp_name} are latest. No need to update." + ) return start_time, end_time = cal[0], cal[-1] @@ -100,7 +103,9 @@ def update_pred(self, recorder: Union[str,Recorder]): recorder.save_objects(**{"pred.pkl": cb_pred}) - self.logger.info(f"Finish updating new {new_pred.shape[0]} predictions in {recorder.info['id']} of {self.exp_name}.") + self.logger.info( + f"Finish updating new {new_pred.shape[0]} predictions in {recorder.info['id']} of {self.exp_name}." + ) def update_all_pred(self, rec_filter_func=None): """update all predictions in this experiment after filter. @@ -126,7 +131,7 @@ def rec_filter_func(recorder): the count of updated record """ - recs = self.list_recorders(rec_filter_func=rec_filter_func,only_have_task=True) + recs = self.list_recorders(rec_filter_func=rec_filter_func, only_have_task=True) for rid, rec in recs.items(): self.update_pred(rec) return len(recs) @@ -150,5 +155,5 @@ def list_online_model(self): dict {rid : recorder of the online model} """ - + return self.list_recorders(rec_filter_func=self.online_filter) diff --git a/qlib/workflow/task/utils.py b/qlib/workflow/task/utils.py index 9445d3c680..091952b81d 100644 --- a/qlib/workflow/task/utils.py +++ b/qlib/workflow/task/utils.py @@ -50,6 +50,7 @@ def get(self, idx: int): if idx >= len(self.cals): return None return self.cals[idx] + def max(self): """ (Deprecated) From e4e8a4abcdb72a2e27d4bc929996795b768381a2 Mon Sep 17 00:00:00 2001 From: Young Date: Fri, 12 Mar 2021 10:17:16 +0000 Subject: [PATCH 19/61] fix task name & add cur_path --- qlib/__init__.py | 7 ++++--- qlib/model/trainer.py | 2 +- qlib/workflow/task/collect.py | 5 +++-- qlib/workflow/task/update.py | 4 ++-- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/qlib/__init__.py b/qlib/__init__.py index 816e5a5852..4fd48f8c19 100644 --- a/qlib/__init__.py +++ b/qlib/__init__.py @@ -154,7 +154,7 @@ def init_from_yaml_conf(conf_path, **kwargs): init(default_conf, **config) -def get_project_path(config_name="config.yaml") -> Path: +def get_project_path(config_name="config.yaml", cur_path=None) -> Path: """ If users are building a project follow the following pattern. - Qlib is a sub folder in project path @@ -181,7 +181,8 @@ def get_project_path(config_name="config.yaml") -> Path: FileNotFoundError: If project path is not found """ - cur_path = Path(__file__).absolute().resolve() + if cur_path is None: + cur_path = Path(__file__).absolute().resolve() while True: if (cur_path / config_name).exists(): return cur_path @@ -199,7 +200,7 @@ def auto_init(**kwargs): """ try: - pp = get_project_path() + pp = get_project_path(cur_path=kwargs.pop("cur_path", None)) except FileNotFoundError: init(**kwargs) else: diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index 5e62a141cd..a4df922185 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -34,7 +34,7 @@ def task_train(task_config: dict, experiment_name: str) -> str: model.fit(dataset) recorder = R.get_recorder() R.save_objects(**{"params.pkl": model}) - R.save_objects(**{"task.pkl": task_config}) # keep the original format and datatype + R.save_objects(task=task_config) # keep the original format and datatype # generate records: prediction, backtest, and analysis records = task_config.get("record", []) diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index b16312ff73..b4a5844947 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -1,5 +1,6 @@ from qlib.workflow import R import pandas as pd +import tqdm.auto import tqdm from typing import Union from qlib import get_module_logger @@ -35,7 +36,7 @@ def collect_predictions( recs_flt = {} for rid, rec in recs.items(): - params = rec.load_object("task.pkl") + params = rec.load_object("task") if rec.status == rec.STATUS_FI: if filter_func is None or filter_func(params): rec.params = params @@ -83,7 +84,7 @@ def __call__(self, exp_name) -> Union[pd.Series, dict]: recs_flt = {} for rid, rec in tqdm(recs.items(), desc="Loading data"): - params = rec.load_object("task.pkl") + params = rec.load_object("task") if rec.status == rec.STATUS_FI: if self.flt_func is None or self.flt_func(params): rec.params = params diff --git a/qlib/workflow/task/update.py b/qlib/workflow/task/update.py index f9d03efbcc..628225a203 100644 --- a/qlib/workflow/task/update.py +++ b/qlib/workflow/task/update.py @@ -74,7 +74,7 @@ def update_pred(self, rid: str): rec = self.exp.get_recorder(recorder_id=rid) old_pred = rec.load_object("pred.pkl") last_end = old_pred.index.get_level_values("datetime").max() - task_config = rec.load_object("task.pkl") + task_config = rec.load_object("task") # updated to the latest trading day cal = D.calendar(start_time=last_end + pd.Timedelta(days=1), end_time=None) @@ -107,7 +107,7 @@ def update_all_pred(self, filter_func=None): .. code-block:: python def record_filter(record): - task_config = record.load_object("task.pkl") + task_config = record.load_object("task") if task_config["model"]["class"]=="LGBModel": return True return False From 8362780e22c2e576f78c8acf30f53dc0ee723e5c Mon Sep 17 00:00:00 2001 From: Young Date: Sun, 14 Mar 2021 15:16:38 +0000 Subject: [PATCH 20/61] fix import bug --- qlib/workflow/task/collect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index b4a5844947..552456e62b 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -1,6 +1,6 @@ from qlib.workflow import R import pandas as pd -import tqdm.auto import tqdm +from tqdm.auto import tqdm from typing import Union from qlib import get_module_logger From 646d899f8de14b42b11c529df511d195d45606a3 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Mon, 15 Mar 2021 03:50:43 +0000 Subject: [PATCH 21/61] update docstring and document --- docs/advanced/task_management.rst | 42 ++++++++++++++++------------ docs/reference/api.rst | 29 +++++++++++++++++++ qlib/workflow/task/collect.py | 27 ++++++++++++------ qlib/workflow/task/gen.py | 46 ++++++++++++++++--------------- qlib/workflow/task/manage.py | 6 ++-- qlib/workflow/task/update.py | 2 +- qlib/workflow/task/utils.py | 11 +++++--- 7 files changed, 106 insertions(+), 57 deletions(-) diff --git a/docs/advanced/task_management.rst b/docs/advanced/task_management.rst index 78ac62410e..230a4e9d1a 100644 --- a/docs/advanced/task_management.rst +++ b/docs/advanced/task_management.rst @@ -9,50 +9,52 @@ Task Management Introduction ============= -The `Workflow <../component/introduction.html>`_ part introduce how to run research workflow in a loosely-coupled way. But it can only execute one ``task`` when you use ``qrun``. To automatically generate and execute different tasks, Task Management module provide a whole process including `Task Generating`_, `Task Storing`_, `Task Running`_ and `Task Collecting`_. -With this module, users can run their ``task`` automatically at different periods, in different losses or even by different models. +The `Workflow <../component/introduction.html>`_ part introduces how to run research workflow in a loosely-coupled way. But it can only execute one ``task`` when you use ``qrun``. +To automatically generate and execute different tasks, ``Task Management`` provides a whole process including `Task Generating`_, `Task Storing`_, `Task Running`_ and `Task Collecting`_. +With this module, users can run their ``task`` automatically at different periods, in different losses, or even by different models. -An example of the entire process is shown `here <>`_. +An example of the entire process is shown `here `_. Task Generating =============== A ``task`` consists of `Model`, `Dataset`, `Record` or anything added by users. -The specific task template can be viewed in +The specific task template(/definition/config) can be viewed in `Task Section <../component/workflow.html#task-section>`_. -Even though the task template is fixed, Users can use ``TaskGen`` to generate different ``task`` by task template. +Even though the task template is fixed, users can customize their ``TaskGen`` to generate different ``task`` by task template. -Here is the base class of TaskGen: +Here is the base class of ``TaskGen``: .. autoclass:: qlib.workflow.task.gen.TaskGen :members: -``Qlib`` provider a class `RollingGen`_ to generate a list of ``task`` of dataset in different date segments. -This allows users to verify the effect of data from different periods on the model in one experiment. +``Qlib`` provider a class `RollingGen `_ to generate a list of ``task`` of the dataset in different date segments. +This class allows users to verify the effect of data from different periods on the model in one experiment. Task Storing =============== -In order to achieve higher efficiency and the possibility of cluster operation, ``Task Manager`` will store all tasks in `MongoDB `_. +To achieve higher efficiency and the possibility of cluster operation, ``Task Manager`` will store all tasks in `MongoDB `_. Users **MUST** finished the configuration of `MongoDB `_ when using this module. -Users need to provide the url and database of ``task`` storing like this. +Users need to provide the URL and database name of ``task`` storing like this. .. code-block:: python from qlib.config import C C["mongo"] = { - "task_url" : "mongodb://localhost:27017/", # maybe you need to change it to your url - "task_db_name" : "rolling_db" # you can custom database name + "task_url" : "mongodb://localhost:27017/", # your MongoDB url + "task_db_name" : "rolling_db" # database name } -The CRUD methods of ``task`` can be found in TaskManager. More methods can be seen in the `Github`_. +The CRUD methods of ``task`` can be found in TaskManager. +More methods can be seen in the `Github `_. .. autoclass:: qlib.workflow.task.manage.TaskManager :members: Task Running =============== -After generating and storing those ``task``, it's time to run the ``task`` in the *WAITING* status. -``qlib`` provide a method to run those ``task`` in task pool, however users can also customize how tasks are executed. +After generating and storing those ``task``, it's time to run the ``task`` which are in the *WAITING* status. +``Qlib`` provides a method called ``run_task`` to run those ``task`` in task pool, however, users can also customize how tasks are executed. An easy way to get the ``task_func`` is using ``qlib.model.trainer.task_train`` directly. It will run the whole workflow defined by ``task``, which includes *Model*, *Dataset*, *Record*. @@ -60,8 +62,12 @@ It will run the whole workflow defined by ``task``, which includes *Model*, *Dat Task Collecting =============== -To see the results of ``task`` after running, ``Qlib`` provide a task collector to collect the tasks by filter condition (optional). -The collector will return a dict of filtered key (users defined by task config) and value (predict scores from ``pred.pkl``). +To see the results of ``task`` after running or to update something, ``Qlib`` provides a ``TaskCollector`` to collect the tasks by filter condition (optional). +Here are some methods in this class. .. autoclass:: qlib.workflow.task.collect.TaskCollector - :members: \ No newline at end of file + :members: + +``Qlib`` provides a concrete `example `_, including a whole process of `Task Generating`_ (using `RollingGen `_), `Task Storing`_, `Task Running`_ and `Task Collecting`_. +Besides, the `example `_ uses a ``ModelUpdater`` inherited from ``TaskCollector``, which can update the inferences and retrain the model if it is out of date. +Actually, the model updating can be viewed as a subset of ``Online Serving``. \ No newline at end of file diff --git a/docs/reference/api.rst b/docs/reference/api.rst index 3167d8a622..691dff7036 100644 --- a/docs/reference/api.rst +++ b/docs/reference/api.rst @@ -155,6 +155,35 @@ Record Template :members: +Task Management +==================== + + +RollingGen +-------------------- +.. autoclass:: qlib.workflow.task.gen.RollingGen + :members: + +TaskManager +-------------------- +.. autoclass:: qlib.workflow.task.manage.TaskManager + :members: + +TaskCollector +-------------------- +.. autoclass:: qlib.workflow.task.collect.TaskCollector + :members: + +ModelUpdater +-------------------- +.. autoclass:: qlib.workflow.task.update.ModelUpdater + :members: + +TimeAdjuster +-------------------- +.. autoclass:: qlib.workflow.task.utils.TimeAdjuster + :members: + Utils ==================== diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index 45e51da360..5d81864cc3 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -18,8 +18,8 @@ def __init__(self, experiment_name: str) -> None: def list_recorders(self, rec_filter_func=None, task_filter_func=None, only_finished=True, only_have_task=False): """ - Return a dict of {rid:Recorder} by recorder filter and task filter. It is not necessary to use those filter. - If you don't train with "task_train", then there is no "task" which includes the task config. + Return a dict of {rid: Recorder} by recorder filter and task filter. It is not necessary to use those filter. + If you don't train with "task_train", then there is no "task"(a file in mlruns/artifacts) which includes the task config. If there is a "task", then it will become rec.task which can be get simply. Parameters @@ -36,12 +36,8 @@ def list_recorders(self, rec_filter_func=None, task_filter_func=None, only_finis Returns ------- dict - a dict of {rid:Recorder} + a dict of {rid: Recorder} - Raises - ------ - OSError - if you use a task filter, but there is no "task" which includes the task config """ recs = self.exp.list_recorders() recs_flt = {} @@ -69,13 +65,14 @@ def collect_predictions( task_filter_func=None, ): """ + Collect predictions using a filter and a key function. Parameters ---------- experiment_name : str - get_key_func : function(task: dict) -> Union[Number, str, tuple] + get_key_func : Callable[[dict], bool] -> Union[Number, str, tuple] get the key of a task when collect it - filter_func : function(task: dict) -> bool + filter_func : Callable[[dict], bool] -> bool to judge a task will be collected or not Returns @@ -108,6 +105,18 @@ def collect_latest_records( self, task_filter_func=None, ): + """Collect latest recorders using a filter. + + Parameters + ---------- + task_filter_func : Callable[[dict], bool], optional + to judge a task will be collected or not, by default None + + Returns + ------- + dict, tuple + a dict of recorders and a tuple of test segments + """ recs_flt = self.list_recorders(task_filter_func=task_filter_func, only_have_task=True) if len(recs_flt) == 0: diff --git a/qlib/workflow/task/gen.py b/qlib/workflow/task/gen.py index 19793c485b..96448cefe2 100644 --- a/qlib/workflow/task/gen.py +++ b/qlib/workflow/task/gen.py @@ -130,30 +130,32 @@ def generate(self, task: dict): task : dict A dict describing a task. For example. - DEFAULT_TASK = { - "model": { - "class": "LGBModel", - "module_path": "qlib.contrib.model.gbdt", - }, - "dataset": { - "class": "DatasetH", - "module_path": "qlib.data.dataset", - "kwargs": { - "handler": { - "class": "Alpha158", - "module_path": "qlib.contrib.data.handler", - "kwargs": data_handler_config, - }, - "segments": { - "train": ("2008-01-01", "2014-12-31"), - "valid": ("2015-01-01", "2016-12-20"), # Please avoid leaking the future test data into validation - "test": ("2017-01-01", "2020-08-01"), + .. code-block:: python + + DEFAULT_TASK = { + "model": { + "class": "LGBModel", + "module_path": "qlib.contrib.model.gbdt", + }, + "dataset": { + "class": "DatasetH", + "module_path": "qlib.data.dataset", + "kwargs": { + "handler": { + "class": "Alpha158", + "module_path": "qlib.contrib.data.handler", + "kwargs": data_handler_config, + }, + "segments": { + "train": ("2008-01-01", "2014-12-31"), + "valid": ("2015-01-01", "2016-12-20"), # Please avoid leaking the future test data into validation + "test": ("2017-01-01", "2020-08-01"), + }, }, }, - }, - # You shoud record the data in specific sequence - # "record": ['SignalRecord', 'SigAnaRecord', 'PortAnaRecord'], - } + # You shoud record the data in specific sequence + # "record": ['SignalRecord', 'SigAnaRecord', 'PortAnaRecord'], + } """ res = [] diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index a5741e3ed1..e97fdb7741 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -18,13 +18,12 @@ import pymongo from qlib.config import C from .utils import get_mongodb -from qlib import auto_init from qlib import get_module_logger class TaskManager: """TaskManager - here is the what will a task looks like + here is what will a task looks like when it created by TaskManager .. code-block:: python @@ -40,7 +39,7 @@ class TaskManager: .. note:: - assumption: the data in MongoDB was encoded and the data out of MongoDB was decoded + Assumption: the data in MongoDB was encoded and the data out of MongoDB was decoded """ STATUS_WAITING = "waiting" @@ -118,6 +117,7 @@ def insert_task_def(self, task_def, task_pool=None): Parameters ---------- task_def: dict + the task definition task_pool: str the name of Collection in MongoDB diff --git a/qlib/workflow/task/update.py b/qlib/workflow/task/update.py index bafb6561b4..73c2f72417 100644 --- a/qlib/workflow/task/update.py +++ b/qlib/workflow/task/update.py @@ -110,7 +110,7 @@ def update_pred(self, recorder: Union[str, Recorder]): def update_all_pred(self, rec_filter_func=None): """update all predictions in this experiment after filter. - An example of filter function: + An example of filter function: .. code-block:: python diff --git a/qlib/workflow/task/utils.py b/qlib/workflow/task/utils.py index 091952b81d..272f219eca 100644 --- a/qlib/workflow/task/utils.py +++ b/qlib/workflow/task/utils.py @@ -107,11 +107,14 @@ def align_seg(self, segment: Union[dict, tuple]): align the given date to trade date for example: - input: {'train': ('2008-01-01', '2014-12-31'), 'valid': ('2015-01-01', '2016-12-31'), 'test': ('2017-01-01', '2020-08-01')} - output: {'train': (Timestamp('2008-01-02 00:00:00'), Timestamp('2014-12-31 00:00:00')), - 'valid': (Timestamp('2015-01-05 00:00:00'), Timestamp('2016-12-30 00:00:00')), - 'test': (Timestamp('2017-01-03 00:00:00'), Timestamp('2020-07-31 00:00:00'))} + .. code-block:: python + + input: {'train': ('2008-01-01', '2014-12-31'), 'valid': ('2015-01-01', '2016-12-31'), 'test': ('2017-01-01', '2020-08-01')} + + output: {'train': (Timestamp('2008-01-02 00:00:00'), Timestamp('2014-12-31 00:00:00')), + 'valid': (Timestamp('2015-01-05 00:00:00'), Timestamp('2016-12-30 00:00:00')), + 'test': (Timestamp('2017-01-03 00:00:00'), Timestamp('2020-07-31 00:00:00'))} Parameters ---------- From 0bc49dab60b202a99706cccb341841e498a6ef9b Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Mon, 15 Mar 2021 03:58:05 +0000 Subject: [PATCH 22/61] add task management to index.rst --- docs/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.rst b/docs/index.rst index 3fa35fc60d..274dc8045e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -50,6 +50,7 @@ Document Structure Building Formulaic Alphas Online & Offline mode Serialization + Task Management .. toctree:: :maxdepth: 3 From e3730b32d716a128f7217e7fd6c80b9e1e77423f Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Tue, 16 Mar 2021 02:23:28 +0000 Subject: [PATCH 23/61] more clearly structure --- examples/taskmanager/update_online_pred.py | 27 ++--- qlib/model/trainer.py | 9 +- qlib/workflow/task/collect.py | 64 +++++------ qlib/workflow/task/online.py | 124 +++++++++++++++++++++ qlib/workflow/task/update.py | 90 +++++---------- 5 files changed, 199 insertions(+), 115 deletions(-) create mode 100644 qlib/workflow/task/online.py diff --git a/examples/taskmanager/update_online_pred.py b/examples/taskmanager/update_online_pred.py index ba4beb8cf0..016336c684 100644 --- a/examples/taskmanager/update_online_pred.py +++ b/examples/taskmanager/update_online_pred.py @@ -1,8 +1,9 @@ import qlib from qlib.model.trainer import task_train -from qlib.workflow.task.update import ModelUpdater +from qlib.workflow.task.online import RollingOnlineManager from qlib.config import REG_CN import fire +from qlib.workflow import R data_handler_config = { "start_time": "2008-01-01", @@ -50,33 +51,33 @@ }, } -provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir - def first_train(experiment_name="online_svr"): - qlib.init(provider_uri=provider_uri, region=REG_CN) - model_updater = ModelUpdater(experiment_name) + rom = RollingOnlineManager(experiment_name) rid = task_train(task_config=task, experiment_name=experiment_name) - model_updater.reset_online_model(rid) + + rom.reset_online_model(rid) def update_online_pred(experiment_name="online_svr"): - qlib.init(provider_uri=provider_uri, region=REG_CN) - model_updater = ModelUpdater(experiment_name) + rom = RollingOnlineManager(experiment_name) print("Here are the online models waiting for update:") - for rid, rec in model_updater.list_online_model().items(): + for rid, rec in rom.list_online_model().items(): print(rid) - model_updater.update_online_pred() + rom.update_online_pred() if __name__ == "__main__": - fire.Fire() - # to train a model and set it to online model, use the command below + ## to train a model and set it to online model, use the command below # python update_online_pred.py first_train - # to update online predictions once a day, use the command below + ## to update online predictions once a day, use the command below # python update_online_pred.py update_online_pred + + provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir + qlib.init(provider_uri=provider_uri, region=REG_CN) + fire.Fire() diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index e901bc2521..b6d4de6e24 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -26,9 +26,11 @@ def task_train(task_config: dict, experiment_name: str) -> str: # model initiaiton model = init_instance_by_config(task_config["model"]) dataset = init_instance_by_config(task_config["dataset"]) - + datahandler = dataset.handler + # start exp with R.start(experiment_name=experiment_name): + # train model R.log_params(**flatten_dict(task_config)) model.fit(dataset) @@ -36,6 +38,10 @@ def task_train(task_config: dict, experiment_name: str) -> str: R.save_objects(**{"params.pkl": model}) R.save_objects(**{"task": task_config}) # keep the original format and datatype + artifact_uri = recorder.get_artifact_uri()[7:] # delete "file://" + dataset.to_pickle(artifact_uri + "/dataset", exclude=["handler"]) + datahandler.to_pickle(artifact_uri + "/datahandler") + # generate records: prediction, backtest, and analysis records = task_config.get("record", []) if isinstance(records, dict): # prevent only one dict @@ -53,4 +59,5 @@ def task_train(task_config: dict, experiment_name: str) -> str: record["kwargs"].update(rconf) ar = init_instance_by_config(record) ar.generate() + return recorder.info["id"] diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index 5d81864cc3..21639e7f8e 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -16,49 +16,39 @@ def __init__(self, experiment_name: str) -> None: self.exp = R.get_exp(experiment_name=experiment_name) self.logger = get_module_logger("TaskCollector") - def list_recorders(self, rec_filter_func=None, task_filter_func=None, only_finished=True, only_have_task=False): - """ - Return a dict of {rid: Recorder} by recorder filter and task filter. It is not necessary to use those filter. - If you don't train with "task_train", then there is no "task"(a file in mlruns/artifacts) which includes the task config. - If there is a "task", then it will become rec.task which can be get simply. - - Parameters - ---------- - rec_filter_func : Callable[[Recorder], bool], optional - judge whether you need this recorder, by default None - task_filter_func : Callable[[dict], bool], optional - judge whether you need this task, by default None - only_finished : bool, optional - whether always use finished recorder, by default True - only_have_task : bool, optional - whether it is necessary to get the task config - - Returns - ------- - dict - a dict of {rid: Recorder} - - """ + def list_recorders(self, rec_filter_func=None): + """""" recs = self.exp.list_recorders() recs_flt = {} - if task_filter_func is not None: - only_have_task = True for rid, rec in recs.items(): - if (only_finished and rec.status == rec.STATUS_FI) or only_finished == False: - if rec_filter_func is None or rec_filter_func(rec): - task = None - try: - task = rec.load_object("task") - except OSError: - pass - if task is None and only_have_task: - continue - if task_filter_func is None or task_filter_func(task): - rec.task = task - recs_flt[rid] = rec + if rec_filter_func is None or rec_filter_func(rec): + recs_flt[rid] = rec return recs_flt + def get_recorder_by_id(self, recorder_id): + return self.exp.get_recorder(recorder_id, create=False) + + def list_recorders_by_task(self, task_filter_func): + """[summary] + + Parameters + ---------- + task_filter_func : [type], optional + [description], by default None + """ + + def rec_filter_func(recorder): + try: + task = recorder.load_object("task") + except OSError: + raise OSError( + f"Can't find task in {recorder.info['id']}, have you trained with model.trainer.task_train?" + ) + return task_filter_func(task) + + return self.list_recorders(rec_filter_func) + def collect_predictions( self, get_key_func, diff --git a/qlib/workflow/task/online.py b/qlib/workflow/task/online.py new file mode 100644 index 0000000000..72d3491220 --- /dev/null +++ b/qlib/workflow/task/online.py @@ -0,0 +1,124 @@ +from typing import Union, List +from qlib import get_module_logger +from qlib.workflow import R +from qlib.model.trainer import task_train +from qlib.workflow.recorder import Recorder +from qlib.workflow.task.collect import TaskCollector +from qlib.workflow.task.update import ModelUpdater + + +class OnlineManagement: + def __init__(self, experiment_name): + pass + + def update_online_pred(self, recorder: Union[str, Recorder]): + """update the predictions of online models + + Parameters + ---------- + recorder : Union[str, Recorder] + the id or the instance of Recorder + + """ + raise NotImplementedError(f"Please implement the `update_pred` method.") + + def prepare_new_models(self, tasks: List[dict]): + """prepare(train) new models + + Parameters + ---------- + tasks : List[dict] + a list of tasks + + """ + raise NotImplementedError(f"Please implement the `prepare_new_models` method.") + + def reset_online_model(self, recorders: List[Union[str, Recorder]]): + """reset online model + + Parameters + ---------- + recorders : List[Union[str, Recorder]] + a list of the recorder id or the instance + + """ + raise NotImplementedError(f"Please implement the `reset_online_model` method.") + + +class RollingOnlineManager(OnlineManagement): + + ONLINE_TAG = "online_model" + ONLINE_TAG_TRUE = "True" + ONLINE_TAG_FALSE = "False" + + def __init__(self, experiment_name: str) -> None: + """ModelUpdater needs experiment name to find the records + + Parameters + ---------- + experiment_name : str + experiment name string + """ + super(RollingOnlineManager, self).__init__(experiment_name) + self.logger = get_module_logger("RollingOnlineManager") + self.exp_name = experiment_name + self.tc = TaskCollector(experiment_name) + + def set_online_model(self, recorder: Union[str, Recorder]): + """online model will be identified at the tags of the record + + Parameters + ---------- + recorder: Union[str,Recorder] + the id of a Recorder or the Recorder instance + """ + if isinstance(recorder, str): + recorder = self.tc.get_recorder_by_id(recorder_id=recorder) + recorder.set_tags(**{self.ONLINE_TAG: self.ONLINE_TAG_TRUE}) + + def cancel_online_model(self, recorder: Union[str, Recorder]): + if isinstance(recorder, str): + recorder = self.tc.get_recorder_by_id(recorder_id=recorder) + recorder.set_tags(**{self.ONLINE_TAG: self.ONLINE_TAG_FALSE}) + + def cancel_all_online_model(self): + recs = self.tc.list_recorders() + for rid, rec in recs.items(): + self.cancel_online_model(rec) + + def reset_online_model(self, recorders: Union[str, List[Union[str, Recorder]]]): + """cancel all online model and reset the given model to online model + + Parameters + ---------- + recorders: List[Union[str,Recorder]] + the list of the id of a Recorder or the Recorder instance + """ + self.cancel_all_online_model() + if isinstance(recorders, str): + recorders = [recorders] + for rec_or_rid in recorders: + self.set_online_model(rec_or_rid) + + def online_filter(self, recorder): + tags = recorder.list_tags() + if tags.get(self.ONLINE_TAG, self.ONLINE_TAG_FALSE) == self.ONLINE_TAG_TRUE: + return True + return False + + def list_online_model(self): + """list the record of online model + + Returns + ------- + dict + {rid : recorder of the online model} + """ + + return self.tc.list_recorders(rec_filter_func=self.online_filter) + + def update_online_pred(self): + """update all online model predictions to the latest day in Calendar.""" + mu = ModelUpdater(self.exp_name) + cnt = mu.update_all_pred(self.online_filter) + self.logger.info(f"Finish updating {cnt} online model predictions of {self.exp_name}.") diff --git a/qlib/workflow/task/update.py b/qlib/workflow/task/update.py index 73c2f72417..9f68dbd0a2 100644 --- a/qlib/workflow/task/update.py +++ b/qlib/workflow/task/update.py @@ -1,9 +1,7 @@ from typing import Union, List from qlib.workflow import R -from tqdm.auto import tqdm from qlib.data import D import pandas as pd -from qlib.utils import init_instance_by_config from qlib import get_module_logger from qlib.workflow import R from qlib.model.trainer import task_train @@ -11,15 +9,11 @@ from qlib.workflow.task.collect import TaskCollector -class ModelUpdater(TaskCollector): +class ModelUpdater: """ - The model updater to re-train model or update predictions + The model updater to update model results in new data. """ - ONLINE_TAG = "online_model" - ONLINE_TAG_TRUE = "True" - ONLINE_TAG_FALSE = "False" - def __init__(self, experiment_name: str) -> None: """ModelUpdater needs experiment name to find the records @@ -29,42 +23,35 @@ def __init__(self, experiment_name: str) -> None: experiment name string """ self.exp_name = experiment_name - self.exp = R.get_exp(experiment_name=experiment_name) self.logger = get_module_logger("ModelUpdater") + self.tc = TaskCollector(experiment_name) - def set_online_model(self, recorder: Union[str, Recorder]): - """online model will be identified at the tags of the record + def _reload_dataset(self, recorder, start_time, end_time): + """reload dataset from pickle file Parameters ---------- - recorder: Union[str,Recorder] - the id of a Recorder or the Recorder instance - """ - if isinstance(recorder, str): - recorder = self.exp.get_recorder(recorder_id=recorder) - recorder.set_tags(**{ModelUpdater.ONLINE_TAG: ModelUpdater.ONLINE_TAG_TRUE}) + recorder : Recorder + the instance of the Recorder + start_time : Timestamp + the start time you want to load + end_time : Timestamp + the end time you want to load - def cancel_online_model(self, recorder: Union[str, Recorder]): - if isinstance(recorder, str): - recorder = self.exp.get_recorder(recorder_id=recorder) - recorder.set_tags(**{ModelUpdater.ONLINE_TAG: ModelUpdater.ONLINE_TAG_FALSE}) - - def cancel_all_online_model(self): - recs = self.exp.list_recorders() - for rid, rec in recs.items(): - self.cancel_online_model(rec) + Returns + ------- + Dataset + the instance of Dataset + """ + segments = {"test": (start_time, end_time)} - def reset_online_model(self, recorders: List[Union[str, Recorder]]): - """cancel all online model and reset the given model to online model + dataset = recorder.load_object("dataset") + datahandler = recorder.load_object("datahandler") - Parameters - ---------- - recorders: List[Union[str,Recorder]] - the list of the id of a Recorder or the Recorder instance - """ - self.cancel_all_online_model() - for rec_or_rid in recorders: - self.set_online_model(rec_or_rid) + datahandler.conf_data(**{"start_time": start_time, "end_time": end_time}) + dataset.setup_data(handler=datahandler, segments=segments) + datahandler.init(datahandler.IT_LS) + return dataset def update_pred(self, recorder: Union[str, Recorder]): """update predictions to the latest day in Calendar based on rid @@ -75,10 +62,9 @@ def update_pred(self, recorder: Union[str, Recorder]): the id of a Recorder or the Recorder instance """ if isinstance(recorder, str): - recorder = self.exp.get_recorder(recorder_id=recorder) + recorder = self.tc.get_recorder_by_id(recorder_id=recorder) old_pred = recorder.load_object("pred.pkl") last_end = old_pred.index.get_level_values("datetime").max() - task_config = recorder.load_object("task") # recorder.task # updated to the latest trading day cal = D.calendar(start_time=last_end + pd.Timedelta(days=1), end_time=None) @@ -90,10 +76,8 @@ def update_pred(self, recorder: Union[str, Recorder]): return start_time, end_time = cal[0], cal[-1] - task_config["dataset"]["kwargs"]["segments"]["test"] = (start_time, end_time) - task_config["dataset"]["kwargs"]["handler"]["kwargs"]["end_time"] = end_time - dataset = init_instance_by_config(task_config["dataset"]) + dataset = self._reload_dataset(recorder, start_time, end_time) model = recorder.load_object("params.pkl") new_pred = model.predict(dataset) @@ -131,29 +115,7 @@ def rec_filter_func(recorder): the count of updated record """ - recs = self.list_recorders(rec_filter_func=rec_filter_func, only_have_task=True) + recs = self.tc.list_recorders(rec_filter_func=rec_filter_func) for rid, rec in recs.items(): self.update_pred(rec) return len(recs) - - def online_filter(self, recorder): - tags = recorder.list_tags() - if tags.get(ModelUpdater.ONLINE_TAG, ModelUpdater.ONLINE_TAG_FALSE) == ModelUpdater.ONLINE_TAG_TRUE: - return True - return False - - def update_online_pred(self): - """update all online model predictions to the latest day in Calendar.""" - cnt = self.update_all_pred(self.online_filter) - self.logger.info(f"Finish updating {cnt} online model predictions of {self.exp_name}.") - - def list_online_model(self): - """list the record of online model - - Returns - ------- - dict - {rid : recorder of the online model} - """ - - return self.list_recorders(rec_filter_func=self.online_filter) From 5953365af30972c82b54cda1fe142f53a47979e5 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Tue, 16 Mar 2021 02:43:12 +0000 Subject: [PATCH 24/61] finished update_online_pred demo --- examples/taskmanager/update_online_pred.py | 7 ++-- qlib/model/trainer.py | 2 +- qlib/workflow/task/online.py | 46 +++++++--------------- 3 files changed, 19 insertions(+), 36 deletions(-) diff --git a/examples/taskmanager/update_online_pred.py b/examples/taskmanager/update_online_pred.py index 016336c684..a24b388894 100644 --- a/examples/taskmanager/update_online_pred.py +++ b/examples/taskmanager/update_online_pred.py @@ -1,6 +1,6 @@ import qlib from qlib.model.trainer import task_train -from qlib.workflow.task.online import RollingOnlineManager +from qlib.workflow.task.online import OnlineManager from qlib.config import REG_CN import fire from qlib.workflow import R @@ -54,16 +54,15 @@ def first_train(experiment_name="online_svr"): - rom = RollingOnlineManager(experiment_name) - rid = task_train(task_config=task, experiment_name=experiment_name) + rom = OnlineManager(experiment_name) rom.reset_online_model(rid) def update_online_pred(experiment_name="online_svr"): - rom = RollingOnlineManager(experiment_name) + rom = OnlineManager(experiment_name) print("Here are the online models waiting for update:") for rid, rec in rom.list_online_model().items(): diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index b6d4de6e24..5c5609eb0e 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -27,7 +27,7 @@ def task_train(task_config: dict, experiment_name: str) -> str: model = init_instance_by_config(task_config["model"]) dataset = init_instance_by_config(task_config["dataset"]) datahandler = dataset.handler - + # start exp with R.start(experiment_name=experiment_name): diff --git a/qlib/workflow/task/online.py b/qlib/workflow/task/online.py index 72d3491220..f2b8e57069 100644 --- a/qlib/workflow/task/online.py +++ b/qlib/workflow/task/online.py @@ -7,21 +7,7 @@ from qlib.workflow.task.update import ModelUpdater -class OnlineManagement: - def __init__(self, experiment_name): - pass - - def update_online_pred(self, recorder: Union[str, Recorder]): - """update the predictions of online models - - Parameters - ---------- - recorder : Union[str, Recorder] - the id or the instance of Recorder - - """ - raise NotImplementedError(f"Please implement the `update_pred` method.") - +class OnlineManager: def prepare_new_models(self, tasks: List[dict]): """prepare(train) new models @@ -33,20 +19,6 @@ def prepare_new_models(self, tasks: List[dict]): """ raise NotImplementedError(f"Please implement the `prepare_new_models` method.") - def reset_online_model(self, recorders: List[Union[str, Recorder]]): - """reset online model - - Parameters - ---------- - recorders : List[Union[str, Recorder]] - a list of the recorder id or the instance - - """ - raise NotImplementedError(f"Please implement the `reset_online_model` method.") - - -class RollingOnlineManager(OnlineManagement): - ONLINE_TAG = "online_model" ONLINE_TAG_TRUE = "True" ONLINE_TAG_FALSE = "False" @@ -59,8 +31,7 @@ def __init__(self, experiment_name: str) -> None: experiment_name : str experiment name string """ - super(RollingOnlineManager, self).__init__(experiment_name) - self.logger = get_module_logger("RollingOnlineManager") + self.logger = get_module_logger("OnlineManagement") self.exp_name = experiment_name self.tc = TaskCollector(experiment_name) @@ -122,3 +93,16 @@ def update_online_pred(self): mu = ModelUpdater(self.exp_name) cnt = mu.update_all_pred(self.online_filter) self.logger.info(f"Finish updating {cnt} online model predictions of {self.exp_name}.") + + +class RollingOnlineManager(OnlineManager): + def prepare_new_models(self, tasks: List[dict]): + """prepare(train) new models + + Parameters + ---------- + tasks : List[dict] + a list of tasks + + """ + pass From d33041dc24abd53e28023095166b0ac3cc984fba Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Tue, 16 Mar 2021 02:52:20 +0000 Subject: [PATCH 25/61] format example --- examples/taskmanager/update_online_pred.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/taskmanager/update_online_pred.py b/examples/taskmanager/update_online_pred.py index a24b388894..5ce963fbc6 100644 --- a/examples/taskmanager/update_online_pred.py +++ b/examples/taskmanager/update_online_pred.py @@ -55,7 +55,7 @@ def first_train(experiment_name="online_svr"): rid = task_train(task_config=task, experiment_name=experiment_name) - + rom = OnlineManager(experiment_name) rom.reset_online_model(rid) From 8abdd63869c9eb329e78e72eef0850ac147b83e7 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Thu, 18 Mar 2021 09:30:01 +0000 Subject: [PATCH 26/61] online_serving V3 --- docs/start/initialization.rst | 11 ++ examples/taskmanager/task_manager_rolling.py | 102 ++++++++---- .../task_manager_rolling_with_updating.py | 156 ++++++++---------- qlib/model/trainer.py | 2 +- qlib/workflow/task/collect.py | 127 +++++++------- qlib/workflow/task/gen.py | 72 ++++---- qlib/workflow/task/manage.py | 5 + qlib/workflow/task/online.py | 129 ++++++++++----- qlib/workflow/task/update.py | 4 +- 9 files changed, 334 insertions(+), 274 deletions(-) diff --git a/docs/start/initialization.rst b/docs/start/initialization.rst index 15aa957d1a..95ab7f77d8 100644 --- a/docs/start/initialization.rst +++ b/docs/start/initialization.rst @@ -75,3 +75,14 @@ Besides `provider_uri` and `region`, `qlib.init` has other parameters. The follo "default_exp_name": "Experiment", } }) +- `mongo` + Type: dict, optional parameter, the setting of `MongoDB `_ which will be used in some features such as `Task Management <../advanced/task_management.html>`_, with high performance and clustered processing. + Users need finished `installatin `_ firstly, and run it in a fixed URL. + + .. code-block:: Python + + # For example, you can initialize qlib below + qlib.init(provider_uri=provider_uri, region=REG_CN, mongo={ + "task_url": "mongodb://localhost:27017/", # your mongo url + "task_db_name": "rolling_db", # the database name of Task Management + }) diff --git a/examples/taskmanager/task_manager_rolling.py b/examples/taskmanager/task_manager_rolling.py index 9223ed8189..ffa88d75ec 100644 --- a/examples/taskmanager/task_manager_rolling.py +++ b/examples/taskmanager/task_manager_rolling.py @@ -3,6 +3,11 @@ from qlib.workflow.task.gen import RollingGen, task_generator from qlib.workflow.task.manage import TaskManager from qlib.config import C +from qlib.workflow.task.manage import run_task +from qlib.workflow.task.collect import RollingCollector +from qlib.model.trainer import task_train +from qlib.workflow import R +from pprint import pprint data_handler_config = { "start_time": "2008-01-01", @@ -60,51 +65,78 @@ "record": record_config, } -provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir -qlib.init(provider_uri=provider_uri, region=REG_CN) +# Reset all things to the first status, be careful to save important data +def reset(): + print("========== reset ==========") + TaskManager(task_pool=task_pool).remove() -C["mongo"] = { - "task_url": "mongodb://localhost:27017/", # maybe you need to change it to your url - "task_db_name": "rolling_db", -} + # exp = R.get_exp(experiment_name=exp_name) -exp_name = "rolling_exp" # experiment name, will be used as the experiment in MLflow -task_pool = "rolling_task" # task pool name, will be used as the document in MongoDB + # for rid in R.list_recorders(): + # exp.delete_recorder(rid) -tasks = task_generator( - task_xgboost_config, # default task name - RollingGen(step=550, rtype=RollingGen.ROLL_SD), # generate different date segment - task_lgb=task_lgb_config, # use "task_lgb" as the task name -) -# Uncomment next two lines to see the generated tasks -# from pprint import pprint -# pprint(tasks) +# This part corresponds to "Task Generating" in the document +def task_generating(): -tm = TaskManager(task_pool=task_pool) -tm.create_task(tasks) # all tasks will be saved to MongoDB + print("========== task_generating ==========") -from qlib.workflow.task.manage import run_task -from qlib.workflow.task.collect import TaskCollector -from qlib.model.trainer import task_train + tasks = task_generator( + tasks=[task_xgboost_config, task_lgb_config], + generators=RollingGen(step=550, rtype=RollingGen.ROLL_SD), # generate different date segment + ) + + pprint(tasks) + + return tasks + + +# This part corresponds to "Task Storing" in the document +def task_storing(tasks): + print("========== task_storing ==========") + tm = TaskManager(task_pool=task_pool) + tm.create_task(tasks) # all tasks will be saved to MongoDB + + +# This part corresponds to "Task Running" in the document +def task_running(): + print("========== task_running ==========") + run_task(task_train, task_pool, experiment_name=exp_name) # all tasks will be trained using "task_train" method + + +# This part corresponds to "Task Collecting" in the document +def task_collecting(): + print("========== task_collecting ==========") -run_task(task_train, task_pool, experiment_name=exp_name) # all tasks will be trained using "task_train" method + def get_task_key(task_config): + return task_config["model"]["class"] + def my_filter(recorder): + # only choose the results of "LGBModel" + task_key = get_task_key(rolling_collector.get_task(recorder)) + if task_key == "LGBModel": + return True + return False -def get_task_key(task_config): - task_key = task_config["task_key"] - rolling_end_timestamp = task_config["dataset"]["kwargs"]["segments"]["test"][1] - return task_key, rolling_end_timestamp.strftime("%Y-%m-%d") + rolling_collector = RollingCollector(exp_name) + # group tasks by "get_task_key" and filter tasks by "my_filter" + pred_rolling = rolling_collector.collect_rolling_predictions(get_task_key, my_filter) + print(pred_rolling) -def my_filter(task_config): - # only choose the results of "task_lgb" and test in 2019 from all tasks - task_key, rolling_end = get_task_key(task_config) - if task_key == "task_lgb" and rolling_end.startswith("2019"): - return True - return False +if __name__ == "__main__": + provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir + mongo_conf = { + "task_url": "mongodb://10.0.0.4:27017/", # maybe you need to change it to your url + "task_db_name": "rolling_db", + } + exp_name = "rolling_exp" # experiment name, will be used as the experiment in MLflow + task_pool = "rolling_task" # task pool name, will be used as the document in MongoDB + qlib.init(provider_uri=provider_uri, region=REG_CN, mongo=mongo_conf) -# name tasks by "get_task_key" and filter tasks by "my_filter" -pred_rolling = TaskCollector.collect_predictions(exp_name, get_task_key, my_filter) -pred_rolling + reset() + tasks = task_generating() + task_storing(tasks) + task_running() + task_collecting() diff --git a/examples/taskmanager/task_manager_rolling_with_updating.py b/examples/taskmanager/task_manager_rolling_with_updating.py index c69b558bc1..27e3ad2694 100644 --- a/examples/taskmanager/task_manager_rolling_with_updating.py +++ b/examples/taskmanager/task_manager_rolling_with_updating.py @@ -3,15 +3,14 @@ import mlflow from qlib.config import C from qlib.workflow import R +from pprint import pprint from qlib.config import REG_CN from qlib.model.trainer import task_train from qlib.workflow.task.manage import run_task from qlib.workflow.task.manage import TaskManager -from qlib.workflow.task.utils import TimeAdjuster -from qlib.workflow.task.update import ModelUpdater -from qlib.workflow.task.collect import TaskCollector +from qlib.workflow.task.collect import RollingCollector from qlib.workflow.task.gen import RollingGen, task_generator - +from qlib.workflow.task.online import RollingOnlineManager data_handler_config = { "start_time": "2013-01-01", @@ -33,7 +32,7 @@ "segments": { "train": ("2013-01-01", "2014-12-31"), "valid": ("2015-01-01", "2015-12-31"), - "test": ("2016-01-01", "2017-01-01"), + "test": ("2016-01-01", "2020-07-10"), }, }, } @@ -69,16 +68,25 @@ "record": record_config, } -# This part corresponds to "Task Generating" in the document -def task_generating(**kwargs): - print("========================================= task_generating =========================================") - rolling_generator = RollingGen(step=rolling_step, rtype=RollingGen.ROLL_EX) +def print_online_model(): + print("Current 'online' model:") + for online in rolling_online_manager.list_online_model().values(): + print(online.info["id"]) + print("Current 'next online' model:") + for online in rolling_online_manager.list_next_online_model().values(): + print(online.info["id"]) + - tasks = task_generator(rolling_generator, **kwargs) +# This part corresponds to "Task Generating" in the document +def task_generating(): + + print("========== task_generating ==========") - # See the generated tasks in a easy way - from pprint import pprint + tasks = task_generator( + tasks=[task_xgboost_config, task_lgb_config], + generators=rolling_gen, # generate different date segment + ) pprint(tasks) @@ -87,49 +95,45 @@ def task_generating(**kwargs): # This part corresponds to "Task Storing" in the document def task_storing(tasks): - print("========================================= task_storing =========================================") + print("========== task_storing ==========") tm = TaskManager(task_pool=task_pool) tm.create_task(tasks) # all tasks will be saved to MongoDB # This part corresponds to "Task Running" in the document def task_running(): - print("========================================= task_running =========================================") + print("========== task_running ==========") run_task(task_train, task_pool, experiment_name=exp_name) # all tasks will be trained using "task_train" method # This part corresponds to "Task Collecting" in the document def task_collecting(): - print("========================================= task_collecting =========================================") + print("========== task_collecting ==========") def get_task_key(task_config): - task_key = task_config["task_key"] - rolling_end_timestamp = task_config["dataset"]["kwargs"]["segments"]["test"][1] - if rolling_end_timestamp == None: - rolling_end_timestamp = TimeAdjuster().last_date() - return task_key, rolling_end_timestamp.strftime("%Y-%m-%d") - - def lgb_filter(task_config): - # only choose the results of "task_lgb" - task_key, rolling_end = get_task_key(task_config) - if task_key == "task_lgb": + return task_config["model"]["class"] + + def my_filter(recorder): + # only choose the results of "LGBModel" + task_key = get_task_key(rolling_collector.get_task(recorder)) + if task_key == "LGBModel": return True return False - task_collector = TaskCollector(exp_name) - pred_rolling = task_collector.collect_predictions( - get_task_key, lgb_filter - ) # name tasks by "get_task_key" and filter tasks by "my_filter" + rolling_collector = RollingCollector(exp_name) + # group tasks by "get_task_key" and filter tasks by "my_filter" + pred_rolling = rolling_collector.collect_rolling_predictions(get_task_key, my_filter) print(pred_rolling) # Reset all things to the first status, be careful to save important data def reset(force_end=False): - print("========================================= reset =========================================") - TaskManager(task_pool=task_pool).remove() - + print("========== reset ==========") + task_manager.remove() + for error in task_manager.query(): + assert False exp = R.get_exp(experiment_name=exp_name) - recs = TaskCollector(exp_name).list_recorders(only_finished=True) + recs = exp.list_recorders() for rid in recs: exp.delete_recorder(rid) @@ -141,82 +145,60 @@ def reset(force_end=False): pass -def set_online_model_to_latest(): - print( - "========================================= set_online_model_to_latest =========================================" - ) - model_updater = ModelUpdater(experiment_name=exp_name) - latest_records, latest_test = model_updater.collect_latest_records() - model_updater.reset_online_model(latest_records.values()) - - # Run this firstly to see the workflow in Task Management def first_run(): - print("========================================= first_run =========================================") + print("========== first_run ==========") reset(force_end=True) - # use "task_lgb" and "task_xgboost" as the task name - tasks = task_generating(**{"task_xgboost": task_xgboost_config, "task_lgb": task_lgb_config}) + tasks = task_generating() task_storing(tasks) task_running() task_collecting() - set_online_model_to_latest() + + rolling_online_manager.set_latest_model_to_next_online() + rolling_online_manager.reset_online_model() # Update the predictions of online model def update_predictions(): - print("========================================= update_predictions =========================================") - model_updater = ModelUpdater(experiment_name=exp_name) - model_updater.update_online_pred() + print("========== update_predictions ==========") + rolling_online_manager.update_online_pred() + task_collecting() + # if there are some next_online_model, then online them. if no, still use current online_model. + print_online_model() + rolling_online_manager.reset_online_model() + print_online_model() # Update the models using the latest date and set them to online model def update_model(): - print("========================================= update_model =========================================") - # get the latest recorders - model_updater = ModelUpdater(experiment_name=exp_name) - latest_records, latest_test = model_updater.collect_latest_records() - # date adjustment based on trade day of Calendar in Qlib - time_adjuster = TimeAdjuster() - calendar_latest = time_adjuster.last_date() - print("The latest date is ", calendar_latest) - if time_adjuster.cal_interval(calendar_latest, latest_test[0]) > rolling_step: - print("Need update models!") - tasks = {} - for rid, rec in latest_records.items(): - old_task = rec.task - test_begin = old_task["dataset"]["kwargs"]["segments"]["test"][0] - # modify the test segment to generate new tasks - old_task["dataset"]["kwargs"]["segments"]["test"] = (test_begin, calendar_latest) - tasks[old_task["task_key"]] = old_task - - # retrain the latest model - new_tasks = task_generating(**tasks) - task_storing(new_tasks) - task_running() - task_collecting() - latest_records, _ = model_updater.collect_latest_records() - - # set the latest model to online model - model_updater.reset_online_model(latest_records.values()) + print("========== update_model ==========") + rolling_online_manager.prepare_new_models() + print_online_model() + rolling_online_manager.set_latest_model_to_next_online() + print_online_model() + + +def after_day(): + rolling_online_manager.prepare_signals() + update_model() + update_predictions() # Run whole workflow completely def whole_workflow(): - print("========================================= whole_workflow =========================================") + print("========== whole_workflow ==========") # run this at the first time first_run() - # run this every day - update_predictions() - # run this every "rolling_steps" day - update_model() + # run this every day after trading + after_day() if __name__ == "__main__": ####### to train the first version's models, use the command below # python task_manager_rolling_with_updating.py first_run - ####### to update the models using the latest date and set them to online model, use the command below + ####### to update the models using the latest date, use the command below # python task_manager_rolling_with_updating.py update_model ####### to update the predictions to the latest date, use the command below @@ -231,8 +213,8 @@ def whole_workflow(): qlib.init(provider_uri=provider_uri, region=REG_CN) C["mongo"] = { - "task_url": "mongodb://localhost:27017/", # your MongoDB url - "task_db_name": "rolling_db", # database name + "task_url": "mongodb://10.0.0.4:27017/", # your MongoDB url + "task_db_name": "online", # database name } exp_name = "rolling_exp" # experiment name, will be used as the experiment in MLflow @@ -240,5 +222,9 @@ def whole_workflow(): rolling_step = 550 ########################################################################################## - + rolling_gen = RollingGen(step=550, rtype=RollingGen.ROLL_SD) + rolling_online_manager = RollingOnlineManager( + experiment_name=exp_name, rolling_gen=rolling_gen, task_pool=task_pool + ) + task_manager = TaskManager(task_pool=task_pool) fire.Fire() diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index 5c5609eb0e..c181450733 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -60,4 +60,4 @@ def task_train(task_config: dict, experiment_name: str) -> str: ar = init_instance_by_config(record) ar.generate() - return recorder.info["id"] + return recorder diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index 21639e7f8e..fb7ff0b0b5 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -8,7 +8,7 @@ class TaskCollector: """ - Collect the record results of the finished tasks with key and filter + Collect the record (or its results) of the tasks """ def __init__(self, experiment_name: str) -> None: @@ -17,7 +17,7 @@ def __init__(self, experiment_name: str) -> None: self.logger = get_module_logger("TaskCollector") def list_recorders(self, rec_filter_func=None): - """""" + recs = self.exp.list_recorders() recs_flt = {} for rid, rec in recs.items(): @@ -26,57 +26,77 @@ def list_recorders(self, rec_filter_func=None): return recs_flt + def list_recorders_by_task(self, task_filter_func=None): + def rec_filter(recorder): + return task_filter_func(self.get_task(recorder)) + + return self.list_recorders(rec_filter) + + def list_latest_recorders(self, rec_filter_func=None): + recs_flt = self.list_recorders(rec_filter_func) + max_test = self.latest_time(recs_flt) + latest_rec = {} + for rid, rec in recs_flt.items(): + if self.get_task(rec)["dataset"]["kwargs"]["segments"]["test"] == max_test: + latest_rec[rid] = rec + return latest_rec + def get_recorder_by_id(self, recorder_id): return self.exp.get_recorder(recorder_id, create=False) - def list_recorders_by_task(self, task_filter_func): - """[summary] + def get_task(self, recorder): + if isinstance(recorder, str): + recorder = self.get_recorder_by_id(recorder_id=recorder) + try: + task = recorder.load_object("task") + except OSError: + raise OSError(f"Can't find task in {recorder.info['id']}, have you trained with model.trainer.task_train?") + return task - Parameters - ---------- - task_filter_func : [type], optional - [description], by default None - """ + def latest_time(self, recorders): + if len(recorders) == 0: + raise Exception(f"Can't find any recorder in {self.exp_name}") + max_test = max(self.get_task(rec)["dataset"]["kwargs"]["segments"]["test"] for rec in recorders.values()) + return max_test - def rec_filter_func(recorder): - try: - task = recorder.load_object("task") - except OSError: - raise OSError( - f"Can't find task in {recorder.info['id']}, have you trained with model.trainer.task_train?" - ) - return task_filter_func(task) - return self.list_recorders(rec_filter_func) +class RollingCollector(TaskCollector): + """ + Collect the record results of the rolling tasks + """ - def collect_predictions( + def __init__( self, - get_key_func, - task_filter_func=None, - ): - """ - Collect predictions using a filter and a key function. + experiment_name: str, + ) -> None: + super().__init__(experiment_name) + self.logger = get_module_logger("RollingCollector") + + def collect_rolling_predictions(self, get_key_func, rec_filter_func=None): + """For rolling tasks, the predictions will be in the diffierent recorder. + To collect and concat the predictions of one rolling task, get_key_func will help this method see which group a recorder will be. Parameters ---------- - experiment_name : str - get_key_func : Callable[[dict], bool] -> Union[Number, str, tuple] - get the key of a task when collect it - filter_func : Callable[[dict], bool] -> bool - to judge a task will be collected or not + get_key_func : Callable[dict,str] + a function that get task config and return its group str + rec_filter_func : Callable[Recorder,bool], optional + a function that decide whether filter a recorder, by default None Returns ------- dict - the dict of predictions + a dict of {group: predictions} """ - recs_flt = self.list_recorders(task_filter_func=task_filter_func, only_have_task=True) + + # filter records + recs_flt = self.list_recorders(rec_filter_func) # group recs_group = {} for _, rec in recs_flt.items(): - params = rec.task - group_key = get_key_func(params) + task = self.get_task(rec) + group_key = get_key_func(task) recs_group.setdefault(group_key, []).append(rec) # reduce group @@ -85,39 +105,12 @@ def collect_predictions( pred_l = [] for rec in rec_l: pred_l.append(rec.load_object("pred.pkl").iloc[:, 0]) - pred = pd.concat(pred_l).sort_index() + # Make sure the pred are sorted according to the rolling start time + pred_l.sort(key=lambda pred: pred.index.get_level_values("datetime").min()) + pred = pd.concat(pred_l) + # If there are duplicated predition, we use the latest perdiction + pred = pred[~pred.index.duplicated(keep="last")] + pred = pred.sort_index() reduce_group[k] = pred - self.logger.info(f"Collect {len(reduce_group)} predictions in {self.exp_name}") - return reduce_group - - def collect_latest_records( - self, - task_filter_func=None, - ): - """Collect latest recorders using a filter. - - Parameters - ---------- - task_filter_func : Callable[[dict], bool], optional - to judge a task will be collected or not, by default None - - Returns - ------- - dict, tuple - a dict of recorders and a tuple of test segments - """ - recs_flt = self.list_recorders(task_filter_func=task_filter_func, only_have_task=True) - - if len(recs_flt) == 0: - self.logger.warning("Can not collect any recorders...") - return None, None - max_test = max(rec.task["dataset"]["kwargs"]["segments"]["test"] for rec in recs_flt.values()) - - latest_record = {} - for rid, rec in recs_flt.items(): - if rec.task["dataset"]["kwargs"]["segments"]["test"] == max_test: - latest_record[rid] = rec - - self.logger.info(f"Collect {len(latest_record)} latest records in {self.exp_name}") - return latest_record, max_test + return reduce_group \ No newline at end of file diff --git a/qlib/workflow/task/gen.py b/qlib/workflow/task/gen.py index 96448cefe2..63000d77d2 100644 --- a/qlib/workflow/task/gen.py +++ b/qlib/workflow/task/gen.py @@ -9,56 +9,40 @@ from .utils import TimeAdjuster -def task_generator(*args, **kwargs) -> list: - """ - Accept the dict of task config and the TaskGen to generate different tasks. - There is no limit to the number and position of input. - The key of input will add to task config. +def task_generator(tasks, generators) -> list: + """Use a list of TaskGen and a list of task templates to generate different tasks. + + For examples: - for example: - There are 3 task_config(a,b,c) and 2 TaskGen(A,B). A will double the task_config and B will triple. - task_generator(a_key=a, b_key=b, c_key=c, A, B) will finally generate 3*2*3 = 18 task_config. + There are 3 task templates a,b,c and 2 TaskGen A,B. A will generates 2 tasks from a template and B will generates 3 tasks from a template. + task_generator([a, b, c], [A, B]) will finally generate 3*2*3 = 18 tasks. Parameters ---------- - args : dict or TaskGen - kwargs : dict or TaskGen + tasks : List[dict] + a list of task templates + generators : List[TaskGen] + a list of TaskGen Returns ------- - gen_task_list : list - a list of task config after generating + list + a list of tasks """ - tasks_list = [] - gen_list = [] - - tmp_id = 1 - for task in args: - if isinstance(task, dict): - task["task_key"] = tmp_id - tmp_id += 1 - tasks_list.append(task) - elif isinstance(task, TaskGen): - gen_list.append(task) - else: - raise NotImplementedError(f"{type(task)} is not supported in task_generator") - - for key, task in kwargs.items(): - if isinstance(task, dict): - task["task_key"] = key - tasks_list.append(task) - elif isinstance(task, TaskGen): - gen_list.append(task) - else: - raise NotImplementedError(f"{type(task)} is not supported in task_generator") + + if isinstance(tasks, dict): + tasks = [tasks] + if isinstance(generators, TaskGen): + generators = [generators] # generate gen_task_list gen_task_list = [] - for gen in gen_list: + for gen in generators: new_task_list = [] - for task in tasks_list: + for task in tasks: new_task_list.extend(gen.generate(task)) gen_task_list = new_task_list + return gen_task_list @@ -144,7 +128,13 @@ def generate(self, task: dict): "handler": { "class": "Alpha158", "module_path": "qlib.contrib.data.handler", - "kwargs": data_handler_config, + "kwargs": { + "start_time": "2008-01-01", + "end_time": "2020-08-01", + "fit_start_time": "2008-01-01", + "fit_end_time": "2014-12-31", + "instruments": "csi100", + }, }, "segments": { "train": ("2008-01-01", "2014-12-31"), @@ -153,8 +143,12 @@ def generate(self, task: dict): }, }, }, - # You shoud record the data in specific sequence - # "record": ['SignalRecord', 'SigAnaRecord', 'PortAnaRecord'], + "record": [ + { + "class": "SignalRecord", + "module_path": "qlib.workflow.record_temp", + }, + ] } """ res = [] diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index e97fdb7741..db4c150383 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -245,6 +245,11 @@ def query(self, query={}, decode=True, task_pool=None): for t in task_pool.find(query): yield self._decode_task(t) + def get_task_result(self, task, task_pool=None): + task_pool = self._get_task_pool(task_pool) + result = task_pool.find_one({"filter": task}) + return self._decode_task(result)["res"] + def commit_task_res(self, task, res, status=None, task_pool=None): task_pool = self._get_task_pool(task_pool) # A workaround to use the class attribute. diff --git a/qlib/workflow/task/online.py b/qlib/workflow/task/online.py index f2b8e57069..8d551e858d 100644 --- a/qlib/workflow/task/online.py +++ b/qlib/workflow/task/online.py @@ -1,10 +1,14 @@ -from typing import Union, List +from typing import Dict, Union, List from qlib import get_module_logger from qlib.workflow import R from qlib.model.trainer import task_train -from qlib.workflow.recorder import Recorder +from qlib.workflow.recorder import MLflowRecorder, Recorder from qlib.workflow.task.collect import TaskCollector from qlib.workflow.task.update import ModelUpdater +from qlib.workflow.task.utils import TimeAdjuster +from qlib.workflow.task.gen import RollingGen, task_generator +from qlib.workflow.task.manage import TaskManager +from qlib.workflow.task.manage import run_task class OnlineManager: @@ -19,9 +23,10 @@ def prepare_new_models(self, tasks: List[dict]): """ raise NotImplementedError(f"Please implement the `prepare_new_models` method.") - ONLINE_TAG = "online_model" - ONLINE_TAG_TRUE = "True" - ONLINE_TAG_FALSE = "False" + ONLINE_KEY = "online_status" # the tag key in recorder + ONLINE_TAG = "online" # the 'online' model + NEXT_ONLINE_TAG = "next_online" # the 'next online' model, which can be 'online' model when call reset_online_model + OFFLINE_TAG = "offline" # the 'offline' model, not for online serving def __init__(self, experiment_name: str) -> None: """ModelUpdater needs experiment name to find the records @@ -35,45 +40,57 @@ def __init__(self, experiment_name: str) -> None: self.exp_name = experiment_name self.tc = TaskCollector(experiment_name) - def set_online_model(self, recorder: Union[str, Recorder]): - """online model will be identified at the tags of the record + def set_next_online_model(self, recorder: MLflowRecorder): + recorder.set_tags(**{self.ONLINE_KEY: self.NEXT_ONLINE_TAG}) - Parameters - ---------- - recorder: Union[str,Recorder] - the id of a Recorder or the Recorder instance - """ - if isinstance(recorder, str): - recorder = self.tc.get_recorder_by_id(recorder_id=recorder) - recorder.set_tags(**{self.ONLINE_TAG: self.ONLINE_TAG_TRUE}) + def set_online_model(self, recorder: MLflowRecorder): + """online model will be identified at the tags of the record""" + recorder.set_tags(**{self.ONLINE_KEY: self.ONLINE_TAG}) - def cancel_online_model(self, recorder: Union[str, Recorder]): - if isinstance(recorder, str): - recorder = self.tc.get_recorder_by_id(recorder_id=recorder) - recorder.set_tags(**{self.ONLINE_TAG: self.ONLINE_TAG_FALSE}) + def set_offline_model(self, recorder: MLflowRecorder): + recorder.set_tags(**{self.ONLINE_KEY: self.OFFLINE_TAG}) - def cancel_all_online_model(self): + def offline_all_model(self): recs = self.tc.list_recorders() for rid, rec in recs.items(): - self.cancel_online_model(rec) + self.set_offline_model(rec) - def reset_online_model(self, recorders: Union[str, List[Union[str, Recorder]]]): - """cancel all online model and reset the given model to online model + def reset_online_model(self, recorders: Union[List, Dict] = None): + """offline all models and set the recorders to 'online'. If no parameter and no 'next online' model, then do nothing. - Parameters - ---------- - recorders: List[Union[str,Recorder]] - the list of the id of a Recorder or the Recorder instance + Args: + recorders (Union[List, Dict], optional): + the recorders you want to reset to 'online'. If don't give, set 'next online' model to 'online' model. If there isn't any 'next online' model, then maintain existing 'online' model. """ - self.cancel_all_online_model() - if isinstance(recorders, str): - recorders = [recorders] - for rec_or_rid in recorders: - self.set_online_model(rec_or_rid) + if recorders is None: + recorders = self.list_next_online_model() + if len(recorders) == 0: + self.logger.info("No 'next online' model, just use current 'online' models.") + return + self.offline_all_model() + if isinstance(recorders, dict): + recorders = recorders.values() + for rec in recorders: + self.set_online_model(rec) + self.logger.info(f"Reset {len(recorders)} models to 'online'.") + + def set_latest_model_to_next_online(self): + latest_rec = self.tc.list_latest_recorders() + for rid, rec in latest_rec.items(): + self.set_next_online_model(rec) + self.logger.info(f"Set {len(latest_rec)} latest models to 'next online'.") + + @staticmethod + def online_filter(recorder): + tags = recorder.list_tags() + if tags.get(OnlineManager.ONLINE_KEY, OnlineManager.OFFLINE_TAG) == OnlineManager.ONLINE_TAG: + return True + return False - def online_filter(self, recorder): + @staticmethod + def next_online_filter(recorder): tags = recorder.list_tags() - if tags.get(self.ONLINE_TAG, self.ONLINE_TAG_FALSE) == self.ONLINE_TAG_TRUE: + if tags.get(OnlineManager.ONLINE_KEY, OnlineManager.OFFLINE_TAG) == OnlineManager.NEXT_ONLINE_TAG: return True return False @@ -88,21 +105,45 @@ def list_online_model(self): return self.tc.list_recorders(rec_filter_func=self.online_filter) + def list_next_online_model(self): + return self.tc.list_recorders(rec_filter_func=self.next_online_filter) + def update_online_pred(self): - """update all online model predictions to the latest day in Calendar.""" + """update all online model predictions to the latest day in Calendar""" mu = ModelUpdater(self.exp_name) cnt = mu.update_all_pred(self.online_filter) self.logger.info(f"Finish updating {cnt} online model predictions of {self.exp_name}.") class RollingOnlineManager(OnlineManager): - def prepare_new_models(self, tasks: List[dict]): - """prepare(train) new models - - Parameters - ---------- - tasks : List[dict] - a list of tasks - - """ + def __init__(self, experiment_name: str, rolling_gen: RollingGen, task_pool) -> None: + super().__init__(experiment_name) + self.ta = TimeAdjuster() + self.rg = rolling_gen + self.tm = TaskManager(task_pool=task_pool) + self.logger = get_module_logger("RollingOnlineManager") + + def prepare_new_models(self): + """prepare(train) new models based on online model""" + latest_records = self.tc.list_latest_recorders(self.online_filter) # if we need online_filter here? + max_test = self.tc.latest_time(latest_records) + calendar_latest = self.ta.last_date() + if self.ta.cal_interval(calendar_latest, max_test[0]) > self.rg.step: + old_tasks = [] + for rid, rec in latest_records.items(): + task = self.tc.get_task(rec) + test_begin = task["dataset"]["kwargs"]["segments"]["test"][0] + # modify the test segment to generate new tasks + task["dataset"]["kwargs"]["segments"]["test"] = (test_begin, calendar_latest) + old_tasks.append(task) + new_tasks = task_generator(old_tasks, self.rg) + self.tm.create_task(new_tasks) + run_task(task_train, self.tm.task_pool, experiment_name=self.exp_name) + self.logger.info(f"Finished prepare {len(new_tasks)} new models.") + return new_tasks + self.logger.info("No need to prepare any new models.") + return [] + + def prepare_signals(self): + # prepare the signals of today pass diff --git a/qlib/workflow/task/update.py b/qlib/workflow/task/update.py index 9f68dbd0a2..fcee843490 100644 --- a/qlib/workflow/task/update.py +++ b/qlib/workflow/task/update.py @@ -53,7 +53,7 @@ def _reload_dataset(self, recorder, start_time, end_time): datahandler.init(datahandler.IT_LS) return dataset - def update_pred(self, recorder: Union[str, Recorder]): + def update_pred(self, recorder: Recorder): """update predictions to the latest day in Calendar based on rid Parameters @@ -61,8 +61,6 @@ def update_pred(self, recorder: Union[str, Recorder]): recorder: Union[str,Recorder] the id of a Recorder or the Recorder instance """ - if isinstance(recorder, str): - recorder = self.tc.get_recorder_by_id(recorder_id=recorder) old_pred = recorder.load_object("pred.pkl") last_end = old_pred.index.get_level_values("datetime").max() From 46cd57688e0c9229067fd028396ecc66cf40a0c0 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Fri, 26 Mar 2021 04:20:25 +0000 Subject: [PATCH 27/61] Online Serving V4 --- docs/start/initialization.rst | 2 +- .../task_manager_rolling.py | 61 +++-- .../task_manager_rolling_with_updating.py | 107 +++----- .../update_online_pred.py | 21 +- qlib/config.py | 7 +- qlib/model/trainer.py | 7 +- qlib/workflow/task/collect.py | 248 +++++++++++------- qlib/workflow/task/gen.py | 8 +- qlib/workflow/task/manage.py | 10 +- qlib/workflow/task/online.py | 183 +++++++------ qlib/workflow/task/update.py | 15 +- qlib/workflow/task/utils.py | 20 ++ 12 files changed, 366 insertions(+), 323 deletions(-) rename examples/{taskmanager => model_rolling}/task_manager_rolling.py (75%) rename examples/{taskmanager => online_svr}/task_manager_rolling_with_updating.py (63%) rename examples/{taskmanager => online_svr}/update_online_pred.py (78%) diff --git a/docs/start/initialization.rst b/docs/start/initialization.rst index 95ab7f77d8..32c17ff837 100644 --- a/docs/start/initialization.rst +++ b/docs/start/initialization.rst @@ -77,7 +77,7 @@ Besides `provider_uri` and `region`, `qlib.init` has other parameters. The follo }) - `mongo` Type: dict, optional parameter, the setting of `MongoDB `_ which will be used in some features such as `Task Management <../advanced/task_management.html>`_, with high performance and clustered processing. - Users need finished `installatin `_ firstly, and run it in a fixed URL. + Users need finished `installation `_ firstly, and run it in a fixed URL. .. code-block:: Python diff --git a/examples/taskmanager/task_manager_rolling.py b/examples/model_rolling/task_manager_rolling.py similarity index 75% rename from examples/taskmanager/task_manager_rolling.py rename to examples/model_rolling/task_manager_rolling.py index ffa88d75ec..70a4f7d7e7 100644 --- a/examples/taskmanager/task_manager_rolling.py +++ b/examples/model_rolling/task_manager_rolling.py @@ -1,13 +1,13 @@ +from pprint import pprint + +import fire import qlib from qlib.config import REG_CN -from qlib.workflow.task.gen import RollingGen, task_generator -from qlib.workflow.task.manage import TaskManager -from qlib.config import C -from qlib.workflow.task.manage import run_task -from qlib.workflow.task.collect import RollingCollector from qlib.model.trainer import task_train from qlib.workflow import R -from pprint import pprint +from qlib.workflow.task.collect import RollingCollector +from qlib.workflow.task.gen import RollingGen, task_generator +from qlib.workflow.task.manage import TaskManager, run_task data_handler_config = { "start_time": "2008-01-01", @@ -66,14 +66,14 @@ } # Reset all things to the first status, be careful to save important data -def reset(): +def reset(task_pool, exp_name): print("========== reset ==========") TaskManager(task_pool=task_pool).remove() - # exp = R.get_exp(experiment_name=exp_name) + exp, _ = R.exp_manager._get_or_create_exp(experiment_name=exp_name) - # for rid in R.list_recorders(): - # exp.delete_recorder(rid) + for rid in exp.list_recorders(): + exp.delete_recorder(rid) # This part corresponds to "Task Generating" in the document @@ -92,51 +92,58 @@ def task_generating(): # This part corresponds to "Task Storing" in the document -def task_storing(tasks): +def task_storing(tasks, task_pool, exp_name): print("========== task_storing ==========") tm = TaskManager(task_pool=task_pool) tm.create_task(tasks) # all tasks will be saved to MongoDB # This part corresponds to "Task Running" in the document -def task_running(): +def task_running(task_pool, exp_name): print("========== task_running ==========") run_task(task_train, task_pool, experiment_name=exp_name) # all tasks will be trained using "task_train" method # This part corresponds to "Task Collecting" in the document -def task_collecting(): +def task_collecting(task_pool, exp_name): print("========== task_collecting ==========") - def get_task_key(task_config): + def get_group_key_func(recorder): + task_config = recorder.load_object("task") return task_config["model"]["class"] def my_filter(recorder): # only choose the results of "LGBModel" - task_key = get_task_key(rolling_collector.get_task(recorder)) + task_key = get_group_key_func(recorder) if task_key == "LGBModel": return True return False rolling_collector = RollingCollector(exp_name) # group tasks by "get_task_key" and filter tasks by "my_filter" - pred_rolling = rolling_collector.collect_rolling_predictions(get_task_key, my_filter) + pred_rolling = rolling_collector.collect(get_group_key_func, my_filter) print(pred_rolling) -if __name__ == "__main__": - - provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir +def main( + provider_uri="~/.qlib/qlib_data/cn_data", + task_url="mongodb://10.0.0.4:27017/", + task_db_name="rolling_db", + exp_name="rolling_exp", + task_pool="rolling_task", +): mongo_conf = { - "task_url": "mongodb://10.0.0.4:27017/", # maybe you need to change it to your url - "task_db_name": "rolling_db", + "task_url": task_url, + "task_db_name": task_db_name, } - exp_name = "rolling_exp" # experiment name, will be used as the experiment in MLflow - task_pool = "rolling_task" # task pool name, will be used as the document in MongoDB qlib.init(provider_uri=provider_uri, region=REG_CN, mongo=mongo_conf) - reset() + reset(task_pool, exp_name) tasks = task_generating() - task_storing(tasks) - task_running() - task_collecting() + task_storing(tasks, task_pool, exp_name) + task_running(task_pool, exp_name) + task_collecting(task_pool, exp_name) + + +if __name__ == "__main__": + fire.Fire() diff --git a/examples/taskmanager/task_manager_rolling_with_updating.py b/examples/online_svr/task_manager_rolling_with_updating.py similarity index 63% rename from examples/taskmanager/task_manager_rolling_with_updating.py rename to examples/online_svr/task_manager_rolling_with_updating.py index 27e3ad2694..24bc38a025 100644 --- a/examples/taskmanager/task_manager_rolling_with_updating.py +++ b/examples/online_svr/task_manager_rolling_with_updating.py @@ -1,16 +1,15 @@ -import qlib -import fire -import mlflow -from qlib.config import C -from qlib.workflow import R from pprint import pprint + +import fire +import qlib from qlib.config import REG_CN from qlib.model.trainer import task_train -from qlib.workflow.task.manage import run_task -from qlib.workflow.task.manage import TaskManager +from qlib.workflow import R from qlib.workflow.task.collect import RollingCollector from qlib.workflow.task.gen import RollingGen, task_generator +from qlib.workflow.task.manage import TaskManager, run_task from qlib.workflow.task.online import RollingOnlineManager +from qlib.workflow.task.utils import list_recorders data_handler_config = { "start_time": "2013-01-01", @@ -70,12 +69,15 @@ def print_online_model(): + print("========== print_online_model ==========") print("Current 'online' model:") - for online in rolling_online_manager.list_online_model().values(): - print(online.info["id"]) + for rid, rec in list_recorders(exp_name).items(): + if rolling_online_manager.get_online_tag(rec) == rolling_online_manager.ONLINE_TAG: + print(rid) print("Current 'next online' model:") - for online in rolling_online_manager.list_next_online_model().values(): - print(online.info["id"]) + for rid, rec in list_recorders(exp_name).items(): + if rolling_online_manager.get_online_tag(rec) == rolling_online_manager.NEXT_ONLINE_TAG: + print(rid) # This part corresponds to "Task Generating" in the document @@ -110,119 +112,76 @@ def task_running(): def task_collecting(): print("========== task_collecting ==========") - def get_task_key(task_config): + def get_group_key_func(recorder): + task_config = recorder.load_object("task") return task_config["model"]["class"] def my_filter(recorder): # only choose the results of "LGBModel" - task_key = get_task_key(rolling_collector.get_task(recorder)) + task_key = get_group_key_func(recorder) if task_key == "LGBModel": return True return False rolling_collector = RollingCollector(exp_name) # group tasks by "get_task_key" and filter tasks by "my_filter" - pred_rolling = rolling_collector.collect_rolling_predictions(get_task_key, my_filter) + pred_rolling = rolling_collector.collect(get_group_key_func, my_filter) print(pred_rolling) # Reset all things to the first status, be careful to save important data -def reset(force_end=False): +def reset(): print("========== reset ==========") task_manager.remove() - for error in task_manager.query(): - assert False - exp = R.get_exp(experiment_name=exp_name) - recs = exp.list_recorders() - - for rid in recs: + exp, _ = R.exp_manager._get_or_create_exp(experiment_name=exp_name) + for rid in exp.list_recorders(): exp.delete_recorder(rid) - try: - if force_end: - mlflow.end_run() - except Exception: - pass - # Run this firstly to see the workflow in Task Management def first_run(): print("========== first_run ==========") - reset(force_end=True) + reset() tasks = task_generating() task_storing(tasks) task_running() task_collecting() - rolling_online_manager.set_latest_model_to_next_online() - rolling_online_manager.reset_online_model() - - -# Update the predictions of online model -def update_predictions(): - print("========== update_predictions ==========") - rolling_online_manager.update_online_pred() - task_collecting() - # if there are some next_online_model, then online them. if no, still use current online_model. - print_online_model() - rolling_online_manager.reset_online_model() - print_online_model() + latest_rec, _ = rolling_online_manager.list_latest_recorders() + rolling_online_manager.reset_online_tag(latest_rec.values()) -# Update the models using the latest date and set them to online model -def update_model(): - print("========== update_model ==========") - rolling_online_manager.prepare_new_models() +def after_day(): + print("========== after_day ==========") print_online_model() - rolling_online_manager.set_latest_model_to_next_online() + rolling_online_manager.after_day() print_online_model() - - -def after_day(): - rolling_online_manager.prepare_signals() - update_model() - update_predictions() - - -# Run whole workflow completely -def whole_workflow(): - print("========== whole_workflow ==========") - # run this at the first time - first_run() - # run this every day after trading - after_day() + task_collecting() if __name__ == "__main__": ####### to train the first version's models, use the command below # python task_manager_rolling_with_updating.py first_run - ####### to update the models using the latest date, use the command below - # python task_manager_rolling_with_updating.py update_model - - ####### to update the predictions to the latest date, use the command below - # python task_manager_rolling_with_updating.py update_predictions - - ####### to run whole workflow completely, use the command below - # python task_manager_rolling_with_updating.py whole_workflow + ####### to update the models and predictions after the trading time, use the command below + # python task_manager_rolling_with_updating.py after_day #################### you need to finish the configurations below ######################### provider_uri = "~/.qlib/qlib_data/cn_data" # data_dir - qlib.init(provider_uri=provider_uri, region=REG_CN) - - C["mongo"] = { + mongo_conf = { "task_url": "mongodb://10.0.0.4:27017/", # your MongoDB url - "task_db_name": "online", # database name + "task_db_name": "rolling_db", # database name } + qlib.init(provider_uri=provider_uri, region=REG_CN, mongo=mongo_conf) exp_name = "rolling_exp" # experiment name, will be used as the experiment in MLflow task_pool = "rolling_task" # task pool name, will be used as the document in MongoDB rolling_step = 550 ########################################################################################## - rolling_gen = RollingGen(step=550, rtype=RollingGen.ROLL_SD) + rolling_gen = RollingGen(step=rolling_step, rtype=RollingGen.ROLL_SD) rolling_online_manager = RollingOnlineManager( experiment_name=exp_name, rolling_gen=rolling_gen, task_pool=task_pool ) diff --git a/examples/taskmanager/update_online_pred.py b/examples/online_svr/update_online_pred.py similarity index 78% rename from examples/taskmanager/update_online_pred.py rename to examples/online_svr/update_online_pred.py index 5ce963fbc6..ac86b48e8e 100644 --- a/examples/taskmanager/update_online_pred.py +++ b/examples/online_svr/update_online_pred.py @@ -1,9 +1,9 @@ +import fire import qlib -from qlib.model.trainer import task_train -from qlib.workflow.task.online import OnlineManager from qlib.config import REG_CN -import fire -from qlib.workflow import R +from qlib.model.trainer import task_train +from qlib.workflow.task.online import OnlineManagerR +from qlib.workflow.task.utils import list_recorders data_handler_config = { "start_time": "2008-01-01", @@ -56,19 +56,20 @@ def first_train(experiment_name="online_svr"): rid = task_train(task_config=task, experiment_name=experiment_name) - rom = OnlineManager(experiment_name) - rom.reset_online_model(rid) + online_manager = OnlineManagerR(experiment_name) + online_manager.reset_online_tag(rid) def update_online_pred(experiment_name="online_svr"): - rom = OnlineManager(experiment_name) + online_manager = OnlineManagerR(experiment_name) print("Here are the online models waiting for update:") - for rid, rec in rom.list_online_model().items(): - print(rid) + for rid, rec in list_recorders(experiment_name).items(): + if online_manager.get_online_tag(rec) == OnlineManagerR.ONLINE_TAG: + print(rid) - rom.update_online_pred() + online_manager.update_online_pred() if __name__ == "__main__": diff --git a/qlib/config.py b/qlib/config.py index b245cc1df5..95fdaf645f 100644 --- a/qlib/config.py +++ b/qlib/config.py @@ -134,7 +134,7 @@ def set_conf_from_C(self, config_c): }, "loggers": {"qlib": {"level": "DEBUG", "handlers": ["console"]}}, }, - # Defatult config for experiment manager + # Default config for experiment manager "exp_manager": { "class": "MLflowExpManager", "module_path": "qlib.workflow.expm", @@ -143,6 +143,11 @@ def set_conf_from_C(self, config_c): "default_exp_name": "Experiment", }, }, + # Default config for MongoDB + "mongo": { + "task_url": "mongodb://localhost:27017/", + "task_db_name": "default_task_db", + } } MODE_CONF = { diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index c181450733..60f56609f9 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -27,6 +27,7 @@ def task_train(task_config: dict, experiment_name: str) -> str: model = init_instance_by_config(task_config["model"]) dataset = init_instance_by_config(task_config["dataset"]) datahandler = dataset.handler + dataset.config(exclude=["handler"]) # start exp with R.start(experiment_name=experiment_name): @@ -37,10 +38,8 @@ def task_train(task_config: dict, experiment_name: str) -> str: recorder = R.get_recorder() R.save_objects(**{"params.pkl": model}) R.save_objects(**{"task": task_config}) # keep the original format and datatype - - artifact_uri = recorder.get_artifact_uri()[7:] # delete "file://" - dataset.to_pickle(artifact_uri + "/dataset", exclude=["handler"]) - datahandler.to_pickle(artifact_uri + "/datahandler") + R.save_objects(**{"dataset": dataset}) + R.save_objects(**{"datahandler": datahandler}) # generate records: prediction, backtest, and analysis records = task_config.get("record", []) diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index fb7ff0b0b5..0a007cc5c0 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -1,116 +1,172 @@ -from qlib.workflow import R -import pandas as pd -from typing import Union -from typing import Callable +from abc import abstractmethod +from typing import Callable, Union +import pandas as pd from qlib import get_module_logger +from qlib.workflow.task.utils import list_recorders -class TaskCollector: +class Collector: """ - Collect the record (or its results) of the tasks + This class will divide disorderly records or anything worth collecting into different groups based on the group_key. + After grouping, we can reduce the useful information from different groups. """ - def __init__(self, experiment_name: str) -> None: - self.exp_name = experiment_name - self.exp = R.get_exp(experiment_name=experiment_name) - self.logger = get_module_logger("TaskCollector") - - def list_recorders(self, rec_filter_func=None): - - recs = self.exp.list_recorders() - recs_flt = {} - for rid, rec in recs.items(): - if rec_filter_func is None or rec_filter_func(rec): - recs_flt[rid] = rec - - return recs_flt - - def list_recorders_by_task(self, task_filter_func=None): - def rec_filter(recorder): - return task_filter_func(self.get_task(recorder)) - - return self.list_recorders(rec_filter) - - def list_latest_recorders(self, rec_filter_func=None): - recs_flt = self.list_recorders(rec_filter_func) - max_test = self.latest_time(recs_flt) - latest_rec = {} - for rid, rec in recs_flt.items(): - if self.get_task(rec)["dataset"]["kwargs"]["segments"]["test"] == max_test: - latest_rec[rid] = rec - return latest_rec - - def get_recorder_by_id(self, recorder_id): - return self.exp.get_recorder(recorder_id, create=False) - - def get_task(self, recorder): - if isinstance(recorder, str): - recorder = self.get_recorder_by_id(recorder_id=recorder) - try: - task = recorder.load_object("task") - except OSError: - raise OSError(f"Can't find task in {recorder.info['id']}, have you trained with model.trainer.task_train?") - return task - - def latest_time(self, recorders): - if len(recorders) == 0: - raise Exception(f"Can't find any recorder in {self.exp_name}") - max_test = max(self.get_task(rec)["dataset"]["kwargs"]["segments"]["test"] for rec in recorders.values()) - return max_test - - -class RollingCollector(TaskCollector): + def group(self, *args, **kwargs): + """ + According to the get_group_key_func, divide disorderly things into different groups. + + For example: + + .. code-block:: python + + input: + [thing1, thing2, thing3, thing4, thing5] + + output: + { + "group_name1": [thing3, thing5, thing1] + "group_name2": [thing2, thing4] + } + + Args: + get_group_key_func (Callable): get a group key based on a thing + things_list (list): a list of things + + Returns: + dict: a dict including the group key and members of the group. + + """ + raise NotImplementedError(f"Please implement the `group` method.") + + def reduce(self, things_group: dict): + """ + Using the dict from `group`, reduce useful information. + + Args: + things_group (dict): a dict after grouping + + Returns: + dict: a dict including the group key, the information key and the information value + + """ + raise NotImplementedError(f"Please implement the `reduce` method.") + + def collect(self, *args, **kwargs): + """group and reduce + + Returns: + dict: a dict including the group key, the information key and the information value + """ + grouped = self.group(*args, **kwargs) + return self.reduce(grouped) + + +class RecorderCollector(Collector): """ - Collect the record results of the rolling tasks + The Recorder's Collector. This class is a implementation of Collector, collecting some artifacts saved by Recorder. """ - def __init__( - self, - experiment_name: str, - ) -> None: - super().__init__(experiment_name) - self.logger = get_module_logger("RollingCollector") - - def collect_rolling_predictions(self, get_key_func, rec_filter_func=None): - """For rolling tasks, the predictions will be in the diffierent recorder. - To collect and concat the predictions of one rolling task, get_key_func will help this method see which group a recorder will be. - - Parameters - ---------- - get_key_func : Callable[dict,str] - a function that get task config and return its group str - rec_filter_func : Callable[Recorder,bool], optional - a function that decide whether filter a recorder, by default None - - Returns - ------- - dict - a dict of {group: predictions} + def __init__(self, experiment_name: str) -> None: + self.exp_name = experiment_name + self.logger = get_module_logger(self.__class__.__name__) + + _artifacts_key_path = {"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"} + _artifacts_key_merge_method = {} + + def default_merge(self, artifact_list): + """Merge disorderly artifacts in artifact list. + + Args: + artifact_list (list): A artifact list. + + Raises: + NotImplementedError: [description] + """ + raise NotImplementedError(f"Please implement the `default_merge` method.") + + def group(self, get_group_key_func, rec_filter_func=None): """ + Filter recorders and group recorders by group key. + Args: + get_group_key_func (Callable): get a group key based on a recorder + rec_filter_func (Callable, optional): filter the recorders in this experiment. Defaults to None. + + Returns: + dict: a dict including the group key and recorders of the group + """ # filter records - recs_flt = self.list_recorders(rec_filter_func) + recs_flt = list_recorders(self.exp_name, rec_filter_func) # group recs_group = {} for _, rec in recs_flt.items(): - task = self.get_task(rec) - group_key = get_key_func(task) + group_key = get_group_key_func(rec) recs_group.setdefault(group_key, []).append(rec) - # reduce group + return recs_group + + def reduce(self, recs_group: dict, artifact_keys_list: list = None): + """ + Reduce artifacts based on the dict of grouped recorder. + The artifacts need be declared by artifact_keys_list. + The artifacts path in recorder need be declared by _artifacts_key_path. + If there is no declartion in _artifacts_key_merge_method, then use default_merge method to merge it. + + Args: + recs_group (dict): The dict grouped by `group` + artifact_keys_list (list): The list of artifact keys. If it is None, then use all artifacts in _artifacts_key_path. + + Returns: + a dict including the group key, the artifact key and the artifact value. + + For example: + + .. code-block:: python + + { + group_key: {"pred": , "IC": } + } + """ + if artifact_keys_list == None: + artifact_keys_list = self._artifacts_key_path.keys() reduce_group = {} - for k, rec_l in recs_group.items(): - pred_l = [] - for rec in rec_l: - pred_l.append(rec.load_object("pred.pkl").iloc[:, 0]) - # Make sure the pred are sorted according to the rolling start time - pred_l.sort(key=lambda pred: pred.index.get_level_values("datetime").min()) - pred = pd.concat(pred_l) - # If there are duplicated predition, we use the latest perdiction - pred = pred[~pred.index.duplicated(keep="last")] - pred = pred.sort_index() - reduce_group[k] = pred - - return reduce_group \ No newline at end of file + for group_key, recorder_list in recs_group.items(): + reduced_artifacts = {} + for artifact_key in artifact_keys_list: + artifact_list = [] + for recorder in recorder_list: + artifact_list.append(recorder.load_object(self._artifacts_key_path[artifact_key])) + merge_method = self._artifacts_key_merge_method.get(artifact_key, self.default_merge) + artifact = merge_method(artifact_list) + reduced_artifacts[artifact_key] = artifact + reduce_group[group_key] = reduced_artifacts + return reduce_group + + +class RollingCollector(RecorderCollector): + """ + Collect the record results of the rolling tasks + """ + + def __init__(self, experiment_name: str): + super().__init__(experiment_name) + self.logger = get_module_logger(self.__class__.__name__) + + def default_merge(self, artifact_list): + """merge disorderly artifacts based on the datetime. + + Args: + artifact_list (list): a list of artifacts from different recorders + + Returns: + merged artifact + """ + # Make sure the pred are sorted according to the rolling start time + artifact_list.sort(key=lambda x: x.index.get_level_values("datetime").min()) + artifact = pd.concat(artifact_list) + # If there are duplicated predition, we use the latest perdiction + artifact = artifact[~artifact.index.duplicated(keep="last")] + artifact = artifact.sort_index() + return artifact diff --git a/qlib/workflow/task/gen.py b/qlib/workflow/task/gen.py index 63000d77d2..1d363d7f11 100644 --- a/qlib/workflow/task/gen.py +++ b/qlib/workflow/task/gen.py @@ -19,10 +19,10 @@ def task_generator(tasks, generators) -> list: Parameters ---------- - tasks : List[dict] - a list of task templates - generators : List[TaskGen] - a list of TaskGen + tasks : List[dict] or dict + a list of task templates or a single task + generators : List[TaskGen] or TaskGen + a list of TaskGen or a single TaskGen Returns ------- diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index db4c150383..6e9fa6571c 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -151,7 +151,8 @@ def create_task(self, task_def_l, task_pool=None, dry_run=False, print_nt=False) if print new task Returns ------- - + int + the length of new tasks """ task_pool = self._get_task_pool(task_pool) new_tasks = [] @@ -173,6 +174,8 @@ def create_task(self, task_def_l, task_pool=None, dry_run=False, print_nt=False) for t in new_tasks: self.insert_task_def(t, task_pool) + + return len(new_tasks) def fetch_task(self, query={}, task_pool=None): task_pool = self._get_task_pool(task_pool) @@ -245,10 +248,9 @@ def query(self, query={}, decode=True, task_pool=None): for t in task_pool.find(query): yield self._decode_task(t) - def get_task_result(self, task, task_pool=None): + def re_query(self, task, task_pool=None): task_pool = self._get_task_pool(task_pool) - result = task_pool.find_one({"filter": task}) - return self._decode_task(result)["res"] + return task_pool.find_one({"_id":ObjectId(task["_id"])}) def commit_task_res(self, task, res, status=None, task_pool=None): task_pool = self._get_task_pool(task_pool) diff --git a/qlib/workflow/task/online.py b/qlib/workflow/task/online.py index 8d551e858d..d23fc88c8c 100644 --- a/qlib/workflow/task/online.py +++ b/qlib/workflow/task/online.py @@ -3,147 +3,140 @@ from qlib.workflow import R from qlib.model.trainer import task_train from qlib.workflow.recorder import MLflowRecorder, Recorder -from qlib.workflow.task.collect import TaskCollector from qlib.workflow.task.update import ModelUpdater from qlib.workflow.task.utils import TimeAdjuster from qlib.workflow.task.gen import RollingGen, task_generator from qlib.workflow.task.manage import TaskManager from qlib.workflow.task.manage import run_task +from qlib.workflow.task.utils import list_recorders +from qlib.utils.serial import Serializable -class OnlineManager: - def prepare_new_models(self, tasks: List[dict]): - """prepare(train) new models +class OnlineManager(Serializable): - Parameters - ---------- - tasks : List[dict] - a list of tasks - - """ - raise NotImplementedError(f"Please implement the `prepare_new_models` method.") - - ONLINE_KEY = "online_status" # the tag key in recorder + ONLINE_KEY = "online_status" # the online status key in recorder ONLINE_TAG = "online" # the 'online' model NEXT_ONLINE_TAG = "next_online" # the 'next online' model, which can be 'online' model when call reset_online_model OFFLINE_TAG = "offline" # the 'offline' model, not for online serving - def __init__(self, experiment_name: str) -> None: - """ModelUpdater needs experiment name to find the records + def prepare_signals(self, *args, **kwargs): + raise NotImplementedError(f"Please implement the `prepare_signals` method.") - Parameters - ---------- - experiment_name : str - experiment name string - """ - self.logger = get_module_logger("OnlineManagement") - self.exp_name = experiment_name - self.tc = TaskCollector(experiment_name) + def prepare_tasks(self, *args, **kwargs): + raise NotImplementedError(f"Please implement the `prepare_tasks` method.") + + def prepare_new_models(self, *args, **kwargs): + raise NotImplementedError(f"Please implement the `prepare_new_models` method.") - def set_next_online_model(self, recorder: MLflowRecorder): - recorder.set_tags(**{self.ONLINE_KEY: self.NEXT_ONLINE_TAG}) + def update_online_pred(self, *args, **kwargs): + raise NotImplementedError(f"Please implement the `update_online_pred` method.") - def set_online_model(self, recorder: MLflowRecorder): - """online model will be identified at the tags of the record""" - recorder.set_tags(**{self.ONLINE_KEY: self.ONLINE_TAG}) + def set_online_tag(self, tag, *args, **kwargs): + raise NotImplementedError(f"Please implement the `set_online_tag` method.") - def set_offline_model(self, recorder: MLflowRecorder): - recorder.set_tags(**{self.ONLINE_KEY: self.OFFLINE_TAG}) + def get_online_tag(self, *args, **kwargs): + raise NotImplementedError(f"Please implement the `get_online_tag` method.") - def offline_all_model(self): - recs = self.tc.list_recorders() - for rid, rec in recs.items(): - self.set_offline_model(rec) - def reset_online_model(self, recorders: Union[List, Dict] = None): +class OnlineManagerR(OnlineManager): + """ + The implementation of OnlineManager based on (R)ecorder. + + """ + + def __init__(self, experiment_name: str) -> None: + self.logger = get_module_logger(self.__class__.__name__) + self.exp_name = experiment_name + + def set_online_tag(self, tag, recorder: Union[Recorder, List]): + if isinstance(recorder, Recorder): + recorder = [recorder] + for rec in recorder: + rec.set_tags(**{self.ONLINE_KEY: tag}) + self.logger.info(f"Set {len(recorder)} models to '{tag}'.") + + def get_online_tag(self, recorder: Recorder): + tags = recorder.list_tags() + return tags.get(OnlineManager.ONLINE_KEY, OnlineManager.OFFLINE_TAG) + + def reset_online_tag(self, recorder: Union[Recorder, List] = None): """offline all models and set the recorders to 'online'. If no parameter and no 'next online' model, then do nothing. Args: recorders (Union[List, Dict], optional): the recorders you want to reset to 'online'. If don't give, set 'next online' model to 'online' model. If there isn't any 'next online' model, then maintain existing 'online' model. """ - if recorders is None: - recorders = self.list_next_online_model() - if len(recorders) == 0: + if recorder is None: + recorder = list_recorders( + self.exp_name, lambda rec: self.get_online_tag(rec) == OnlineManager.NEXT_ONLINE_TAG + ).values() + if isinstance(recorder, Recorder): + recorder = [recorder] + if len(recorder) == 0: self.logger.info("No 'next online' model, just use current 'online' models.") return - self.offline_all_model() - if isinstance(recorders, dict): - recorders = recorders.values() - for rec in recorders: - self.set_online_model(rec) - self.logger.info(f"Reset {len(recorders)} models to 'online'.") - - def set_latest_model_to_next_online(self): - latest_rec = self.tc.list_latest_recorders() - for rid, rec in latest_rec.items(): - self.set_next_online_model(rec) - self.logger.info(f"Set {len(latest_rec)} latest models to 'next online'.") - - @staticmethod - def online_filter(recorder): - tags = recorder.list_tags() - if tags.get(OnlineManager.ONLINE_KEY, OnlineManager.OFFLINE_TAG) == OnlineManager.ONLINE_TAG: - return True - return False - - @staticmethod - def next_online_filter(recorder): - tags = recorder.list_tags() - if tags.get(OnlineManager.ONLINE_KEY, OnlineManager.OFFLINE_TAG) == OnlineManager.NEXT_ONLINE_TAG: - return True - return False - - def list_online_model(self): - """list the record of online model - - Returns - ------- - dict - {rid : recorder of the online model} - """ - - return self.tc.list_recorders(rec_filter_func=self.online_filter) - - def list_next_online_model(self): - return self.tc.list_recorders(rec_filter_func=self.next_online_filter) + recs = list_recorders(self.exp_name) + self.set_online_tag(OnlineManager.OFFLINE_TAG, recs.values()) + self.set_online_tag(OnlineManager.ONLINE_TAG, recorder) + self.logger.info(f"Reset {len(recorder)} models to 'online'.") def update_online_pred(self): """update all online model predictions to the latest day in Calendar""" mu = ModelUpdater(self.exp_name) - cnt = mu.update_all_pred(self.online_filter) + cnt = mu.update_all_pred(lambda rec: self.get_online_tag(rec) == OnlineManager.ONLINE_TAG) self.logger.info(f"Finish updating {cnt} online model predictions of {self.exp_name}.") + def after_day(self, *args, **kwargs): + self.prepare_signals(*args, **kwargs) + self.prepare_tasks(*args, **kwargs) + self.prepare_new_models(*args, **kwargs) + self.update_online_pred(*args, **kwargs) + self.reset_online_tag() -class RollingOnlineManager(OnlineManager): + +class RollingOnlineManager(OnlineManagerR): def __init__(self, experiment_name: str, rolling_gen: RollingGen, task_pool) -> None: super().__init__(experiment_name) self.ta = TimeAdjuster() self.rg = rolling_gen self.tm = TaskManager(task_pool=task_pool) - self.logger = get_module_logger("RollingOnlineManager") + self.logger = get_module_logger(self.__class__.__name__) - def prepare_new_models(self): - """prepare(train) new models based on online model""" - latest_records = self.tc.list_latest_recorders(self.online_filter) # if we need online_filter here? - max_test = self.tc.latest_time(latest_records) + def prepare_signals(self): + pass + + def prepare_tasks(self): + latest_records, max_test = self.list_latest_recorders(lambda rec: self.get_online_tag(rec) == OnlineManager.ONLINE_TAG) + if max_test is None: + self.logger.warn(f"No latest_recorders.") + return calendar_latest = self.ta.last_date() if self.ta.cal_interval(calendar_latest, max_test[0]) > self.rg.step: old_tasks = [] for rid, rec in latest_records.items(): - task = self.tc.get_task(rec) + task = rec.load_object("task") test_begin = task["dataset"]["kwargs"]["segments"]["test"][0] # modify the test segment to generate new tasks task["dataset"]["kwargs"]["segments"]["test"] = (test_begin, calendar_latest) old_tasks.append(task) new_tasks = task_generator(old_tasks, self.rg) - self.tm.create_task(new_tasks) - run_task(task_train, self.tm.task_pool, experiment_name=self.exp_name) - self.logger.info(f"Finished prepare {len(new_tasks)} new models.") - return new_tasks - self.logger.info("No need to prepare any new models.") - return [] + new_num = self.tm.create_task(new_tasks) + self.logger.info(f"Finished prepare {new_num} tasks.") - def prepare_signals(self): - # prepare the signals of today - pass + def prepare_new_models(self): + """prepare(train) new models based on online model""" + run_task(task_train, self.tm.task_pool, experiment_name=self.exp_name) + latest_records, _ = self.list_latest_recorders() + self.set_online_tag(OnlineManager.NEXT_ONLINE_TAG, latest_records.values()) + self.logger.info(f"Finished prepare {len(latest_records)} new models and set them to next_online.") + + def list_latest_recorders(self, rec_filter_func=None): + recs_flt = list_recorders(self.exp_name, rec_filter_func) + if len(recs_flt) == 0: + return recs_flt, None + max_test = max(rec.load_object("task")["dataset"]["kwargs"]["segments"]["test"] for rec in recs_flt.values()) + latest_rec = {} + for rid, rec in recs_flt.items(): + if rec.load_object("task")["dataset"]["kwargs"]["segments"]["test"] == max_test: + latest_rec[rid] = rec + return latest_rec, max_test \ No newline at end of file diff --git a/qlib/workflow/task/update.py b/qlib/workflow/task/update.py index fcee843490..b8190bca0f 100644 --- a/qlib/workflow/task/update.py +++ b/qlib/workflow/task/update.py @@ -6,8 +6,7 @@ from qlib.workflow import R from qlib.model.trainer import task_train from qlib.workflow.recorder import Recorder -from qlib.workflow.task.collect import TaskCollector - +from qlib.workflow.task.utils import list_recorders class ModelUpdater: """ @@ -23,8 +22,7 @@ def __init__(self, experiment_name: str) -> None: experiment name string """ self.exp_name = experiment_name - self.logger = get_module_logger("ModelUpdater") - self.tc = TaskCollector(experiment_name) + self.logger = get_module_logger(self.__class__.__name__) def _reload_dataset(self, recorder, start_time, end_time): """reload dataset from pickle file @@ -53,7 +51,7 @@ def _reload_dataset(self, recorder, start_time, end_time): datahandler.init(datahandler.IT_LS) return dataset - def update_pred(self, recorder: Recorder): + def update_pred(self, recorder: Recorder, frequency='day'): """update predictions to the latest day in Calendar based on rid Parameters @@ -65,7 +63,10 @@ def update_pred(self, recorder: Recorder): last_end = old_pred.index.get_level_values("datetime").max() # updated to the latest trading day - cal = D.calendar(start_time=last_end + pd.Timedelta(days=1), end_time=None) + if frequency=='day': + cal = D.calendar(start_time=last_end + pd.Timedelta(days=1), end_time=None) + else: + raise NotImplementedError("Now Qlib only support update daily frequency prediction") if len(cal) == 0: self.logger.info( @@ -113,7 +114,7 @@ def rec_filter_func(recorder): the count of updated record """ - recs = self.tc.list_recorders(rec_filter_func=rec_filter_func) + recs = list_recorders(self.exp_name, rec_filter_func=rec_filter_func) for rid, rec in recs.items(): self.update_pred(rec) return len(recs) diff --git a/qlib/workflow/task/utils.py b/qlib/workflow/task/utils.py index 272f219eca..15123a291c 100644 --- a/qlib/workflow/task/utils.py +++ b/qlib/workflow/task/utils.py @@ -3,6 +3,7 @@ import bisect import pandas as pd from qlib.data import D +from qlib.workflow import R from qlib.config import C from qlib.log import get_module_logger from pymongo import MongoClient @@ -29,6 +30,25 @@ def get_mongodb(): client = MongoClient(cfg["task_url"]) return client.get_database(name=cfg["task_db_name"]) +def list_recorders(experiment, rec_filter_func=None): + """list all recorders which can pass the filter in a experiment. + + Args: + experiment (str or Experiment): the name of a Experiment or a instance + rec_filter_func (Callable, optional): return True to retain the given recorder. Defaults to None. + + Returns: + dict: a dict {rid: recorder} after filtering. + """ + if isinstance(experiment, str): + experiment, _ = R.exp_manager._get_or_create_exp(experiment_name=experiment) + recs = experiment.list_recorders() + recs_flt = {} + for rid, rec in recs.items(): + if rec_filter_func is None or rec_filter_func(rec): + recs_flt[rid] = rec + + return recs_flt class TimeAdjuster: """ From 1f2d2c9b69782524070f4d356353b2275b340cf5 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Tue, 30 Mar 2021 06:56:04 +0000 Subject: [PATCH 28/61] online debug --- qlib/workflow/task/gen.py | 5 ++--- qlib/workflow/task/utils.py | 4 +++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/qlib/workflow/task/gen.py b/qlib/workflow/task/gen.py index 1d363d7f11..c64939e822 100644 --- a/qlib/workflow/task/gen.py +++ b/qlib/workflow/task/gen.py @@ -36,14 +36,13 @@ def task_generator(tasks, generators) -> list: generators = [generators] # generate gen_task_list - gen_task_list = [] for gen in generators: new_task_list = [] for task in tasks: new_task_list.extend(gen.generate(task)) - gen_task_list = new_task_list + tasks = new_task_list - return gen_task_list + return tasks class TaskGen(metaclass=abc.ABCMeta): diff --git a/qlib/workflow/task/utils.py b/qlib/workflow/task/utils.py index 15123a291c..29d7a495ca 100644 --- a/qlib/workflow/task/utils.py +++ b/qlib/workflow/task/utils.py @@ -30,6 +30,7 @@ def get_mongodb(): client = MongoClient(cfg["task_url"]) return client.get_database(name=cfg["task_db_name"]) + def list_recorders(experiment, rec_filter_func=None): """list all recorders which can pass the filter in a experiment. @@ -50,6 +51,7 @@ def list_recorders(experiment, rec_filter_func=None): return recs_flt + class TimeAdjuster: """ find appropriate date and adjust date. @@ -146,7 +148,7 @@ def align_seg(self, segment: Union[dict, tuple]): """ if isinstance(segment, dict): return {k: self.align_seg(seg) for k, seg in segment.items()} - elif isinstance(segment, tuple): + elif isinstance(segment, tuple) or isinstance(segment, list): return self.align_time(segment[0], tp_type="start"), self.align_time(segment[1], tp_type="end") else: raise NotImplementedError(f"This type of input is not supported") From 544365f3a9ac13860ed1075d0211abeb5fbf8e42 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Wed, 31 Mar 2021 02:39:14 +0000 Subject: [PATCH 29/61] ensemble & get_exp & dataset_pickle --- .../model_rolling/task_manager_rolling.py | 21 +- .../task_manager_rolling_with_updating.py | 19 +- qlib/model/trainer.py | 3 - qlib/workflow/task/collect.py | 179 ++++------------- qlib/workflow/task/ensemble.py | 180 ++++++++++++++++++ qlib/workflow/task/update.py | 9 +- qlib/workflow/task/utils.py | 2 +- 7 files changed, 249 insertions(+), 164 deletions(-) create mode 100644 qlib/workflow/task/ensemble.py diff --git a/examples/model_rolling/task_manager_rolling.py b/examples/model_rolling/task_manager_rolling.py index 70a4f7d7e7..e5de1ef605 100644 --- a/examples/model_rolling/task_manager_rolling.py +++ b/examples/model_rolling/task_manager_rolling.py @@ -5,9 +5,12 @@ from qlib.config import REG_CN from qlib.model.trainer import task_train from qlib.workflow import R -from qlib.workflow.task.collect import RollingCollector from qlib.workflow.task.gen import RollingGen, task_generator from qlib.workflow.task.manage import TaskManager, run_task +from qlib.workflow.task.collect import RecorderCollector +from qlib.workflow.task.ensemble import RollingEnsemble +import pandas as pd +from qlib.workflow.task.utils import list_recorders data_handler_config = { "start_time": "2008-01-01", @@ -70,7 +73,7 @@ def reset(task_pool, exp_name): print("========== reset ==========") TaskManager(task_pool=task_pool).remove() - exp, _ = R.exp_manager._get_or_create_exp(experiment_name=exp_name) + exp, _ = R.get_exp(experiment_name=exp_name) for rid in exp.list_recorders(): exp.delete_recorder(rid) @@ -110,19 +113,21 @@ def task_collecting(task_pool, exp_name): def get_group_key_func(recorder): task_config = recorder.load_object("task") - return task_config["model"]["class"] + model_key = task_config["model"]["class"] + rolling_key = task_config["dataset"]["kwargs"]["segments"]["test"] + return model_key, model_key, rolling_key def my_filter(recorder): # only choose the results of "LGBModel" - task_key = get_group_key_func(recorder) - if task_key == "LGBModel": + model_key, rolling_key = get_group_key_func(recorder) + if model_key == "LGBModel": return True return False - rolling_collector = RollingCollector(exp_name) + collector = RecorderCollector(exp_name) # group tasks by "get_task_key" and filter tasks by "my_filter" - pred_rolling = rolling_collector.collect(get_group_key_func, my_filter) - print(pred_rolling) + artifact = collector.collect(RollingEnsemble(), get_group_key_func, rec_filter_func=my_filter) + print(artifact) def main( diff --git a/examples/online_svr/task_manager_rolling_with_updating.py b/examples/online_svr/task_manager_rolling_with_updating.py index 24bc38a025..4e9fdd3364 100644 --- a/examples/online_svr/task_manager_rolling_with_updating.py +++ b/examples/online_svr/task_manager_rolling_with_updating.py @@ -5,7 +5,8 @@ from qlib.config import REG_CN from qlib.model.trainer import task_train from qlib.workflow import R -from qlib.workflow.task.collect import RollingCollector +from qlib.workflow.task.collect import RecorderCollector +from qlib.workflow.task.ensemble import RollingEnsemble from qlib.workflow.task.gen import RollingGen, task_generator from qlib.workflow.task.manage import TaskManager, run_task from qlib.workflow.task.online import RollingOnlineManager @@ -114,26 +115,28 @@ def task_collecting(): def get_group_key_func(recorder): task_config = recorder.load_object("task") - return task_config["model"]["class"] + model_key = task_config["model"]["class"] + rolling_key = task_config["dataset"]["kwargs"]["segments"]["test"] + return model_key, model_key, rolling_key def my_filter(recorder): # only choose the results of "LGBModel" - task_key = get_group_key_func(recorder) - if task_key == "LGBModel": + model_key, rolling_key = get_group_key_func(recorder) + if model_key == "LGBModel": return True return False - rolling_collector = RollingCollector(exp_name) + collector = RecorderCollector(exp_name) # group tasks by "get_task_key" and filter tasks by "my_filter" - pred_rolling = rolling_collector.collect(get_group_key_func, my_filter) - print(pred_rolling) + artifact = collector.collect(RollingEnsemble(), get_group_key_func, rec_filter_func=my_filter) + print(artifact) # Reset all things to the first status, be careful to save important data def reset(): print("========== reset ==========") task_manager.remove() - exp, _ = R.exp_manager._get_or_create_exp(experiment_name=exp_name) + exp, _ = R.get_exp(experiment_name=exp_name) for rid in exp.list_recorders(): exp.delete_recorder(rid) diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index 60f56609f9..45650c0c7c 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -26,8 +26,6 @@ def task_train(task_config: dict, experiment_name: str) -> str: # model initiaiton model = init_instance_by_config(task_config["model"]) dataset = init_instance_by_config(task_config["dataset"]) - datahandler = dataset.handler - dataset.config(exclude=["handler"]) # start exp with R.start(experiment_name=experiment_name): @@ -39,7 +37,6 @@ def task_train(task_config: dict, experiment_name: str) -> str: R.save_objects(**{"params.pkl": model}) R.save_objects(**{"task": task_config}) # keep the original format and datatype R.save_objects(**{"dataset": dataset}) - R.save_objects(**{"datahandler": datahandler}) # generate records: prediction, backtest, and analysis records = task_config.get("record", []) diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index 0a007cc5c0..a7a6ce4bb8 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -7,166 +7,69 @@ class Collector: + """The collector to collect different results based on experiment backend and ensemble method """ - This class will divide disorderly records or anything worth collecting into different groups based on the group_key. - After grouping, we can reduce the useful information from different groups. - """ - - def group(self, *args, **kwargs): - """ - According to the get_group_key_func, divide disorderly things into different groups. - - For example: - - .. code-block:: python - - input: - [thing1, thing2, thing3, thing4, thing5] - output: - { - "group_name1": [thing3, thing5, thing1] - "group_name2": [thing2, thing4] - } + def collect(self, ensemble, get_group_key_func, *args, **kwargs): + """To collect the results, we need to get the experiment record firstly and divided them into + different groups. Then use ensemble methods to merge the group. Args: - get_group_key_func (Callable): get a group key based on a thing - things_list (list): a list of things + ensemble (Ensemble): an instance of Ensemble + get_group_key_func (Callable): a function to get the group of a experiment record - Returns: - dict: a dict including the group key and members of the group. - - """ - raise NotImplementedError(f"Please implement the `group` method.") - - def reduce(self, things_group: dict): """ - Using the dict from `group`, reduce useful information. - - Args: - things_group (dict): a dict after grouping - - Returns: - dict: a dict including the group key, the information key and the information value - - """ - raise NotImplementedError(f"Please implement the `reduce` method.") - - def collect(self, *args, **kwargs): - """group and reduce - - Returns: - dict: a dict including the group key, the information key and the information value - """ - grouped = self.group(*args, **kwargs) - return self.reduce(grouped) + raise NotImplementedError(f"Please implement the `collect` method.") class RecorderCollector(Collector): - """ - The Recorder's Collector. This class is a implementation of Collector, collecting some artifacts saved by Recorder. - """ - - def __init__(self, experiment_name: str) -> None: - self.exp_name = experiment_name - self.logger = get_module_logger(self.__class__.__name__) - - _artifacts_key_path = {"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"} - _artifacts_key_merge_method = {} - - def default_merge(self, artifact_list): - """Merge disorderly artifacts in artifact list. + def __init__(self, exp_name, artifacts_path = {"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"}) -> None: + """init RecorderCollector Args: - artifact_list (list): A artifact list. - - Raises: - NotImplementedError: [description] + exp_name (str): the name of Experiment + artifacts_path (dict, optional): The artifacts name and its path in Recorder. Defaults to {"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"}. """ - raise NotImplementedError(f"Please implement the `default_merge` method.") + self.exp_name = exp_name + self.artifacts_path = artifacts_path - def group(self, get_group_key_func, rec_filter_func=None): - """ - Filter recorders and group recorders by group key. + def collect(self, ensemble, get_group_key_func, artifacts_key=None, rec_filter_func=None): + """Collect different artifacts based on recorder after filtering and ensemble method. + Group recorder by get_group_key_func. Args: - get_group_key_func (Callable): get a group key based on a recorder - rec_filter_func (Callable, optional): filter the recorders in this experiment. Defaults to None. + ensemble (Ensemble): an instance of Ensemble + get_group_key_func (Callable): a function to get the group of a experiment record + artifacts_key (str or List, optional): the artifacts key you want to get. Defaults to None. + rec_filter_func (Callable, optional): filter the recorder by return True or False. Defaults to None. Returns: - dict: a dict including the group key and recorders of the group + dict: the dict after collected. """ + if artifacts_key is None: + artifacts_key = self.artifacts_path.keys() + + if isinstance(artifacts_key, str): + artifacts_key = [artifacts_key] + + # prepare_ensemble + ensemble_dict = {} + for key in artifacts_key: + ensemble_dict.setdefault(key,{}) # filter records recs_flt = list_recorders(self.exp_name, rec_filter_func) - - # group - recs_group = {} for _, rec in recs_flt.items(): group_key = get_group_key_func(rec) - recs_group.setdefault(group_key, []).append(rec) - - return recs_group - - def reduce(self, recs_group: dict, artifact_keys_list: list = None): - """ - Reduce artifacts based on the dict of grouped recorder. - The artifacts need be declared by artifact_keys_list. - The artifacts path in recorder need be declared by _artifacts_key_path. - If there is no declartion in _artifacts_key_merge_method, then use default_merge method to merge it. - - Args: - recs_group (dict): The dict grouped by `group` - artifact_keys_list (list): The list of artifact keys. If it is None, then use all artifacts in _artifacts_key_path. - - Returns: - a dict including the group key, the artifact key and the artifact value. - - For example: - - .. code-block:: python + for key in artifacts_key: + artifact = rec.load_object(self.artifacts_path[key]) + ensemble_dict[key][group_key] = artifact - { - group_key: {"pred": , "IC": } - } - """ - if artifact_keys_list == None: - artifact_keys_list = self._artifacts_key_path.keys() - reduce_group = {} - for group_key, recorder_list in recs_group.items(): - reduced_artifacts = {} - for artifact_key in artifact_keys_list: - artifact_list = [] - for recorder in recorder_list: - artifact_list.append(recorder.load_object(self._artifacts_key_path[artifact_key])) - merge_method = self._artifacts_key_merge_method.get(artifact_key, self.default_merge) - artifact = merge_method(artifact_list) - reduced_artifacts[artifact_key] = artifact - reduce_group[group_key] = reduced_artifacts - return reduce_group - - -class RollingCollector(RecorderCollector): - """ - Collect the record results of the rolling tasks - """ - def __init__(self, experiment_name: str): - super().__init__(experiment_name) - self.logger = get_module_logger(self.__class__.__name__) + if isinstance(artifacts_key, str): + return ensemble(ensemble_dict[artifacts_key]) - def default_merge(self, artifact_list): - """merge disorderly artifacts based on the datetime. - - Args: - artifact_list (list): a list of artifacts from different recorders - - Returns: - merged artifact - """ - # Make sure the pred are sorted according to the rolling start time - artifact_list.sort(key=lambda x: x.index.get_level_values("datetime").min()) - artifact = pd.concat(artifact_list) - # If there are duplicated predition, we use the latest perdiction - artifact = artifact[~artifact.index.duplicated(keep="last")] - artifact = artifact.sort_index() - return artifact + collect_dict = {} + for key in artifacts_key: + collect_dict[key] = ensemble(ensemble_dict[key]) + return collect_dict + \ No newline at end of file diff --git a/qlib/workflow/task/ensemble.py b/qlib/workflow/task/ensemble.py new file mode 100644 index 0000000000..649ce94151 --- /dev/null +++ b/qlib/workflow/task/ensemble.py @@ -0,0 +1,180 @@ +from abc import abstractmethod +from typing import Callable, Union + +import pandas as pd +from qlib import get_module_logger +from qlib.workflow.task.utils import list_recorders +from typing import Dict + + + +class Ensemble: + """Merge the objects in an Ensemble. + """ + + def __init__(self, merge_func = None, get_grouped_key_func = None) -> None: + """init Ensemble + + Args: + merge_func (Callable, optional): The specific merge function. Defaults to None. + get_grouped_key_func (Callable, optional): Get group_inner_key and group_outer_key by group_key. Defaults to None. + """ + self.logger = get_module_logger(self.__class__.__name__) + if merge_func is not None: + self.merge_func = merge_func + if get_grouped_key_func is not None: + self.get_grouped_key_func = get_grouped_key_func + + def merge_func(self, group_inner_dict): + """Given a group_inner_dict such as {Rollinga_b: object, Rollingb_c: object}, + merge it to object + + Args: + group_inner_dict (dict): the inner group dict + + """ + raise NotImplementedError(f"Please implement the `merge_func` method.") + + def get_grouped_key_func(self, group_key): + """Given a group_key and return the group_outer_key, group_inner_key. + + For example: + (A,B,Rolling) -> (A,B):Rolling + (A,B) -> C:(A,B) + + Args: + group_key (tuple or str): the group key + """ + raise NotImplementedError(f"Please implement the `get_grouped_key_func` method.") + + def group(self, group_dict: Dict[tuple or str, object]) -> Dict[tuple or str, Dict[tuple or str, object]]: + """In a group of dict, further divide them into outgroups and innergroup. + + For example: + + .. code-block:: python + + RollingEnsemble: + input: + { + (ModelA,Horizon5,Rollinga_b): object + (ModelA,Horizon5,Rollingb_c): object + (ModelA,Horizon10,Rollinga_b): object + (ModelA,Horizon10,Rollingb_c): object + (ModelB,Horizon5,Rollinga_b): object + (ModelB,Horizon5,Rollingb_c): object + (ModelB,Horizon10,Rollinga_b): object + (ModelB,Horizon10,Rollingb_c): object + } + + output: + { + (ModelA,Horizon5): {Rollinga_b: object, Rollingb_c: object} + (ModelA,Horizon10): {Rollinga_b: object, Rollingb_c: object} + (ModelB,Horizon5): {Rollinga_b: object, Rollingb_c: object} + (ModelB,Horizon10): {Rollinga_b: object, Rollingb_c: object} + } + + Args: + group_dict (Dict[tuple or str, object]): a group of dict + + Returns: + Dict[tuple or str, Dict[tuple or str, object]]: the dict after `group` + """ + grouped_dict = {} + for group_key, artifact in group_dict.items(): + group_outer_key, group_inner_key = self.get_grouped_key_func(group_key) # (A,B,Rolling) -> (A,B):Rolling + grouped_dict.setdefault(group_outer_key, {})[group_inner_key] = artifact + return grouped_dict + + def reduce(self, grouped_dict: dict): + """After grouping, reduce the innergroup. + + For example: + + .. code-block:: python + + RollingEnsemble: + input: + { + (ModelA,Horizon5): {Rollinga_b: object, Rollingb_c: object} + (ModelA,Horizon10): {Rollinga_b: object, Rollingb_c: object} + (ModelB,Horizon5): {Rollinga_b: object, Rollingb_c: object} + (ModelB,Horizon10): {Rollinga_b: object, Rollingb_c: object} + } + + output: + { + (ModelA,Horizon5): object + (ModelA,Horizon10): object + (ModelB,Horizon5): object + (ModelB,Horizon10): object + } + + Args: + grouped_dict (dict): the dict after `group` + + Returns: + dict: the dict after `reduce` + """ + reduce_group = {} + for group_outer_key, group_inner_dict in grouped_dict.items(): + artifact = self.merge_func(group_inner_dict) + reduce_group[group_outer_key] = artifact + return reduce_group + + def __call__(self, group_dict): + """The process of Ensemble is group it firstly and then reduce it, like MapReduce. + + Args: + group_dict (Dict[tuple or str, object]): a group of dict + + Returns: + dict: the dict after `reduce` + """ + grouped_dict = self.group(group_dict) + return self.reduce(grouped_dict) + +class RollingEnsemble(Ensemble): + """A specific implementation of Ensemble for Rolling. + + """ + def merge_func(self, group_inner_dict): + """merge group_inner_dict by datetime. + + Args: + group_inner_dict (dict): the inner group dict + + Returns: + object: the artifact after merging + """ + artifact_list = list(group_inner_dict.values()) + artifact_list.sort(key=lambda x: x.index.get_level_values("datetime").min()) + artifact = pd.concat(artifact_list) + # If there are duplicated predition, use the latest perdiction + artifact = artifact[~artifact.index.duplicated(keep="last")] + artifact = artifact.sort_index() + return artifact + + def get_grouped_key_func(self, group_key): + """The final axis of group_key must be the Rolling key. + When `collect`, get_group_key_func can add the statement below. + + .. code-block:: python + + def get_group_key_func(recorder): + task_config = recorder.load_object("task") + ...... + rolling_key = task_config["dataset"]["kwargs"]["segments"]["test"] + return ......, rolling_key + + Args: + group_key (tuple or str): the group key + + Returns: + tuple or str, tuple or str: group_outer_key, group_inner_key + """ + assert len(group_key)>=2 + return group_key[:-1], group_key[-1] + + diff --git a/qlib/workflow/task/update.py b/qlib/workflow/task/update.py index bab7df3c86..43c3042392 100644 --- a/qlib/workflow/task/update.py +++ b/qlib/workflow/task/update.py @@ -7,6 +7,7 @@ from qlib.model.trainer import task_train from qlib.workflow.recorder import Recorder from qlib.workflow.task.utils import list_recorders +from qlib.data.dataset.handler import DataHandlerLP class ModelUpdater: """ @@ -42,13 +43,9 @@ def _reload_dataset(self, recorder, start_time, end_time): the instance of Dataset """ segments = {"test": (start_time, end_time)} - dataset = recorder.load_object("dataset") - datahandler = recorder.load_object("datahandler") - - datahandler.conf_data(**{"start_time": start_time, "end_time": end_time}) - dataset.setup_data(handler=datahandler, segments=segments) - datahandler.init(datahandler.IT_LS) + dataset.config(handler_kwargs={"start_time": start_time, "end_time": end_time}) + dataset.setup_data(handler_kwargs={"init_type": DataHandlerLP.IT_LS}, segments=segments) return dataset def update_pred(self, recorder: Recorder, frequency='day'): diff --git a/qlib/workflow/task/utils.py b/qlib/workflow/task/utils.py index 29d7a495ca..b34b753068 100644 --- a/qlib/workflow/task/utils.py +++ b/qlib/workflow/task/utils.py @@ -42,7 +42,7 @@ def list_recorders(experiment, rec_filter_func=None): dict: a dict {rid: recorder} after filtering. """ if isinstance(experiment, str): - experiment, _ = R.exp_manager._get_or_create_exp(experiment_name=experiment) + experiment, _ = R.get_exp(experiment_name=experiment) recs = experiment.list_recorders() recs_flt = {} for rid, rec in recs.items(): From edcd7b1ff9b38b2ce8ea13a96276eed3964ac119 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Wed, 31 Mar 2021 03:08:48 +0000 Subject: [PATCH 30/61] bug fixed & code format --- .../model_rolling/task_manager_rolling.py | 4 ++-- .../task_manager_rolling_with_updating.py | 4 ++-- qlib/workflow/task/collect.py | 11 ++++------ qlib/workflow/task/ensemble.py | 22 ++++++++----------- qlib/workflow/task/manage.py | 4 ++-- qlib/workflow/task/online.py | 4 +++- qlib/workflow/task/update.py | 7 +++--- qlib/workflow/task/utils.py | 2 +- 8 files changed, 27 insertions(+), 31 deletions(-) diff --git a/examples/model_rolling/task_manager_rolling.py b/examples/model_rolling/task_manager_rolling.py index e5de1ef605..75d360fa16 100644 --- a/examples/model_rolling/task_manager_rolling.py +++ b/examples/model_rolling/task_manager_rolling.py @@ -73,7 +73,7 @@ def reset(task_pool, exp_name): print("========== reset ==========") TaskManager(task_pool=task_pool).remove() - exp, _ = R.get_exp(experiment_name=exp_name) + exp = R.get_exp(experiment_name=exp_name) for rid in exp.list_recorders(): exp.delete_recorder(rid) @@ -115,7 +115,7 @@ def get_group_key_func(recorder): task_config = recorder.load_object("task") model_key = task_config["model"]["class"] rolling_key = task_config["dataset"]["kwargs"]["segments"]["test"] - return model_key, model_key, rolling_key + return model_key, rolling_key def my_filter(recorder): # only choose the results of "LGBModel" diff --git a/examples/online_svr/task_manager_rolling_with_updating.py b/examples/online_svr/task_manager_rolling_with_updating.py index 4e9fdd3364..fff470c86f 100644 --- a/examples/online_svr/task_manager_rolling_with_updating.py +++ b/examples/online_svr/task_manager_rolling_with_updating.py @@ -117,7 +117,7 @@ def get_group_key_func(recorder): task_config = recorder.load_object("task") model_key = task_config["model"]["class"] rolling_key = task_config["dataset"]["kwargs"]["segments"]["test"] - return model_key, model_key, rolling_key + return model_key, rolling_key def my_filter(recorder): # only choose the results of "LGBModel" @@ -136,7 +136,7 @@ def my_filter(recorder): def reset(): print("========== reset ==========") task_manager.remove() - exp, _ = R.get_exp(experiment_name=exp_name) + exp = R.get_exp(experiment_name=exp_name) for rid in exp.list_recorders(): exp.delete_recorder(rid) diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index a7a6ce4bb8..91b713ef8d 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -7,8 +7,7 @@ class Collector: - """The collector to collect different results based on experiment backend and ensemble method - """ + """The collector to collect different results based on experiment backend and ensemble method""" def collect(self, ensemble, get_group_key_func, *args, **kwargs): """To collect the results, we need to get the experiment record firstly and divided them into @@ -23,7 +22,7 @@ def collect(self, ensemble, get_group_key_func, *args, **kwargs): class RecorderCollector(Collector): - def __init__(self, exp_name, artifacts_path = {"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"}) -> None: + def __init__(self, exp_name, artifacts_path={"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"}) -> None: """init RecorderCollector Args: @@ -48,14 +47,14 @@ def collect(self, ensemble, get_group_key_func, artifacts_key=None, rec_filter_f """ if artifacts_key is None: artifacts_key = self.artifacts_path.keys() - + if isinstance(artifacts_key, str): artifacts_key = [artifacts_key] # prepare_ensemble ensemble_dict = {} for key in artifacts_key: - ensemble_dict.setdefault(key,{}) + ensemble_dict.setdefault(key, {}) # filter records recs_flt = list_recorders(self.exp_name, rec_filter_func) for _, rec in recs_flt.items(): @@ -64,7 +63,6 @@ def collect(self, ensemble, get_group_key_func, artifacts_key=None, rec_filter_f artifact = rec.load_object(self.artifacts_path[key]) ensemble_dict[key][group_key] = artifact - if isinstance(artifacts_key, str): return ensemble(ensemble_dict[artifacts_key]) @@ -72,4 +70,3 @@ def collect(self, ensemble, get_group_key_func, artifacts_key=None, rec_filter_f for key in artifacts_key: collect_dict[key] = ensemble(ensemble_dict[key]) return collect_dict - \ No newline at end of file diff --git a/qlib/workflow/task/ensemble.py b/qlib/workflow/task/ensemble.py index 649ce94151..dca0dee3e0 100644 --- a/qlib/workflow/task/ensemble.py +++ b/qlib/workflow/task/ensemble.py @@ -7,12 +7,10 @@ from typing import Dict - class Ensemble: - """Merge the objects in an Ensemble. - """ + """Merge the objects in an Ensemble.""" - def __init__(self, merge_func = None, get_grouped_key_func = None) -> None: + def __init__(self, merge_func=None, get_grouped_key_func=None) -> None: """init Ensemble Args: @@ -26,7 +24,7 @@ def __init__(self, merge_func = None, get_grouped_key_func = None) -> None: self.get_grouped_key_func = get_grouped_key_func def merge_func(self, group_inner_dict): - """Given a group_inner_dict such as {Rollinga_b: object, Rollingb_c: object}, + """Given a group_inner_dict such as {Rollinga_b: object, Rollingb_c: object}, merge it to object Args: @@ -34,10 +32,10 @@ def merge_func(self, group_inner_dict): """ raise NotImplementedError(f"Please implement the `merge_func` method.") - + def get_grouped_key_func(self, group_key): """Given a group_key and return the group_outer_key, group_inner_key. - + For example: (A,B,Rolling) -> (A,B):Rolling (A,B) -> C:(A,B) @@ -135,10 +133,10 @@ def __call__(self, group_dict): grouped_dict = self.group(group_dict) return self.reduce(grouped_dict) + class RollingEnsemble(Ensemble): - """A specific implementation of Ensemble for Rolling. + """A specific implementation of Ensemble for Rolling.""" - """ def merge_func(self, group_inner_dict): """merge group_inner_dict by datetime. @@ -155,7 +153,7 @@ def merge_func(self, group_inner_dict): artifact = artifact[~artifact.index.duplicated(keep="last")] artifact = artifact.sort_index() return artifact - + def get_grouped_key_func(self, group_key): """The final axis of group_key must be the Rolling key. When `collect`, get_group_key_func can add the statement below. @@ -174,7 +172,5 @@ def get_group_key_func(recorder): Returns: tuple or str, tuple or str: group_outer_key, group_inner_key """ - assert len(group_key)>=2 + assert len(group_key) >= 2 return group_key[:-1], group_key[-1] - - diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 6e9fa6571c..a621642078 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -174,7 +174,7 @@ def create_task(self, task_def_l, task_pool=None, dry_run=False, print_nt=False) for t in new_tasks: self.insert_task_def(t, task_pool) - + return len(new_tasks) def fetch_task(self, query={}, task_pool=None): @@ -250,7 +250,7 @@ def query(self, query={}, decode=True, task_pool=None): def re_query(self, task, task_pool=None): task_pool = self._get_task_pool(task_pool) - return task_pool.find_one({"_id":ObjectId(task["_id"])}) + return task_pool.find_one({"_id": ObjectId(task["_id"])}) def commit_task_res(self, task, res, status=None, task_pool=None): task_pool = self._get_task_pool(task_pool) diff --git a/qlib/workflow/task/online.py b/qlib/workflow/task/online.py index d23fc88c8c..f7ffbd18a9 100644 --- a/qlib/workflow/task/online.py +++ b/qlib/workflow/task/online.py @@ -106,7 +106,9 @@ def prepare_signals(self): pass def prepare_tasks(self): - latest_records, max_test = self.list_latest_recorders(lambda rec: self.get_online_tag(rec) == OnlineManager.ONLINE_TAG) + latest_records, max_test = self.list_latest_recorders( + lambda rec: self.get_online_tag(rec) == OnlineManager.ONLINE_TAG + ) if max_test is None: self.logger.warn(f"No latest_recorders.") return diff --git a/qlib/workflow/task/update.py b/qlib/workflow/task/update.py index 43c3042392..002f1128f8 100644 --- a/qlib/workflow/task/update.py +++ b/qlib/workflow/task/update.py @@ -9,6 +9,7 @@ from qlib.workflow.task.utils import list_recorders from qlib.data.dataset.handler import DataHandlerLP + class ModelUpdater: """ The model updater to update model results in new data. @@ -48,7 +49,7 @@ def _reload_dataset(self, recorder, start_time, end_time): dataset.setup_data(handler_kwargs={"init_type": DataHandlerLP.IT_LS}, segments=segments) return dataset - def update_pred(self, recorder: Recorder, frequency='day'): + def update_pred(self, recorder: Recorder, frequency="day"): """update predictions to the latest day in Calendar based on rid Parameters @@ -60,10 +61,10 @@ def update_pred(self, recorder: Recorder, frequency='day'): last_end = old_pred.index.get_level_values("datetime").max() # updated to the latest trading day - if frequency=='day': + if frequency == "day": cal = D.calendar(start_time=last_end + pd.Timedelta(days=1), end_time=None) else: - raise NotImplementedError("Now Qlib only support update daily frequency prediction") + raise NotImplementedError("Now `ModelUpdater` only support update daily frequency prediction") if len(cal) == 0: self.logger.info( diff --git a/qlib/workflow/task/utils.py b/qlib/workflow/task/utils.py index b34b753068..b6287abc2b 100644 --- a/qlib/workflow/task/utils.py +++ b/qlib/workflow/task/utils.py @@ -42,7 +42,7 @@ def list_recorders(experiment, rec_filter_func=None): dict: a dict {rid: recorder} after filtering. """ if isinstance(experiment, str): - experiment, _ = R.get_exp(experiment_name=experiment) + experiment = R.get_exp(experiment_name=experiment) recs = experiment.list_recorders() recs_flt = {} for rid, rec in recs.items(): From bd7a1c11b981099cdbe9a69429e3566be36854be Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Fri, 2 Apr 2021 04:27:14 +0000 Subject: [PATCH 31/61] trainer & group & collect & ensemble --- .../model_rolling/task_manager_rolling.py | 35 ++-- .../task_manager_rolling_with_updating.py | 8 +- .../update_online_pred.py | 6 +- qlib/model/ens/ensemble.py | 98 ++++++++++ qlib/model/ens/group.py | 68 +++++++ qlib/model/trainer.py | 68 +++++++ qlib/workflow/online/__init__.py | 0 .../{task/online.py => online/manager.py} | 33 ++-- qlib/workflow/{task => online}/update.py | 4 +- qlib/workflow/task/collect.py | 58 +++--- qlib/workflow/task/ensemble.py | 176 ------------------ qlib/workflow/task/manage.py | 26 +-- 12 files changed, 319 insertions(+), 261 deletions(-) rename examples/{online_svr => online_srv}/task_manager_rolling_with_updating.py (97%) rename examples/{online_svr => online_srv}/update_online_pred.py (90%) create mode 100644 qlib/model/ens/ensemble.py create mode 100644 qlib/model/ens/group.py create mode 100644 qlib/workflow/online/__init__.py rename qlib/workflow/{task/online.py => online/manager.py} (87%) rename qlib/workflow/{task => online}/update.py (98%) delete mode 100644 qlib/workflow/task/ensemble.py diff --git a/examples/model_rolling/task_manager_rolling.py b/examples/model_rolling/task_manager_rolling.py index 75d360fa16..3eb05de728 100644 --- a/examples/model_rolling/task_manager_rolling.py +++ b/examples/model_rolling/task_manager_rolling.py @@ -8,9 +8,11 @@ from qlib.workflow.task.gen import RollingGen, task_generator from qlib.workflow.task.manage import TaskManager, run_task from qlib.workflow.task.collect import RecorderCollector -from qlib.workflow.task.ensemble import RollingEnsemble +from qlib.model.ens.ensemble import RollingEnsemble, ens_workflow import pandas as pd from qlib.workflow.task.utils import list_recorders +from qlib.model.ens.group import RollingGroup +from qlib.model.trainer import TrainerRM data_handler_config = { "start_time": "2008-01-01", @@ -94,24 +96,16 @@ def task_generating(): return tasks -# This part corresponds to "Task Storing" in the document -def task_storing(tasks, task_pool, exp_name): - print("========== task_storing ==========") - tm = TaskManager(task_pool=task_pool) - tm.create_task(tasks) # all tasks will be saved to MongoDB - - -# This part corresponds to "Task Running" in the document -def task_running(task_pool, exp_name): - print("========== task_running ==========") - run_task(task_train, task_pool, experiment_name=exp_name) # all tasks will be trained using "task_train" method +def task_training(tasks, task_pool, exp_name): + trainer = TrainerRM() + trainer.train(tasks, exp_name, task_pool) # This part corresponds to "Task Collecting" in the document def task_collecting(task_pool, exp_name): print("========== task_collecting ==========") - def get_group_key_func(recorder): + def rec_key(recorder): task_config = recorder.load_object("task") model_key = task_config["model"]["class"] rolling_key = task_config["dataset"]["kwargs"]["segments"]["test"] @@ -119,14 +113,14 @@ def get_group_key_func(recorder): def my_filter(recorder): # only choose the results of "LGBModel" - model_key, rolling_key = get_group_key_func(recorder) + model_key, rolling_key = rec_key(recorder) if model_key == "LGBModel": return True return False - collector = RecorderCollector(exp_name) - # group tasks by "get_task_key" and filter tasks by "my_filter" - artifact = collector.collect(RollingEnsemble(), get_group_key_func, rec_filter_func=my_filter) + artifact = ens_workflow( + RecorderCollector(exp_name=exp_name, rec_key_func=rec_key), RollingGroup(), rec_filter_func=my_filter + ) print(artifact) @@ -143,10 +137,9 @@ def main( } qlib.init(provider_uri=provider_uri, region=REG_CN, mongo=mongo_conf) - reset(task_pool, exp_name) - tasks = task_generating() - task_storing(tasks, task_pool, exp_name) - task_running(task_pool, exp_name) + # reset(task_pool, exp_name) + # tasks = task_generating() + # task_training(tasks, task_pool, exp_name) task_collecting(task_pool, exp_name) diff --git a/examples/online_svr/task_manager_rolling_with_updating.py b/examples/online_srv/task_manager_rolling_with_updating.py similarity index 97% rename from examples/online_svr/task_manager_rolling_with_updating.py rename to examples/online_srv/task_manager_rolling_with_updating.py index fff470c86f..32f582b4c9 100644 --- a/examples/online_svr/task_manager_rolling_with_updating.py +++ b/examples/online_srv/task_manager_rolling_with_updating.py @@ -6,10 +6,10 @@ from qlib.model.trainer import task_train from qlib.workflow import R from qlib.workflow.task.collect import RecorderCollector -from qlib.workflow.task.ensemble import RollingEnsemble +from qlib.model.ens.ensemble import RollingEnsemble from qlib.workflow.task.gen import RollingGen, task_generator from qlib.workflow.task.manage import TaskManager, run_task -from qlib.workflow.task.online import RollingOnlineManager +from qlib.workflow.online.manager import RollingOnlineManager from qlib.workflow.task.utils import list_recorders data_handler_config = { @@ -155,10 +155,10 @@ def first_run(): rolling_online_manager.reset_online_tag(latest_rec.values()) -def after_day(): +def routine(): print("========== after_day ==========") print_online_model() - rolling_online_manager.after_day() + rolling_online_manager.routine() print_online_model() task_collecting() diff --git a/examples/online_svr/update_online_pred.py b/examples/online_srv/update_online_pred.py similarity index 90% rename from examples/online_svr/update_online_pred.py rename to examples/online_srv/update_online_pred.py index ac86b48e8e..7bce82ac83 100644 --- a/examples/online_svr/update_online_pred.py +++ b/examples/online_srv/update_online_pred.py @@ -2,7 +2,7 @@ import qlib from qlib.config import REG_CN from qlib.model.trainer import task_train -from qlib.workflow.task.online import OnlineManagerR +from qlib.workflow.online.manager import OnlineManagerR from qlib.workflow.task.utils import list_recorders data_handler_config = { @@ -52,7 +52,7 @@ } -def first_train(experiment_name="online_svr"): +def first_train(experiment_name="online_srv"): rid = task_train(task_config=task, experiment_name=experiment_name) @@ -60,7 +60,7 @@ def first_train(experiment_name="online_svr"): online_manager.reset_online_tag(rid) -def update_online_pred(experiment_name="online_svr"): +def update_online_pred(experiment_name="online_srv"): online_manager = OnlineManagerR(experiment_name) diff --git a/qlib/model/ens/ensemble.py b/qlib/model/ens/ensemble.py new file mode 100644 index 0000000000..dcc4ba5d33 --- /dev/null +++ b/qlib/model/ens/ensemble.py @@ -0,0 +1,98 @@ +from abc import abstractmethod +from typing import Callable, Union + +import pandas as pd +from qlib.workflow.task.collect import Collector + + +def ens_workflow(collector: Collector, process_list, artifacts_key=None, rec_filter_func=None, *args, **kwargs): + """the ensemble workflow based on collector and different dict processors. + + Args: + collector (Collector): the collector to collect the result into {result_key: things} + process_list (list or Callable): the list of processors or the instance of processor to process dict. + The processor order is same as the list order. + + For example: [Group1(..., Ensemble1()), Group2(..., Ensemble2())] + + artifacts_key (list, optional): the artifacts key you want to get. If None, get all artifacts. + rec_filter_func (Callable, optional): filter the recorder by return True or False. Defaults to None. + + Returns: + dict: the ensemble dict + """ + collect_dict = collector.collect(artifacts_key=artifacts_key, rec_filter_func=rec_filter_func) + if not isinstance(process_list, list): + process_list = [process_list] + + ensemble = {} + for artifact in collect_dict: + value = collect_dict[artifact] + for process in process_list: + if not callable(process): + raise NotImplementedError(f"{type(process)} is not supported in `ens_workflow`.") + value = process(value, *args, **kwargs) + ensemble[artifact] = value + + return ensemble + + +class Ensemble: + """Merge the objects in an Ensemble.""" + + def __init__(self, merge_func=None): + """init Ensemble + + Args: + merge_func (Callable, optional): Given a dict and return the ensemble. + + For example: {Rollinga_b: object, Rollingb_c: object} -> object + + Defaults to None. + """ + self._merge = merge_func + + def __call__(self, ensemble_dict: dict, *args, **kwargs): + """Merge the ensemble_dict into an ensemble object. + + Args: + ensemble_dict (dict): the ensemble dict waiting for merging like {name: things} + + Returns: + object: the ensemble object + """ + if isinstance(getattr(self, "_merge", None), Callable): + return self._merge(ensemble_dict, *args, **kwargs) + else: + raise NotImplementedError(f"Please specify valid merge_func.") + + +class RollingEnsemble(Ensemble): + + """Merge the rolling objects in an Ensemble""" + + @staticmethod + def rolling_merge(rolling_dict: dict): + """Merge a dict of rolling dataframe like `prediction` or `IC` into an ensemble. + + NOTE: The values of dict must be pd.Dataframe, and have the index "datetime" + + Args: + rolling_dict (dict): a dict like {"A": pd.Dataframe, "B": pd.Dataframe}. + The key of the dict will be ignored. + + Returns: + pd.Dataframe: the complete result of rolling. + """ + artifact_list = list(rolling_dict.values()) + artifact_list.sort(key=lambda x: x.index.get_level_values("datetime").min()) + artifact = pd.concat(artifact_list) + # If there are duplicated predition, use the latest perdiction + artifact = artifact[~artifact.index.duplicated(keep="last")] + artifact = artifact.sort_index() + return artifact + + def __init__(self, merge_func=None): + super().__init__(merge_func=merge_func) + if merge_func is None: + self._merge = RollingEnsemble.rolling_merge \ No newline at end of file diff --git a/qlib/model/ens/group.py b/qlib/model/ens/group.py new file mode 100644 index 0000000000..1ef3da77f6 --- /dev/null +++ b/qlib/model/ens/group.py @@ -0,0 +1,68 @@ +from qlib.model.ens.ensemble import Ensemble, RollingEnsemble +from typing import Callable, Union + + +class Group: + """Group the objects based on dict""" + + def __init__(self, group_func=None, ens: Ensemble = None): + """init Group. + + Args: + group_func (Callable, optional): Given a dict and return the group key and one of group elements. + + For example: {(A,B,C1): object, (A,B,C2): object} -> {(A,B): {C1: object, C2: object}} + + Defaults to None. + + ens (Ensemble, optional): If not None, do ensemble for grouped value after grouping. + """ + self._group = group_func + self._ens = ens + + def __call__(self, ungrouped_dict: dict, *args, **kwargs): + """Group the ungrouped_dict into different groups. + + Args: + ungrouped_dict (dict): the ungrouped dict waiting for grouping like {name: things} + + Returns: + dict: grouped_dict like {G1: object, G2: object} + """ + if isinstance(getattr(self, "_group", None), Callable): + grouped_dict = self._group(ungrouped_dict, *args, **kwargs) + if self._ens is not None: + ens_dict = {} + for key, value in grouped_dict.items(): + ens_dict[key] = self._ens(value) + grouped_dict = ens_dict + return grouped_dict + else: + raise NotImplementedError(f"Please specify valid merge_func.") + + +class RollingGroup(Group): + """group the rolling dict""" + + @staticmethod + def rolling_group(rolling_dict: dict): + """Given an rolling dict likes {(A,B,R): things}, return the grouped dict likes {(A,B): {R:things}} + + NOTE: There is a assumption which is the rolling key is at the end of key tuple, because the rolling results always need to be ensemble firstly. + + Args: + rolling_dict (dict): an rolling dict. If the key is not a tuple, then do nothing. + + Returns: + dict: grouped dict + """ + grouped_dict = {} + for key, values in rolling_dict.items(): + if isinstance(key, tuple): + grouped_dict.setdefault(key[:-1], {})[key[-1]] = values + return grouped_dict + + def __init__(self, group_func=None, ens: Ensemble = RollingEnsemble()): + super().__init__(group_func=group_func, ens=ens) + if group_func is None: + self._group = RollingGroup.rolling_group \ No newline at end of file diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index 45650c0c7c..e128e700dc 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -4,6 +4,7 @@ from qlib.utils import init_instance_by_config, flatten_dict from qlib.workflow import R from qlib.workflow.record_temp import SignalRecord +from qlib.workflow.task.manage import TaskManager, run_task def task_train(task_config: dict, experiment_name: str) -> str: @@ -57,3 +58,70 @@ def task_train(task_config: dict, experiment_name: str) -> str: ar.generate() return recorder + + +class Trainer: + """ + The trainer which can train a list of model + """ + + def train(self, *args, **kwargs): + """Given a list of model definition, finished training and return the results of them. + + Returns: + list: a list of trained results + """ + raise NotImplementedError(f"Please implement the `train` method.") + + +class TrainerR(Trainer): + """Trainer based on (R)ecorder. + + Assumption: models were defined by `task` and the results will saved to `Recorder` + """ + + def train(self, tasks: list, experiment_name: str, train_func=task_train, *args, **kwargs): + """Given a list of `task`s and return a list of trained Recorder. The order can be guaranteed. + + Args: + tasks (list): a list of definition based on `task` dict + experiment_name (str): the experiment name + train_func (Callable): the train method which need at least `task` and `experiment_name` + + Returns: + list: a list of Recorders + """ + recs = [] + for task in tasks: + recs.append(train_func(task, experiment_name, *args, **kwargs)) + return recs + + +class TrainerRM(TrainerR): + """Trainer based on (R)ecorder and Task(M)anager + + Assumption: `task` will be saved to TaskManager and `task` will be fetched and trained from TaskManager + """ + + def train(self, tasks: list, experiment_name: str, task_pool: str, train_func=task_train, *args, **kwargs): + """Given a list of `task`s and return a list of trained Recorder. The order can be guaranteed. + + This method defaults to a single process, but TaskManager offered a great way to parallel training. + Users can customize their train_func to realize multiple processes or even multiple machines. + + Args: + tasks (list): a list of definition based on `task` dict + experiment_name (str): the experiment name + train_func (Callable): the train method which need at least `task` and `experiment_name` + + Returns: + list: a list of Recorders + """ + tm = TaskManager(task_pool=task_pool) + _id_list = tm.create_task(tasks) # all tasks will be saved to MongoDB + run_task(train_func, task_pool, experiment_name=experiment_name, *args, **kwargs) + + recs = [] + for _id in _id_list: + recs.append(tm.re_query(_id)["res"]) + return recs \ No newline at end of file diff --git a/qlib/workflow/online/__init__.py b/qlib/workflow/online/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/qlib/workflow/task/online.py b/qlib/workflow/online/manager.py similarity index 87% rename from qlib/workflow/task/online.py rename to qlib/workflow/online/manager.py index f7ffbd18a9..fbee0d707b 100644 --- a/qlib/workflow/task/online.py +++ b/qlib/workflow/online/manager.py @@ -3,7 +3,7 @@ from qlib.workflow import R from qlib.model.trainer import task_train from qlib.workflow.recorder import MLflowRecorder, Recorder -from qlib.workflow.task.update import ModelUpdater +from qlib.workflow.online.update import ModelUpdater from qlib.workflow.task.utils import TimeAdjuster from qlib.workflow.task.gen import RollingGen, task_generator from qlib.workflow.task.manage import TaskManager @@ -37,6 +37,16 @@ def set_online_tag(self, tag, *args, **kwargs): def get_online_tag(self, *args, **kwargs): raise NotImplementedError(f"Please implement the `get_online_tag` method.") + def reset_online_tag(self, *args, **kwargs): + raise NotImplementedError(f"Please implement the `reset_online_tag` method.") + + def routine(self, *args, **kwargs): + self.prepare_signals(*args, **kwargs) + self.prepare_tasks(*args, **kwargs) + self.prepare_new_models(*args, **kwargs) + self.update_online_pred(*args, **kwargs) + self.reset_online_tag(*args, **kwargs) + class OnlineManagerR(OnlineManager): """ @@ -86,21 +96,18 @@ def update_online_pred(self): cnt = mu.update_all_pred(lambda rec: self.get_online_tag(rec) == OnlineManager.ONLINE_TAG) self.logger.info(f"Finish updating {cnt} online model predictions of {self.exp_name}.") - def after_day(self, *args, **kwargs): - self.prepare_signals(*args, **kwargs) - self.prepare_tasks(*args, **kwargs) - self.prepare_new_models(*args, **kwargs) - self.update_online_pred(*args, **kwargs) - self.reset_online_tag() - class RollingOnlineManager(OnlineManagerR): - def __init__(self, experiment_name: str, rolling_gen: RollingGen, task_pool) -> None: + # FIXME: TaskManager不应该与onlinemanager强耦合 + def __init__( + self, experiment_name: str, rolling_gen: RollingGen, task_manager: TaskManager, trainer=run_task + ) -> None: super().__init__(experiment_name) self.ta = TimeAdjuster() self.rg = rolling_gen - self.tm = TaskManager(task_pool=task_pool) + self.tm = task_manager self.logger = get_module_logger(self.__class__.__name__) + self.trainer = trainer def prepare_signals(self): pass @@ -122,13 +129,13 @@ def prepare_tasks(self): task["dataset"]["kwargs"]["segments"]["test"] = (test_begin, calendar_latest) old_tasks.append(task) new_tasks = task_generator(old_tasks, self.rg) - new_num = self.tm.create_task(new_tasks) - self.logger.info(f"Finished prepare {new_num} tasks.") + self.tm.create_task(new_tasks) def prepare_new_models(self): """prepare(train) new models based on online model""" - run_task(task_train, self.tm.task_pool, experiment_name=self.exp_name) + run_task(task_train, task_pool=self.tm.task_pool, experiment_name=self.exp_name) latest_records, _ = self.list_latest_recorders() + # FIXME: 现有的流程，如果没有可更新的模型，仍会调用这个，导致会先将以前的模型设置成nextonline再去更新pred，但这个时候online已经没有了，pred无法更新 self.set_online_tag(OnlineManager.NEXT_ONLINE_TAG, latest_records.values()) self.logger.info(f"Finished prepare {len(latest_records)} new models and set them to next_online.") diff --git a/qlib/workflow/task/update.py b/qlib/workflow/online/update.py similarity index 98% rename from qlib/workflow/task/update.py rename to qlib/workflow/online/update.py index 002f1128f8..1a6897d02c 100644 --- a/qlib/workflow/task/update.py +++ b/qlib/workflow/online/update.py @@ -45,8 +45,8 @@ def _reload_dataset(self, recorder, start_time, end_time): """ segments = {"test": (start_time, end_time)} dataset = recorder.load_object("dataset") - dataset.config(handler_kwargs={"start_time": start_time, "end_time": end_time}) - dataset.setup_data(handler_kwargs={"init_type": DataHandlerLP.IT_LS}, segments=segments) + dataset.config(handler_kwargs={"start_time": start_time, "end_time": end_time}, segments=segments) + dataset.setup_data(handler_kwargs={"init_type": DataHandlerLP.IT_LS}) return dataset def update_pred(self, recorder: Recorder, frequency="day"): diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index 91b713ef8d..7e555ed067 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -1,49 +1,54 @@ from abc import abstractmethod from typing import Callable, Union - -import pandas as pd -from qlib import get_module_logger from qlib.workflow.task.utils import list_recorders class Collector: - """The collector to collect different results based on experiment backend and ensemble method""" + """The collector to collect different results""" - def collect(self, ensemble, get_group_key_func, *args, **kwargs): - """To collect the results, we need to get the experiment record firstly and divided them into - different groups. Then use ensemble methods to merge the group. + def collect(self, *args, **kwargs): + """Collect the results and return a dict like {key: things} - Args: - ensemble (Ensemble): an instance of Ensemble - get_group_key_func (Callable): a function to get the group of a experiment record + Returns: + dict: the dict after collected. + + For example: + + {"prediction": pd.Series} + {"IC": {"Xgboost": pd.Series, "LSTM": pd.Series}} + + ...... """ raise NotImplementedError(f"Please implement the `collect` method.") class RecorderCollector(Collector): - def __init__(self, exp_name, artifacts_path={"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"}) -> None: + def __init__( + self, exp_name, artifacts_path={"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"}, rec_key_func=None + ) -> None: """init RecorderCollector Args: exp_name (str): the name of Experiment artifacts_path (dict, optional): The artifacts name and its path in Recorder. Defaults to {"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"}. + rec_key_func (Callable): a function to get the key of a recorder. If None, use recorder id. """ self.exp_name = exp_name self.artifacts_path = artifacts_path + if rec_key_func is None: + rec_key_func = lambda rec: rec.info["id"] + self._get_key = rec_key_func - def collect(self, ensemble, get_group_key_func, artifacts_key=None, rec_filter_func=None): - """Collect different artifacts based on recorder after filtering and ensemble method. - Group recorder by get_group_key_func. + def collect(self, artifacts_key=None, rec_filter_func=None): # ensemble, get_group_key_func, + """Collect different artifacts based on recorder after filtering. Args: - ensemble (Ensemble): an instance of Ensemble - get_group_key_func (Callable): a function to get the group of a experiment record - artifacts_key (str or List, optional): the artifacts key you want to get. Defaults to None. + artifacts_key (str or List, optional): the artifacts key you want to get. If None, get all artifacts. rec_filter_func (Callable, optional): filter the recorder by return True or False. Defaults to None. Returns: - dict: the dict after collected. + dict: the dict after collected like {artifact: {rec_key: object}} """ if artifacts_key is None: artifacts_key = self.artifacts_path.keys() @@ -51,22 +56,13 @@ def collect(self, ensemble, get_group_key_func, artifacts_key=None, rec_filter_f if isinstance(artifacts_key, str): artifacts_key = [artifacts_key] - # prepare_ensemble - ensemble_dict = {} - for key in artifacts_key: - ensemble_dict.setdefault(key, {}) + collect_dict = {} # filter records recs_flt = list_recorders(self.exp_name, rec_filter_func) for _, rec in recs_flt.items(): - group_key = get_group_key_func(rec) + rec_key = self._get_key(rec) for key in artifacts_key: artifact = rec.load_object(self.artifacts_path[key]) - ensemble_dict[key][group_key] = artifact + collect_dict.setdefault(key, {})[rec_key] = artifact - if isinstance(artifacts_key, str): - return ensemble(ensemble_dict[artifacts_key]) - - collect_dict = {} - for key in artifacts_key: - collect_dict[key] = ensemble(ensemble_dict[key]) - return collect_dict + return collect_dict \ No newline at end of file diff --git a/qlib/workflow/task/ensemble.py b/qlib/workflow/task/ensemble.py deleted file mode 100644 index dca0dee3e0..0000000000 --- a/qlib/workflow/task/ensemble.py +++ /dev/null @@ -1,176 +0,0 @@ -from abc import abstractmethod -from typing import Callable, Union - -import pandas as pd -from qlib import get_module_logger -from qlib.workflow.task.utils import list_recorders -from typing import Dict - - -class Ensemble: - """Merge the objects in an Ensemble.""" - - def __init__(self, merge_func=None, get_grouped_key_func=None) -> None: - """init Ensemble - - Args: - merge_func (Callable, optional): The specific merge function. Defaults to None. - get_grouped_key_func (Callable, optional): Get group_inner_key and group_outer_key by group_key. Defaults to None. - """ - self.logger = get_module_logger(self.__class__.__name__) - if merge_func is not None: - self.merge_func = merge_func - if get_grouped_key_func is not None: - self.get_grouped_key_func = get_grouped_key_func - - def merge_func(self, group_inner_dict): - """Given a group_inner_dict such as {Rollinga_b: object, Rollingb_c: object}, - merge it to object - - Args: - group_inner_dict (dict): the inner group dict - - """ - raise NotImplementedError(f"Please implement the `merge_func` method.") - - def get_grouped_key_func(self, group_key): - """Given a group_key and return the group_outer_key, group_inner_key. - - For example: - (A,B,Rolling) -> (A,B):Rolling - (A,B) -> C:(A,B) - - Args: - group_key (tuple or str): the group key - """ - raise NotImplementedError(f"Please implement the `get_grouped_key_func` method.") - - def group(self, group_dict: Dict[tuple or str, object]) -> Dict[tuple or str, Dict[tuple or str, object]]: - """In a group of dict, further divide them into outgroups and innergroup. - - For example: - - .. code-block:: python - - RollingEnsemble: - input: - { - (ModelA,Horizon5,Rollinga_b): object - (ModelA,Horizon5,Rollingb_c): object - (ModelA,Horizon10,Rollinga_b): object - (ModelA,Horizon10,Rollingb_c): object - (ModelB,Horizon5,Rollinga_b): object - (ModelB,Horizon5,Rollingb_c): object - (ModelB,Horizon10,Rollinga_b): object - (ModelB,Horizon10,Rollingb_c): object - } - - output: - { - (ModelA,Horizon5): {Rollinga_b: object, Rollingb_c: object} - (ModelA,Horizon10): {Rollinga_b: object, Rollingb_c: object} - (ModelB,Horizon5): {Rollinga_b: object, Rollingb_c: object} - (ModelB,Horizon10): {Rollinga_b: object, Rollingb_c: object} - } - - Args: - group_dict (Dict[tuple or str, object]): a group of dict - - Returns: - Dict[tuple or str, Dict[tuple or str, object]]: the dict after `group` - """ - grouped_dict = {} - for group_key, artifact in group_dict.items(): - group_outer_key, group_inner_key = self.get_grouped_key_func(group_key) # (A,B,Rolling) -> (A,B):Rolling - grouped_dict.setdefault(group_outer_key, {})[group_inner_key] = artifact - return grouped_dict - - def reduce(self, grouped_dict: dict): - """After grouping, reduce the innergroup. - - For example: - - .. code-block:: python - - RollingEnsemble: - input: - { - (ModelA,Horizon5): {Rollinga_b: object, Rollingb_c: object} - (ModelA,Horizon10): {Rollinga_b: object, Rollingb_c: object} - (ModelB,Horizon5): {Rollinga_b: object, Rollingb_c: object} - (ModelB,Horizon10): {Rollinga_b: object, Rollingb_c: object} - } - - output: - { - (ModelA,Horizon5): object - (ModelA,Horizon10): object - (ModelB,Horizon5): object - (ModelB,Horizon10): object - } - - Args: - grouped_dict (dict): the dict after `group` - - Returns: - dict: the dict after `reduce` - """ - reduce_group = {} - for group_outer_key, group_inner_dict in grouped_dict.items(): - artifact = self.merge_func(group_inner_dict) - reduce_group[group_outer_key] = artifact - return reduce_group - - def __call__(self, group_dict): - """The process of Ensemble is group it firstly and then reduce it, like MapReduce. - - Args: - group_dict (Dict[tuple or str, object]): a group of dict - - Returns: - dict: the dict after `reduce` - """ - grouped_dict = self.group(group_dict) - return self.reduce(grouped_dict) - - -class RollingEnsemble(Ensemble): - """A specific implementation of Ensemble for Rolling.""" - - def merge_func(self, group_inner_dict): - """merge group_inner_dict by datetime. - - Args: - group_inner_dict (dict): the inner group dict - - Returns: - object: the artifact after merging - """ - artifact_list = list(group_inner_dict.values()) - artifact_list.sort(key=lambda x: x.index.get_level_values("datetime").min()) - artifact = pd.concat(artifact_list) - # If there are duplicated predition, use the latest perdiction - artifact = artifact[~artifact.index.duplicated(keep="last")] - artifact = artifact.sort_index() - return artifact - - def get_grouped_key_func(self, group_key): - """The final axis of group_key must be the Rolling key. - When `collect`, get_group_key_func can add the statement below. - - .. code-block:: python - - def get_group_key_func(recorder): - task_config = recorder.load_object("task") - ...... - rolling_key = task_config["dataset"]["kwargs"]["segments"]["test"] - return ......, rolling_key - - Args: - group_key (tuple or str): the group key - - Returns: - tuple or str, tuple or str: group_outer_key, group_inner_key - """ - assert len(group_key) >= 2 - return group_key[:-1], group_key[-1] diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index a621642078..ddd833aa40 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -60,7 +60,7 @@ def __init__(self, task_pool=None): """ self.mdb = get_mongodb() self.task_pool = task_pool - self.logger = get_module_logger("TaskManager") + self.logger = get_module_logger(self.__class__.__name__) def list(self): return self.mdb.list_collection_names() @@ -105,10 +105,11 @@ def replace_task(self, task, new_task, task_pool=None): def insert_task(self, task, task_pool=None): task_pool = self._get_task_pool(task_pool) try: - task_pool.insert_one(task) + insert_result = task_pool.insert_one(task) except InvalidDocument: task["filter"] = self._dict_to_str(task["filter"]) - task_pool.insert_one(task) + insert_result = task_pool.insert_one(task) + return insert_result def insert_task_def(self, task_def, task_pool=None): """ @@ -133,7 +134,8 @@ def insert_task_def(self, task_def, task_pool=None): "status": self.STATUS_WAITING, } ) - self.insert_task(task, task_pool) + insert_result = self.insert_task(task, task_pool) + return insert_result def create_task(self, task_def_l, task_pool=None, dry_run=False, print_nt=False): """ @@ -151,8 +153,8 @@ def create_task(self, task_def_l, task_pool=None, dry_run=False, print_nt=False) if print new task Returns ------- - int - the length of new tasks + list + a list of the _id of new tasks """ task_pool = self._get_task_pool(task_pool) new_tasks = [] @@ -163,7 +165,7 @@ def create_task(self, task_def_l, task_pool=None, dry_run=False, print_nt=False) r = task_pool.find_one({"filter": self._dict_to_str(t)}) if r is None: new_tasks.append(t) - print("Total Tasks, New Tasks:", len(task_def_l), len(new_tasks)) + self.logger.info(f"Total Tasks: {len(task_def_l)}, New Tasks: {len(new_tasks)}") if print_nt: # print new task for t in new_tasks: @@ -172,10 +174,12 @@ def create_task(self, task_def_l, task_pool=None, dry_run=False, print_nt=False) if dry_run: return + _id_list = [] for t in new_tasks: - self.insert_task_def(t, task_pool) + insert_result = self.insert_task_def(t, task_pool) + _id_list.append(insert_result.inserted_id) - return len(new_tasks) + return _id_list def fetch_task(self, query={}, task_pool=None): task_pool = self._get_task_pool(task_pool) @@ -248,9 +252,9 @@ def query(self, query={}, decode=True, task_pool=None): for t in task_pool.find(query): yield self._decode_task(t) - def re_query(self, task, task_pool=None): + def re_query(self, _id, task_pool=None): task_pool = self._get_task_pool(task_pool) - return task_pool.find_one({"_id": ObjectId(task["_id"])}) + return task_pool.find_one({"_id": ObjectId(_id)}) def commit_task_res(self, task, res, status=None, task_pool=None): task_pool = self._get_task_pool(task_pool) From 431a9c92c1654b132e00211361713b8edcbfd5eb Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Fri, 2 Apr 2021 07:09:29 +0000 Subject: [PATCH 32/61] online serving v5 --- .../task_manager_rolling_with_updating.py | 38 ++++----- qlib/workflow/online/manager.py | 83 ++++++++++++++----- 2 files changed, 77 insertions(+), 44 deletions(-) diff --git a/examples/online_srv/task_manager_rolling_with_updating.py b/examples/online_srv/task_manager_rolling_with_updating.py index 32f582b4c9..d8bd959274 100644 --- a/examples/online_srv/task_manager_rolling_with_updating.py +++ b/examples/online_srv/task_manager_rolling_with_updating.py @@ -6,11 +6,13 @@ from qlib.model.trainer import task_train from qlib.workflow import R from qlib.workflow.task.collect import RecorderCollector -from qlib.model.ens.ensemble import RollingEnsemble +from qlib.model.ens.ensemble import RollingEnsemble, ens_workflow from qlib.workflow.task.gen import RollingGen, task_generator from qlib.workflow.task.manage import TaskManager, run_task from qlib.workflow.online.manager import RollingOnlineManager from qlib.workflow.task.utils import list_recorders +from qlib.model.trainer import TrainerRM +from qlib.model.ens.group import RollingGroup data_handler_config = { "start_time": "2013-01-01", @@ -96,24 +98,15 @@ def task_generating(): return tasks -# This part corresponds to "Task Storing" in the document -def task_storing(tasks): - print("========== task_storing ==========") - tm = TaskManager(task_pool=task_pool) - tm.create_task(tasks) # all tasks will be saved to MongoDB - - -# This part corresponds to "Task Running" in the document -def task_running(): - print("========== task_running ==========") - run_task(task_train, task_pool, experiment_name=exp_name) # all tasks will be trained using "task_train" method +def task_training(tasks): + trainer.train(tasks, exp_name, task_pool) # This part corresponds to "Task Collecting" in the document def task_collecting(): print("========== task_collecting ==========") - def get_group_key_func(recorder): + def rec_key(recorder): task_config = recorder.load_object("task") model_key = task_config["model"]["class"] rolling_key = task_config["dataset"]["kwargs"]["segments"]["test"] @@ -121,14 +114,14 @@ def get_group_key_func(recorder): def my_filter(recorder): # only choose the results of "LGBModel" - model_key, rolling_key = get_group_key_func(recorder) + model_key, rolling_key = rec_key(recorder) if model_key == "LGBModel": return True return False - collector = RecorderCollector(exp_name) - # group tasks by "get_task_key" and filter tasks by "my_filter" - artifact = collector.collect(RollingEnsemble(), get_group_key_func, rec_filter_func=my_filter) + artifact = ens_workflow( + RecorderCollector(exp_name=exp_name, rec_key_func=rec_key), RollingGroup(), rec_filter_func=my_filter + ) print(artifact) @@ -147,8 +140,7 @@ def first_run(): reset() tasks = task_generating() - task_storing(tasks) - task_running() + task_training(tasks) task_collecting() latest_rec, _ = rolling_online_manager.list_latest_recorders() @@ -156,7 +148,7 @@ def first_run(): def routine(): - print("========== after_day ==========") + print("========== routine ==========") print_online_model() rolling_online_manager.routine() print_online_model() @@ -185,8 +177,10 @@ def routine(): ########################################################################################## rolling_gen = RollingGen(step=rolling_step, rtype=RollingGen.ROLL_SD) + task_manager = TaskManager(task_pool=task_pool) + trainer = TrainerRM() rolling_online_manager = RollingOnlineManager( - experiment_name=exp_name, rolling_gen=rolling_gen, task_pool=task_pool + experiment_name=exp_name, rolling_gen=rolling_gen, task_manager=task_manager, trainer=trainer ) - task_manager = TaskManager(task_pool=task_pool) + fire.Fire() diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index fbee0d707b..25a3682696 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -10,6 +10,7 @@ from qlib.workflow.task.manage import run_task from qlib.workflow.task.utils import list_recorders from qlib.utils.serial import Serializable +from qlib.model.trainer import Trainer, TrainerR class OnlineManager(Serializable): @@ -19,31 +20,57 @@ class OnlineManager(Serializable): NEXT_ONLINE_TAG = "next_online" # the 'next online' model, which can be 'online' model when call reset_online_model OFFLINE_TAG = "offline" # the 'offline' model, not for online serving + def __init__(self, trainer: Trainer = None) -> None: + self._trainer = trainer + self.logger = get_module_logger(self.__class__.__name__) + def prepare_signals(self, *args, **kwargs): raise NotImplementedError(f"Please implement the `prepare_signals` method.") def prepare_tasks(self, *args, **kwargs): + """return the new tasks waiting for training.""" raise NotImplementedError(f"Please implement the `prepare_tasks` method.") - def prepare_new_models(self, *args, **kwargs): - raise NotImplementedError(f"Please implement the `prepare_new_models` method.") + def prepare_new_models(self, tasks, *args, **kwargs): + """Use trainer to train a list of tasks and set the trained model to next_online. + + Args: + tasks (list): a list of tasks. + """ + if not (tasks is None or len(tasks) == 0): + if self._trainer is not None: + new_models = self._trainer.train(tasks, *args, **kwargs) + self.set_online_tag(self.NEXT_ONLINE_TAG, new_models) + self.logger.info( + f"Finished prepare {len(new_models)} new models and set them to `{self.NEXT_ONLINE_TAG}`." + ) + else: + self.logger.warn("No trainer to train new tasks.") def update_online_pred(self, *args, **kwargs): raise NotImplementedError(f"Please implement the `update_online_pred` method.") def set_online_tag(self, tag, *args, **kwargs): + """set `tag` to the model to sign whether online + + Args: + tag (str): the tags in ONLINE_TAG, NEXT_ONLINE_TAG, OFFLINE_TAG + """ raise NotImplementedError(f"Please implement the `set_online_tag` method.") def get_online_tag(self, *args, **kwargs): + """given a model and return its online tag""" raise NotImplementedError(f"Please implement the `get_online_tag` method.") def reset_online_tag(self, *args, **kwargs): + """offline all models and set the recorders to 'online'. If no parameter and no 'next online' model, then do nothing.""" raise NotImplementedError(f"Please implement the `reset_online_tag` method.") def routine(self, *args, **kwargs): + """The typical update process in a routine such as day by day or month by month""" self.prepare_signals(*args, **kwargs) - self.prepare_tasks(*args, **kwargs) - self.prepare_new_models(*args, **kwargs) + tasks = self.prepare_tasks(*args, **kwargs) + self.prepare_new_models(tasks, *args, **kwargs) self.update_online_pred(*args, **kwargs) self.reset_online_tag(*args, **kwargs) @@ -54,7 +81,8 @@ class OnlineManagerR(OnlineManager): """ - def __init__(self, experiment_name: str) -> None: + def __init__(self, experiment_name: str, trainer: Trainer = TrainerR()) -> None: + super().__init__(trainer) self.logger = get_module_logger(self.__class__.__name__) self.exp_name = experiment_name @@ -98,27 +126,36 @@ def update_online_pred(self): class RollingOnlineManager(OnlineManagerR): - # FIXME: TaskManager不应该与onlinemanager强耦合 + """An implementation of OnlineManager based on Rolling. + + """ + def __init__( - self, experiment_name: str, rolling_gen: RollingGen, task_manager: TaskManager, trainer=run_task + self, + experiment_name: str, + rolling_gen: RollingGen, + trainer: Trainer = TrainerR(), ) -> None: - super().__init__(experiment_name) + super().__init__(experiment_name, trainer) self.ta = TimeAdjuster() self.rg = rolling_gen - self.tm = task_manager self.logger = get_module_logger(self.__class__.__name__) - self.trainer = trainer def prepare_signals(self): pass def prepare_tasks(self): + """prepare new tasks based on new date. + + Returns: + list: a list of new tasks. + """ latest_records, max_test = self.list_latest_recorders( lambda rec: self.get_online_tag(rec) == OnlineManager.ONLINE_TAG ) if max_test is None: - self.logger.warn(f"No latest_recorders.") - return + self.logger.warn(f"No latest online recorders, no new tasks.") + return None calendar_latest = self.ta.last_date() if self.ta.cal_interval(calendar_latest, max_test[0]) > self.rg.step: old_tasks = [] @@ -128,18 +165,20 @@ def prepare_tasks(self): # modify the test segment to generate new tasks task["dataset"]["kwargs"]["segments"]["test"] = (test_begin, calendar_latest) old_tasks.append(task) - new_tasks = task_generator(old_tasks, self.rg) - self.tm.create_task(new_tasks) - - def prepare_new_models(self): - """prepare(train) new models based on online model""" - run_task(task_train, task_pool=self.tm.task_pool, experiment_name=self.exp_name) - latest_records, _ = self.list_latest_recorders() - # FIXME: 现有的流程，如果没有可更新的模型，仍会调用这个，导致会先将以前的模型设置成nextonline再去更新pred，但这个时候online已经没有了，pred无法更新 - self.set_online_tag(OnlineManager.NEXT_ONLINE_TAG, latest_records.values()) - self.logger.info(f"Finished prepare {len(latest_records)} new models and set them to next_online.") + new_tasks_tmp = task_generator(old_tasks, self.rg) + new_tasks = [task for task in new_tasks_tmp if task not in old_tasks] + return new_tasks + return None def list_latest_recorders(self, rec_filter_func=None): + """find latest recorders based on test segments. + + Args: + rec_filter_func (Callable, optional): recorder filter. Defaults to None. + + Returns: + dict, tuple: the latest recorders and the latest date of them + """ recs_flt = list_recorders(self.exp_name, rec_filter_func) if len(recs_flt) == 0: return recs_flt, None From cb42e99bee83e494536538340e88f894a5dbf257 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Wed, 7 Apr 2021 03:33:27 +0000 Subject: [PATCH 33/61] bug fixed & examples fire --- .../model_rolling/task_manager_rolling.py | 18 +- .../task_manager_rolling_with_updating.py | 166 +++++++++--------- examples/online_srv/update_online_pred.py | 14 +- qlib/config.py | 2 +- qlib/model/ens/ensemble.py | 37 +--- qlib/model/ens/group.py | 19 +- qlib/model/trainer.py | 39 ++-- qlib/workflow/online/manager.py | 36 ++-- qlib/workflow/task/collect.py | 34 ++-- qlib/workflow/task/manage.py | 117 ++++++------ 10 files changed, 250 insertions(+), 232 deletions(-) diff --git a/examples/model_rolling/task_manager_rolling.py b/examples/model_rolling/task_manager_rolling.py index 3eb05de728..9070864870 100644 --- a/examples/model_rolling/task_manager_rolling.py +++ b/examples/model_rolling/task_manager_rolling.py @@ -97,8 +97,8 @@ def task_generating(): def task_training(tasks, task_pool, exp_name): - trainer = TrainerRM() - trainer.train(tasks, exp_name, task_pool) + trainer = TrainerRM(exp_name, task_pool) + trainer.train(tasks) # This part corresponds to "Task Collecting" in the document @@ -119,7 +119,7 @@ def my_filter(recorder): return False artifact = ens_workflow( - RecorderCollector(exp_name=exp_name, rec_key_func=rec_key), RollingGroup(), rec_filter_func=my_filter + RecorderCollector(exp_name=exp_name, rec_key_func=rec_key, rec_filter_func=my_filter), RollingGroup(), ) print(artifact) @@ -128,7 +128,7 @@ def main( provider_uri="~/.qlib/qlib_data/cn_data", task_url="mongodb://10.0.0.4:27017/", task_db_name="rolling_db", - exp_name="rolling_exp", + experiment_name="rolling_exp", task_pool="rolling_task", ): mongo_conf = { @@ -137,11 +137,13 @@ def main( } qlib.init(provider_uri=provider_uri, region=REG_CN, mongo=mongo_conf) - # reset(task_pool, exp_name) - # tasks = task_generating() - # task_training(tasks, task_pool, exp_name) - task_collecting(task_pool, exp_name) + reset(task_pool, experiment_name) + tasks = task_generating() + task_training(tasks, task_pool, experiment_name) + task_collecting(task_pool, experiment_name) if __name__ == "__main__": + ## to see the whole process with your own parameters, use the command below + # python update_online_pred.py main --experiment_name="your_exp_name" fire.Fire() diff --git a/examples/online_srv/task_manager_rolling_with_updating.py b/examples/online_srv/task_manager_rolling_with_updating.py index d8bd959274..5b80f9133e 100644 --- a/examples/online_srv/task_manager_rolling_with_updating.py +++ b/examples/online_srv/task_manager_rolling_with_updating.py @@ -70,89 +70,106 @@ "record": record_config, } +class RollingOnlineExample: -def print_online_model(): - print("========== print_online_model ==========") - print("Current 'online' model:") - for rid, rec in list_recorders(exp_name).items(): - if rolling_online_manager.get_online_tag(rec) == rolling_online_manager.ONLINE_TAG: - print(rid) - print("Current 'next online' model:") - for rid, rec in list_recorders(exp_name).items(): - if rolling_online_manager.get_online_tag(rec) == rolling_online_manager.NEXT_ONLINE_TAG: - print(rid) + def __init__(self, exp_name="rolling_exp", task_pool="rolling_task", provider_uri="~/.qlib/qlib_data/cn_data", region="cn", task_url="mongodb://10.0.0.4:27017/", task_db_name="rolling_db", rolling_step=550): + self.exp_name = exp_name + self.task_pool = task_pool + mongo_conf = { + "task_url": task_url, # your MongoDB url + "task_db_name": task_db_name, # database name + } + qlib.init(provider_uri=provider_uri, region=region, mongo=mongo_conf) + self.rolling_gen = RollingGen(step=rolling_step, rtype=RollingGen.ROLL_SD) + self.trainer = TrainerRM(self.exp_name, self.task_pool) + self.task_manager = TaskManager(self.task_pool) + self.rolling_online_manager = RollingOnlineManager(experiment_name=exp_name, rolling_gen=self.rolling_gen, trainer=self.trainer) -# This part corresponds to "Task Generating" in the document -def task_generating(): + - print("========== task_generating ==========") + def print_online_model(self): + print("========== print_online_model ==========") + print("Current 'online' model:") + for rid, rec in list_recorders(self.exp_name).items(): + if self.rolling_online_manager.get_online_tag(rec) == self.rolling_online_manager.ONLINE_TAG: + print(rid) + print("Current 'next online' model:") + for rid, rec in list_recorders(self.exp_name).items(): + if self.rolling_online_manager.get_online_tag(rec) == self.rolling_online_manager.NEXT_ONLINE_TAG: + print(rid) - tasks = task_generator( - tasks=[task_xgboost_config, task_lgb_config], - generators=rolling_gen, # generate different date segment - ) - pprint(tasks) + # This part corresponds to "Task Generating" in the document + def task_generating(self): - return tasks + print("========== task_generating ==========") + tasks = task_generator( + tasks=[task_xgboost_config, task_lgb_config], + generators=self.rolling_gen, # generate different date segment + ) -def task_training(tasks): - trainer.train(tasks, exp_name, task_pool) + pprint(tasks) + return tasks -# This part corresponds to "Task Collecting" in the document -def task_collecting(): - print("========== task_collecting ==========") - def rec_key(recorder): - task_config = recorder.load_object("task") - model_key = task_config["model"]["class"] - rolling_key = task_config["dataset"]["kwargs"]["segments"]["test"] - return model_key, rolling_key + def task_training(self, tasks): + self.trainer.train(tasks) - def my_filter(recorder): - # only choose the results of "LGBModel" - model_key, rolling_key = rec_key(recorder) - if model_key == "LGBModel": - return True - return False - artifact = ens_workflow( - RecorderCollector(exp_name=exp_name, rec_key_func=rec_key), RollingGroup(), rec_filter_func=my_filter - ) - print(artifact) + # This part corresponds to "Task Collecting" in the document + def task_collecting(self): + print("========== task_collecting ==========") + def rec_key(recorder): + task_config = recorder.load_object("task") + model_key = task_config["model"]["class"] + rolling_key = task_config["dataset"]["kwargs"]["segments"]["test"] + return model_key, rolling_key -# Reset all things to the first status, be careful to save important data -def reset(): - print("========== reset ==========") - task_manager.remove() - exp = R.get_exp(experiment_name=exp_name) - for rid in exp.list_recorders(): - exp.delete_recorder(rid) + def my_filter(recorder): + # only choose the results of "LGBModel" + model_key, rolling_key = rec_key(recorder) + if model_key == "LGBModel": + return True + return False + artifact = ens_workflow( + RecorderCollector(exp_name=self.exp_name, rec_key_func=rec_key, rec_filter_func=my_filter), RollingGroup() + ) + print(artifact) -# Run this firstly to see the workflow in Task Management -def first_run(): - print("========== first_run ==========") - reset() - tasks = task_generating() - task_training(tasks) - task_collecting() + # Reset all things to the first status, be careful to save important data + def reset(self): + print("========== reset ==========") + self.task_manager.remove() + exp = R.get_exp(experiment_name=self.exp_name) + for rid in exp.list_recorders(): + exp.delete_recorder(rid) - latest_rec, _ = rolling_online_manager.list_latest_recorders() - rolling_online_manager.reset_online_tag(latest_rec.values()) + # Run this firstly to see the workflow in Task Management + def first_run(self): + print("========== first_run ==========") + self.reset() -def routine(): - print("========== routine ==========") - print_online_model() - rolling_online_manager.routine() - print_online_model() - task_collecting() + tasks = self.task_generating() + self.task_training(tasks) + self.task_collecting() + + latest_rec, _ = self.rolling_online_manager.list_latest_recorders() + self.rolling_online_manager.reset_online_tag(list(latest_rec.values())) + + + def routine(self): + print("========== routine ==========") + self.print_online_model() + self.rolling_online_manager.routine() + self.print_online_model() + self.task_collecting() if __name__ == "__main__": @@ -161,26 +178,7 @@ def routine(): ####### to update the models and predictions after the trading time, use the command below # python task_manager_rolling_with_updating.py after_day - - #################### you need to finish the configurations below ######################### - - provider_uri = "~/.qlib/qlib_data/cn_data" # data_dir - mongo_conf = { - "task_url": "mongodb://10.0.0.4:27017/", # your MongoDB url - "task_db_name": "rolling_db", # database name - } - qlib.init(provider_uri=provider_uri, region=REG_CN, mongo=mongo_conf) - - exp_name = "rolling_exp" # experiment name, will be used as the experiment in MLflow - task_pool = "rolling_task" # task pool name, will be used as the document in MongoDB - rolling_step = 550 - - ########################################################################################## - rolling_gen = RollingGen(step=rolling_step, rtype=RollingGen.ROLL_SD) - task_manager = TaskManager(task_pool=task_pool) - trainer = TrainerRM() - rolling_online_manager = RollingOnlineManager( - experiment_name=exp_name, rolling_gen=rolling_gen, task_manager=task_manager, trainer=trainer - ) - - fire.Fire() + + ####### to define your own parameters, use `--` + # python task_manager_rolling_with_updating.py first_run --exp_name='your_exp_name' --rolling_step=40 + fire.Fire(RollingOnlineExample) diff --git a/examples/online_srv/update_online_pred.py b/examples/online_srv/update_online_pred.py index 7bce82ac83..84472bc3b6 100644 --- a/examples/online_srv/update_online_pred.py +++ b/examples/online_srv/update_online_pred.py @@ -54,10 +54,10 @@ def first_train(experiment_name="online_srv"): - rid = task_train(task_config=task, experiment_name=experiment_name) + rec = task_train(task_config=task, experiment_name=experiment_name) online_manager = OnlineManagerR(experiment_name) - online_manager.reset_online_tag(rid) + online_manager.reset_online_tag(rec) def update_online_pred(experiment_name="online_srv"): @@ -71,13 +71,17 @@ def update_online_pred(experiment_name="online_srv"): online_manager.update_online_pred() +def main(provider_uri = "~/.qlib/qlib_data/cn_data", region=REG_CN, experiment_name="online_srv"): + provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir + qlib.init(provider_uri=provider_uri, region=region) + first_train(experiment_name) + update_online_pred(experiment_name) if __name__ == "__main__": ## to train a model and set it to online model, use the command below # python update_online_pred.py first_train ## to update online predictions once a day, use the command below # python update_online_pred.py update_online_pred - - provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir - qlib.init(provider_uri=provider_uri, region=REG_CN) + ## to see the whole process with your own parameters, use the command below + # python update_online_pred.py main --experiment_name="your_exp_name" fire.Fire() diff --git a/qlib/config.py b/qlib/config.py index d0479a345c..4dedf75d06 100644 --- a/qlib/config.py +++ b/qlib/config.py @@ -147,7 +147,7 @@ def set_conf_from_C(self, config_c): "mongo": { "task_url": "mongodb://localhost:27017/", "task_db_name": "default_task_db", - } + }, } MODE_CONF = { diff --git a/qlib/model/ens/ensemble.py b/qlib/model/ens/ensemble.py index dcc4ba5d33..a2333cfeba 100644 --- a/qlib/model/ens/ensemble.py +++ b/qlib/model/ens/ensemble.py @@ -3,9 +3,10 @@ import pandas as pd from qlib.workflow.task.collect import Collector +from qlib.utils.serial import Serializable -def ens_workflow(collector: Collector, process_list, artifacts_key=None, rec_filter_func=None, *args, **kwargs): +def ens_workflow(collector: Collector, process_list, *args, **kwargs): """the ensemble workflow based on collector and different dict processors. Args: @@ -21,7 +22,7 @@ def ens_workflow(collector: Collector, process_list, artifacts_key=None, rec_fil Returns: dict: the ensemble dict """ - collect_dict = collector.collect(artifacts_key=artifacts_key, rec_filter_func=rec_filter_func) + collect_dict = collector.collect() if not isinstance(process_list, list): process_list = [process_list] @@ -37,23 +38,12 @@ def ens_workflow(collector: Collector, process_list, artifacts_key=None, rec_fil return ensemble -class Ensemble: +class Ensemble(Serializable): """Merge the objects in an Ensemble.""" - def __init__(self, merge_func=None): - """init Ensemble - - Args: - merge_func (Callable, optional): Given a dict and return the ensemble. - - For example: {Rollinga_b: object, Rollingb_c: object} -> object - - Defaults to None. - """ - self._merge = merge_func - def __call__(self, ensemble_dict: dict, *args, **kwargs): """Merge the ensemble_dict into an ensemble object. + For example: {Rollinga_b: object, Rollingb_c: object} -> object Args: ensemble_dict (dict): the ensemble dict waiting for merging like {name: things} @@ -61,38 +51,29 @@ def __call__(self, ensemble_dict: dict, *args, **kwargs): Returns: object: the ensemble object """ - if isinstance(getattr(self, "_merge", None), Callable): - return self._merge(ensemble_dict, *args, **kwargs) - else: - raise NotImplementedError(f"Please specify valid merge_func.") + raise NotImplementedError(f"Please implement the `__call__` method.") class RollingEnsemble(Ensemble): """Merge the rolling objects in an Ensemble""" - @staticmethod - def rolling_merge(rolling_dict: dict): + def __call__(self, ensemble_dict: dict, *args, **kwargs): """Merge a dict of rolling dataframe like `prediction` or `IC` into an ensemble. NOTE: The values of dict must be pd.Dataframe, and have the index "datetime" Args: - rolling_dict (dict): a dict like {"A": pd.Dataframe, "B": pd.Dataframe}. + ensemble_dict (dict): a dict like {"A": pd.Dataframe, "B": pd.Dataframe}. The key of the dict will be ignored. Returns: pd.Dataframe: the complete result of rolling. """ - artifact_list = list(rolling_dict.values()) + artifact_list = list(ensemble_dict.values()) artifact_list.sort(key=lambda x: x.index.get_level_values("datetime").min()) artifact = pd.concat(artifact_list) # If there are duplicated predition, use the latest perdiction artifact = artifact[~artifact.index.duplicated(keep="last")] artifact = artifact.sort_index() return artifact - - def __init__(self, merge_func=None): - super().__init__(merge_func=merge_func) - if merge_func is None: - self._merge = RollingEnsemble.rolling_merge \ No newline at end of file diff --git a/qlib/model/ens/group.py b/qlib/model/ens/group.py index 1ef3da77f6..9cc5db9714 100644 --- a/qlib/model/ens/group.py +++ b/qlib/model/ens/group.py @@ -1,8 +1,9 @@ from qlib.model.ens.ensemble import Ensemble, RollingEnsemble from typing import Callable, Union +from qlib.utils.serial import Serializable -class Group: +class Group(Serializable): """Group the objects based on dict""" def __init__(self, group_func=None, ens: Ensemble = None): @@ -17,8 +18,8 @@ def __init__(self, group_func=None, ens: Ensemble = None): ens (Ensemble, optional): If not None, do ensemble for grouped value after grouping. """ - self._group = group_func - self._ens = ens + self.group = group_func + self.ens = ens def __call__(self, ungrouped_dict: dict, *args, **kwargs): """Group the ungrouped_dict into different groups. @@ -29,16 +30,16 @@ def __call__(self, ungrouped_dict: dict, *args, **kwargs): Returns: dict: grouped_dict like {G1: object, G2: object} """ - if isinstance(getattr(self, "_group", None), Callable): - grouped_dict = self._group(ungrouped_dict, *args, **kwargs) - if self._ens is not None: + if isinstance(getattr(self, "group", None), Callable): + grouped_dict = self.group(ungrouped_dict, *args, **kwargs) + if self.ens is not None: ens_dict = {} for key, value in grouped_dict.items(): - ens_dict[key] = self._ens(value) + ens_dict[key] = self.ens(value) grouped_dict = ens_dict return grouped_dict else: - raise NotImplementedError(f"Please specify valid merge_func.") + raise NotImplementedError(f"Please specify valid group_func.") class RollingGroup(Group): @@ -65,4 +66,4 @@ def rolling_group(rolling_dict: dict): def __init__(self, group_func=None, ens: Ensemble = RollingEnsemble()): super().__init__(group_func=group_func, ens=ens) if group_func is None: - self._group = RollingGroup.rolling_group \ No newline at end of file + self.group = RollingGroup.rolling_group diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index e128e700dc..f087cc2485 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -3,11 +3,12 @@ from qlib.utils import init_instance_by_config, flatten_dict from qlib.workflow import R +from qlib.workflow.recorder import Recorder from qlib.workflow.record_temp import SignalRecord from qlib.workflow.task.manage import TaskManager, run_task -def task_train(task_config: dict, experiment_name: str) -> str: +def task_train(task_config: dict, experiment_name: str) -> Recorder: """ task based training @@ -20,8 +21,7 @@ def task_train(task_config: dict, experiment_name: str) -> str: Returns ---------- - rid : str - The id of the recorder of this task + Recorder : The instance of the recorder """ # model initiaiton @@ -80,30 +80,40 @@ class TrainerR(Trainer): Assumption: models were defined by `task` and the results will saved to `Recorder` """ - def train(self, tasks: list, experiment_name: str, train_func=task_train, *args, **kwargs): + def __init__(self, experiment_name, train_func=task_train): + self.experiment_name = experiment_name + self.train_func = train_func + + def train(self, tasks: list, train_func=None, *args, **kwargs): """Given a list of `task`s and return a list of trained Recorder. The order can be guaranteed. Args: tasks (list): a list of definition based on `task` dict - experiment_name (str): the experiment name - train_func (Callable): the train method which need at least `task` and `experiment_name` + train_func (Callable): the train method which need at least `task` and `experiment_name`. None for default. Returns: list: a list of Recorders """ + if train_func is None: + train_func = self.train_func recs = [] for task in tasks: - recs.append(train_func(task, experiment_name, *args, **kwargs)) + recs.append(train_func(task, self.experiment_name, *args, **kwargs)) return recs -class TrainerRM(TrainerR): +class TrainerRM(Trainer): """Trainer based on (R)ecorder and Task(M)anager Assumption: `task` will be saved to TaskManager and `task` will be fetched and trained from TaskManager """ - def train(self, tasks: list, experiment_name: str, task_pool: str, train_func=task_train, *args, **kwargs): + def __init__(self, experiment_name: str, task_pool: str, train_func=task_train): + self.experiment_name = experiment_name + self.task_pool = task_pool + self.train_func = train_func + + def train(self, tasks: list, train_func=None, *args, **kwargs): """Given a list of `task`s and return a list of trained Recorder. The order can be guaranteed. This method defaults to a single process, but TaskManager offered a great way to parallel training. @@ -111,17 +121,18 @@ def train(self, tasks: list, experiment_name: str, task_pool: str, train_func=ta Args: tasks (list): a list of definition based on `task` dict - experiment_name (str): the experiment name - train_func (Callable): the train method which need at least `task` and `experiment_name` + train_func (Callable): the train method which need at least `task` and `experiment_name`. None for default. Returns: list: a list of Recorders """ - tm = TaskManager(task_pool=task_pool) + if train_func is None: + train_func = self.train_func + tm = TaskManager(task_pool=self.task_pool) _id_list = tm.create_task(tasks) # all tasks will be saved to MongoDB - run_task(train_func, task_pool, experiment_name=experiment_name, *args, **kwargs) + run_task(train_func, self.task_pool, experiment_name=self.experiment_name, *args, **kwargs) recs = [] for _id in _id_list: recs.append(tm.re_query(_id)["res"]) - return recs \ No newline at end of file + return recs diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index 25a3682696..0676bfb6b5 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -20,7 +20,7 @@ class OnlineManager(Serializable): NEXT_ONLINE_TAG = "next_online" # the 'next online' model, which can be 'online' model when call reset_online_model OFFLINE_TAG = "offline" # the 'offline' model, not for online serving - def __init__(self, trainer: Trainer = None) -> None: + def __init__(self, trainer: Trainer = None): self._trainer = trainer self.logger = get_module_logger(self.__class__.__name__) @@ -81,7 +81,8 @@ class OnlineManagerR(OnlineManager): """ - def __init__(self, experiment_name: str, trainer: Trainer = TrainerR()) -> None: + def __init__(self, experiment_name: str, trainer: Trainer = None): + trainer = TrainerR(experiment_name) super().__init__(trainer) self.logger = get_module_logger(self.__class__.__name__) self.exp_name = experiment_name @@ -105,20 +106,22 @@ def reset_online_tag(self, recorder: Union[Recorder, List] = None): the recorders you want to reset to 'online'. If don't give, set 'next online' model to 'online' model. If there isn't any 'next online' model, then maintain existing 'online' model. """ if recorder is None: - recorder = list_recorders( - self.exp_name, lambda rec: self.get_online_tag(rec) == OnlineManager.NEXT_ONLINE_TAG - ).values() + recorder = list( + list_recorders( + self.exp_name, lambda rec: self.get_online_tag(rec) == OnlineManager.NEXT_ONLINE_TAG + ).values() + ) if isinstance(recorder, Recorder): recorder = [recorder] if len(recorder) == 0: self.logger.info("No 'next online' model, just use current 'online' models.") return recs = list_recorders(self.exp_name) - self.set_online_tag(OnlineManager.OFFLINE_TAG, recs.values()) + self.set_online_tag(OnlineManager.OFFLINE_TAG, list(recs.values())) self.set_online_tag(OnlineManager.ONLINE_TAG, recorder) self.logger.info(f"Reset {len(recorder)} models to 'online'.") - def update_online_pred(self): + def update_online_pred(self, *args, **kwargs): """update all online model predictions to the latest day in Calendar""" mu = ModelUpdater(self.exp_name) cnt = mu.update_all_pred(lambda rec: self.get_online_tag(rec) == OnlineManager.ONLINE_TAG) @@ -126,25 +129,24 @@ def update_online_pred(self): class RollingOnlineManager(OnlineManagerR): - """An implementation of OnlineManager based on Rolling. - - """ + """An implementation of OnlineManager based on Rolling.""" def __init__( self, experiment_name: str, rolling_gen: RollingGen, - trainer: Trainer = TrainerR(), - ) -> None: + trainer: Trainer = None, + ): + trainer = TrainerR(experiment_name) super().__init__(experiment_name, trainer) self.ta = TimeAdjuster() self.rg = rolling_gen self.logger = get_module_logger(self.__class__.__name__) - def prepare_signals(self): + def prepare_signals(self, *args, **kwargs): pass - def prepare_tasks(self): + def prepare_tasks(self, *args, **kwargs): """prepare new tasks based on new date. Returns: @@ -155,7 +157,7 @@ def prepare_tasks(self): ) if max_test is None: self.logger.warn(f"No latest online recorders, no new tasks.") - return None + return [] calendar_latest = self.ta.last_date() if self.ta.cal_interval(calendar_latest, max_test[0]) > self.rg.step: old_tasks = [] @@ -168,7 +170,7 @@ def prepare_tasks(self): new_tasks_tmp = task_generator(old_tasks, self.rg) new_tasks = [task for task in new_tasks_tmp if task not in old_tasks] return new_tasks - return None + return [] def list_latest_recorders(self, rec_filter_func=None): """find latest recorders based on test segments. @@ -187,4 +189,4 @@ def list_latest_recorders(self, rec_filter_func=None): for rid, rec in recs_flt.items(): if rec.load_object("task")["dataset"]["kwargs"]["segments"]["test"] == max_test: latest_rec[rid] = rec - return latest_rec, max_test \ No newline at end of file + return latest_rec, max_test diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index 7e555ed067..63d4a6a040 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -1,9 +1,10 @@ from abc import abstractmethod from typing import Callable, Union from qlib.workflow.task.utils import list_recorders +from qlib.utils.serial import Serializable -class Collector: +class Collector(Serializable): """The collector to collect different results""" def collect(self, *args, **kwargs): @@ -25,33 +26,46 @@ def collect(self, *args, **kwargs): class RecorderCollector(Collector): def __init__( - self, exp_name, artifacts_path={"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"}, rec_key_func=None - ) -> None: + self, + exp_name, + artifacts_path={"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"}, + rec_key_func=None, + artifacts_key=None, + rec_filter_func=None, + ): """init RecorderCollector Args: exp_name (str): the name of Experiment artifacts_path (dict, optional): The artifacts name and its path in Recorder. Defaults to {"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"}. rec_key_func (Callable): a function to get the key of a recorder. If None, use recorder id. + artifacts_key (str or List, optional): the artifacts key you want to get. If None, get all artifacts. + rec_filter_func (Callable, optional): filter the recorder by return True or False. Defaults to None. """ self.exp_name = exp_name self.artifacts_path = artifacts_path if rec_key_func is None: rec_key_func = lambda rec: rec.info["id"] - self._get_key = rec_key_func + if artifacts_key is None: + artifacts_key = self.artifacts_path.keys() + self.rec_key = rec_key_func + self.artifacts_key = artifacts_key + self.rec_filter = rec_filter_func - def collect(self, artifacts_key=None, rec_filter_func=None): # ensemble, get_group_key_func, + def collect(self, artifacts_key=None, rec_filter_func=None): """Collect different artifacts based on recorder after filtering. Args: - artifacts_key (str or List, optional): the artifacts key you want to get. If None, get all artifacts. - rec_filter_func (Callable, optional): filter the recorder by return True or False. Defaults to None. + artifacts_key (str or List, optional): the artifacts key you want to get. If None, use default. + rec_filter_func (Callable, optional): filter the recorder by return True or False. If None, use default. Returns: dict: the dict after collected like {artifact: {rec_key: object}} """ if artifacts_key is None: - artifacts_key = self.artifacts_path.keys() + artifacts_key = self.artifacts_key + if rec_filter_func is None: + rec_filter_func = self.rec_filter if isinstance(artifacts_key, str): artifacts_key = [artifacts_key] @@ -60,9 +74,9 @@ def collect(self, artifacts_key=None, rec_filter_func=None): # ensemble, get_gr # filter records recs_flt = list_recorders(self.exp_name, rec_filter_func) for _, rec in recs_flt.items(): - rec_key = self._get_key(rec) + rec_key = self.rec_key(rec) for key in artifacts_key: artifact = rec.load_object(self.artifacts_path[key]) collect_dict.setdefault(key, {})[rec_key] = artifact - return collect_dict \ No newline at end of file + return collect_dict diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index ddd833aa40..0d6f8c0de3 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -49,7 +49,7 @@ class TaskManager: ENCODE_FIELDS_PREFIX = ["def", "res"] - def __init__(self, task_pool=None): + def __init__(self, task_pool: str): """ init Task Manager, remember to make the statement of MongoDB url and database name firstly. @@ -59,9 +59,13 @@ def __init__(self, task_pool=None): the name of Collection in MongoDB """ self.mdb = get_mongodb() - self.task_pool = task_pool + self.task_pool = getattr(self.mdb, task_pool) self.logger = get_module_logger(self.__class__.__name__) + # @property + # def task_pool(self): + # return self._task_pool + def list(self): return self.mdb.list_collection_names() @@ -79,39 +83,39 @@ def _decode_task(self, task): task[k] = pickle.loads(task[k]) return task - def _get_task_pool(self, task_pool=None): - if task_pool is None: - task_pool = self.task_pool - if task_pool is None: - raise ValueError("You must specify a task pool.") - if isinstance(task_pool, str): - return getattr(self.mdb, task_pool) - return task_pool + # def _get_task_pool(self, task_pool=None): + # if task_pool is None: + # task_pool = self.task_pool + # if task_pool is None: + # raise ValueError("You must specify a task pool.") + # if isinstance(task_pool, str): + # return getattr(self.mdb, task_pool) + # return task_pool def _dict_to_str(self, flt): return {k: str(v) for k, v in flt.items()} - def replace_task(self, task, new_task, task_pool=None): + def replace_task(self, task, new_task): # assume that the data out of interface was decoded and the data in interface was encoded new_task = self._encode_task(new_task) - task_pool = self._get_task_pool(task_pool) + # task_pool = self._get_task_pool(task_pool) query = {"_id": ObjectId(task["_id"])} try: - task_pool.replace_one(query, new_task) + self.task_pool.replace_one(query, new_task) except InvalidDocument: task["filter"] = self._dict_to_str(task["filter"]) - task_pool.replace_one(query, new_task) + self.task_pool.replace_one(query, new_task) - def insert_task(self, task, task_pool=None): - task_pool = self._get_task_pool(task_pool) + def insert_task(self, task): + # task_pool = self._get_task_pool(task_pool) try: - insert_result = task_pool.insert_one(task) + insert_result = self.task_pool.insert_one(task) except InvalidDocument: task["filter"] = self._dict_to_str(task["filter"]) - insert_result = task_pool.insert_one(task) + insert_result = self.task_pool.insert_one(task) return insert_result - def insert_task_def(self, task_def, task_pool=None): + def insert_task_def(self, task_def): """ insert a task to task_pool @@ -126,7 +130,7 @@ def insert_task_def(self, task_def, task_pool=None): ------- """ - task_pool = self._get_task_pool(task_pool) + # task_pool = self._get_task_pool(task_pool) task = self._encode_task( { "def": task_def, @@ -134,10 +138,10 @@ def insert_task_def(self, task_def, task_pool=None): "status": self.STATUS_WAITING, } ) - insert_result = self.insert_task(task, task_pool) + insert_result = self.insert_task(task) return insert_result - def create_task(self, task_def_l, task_pool=None, dry_run=False, print_nt=False): + def create_task(self, task_def_l, dry_run=False, print_nt=False): """ if the tasks in task_def_l is new, then insert new tasks into the task_pool @@ -156,13 +160,13 @@ def create_task(self, task_def_l, task_pool=None, dry_run=False, print_nt=False) list a list of the _id of new tasks """ - task_pool = self._get_task_pool(task_pool) + # task_pool = self._get_task_pool(task_pool) new_tasks = [] for t in task_def_l: try: - r = task_pool.find_one({"filter": t}) + r = self.task_pool.find_one({"filter": t}) except InvalidDocument: - r = task_pool.find_one({"filter": self._dict_to_str(t)}) + r = self.task_pool.find_one({"filter": self._dict_to_str(t)}) if r is None: new_tasks.append(t) self.logger.info(f"Total Tasks: {len(task_def_l)}, New Tasks: {len(new_tasks)}") @@ -176,18 +180,18 @@ def create_task(self, task_def_l, task_pool=None, dry_run=False, print_nt=False) _id_list = [] for t in new_tasks: - insert_result = self.insert_task_def(t, task_pool) + insert_result = self.insert_task_def(t) _id_list.append(insert_result.inserted_id) return _id_list - def fetch_task(self, query={}, task_pool=None): - task_pool = self._get_task_pool(task_pool) + def fetch_task(self, query={}): + # task_pool = self._get_task_pool(task_pool) query = query.copy() if "_id" in query: query["_id"] = ObjectId(query["_id"]) query.update({"status": self.STATUS_WAITING}) - task = task_pool.find_one_and_update( + task = self.task_pool.find_one_and_update( query, {"$set": {"status": self.STATUS_RUNNING}}, sort=[("priority", pymongo.DESCENDING)] ) # null will be at the top after sorting when using ASCENDING, so the larger the number higher, the higher the priority @@ -197,7 +201,7 @@ def fetch_task(self, query={}, task_pool=None): return self._decode_task(task) @contextmanager - def safe_fetch_task(self, query={}, task_pool=None): + def safe_fetch_task(self, query={}): """ fetch task from task_pool using query with contextmanager @@ -212,7 +216,7 @@ def safe_fetch_task(self, query={}, task_pool=None): ------- """ - task = self.fetch_task(query=query, task_pool=task_pool) + task = self.fetch_task(query=query) try: yield task except Exception: @@ -229,7 +233,7 @@ def task_fetcher_iter(self, query={}, task_pool=None): break yield task - def query(self, query={}, decode=True, task_pool=None): + def query(self, query={}, decode=True): """ This function may raise exception `pymongo.errors.CursorNotFound: cursor id not found` if it takes too long to iterate the generator @@ -248,29 +252,30 @@ def query(self, query={}, decode=True, task_pool=None): query = query.copy() if "_id" in query: query["_id"] = ObjectId(query["_id"]) - task_pool = self._get_task_pool(task_pool) - for t in task_pool.find(query): + # task_pool = self._get_task_pool(task_pool) + for t in self.task_pool.find(query): yield self._decode_task(t) - def re_query(self, _id, task_pool=None): - task_pool = self._get_task_pool(task_pool) - return task_pool.find_one({"_id": ObjectId(_id)}) + def re_query(self, _id): + # task_pool = self._get_task_pool(task_pool) + t = self.task_pool.find_one({"_id": ObjectId(_id)}) + return self._decode_task(t) - def commit_task_res(self, task, res, status=None, task_pool=None): - task_pool = self._get_task_pool(task_pool) + def commit_task_res(self, task, res, status=None): + # task_pool = self._get_task_pool(task_pool) # A workaround to use the class attribute. if status is None: status = TaskManager.STATUS_DONE - task_pool.update_one({"_id": task["_id"]}, {"$set": {"status": status, "res": Binary(pickle.dumps(res))}}) + self.task_pool.update_one({"_id": task["_id"]}, {"$set": {"status": status, "res": Binary(pickle.dumps(res))}}) - def return_task(self, task, status=None, task_pool=None): - task_pool = self._get_task_pool(task_pool) + def return_task(self, task, status=None): + # task_pool = self._get_task_pool(task_pool) if status is None: status = TaskManager.STATUS_WAITING update_dict = {"$set": {"status": status}} - task_pool.update_one({"_id": task["_id"]}, update_dict) + self.task_pool.update_one({"_id": task["_id"]}, update_dict) - def remove(self, query={}, task_pool=None): + def remove(self, query={}): """ remove the task using query @@ -286,16 +291,16 @@ def remove(self, query={}, task_pool=None): """ query = query.copy() - task_pool = self._get_task_pool(task_pool) + # task_pool = self._get_task_pool(task_pool) if "_id" in query: query["_id"] = ObjectId(query["_id"]) - task_pool.delete_many(query) + self.task_pool.delete_many(query) - def task_stat(self, query={}, task_pool=None): + def task_stat(self, query={}): query = query.copy() if "_id" in query: query["_id"] = ObjectId(query["_id"]) - tasks = self.query(task_pool=task_pool, query=query, decode=False) + tasks = self.query(query=query, decode=False) status_stat = {} for t in tasks: status_stat[t["status"]] = status_stat.get(t["status"], 0) + 1 @@ -306,14 +311,14 @@ def reset_waiting(self, query={}, task_pool=None): # default query if "status" not in query: query["status"] = self.STATUS_RUNNING - return self.reset_status(query=query, status=self.STATUS_WAITING, task_pool=task_pool) + return self.reset_status(query=query, status=self.STATUS_WAITING) - def reset_status(self, query, status, task_pool=None): + def reset_status(self, query, status): query = query.copy() - task_pool = self._get_task_pool(task_pool) + # task_pool = self._get_task_pool(task_pool) if "_id" in query: query["_id"] = ObjectId(query["_id"]) - print(task_pool.update_many(query, {"$set": {"status": status}})) + print(self.task_pool.update_many(query, {"$set": {"status": status}})) def _get_undone_n(self, task_stat): return task_stat.get(self.STATUS_WAITING, 0) + task_stat.get(self.STATUS_RUNNING, 0) @@ -321,14 +326,14 @@ def _get_undone_n(self, task_stat): def _get_total(self, task_stat): return sum(task_stat.values()) - def wait(self, query={}, task_pool=None): - task_stat = self.task_stat(query, task_pool) + def wait(self, query={}): + task_stat = self.task_stat(query) total = self._get_total(task_stat) last_undone_n = self._get_undone_n(task_stat) with tqdm(total=total, initial=total - last_undone_n) as pbar: while True: time.sleep(10) - undone_n = self._get_undone_n(self.task_stat(query, task_pool)) + undone_n = self._get_undone_n(self.task_stat(query)) pbar.update(last_undone_n - undone_n) last_undone_n = undone_n if undone_n == 0: @@ -365,7 +370,7 @@ def run_task(task_func, task_pool, force_release=False, *args, **kwargs): break get_module_logger("run_task").info(task["def"]) if force_release: - with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: + with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: # what this means? res = executor.submit(task_func, task["def"], *args, **kwargs).result() else: res = task_func(task["def"], *args, **kwargs) From 1dbb56174450ec5cceb443ee41ecbcb4bd60a3af Mon Sep 17 00:00:00 2001 From: Young Date: Wed, 7 Apr 2021 03:31:50 +0000 Subject: [PATCH 34/61] Fix some API(for lb nn) --- qlib/contrib/data/handler.py | 1 + qlib/data/dataset/processor.py | 3 ++- qlib/model/trainer.py | 17 +++++-------- qlib/utils/__init__.py | 44 ++++++++++++++++++++-------------- qlib/workflow/record_temp.py | 4 ++++ 5 files changed, 39 insertions(+), 30 deletions(-) diff --git a/qlib/contrib/data/handler.py b/qlib/contrib/data/handler.py index 970b032d6b..be2016ea32 100644 --- a/qlib/contrib/data/handler.py +++ b/qlib/contrib/data/handler.py @@ -26,6 +26,7 @@ def check_transform_proc(proc_l, fit_start_time, fit_end_time): "fit_end_time": fit_end_time, } ) + # FIXME: the `module_path` parameter is missed. new_l.append({"class": klass.__name__, "kwargs": pkwargs}) else: new_l.append(p) diff --git a/qlib/data/dataset/processor.py b/qlib/data/dataset/processor.py index 5a06f66bef..972a1294aa 100755 --- a/qlib/data/dataset/processor.py +++ b/qlib/data/dataset/processor.py @@ -2,6 +2,7 @@ # Licensed under the MIT License. import abc +from typing import Union, Text import numpy as np import pandas as pd import copy @@ -14,7 +15,7 @@ EPS = 1e-12 -def get_group_columns(df: pd.DataFrame, group: str): +def get_group_columns(df: pd.DataFrame, group: Union[Text, None]): """ get a group of columns from multi-index columns DataFrame diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index a4df922185..57132a33ec 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -from qlib.utils import init_instance_by_config, flatten_dict +from qlib.utils import init_instance_by_config, flatten_dict, get_cls_kwargs from qlib.workflow import R from qlib.workflow.record_temp import SignalRecord @@ -41,16 +41,11 @@ def task_train(task_config: dict, experiment_name: str) -> str: if isinstance(records, dict): # prevent only one dict records = [records] for record in records: - if record["class"] == SignalRecord.__name__: - srconf = {"model": model, "dataset": dataset, "recorder": recorder} - record.setdefault("kwargs", {}) - record["kwargs"].update(srconf) - sr = init_instance_by_config(record) - sr.generate() + cls, kwargs = get_cls_kwargs(record, default_module="qlib.workflow.record_temp") + if cls is SignalRecord: + rconf = {"model": model, "dataset": dataset, "recorder": recorder} else: rconf = {"recorder": recorder} - record.setdefault("kwargs", {}) - record["kwargs"].update(rconf) - ar = init_instance_by_config(record) - ar.generate() + r = cls(**kwargs, **rconf) + r.generate() return recorder.info["id"] diff --git a/qlib/utils/__init__.py b/qlib/utils/__init__.py index be7969b658..606d007a8d 100644 --- a/qlib/utils/__init__.py +++ b/qlib/utils/__init__.py @@ -24,7 +24,8 @@ import numpy as np import pandas as pd from pathlib import Path -from typing import Union, Tuple +from typing import Union, Tuple, Any +from types import ModuleType from ..config import C from ..log import get_module_logger, set_log_with_config @@ -165,24 +166,25 @@ def parse_field(field): return re.sub(r"\$(\w+)", r'Feature("\1")', re.sub(r"(\w+\s*)\(", r"Operators.\1(", field)) -def get_module_by_module_path(module_path): +def get_module_by_module_path(module_path: Union[str, ModuleType]): """Load module path :param module_path: :return: """ - - if module_path.endswith(".py"): - module_spec = importlib.util.spec_from_file_location("", module_path) - module = importlib.util.module_from_spec(module_spec) - module_spec.loader.exec_module(module) + if isinstance(module_path, ModuleType): + module = module_path else: - module = importlib.import_module(module_path) - + if module_path.endswith(".py"): + module_spec = importlib.util.spec_from_file_location("", module_path) + module = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(module) + else: + module = importlib.import_module(module_path) return module -def get_cls_kwargs(config: Union[dict, str], module) -> (type, dict): +def get_cls_kwargs(config: Union[dict, str], default_module: Union[str, ModuleType]=None) -> (type, dict): """ extract class and kwargs from config info @@ -191,8 +193,10 @@ def get_cls_kwargs(config: Union[dict, str], module) -> (type, dict): config : [dict, str] similar to config - module : Python module + default_module : Python module or str It should be a python module to load the class type + This function will load class from the config['module_path'] first. + If config['module_path'] doesn't exists, it will load the class from default_module. Returns ------- @@ -200,10 +204,14 @@ def get_cls_kwargs(config: Union[dict, str], module) -> (type, dict): the class object and it's arguments. """ if isinstance(config, dict): + module = get_module_by_module_path(config.get("module_path", default_module)) + # raise AttributeError klass = getattr(module, config["class"]) kwargs = config.get("kwargs", {}) elif isinstance(config, str): + module = get_module_by_module_path(default_module) + klass = getattr(module, config) kwargs = {} else: @@ -212,8 +220,8 @@ def get_cls_kwargs(config: Union[dict, str], module) -> (type, dict): def init_instance_by_config( - config: Union[str, dict, object], module=None, accept_types: Union[type, Tuple[type]] = tuple([]), **kwargs -) -> object: + config: Union[str, dict, object], default_module=None, accept_types: Union[type, Tuple[type]] = tuple([]), **kwargs +) -> Any: """ get initialized instance with config @@ -230,10 +238,13 @@ def init_instance_by_config( "ClassName": getattr(module, config)() will be used. object example: instance of accept_types - module : Python module + default_module : Python module Optional. It should be a python module. NOTE: the "module_path" will be override by `module` arguments + This function will load class from the config['module_path'] first. + If config['module_path'] doesn't exists, it will load the class from default_module. + accept_types: Union[type, Tuple[type]] Optional. If the config is a instance of specific type, return the config directly. This will be passed into the second parameter of isinstance. @@ -246,10 +257,7 @@ def init_instance_by_config( if isinstance(config, accept_types): return config - if module is None: - module = get_module_by_module_path(config["module_path"]) - - klass, cls_kwargs = get_cls_kwargs(config, module) + klass, cls_kwargs = get_cls_kwargs(config, default_module=default_module) return klass(**cls_kwargs, **kwargs) diff --git a/qlib/workflow/record_temp.py b/qlib/workflow/record_temp.py index be458a24d2..c54a6f7001 100644 --- a/qlib/workflow/record_temp.py +++ b/qlib/workflow/record_temp.py @@ -145,6 +145,10 @@ def generate(self, **kwargs): del params["data_key"] # The backend handler should be DataHandler raw_label = self.dataset.prepare(**params) + except AttributeError: + # The data handler is initialize with `drop_raw=True`... + # So raw_label is not available + raw_label = None self.recorder.save_objects(**{"label.pkl": raw_label}) self.dataset.__class__ = orig_cls From c20eb5c8a61323a11589333b625ab09daed3c2bd Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Thu, 8 Apr 2021 03:30:24 +0000 Subject: [PATCH 35/61] format code --- .../model_rolling/task_manager_rolling.py | 3 +- .../task_manager_rolling_with_updating.py | 27 ++++++------ examples/online_srv/update_online_pred.py | 4 +- qlib/utils/__init__.py | 2 +- qlib/workflow/online/manager.py | 2 + qlib/workflow/task/gen.py | 3 +- qlib/workflow/task/manage.py | 41 ++----------------- 7 files changed, 29 insertions(+), 53 deletions(-) diff --git a/examples/model_rolling/task_manager_rolling.py b/examples/model_rolling/task_manager_rolling.py index 9070864870..3e914cc636 100644 --- a/examples/model_rolling/task_manager_rolling.py +++ b/examples/model_rolling/task_manager_rolling.py @@ -119,7 +119,8 @@ def my_filter(recorder): return False artifact = ens_workflow( - RecorderCollector(exp_name=exp_name, rec_key_func=rec_key, rec_filter_func=my_filter), RollingGroup(), + RecorderCollector(exp_name=exp_name, rec_key_func=rec_key, rec_filter_func=my_filter), + RollingGroup(), ) print(artifact) diff --git a/examples/online_srv/task_manager_rolling_with_updating.py b/examples/online_srv/task_manager_rolling_with_updating.py index 5b80f9133e..bfdc5f3c03 100644 --- a/examples/online_srv/task_manager_rolling_with_updating.py +++ b/examples/online_srv/task_manager_rolling_with_updating.py @@ -70,9 +70,18 @@ "record": record_config, } -class RollingOnlineExample: - def __init__(self, exp_name="rolling_exp", task_pool="rolling_task", provider_uri="~/.qlib/qlib_data/cn_data", region="cn", task_url="mongodb://10.0.0.4:27017/", task_db_name="rolling_db", rolling_step=550): +class RollingOnlineExample: + def __init__( + self, + exp_name="rolling_exp", + task_pool="rolling_task", + provider_uri="~/.qlib/qlib_data/cn_data", + region="cn", + task_url="mongodb://10.0.0.4:27017/", + task_db_name="rolling_db", + rolling_step=550, + ): self.exp_name = exp_name self.task_pool = task_pool mongo_conf = { @@ -84,9 +93,9 @@ def __init__(self, exp_name="rolling_exp", task_pool="rolling_task", provider_ur self.rolling_gen = RollingGen(step=rolling_step, rtype=RollingGen.ROLL_SD) self.trainer = TrainerRM(self.exp_name, self.task_pool) self.task_manager = TaskManager(self.task_pool) - self.rolling_online_manager = RollingOnlineManager(experiment_name=exp_name, rolling_gen=self.rolling_gen, trainer=self.trainer) - - + self.rolling_online_manager = RollingOnlineManager( + experiment_name=exp_name, rolling_gen=self.rolling_gen, trainer=self.trainer + ) def print_online_model(self): print("========== print_online_model ==========") @@ -99,7 +108,6 @@ def print_online_model(self): if self.rolling_online_manager.get_online_tag(rec) == self.rolling_online_manager.NEXT_ONLINE_TAG: print(rid) - # This part corresponds to "Task Generating" in the document def task_generating(self): @@ -114,11 +122,9 @@ def task_generating(self): return tasks - def task_training(self, tasks): self.trainer.train(tasks) - # This part corresponds to "Task Collecting" in the document def task_collecting(self): print("========== task_collecting ==========") @@ -141,7 +147,6 @@ def my_filter(recorder): ) print(artifact) - # Reset all things to the first status, be careful to save important data def reset(self): print("========== reset ==========") @@ -150,7 +155,6 @@ def reset(self): for rid in exp.list_recorders(): exp.delete_recorder(rid) - # Run this firstly to see the workflow in Task Management def first_run(self): print("========== first_run ==========") @@ -163,7 +167,6 @@ def first_run(self): latest_rec, _ = self.rolling_online_manager.list_latest_recorders() self.rolling_online_manager.reset_online_tag(list(latest_rec.values())) - def routine(self): print("========== routine ==========") self.print_online_model() @@ -178,7 +181,7 @@ def routine(self): ####### to update the models and predictions after the trading time, use the command below # python task_manager_rolling_with_updating.py after_day - + ####### to define your own parameters, use `--` # python task_manager_rolling_with_updating.py first_run --exp_name='your_exp_name' --rolling_step=40 fire.Fire(RollingOnlineExample) diff --git a/examples/online_srv/update_online_pred.py b/examples/online_srv/update_online_pred.py index 84472bc3b6..0f075abcd0 100644 --- a/examples/online_srv/update_online_pred.py +++ b/examples/online_srv/update_online_pred.py @@ -71,12 +71,14 @@ def update_online_pred(experiment_name="online_srv"): online_manager.update_online_pred() -def main(provider_uri = "~/.qlib/qlib_data/cn_data", region=REG_CN, experiment_name="online_srv"): + +def main(provider_uri="~/.qlib/qlib_data/cn_data", region=REG_CN, experiment_name="online_srv"): provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir qlib.init(provider_uri=provider_uri, region=region) first_train(experiment_name) update_online_pred(experiment_name) + if __name__ == "__main__": ## to train a model and set it to online model, use the command below # python update_online_pred.py first_train diff --git a/qlib/utils/__init__.py b/qlib/utils/__init__.py index 35f0673043..2a34035f3b 100644 --- a/qlib/utils/__init__.py +++ b/qlib/utils/__init__.py @@ -184,7 +184,7 @@ def get_module_by_module_path(module_path: Union[str, ModuleType]): return module -def get_cls_kwargs(config: Union[dict, str], default_module: Union[str, ModuleType]=None) -> (type, dict): +def get_cls_kwargs(config: Union[dict, str], default_module: Union[str, ModuleType] = None) -> (type, dict): """ extract class and kwargs from config info diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index 0676bfb6b5..66df160cdc 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -74,6 +74,8 @@ def routine(self, *args, **kwargs): self.update_online_pred(*args, **kwargs) self.reset_online_tag(*args, **kwargs) + # TODO: first_train? + class OnlineManagerR(OnlineManager): """ diff --git a/qlib/workflow/task/gen.py b/qlib/workflow/task/gen.py index c64939e822..a8426d920e 100644 --- a/qlib/workflow/task/gen.py +++ b/qlib/workflow/task/gen.py @@ -106,7 +106,8 @@ def __init__(self, step: int = 40, rtype: str = ROLL_EX): def generate(self, task: dict): """ - Converting the task into a rolling task + Converting the task into a rolling task. + # FIXME: only modify dataset layer, user need to change datahandler firstly. Parameters ---------- diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 0d6f8c0de3..720eeb12f1 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -62,10 +62,6 @@ def __init__(self, task_pool: str): self.task_pool = getattr(self.mdb, task_pool) self.logger = get_module_logger(self.__class__.__name__) - # @property - # def task_pool(self): - # return self._task_pool - def list(self): return self.mdb.list_collection_names() @@ -83,22 +79,12 @@ def _decode_task(self, task): task[k] = pickle.loads(task[k]) return task - # def _get_task_pool(self, task_pool=None): - # if task_pool is None: - # task_pool = self.task_pool - # if task_pool is None: - # raise ValueError("You must specify a task pool.") - # if isinstance(task_pool, str): - # return getattr(self.mdb, task_pool) - # return task_pool - def _dict_to_str(self, flt): return {k: str(v) for k, v in flt.items()} def replace_task(self, task, new_task): # assume that the data out of interface was decoded and the data in interface was encoded new_task = self._encode_task(new_task) - # task_pool = self._get_task_pool(task_pool) query = {"_id": ObjectId(task["_id"])} try: self.task_pool.replace_one(query, new_task) @@ -107,7 +93,7 @@ def replace_task(self, task, new_task): self.task_pool.replace_one(query, new_task) def insert_task(self, task): - # task_pool = self._get_task_pool(task_pool) + try: insert_result = self.task_pool.insert_one(task) except InvalidDocument: @@ -123,14 +109,11 @@ def insert_task_def(self, task_def): ---------- task_def: dict the task definition - task_pool: str - the name of Collection in MongoDB Returns ------- """ - # task_pool = self._get_task_pool(task_pool) task = self._encode_task( { "def": task_def, @@ -149,8 +132,6 @@ def create_task(self, task_def_l, dry_run=False, print_nt=False): ---------- task_def_l: list a list of task - task_pool: str - the name of task_pool (collection name of MongoDB) dry_run: bool if insert those new tasks to task pool print_nt: bool @@ -160,7 +141,6 @@ def create_task(self, task_def_l, dry_run=False, print_nt=False): list a list of the _id of new tasks """ - # task_pool = self._get_task_pool(task_pool) new_tasks = [] for t in task_def_l: try: @@ -186,7 +166,6 @@ def create_task(self, task_def_l, dry_run=False, print_nt=False): return _id_list def fetch_task(self, query={}): - # task_pool = self._get_task_pool(task_pool) query = query.copy() if "_id" in query: query["_id"] = ObjectId(query["_id"]) @@ -209,8 +188,6 @@ def safe_fetch_task(self, query={}): ---------- query: dict the dict of query - task_pool: str - the name of Collection in MongoDB Returns ------- @@ -226,9 +203,9 @@ def safe_fetch_task(self, query={}): self.logger.info("Task returned") raise - def task_fetcher_iter(self, query={}, task_pool=None): + def task_fetcher_iter(self, query={}): while True: - with self.safe_fetch_task(query=query, task_pool=task_pool) as task: + with self.safe_fetch_task(query=query) as task: if task is None: break yield task @@ -242,8 +219,6 @@ def query(self, query={}, decode=True): query: dict the dict of query decode: bool - task_pool: str - the name of Collection in MongoDB Returns ------- @@ -252,24 +227,20 @@ def query(self, query={}, decode=True): query = query.copy() if "_id" in query: query["_id"] = ObjectId(query["_id"]) - # task_pool = self._get_task_pool(task_pool) for t in self.task_pool.find(query): yield self._decode_task(t) def re_query(self, _id): - # task_pool = self._get_task_pool(task_pool) t = self.task_pool.find_one({"_id": ObjectId(_id)}) return self._decode_task(t) def commit_task_res(self, task, res, status=None): - # task_pool = self._get_task_pool(task_pool) # A workaround to use the class attribute. if status is None: status = TaskManager.STATUS_DONE self.task_pool.update_one({"_id": task["_id"]}, {"$set": {"status": status, "res": Binary(pickle.dumps(res))}}) def return_task(self, task, status=None): - # task_pool = self._get_task_pool(task_pool) if status is None: status = TaskManager.STATUS_WAITING update_dict = {"$set": {"status": status}} @@ -283,15 +254,12 @@ def remove(self, query={}): ---------- query: dict the dict of query - task_pool: str - the name of Collection in MongoDB Returns ------- """ query = query.copy() - # task_pool = self._get_task_pool(task_pool) if "_id" in query: query["_id"] = ObjectId(query["_id"]) self.task_pool.delete_many(query) @@ -306,7 +274,7 @@ def task_stat(self, query={}): status_stat[t["status"]] = status_stat.get(t["status"], 0) + 1 return status_stat - def reset_waiting(self, query={}, task_pool=None): + def reset_waiting(self, query={}): query = query.copy() # default query if "status" not in query: @@ -315,7 +283,6 @@ def reset_waiting(self, query={}, task_pool=None): def reset_status(self, query, status): query = query.copy() - # task_pool = self._get_task_pool(task_pool) if "_id" in query: query["_id"] = ObjectId(query["_id"]) print(self.task_pool.update_many(query, {"$set": {"status": status}})) From 18bf4b547731c946d6c7cfc05955c039e8943706 Mon Sep 17 00:00:00 2001 From: Young Date: Thu, 8 Apr 2021 03:52:58 +0000 Subject: [PATCH 36/61] parameter adjustment --- qlib/model/ens/group.py | 4 ++-- qlib/workflow/task/collect.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/qlib/model/ens/group.py b/qlib/model/ens/group.py index 9cc5db9714..d138b917c6 100644 --- a/qlib/model/ens/group.py +++ b/qlib/model/ens/group.py @@ -63,7 +63,7 @@ def rolling_group(rolling_dict: dict): grouped_dict.setdefault(key[:-1], {})[key[-1]] = values return grouped_dict - def __init__(self, group_func=None, ens: Ensemble = RollingEnsemble()): - super().__init__(group_func=group_func, ens=ens) + def __init__(self, group_func=None): + super().__init__(group_func=group_func, ens=RollingEnsemble()) if group_func is None: self.group = RollingGroup.rolling_group diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index 63d4a6a040..6b9418daf9 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -28,19 +28,19 @@ class RecorderCollector(Collector): def __init__( self, exp_name, - artifacts_path={"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"}, rec_key_func=None, - artifacts_key=None, rec_filter_func=None, + artifacts_path={"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"}, + artifacts_key=None, ): """init RecorderCollector Args: exp_name (str): the name of Experiment - artifacts_path (dict, optional): The artifacts name and its path in Recorder. Defaults to {"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"}. rec_key_func (Callable): a function to get the key of a recorder. If None, use recorder id. - artifacts_key (str or List, optional): the artifacts key you want to get. If None, get all artifacts. rec_filter_func (Callable, optional): filter the recorder by return True or False. Defaults to None. + artifacts_path (dict, optional): The artifacts name and its path in Recorder. Defaults to {"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"}. + artifacts_key (str or List, optional): the artifacts key you want to get. If None, get all artifacts. """ self.exp_name = exp_name self.artifacts_path = artifacts_path From a366c11d67cc6be9c4bd853357e5a886a8798d35 Mon Sep 17 00:00:00 2001 From: Young Date: Fri, 9 Apr 2021 13:48:01 +0000 Subject: [PATCH 37/61] Update features for hyb nn --- qlib/data/dataset/__init__.py | 36 +++++++++++++++++++++++++++-------- qlib/data/dataset/handler.py | 27 ++++++++++++++++++++++++-- qlib/data/dataset/loader.py | 7 ++++++- qlib/model/trainer.py | 8 ++++++-- qlib/utils/__init__.py | 14 +++++++++++++- qlib/utils/serial.py | 36 +++++++++++++++++++++++++++++------ qlib/workflow/record_temp.py | 3 +++ 7 files changed, 111 insertions(+), 20 deletions(-) diff --git a/qlib/data/dataset/__init__.py b/qlib/data/dataset/__init__.py index b3eaac7a33..ef30c634e6 100644 --- a/qlib/data/dataset/__init__.py +++ b/qlib/data/dataset/__init__.py @@ -112,7 +112,7 @@ def __init__(self, handler: Union[Dict, DataHandler], segments: Dict[Text, Tuple 'outsample': ("2017-01-01", "2020-08-01",), } """ - self.handler = init_instance_by_config(handler, accept_types=DataHandler) + self.handler: DataHandler = init_instance_by_config(handler, accept_types=DataHandler) self.segments = segments.copy() super().__init__(**kwargs) @@ -243,7 +243,7 @@ class TSDataSampler: """ - def __init__(self, data: pd.DataFrame, start, end, step_len: int, fillna_type: str = "none"): + def __init__(self, data: pd.DataFrame, start, end, step_len: int, fillna_type: str = "none", dtype=None): """ Build a dataset which looks like torch.data.utils.Dataset. @@ -272,9 +272,18 @@ def __init__(self, data: pd.DataFrame, start, end, step_len: int, fillna_type: s self.fillna_type = fillna_type assert get_level_index(data, "datetime") == 0 self.data = lazy_sort_index(data) - self.data_arr = np.array(self.data) # Get index from numpy.array will much faster than DataFrame.values! - # NOTE: append last line with full NaN for better performance in `__getitem__` - self.data_arr = np.append(self.data_arr, np.full((1, self.data_arr.shape[1]), np.nan), axis=0) + + kwargs = {"object": self.data} + if dtype is not None: + kwargs["dtype"] = dtype + + self.data_arr = np.array(**kwargs) # Get index from numpy.array will much faster than DataFrame.values! + # NOTE: + # - append last line with full NaN for better performance in `__getitem__` + # - Keep the same dtype will result in a better performance + self.data_arr = np.append( + self.data_arr, np.full((1, self.data_arr.shape[1]), np.nan, dtype=self.data_arr.dtype), axis=0 + ) self.nan_idx = -1 # The last line is all NaN # the data type will be changed @@ -282,13 +291,16 @@ def __init__(self, data: pd.DataFrame, start, end, step_len: int, fillna_type: s self.start_idx, self.end_idx = self.data.index.slice_locs(start=pd.Timestamp(start), end=pd.Timestamp(end)) self.idx_df, self.idx_map = self.build_index(self.data) self.idx_arr = np.array(self.idx_df.values, dtype=np.float64) # for better performance + self.data_idx = deepcopy(self.data.index) + + del self.data # save memory def get_index(self): """ Get the pandas index of the data, it will be useful in following scenarios - Special sampler will be used (e.g. user want to sample day by day) """ - return self.data.index[self.start_idx : self.end_idx] + return self.data_idx[self.start_idx : self.end_idx] def config(self, **kwargs): # Config the attributes @@ -461,7 +473,7 @@ def setup_data(self, **kwargs): cal = sorted(cal) self.cal = cal - def _prepare_seg(self, slc: slice, **kwargs) -> TSDataSampler: + def _prepare_raw_seg(self, slc: slice, **kwargs) -> pd.DataFrame: # Dataset decide how to slice data(Get more data for timeseries). start, end = slc.start, slc.stop start_idx = bisect.bisect_left(self.cal, pd.Timestamp(start)) @@ -470,6 +482,14 @@ def _prepare_seg(self, slc: slice, **kwargs) -> TSDataSampler: # TSDatasetH will retrieve more data for complete data = super()._prepare_seg(slice(pad_start, end), **kwargs) + return data - tsds = TSDataSampler(data=data, start=start, end=end, step_len=self.step_len) + def _prepare_seg(self, slc: slice, **kwargs) -> TSDataSampler: + """ + split the _prepare_raw_seg is to leave a hook for data preprocessing before creating processing data + """ + dtype = kwargs.pop("dtype") + start, end = slc.start, slc.stop + data = self._prepare_raw_seg(slc=slc, **kwargs) + tsds = TSDataSampler(data=data, start=start, end=end, step_len=self.step_len, dtype=dtype) return tsds diff --git a/qlib/data/dataset/handler.py b/qlib/data/dataset/handler.py index 3c321ed9e2..f1fa39c3bd 100644 --- a/qlib/data/dataset/handler.py +++ b/qlib/data/dataset/handler.py @@ -7,7 +7,7 @@ import logging import warnings from inspect import getfullargspec -from typing import Union, Tuple, List, Iterator, Optional +from typing import Callable, Union, Tuple, List, Iterator, Optional import pandas as pd import numpy as np @@ -166,6 +166,7 @@ def fetch( level: Union[str, int] = "datetime", col_set: Union[str, List[str]] = CS_ALL, squeeze: bool = False, + proc_func: Callable = None, ) -> pd.DataFrame: """ fetch data from underlying data source @@ -188,6 +189,14 @@ def fetch( - if isinstance(col_set, List[str]): select several sets of meaningful columns, the returned data has multiple levels + proc_func: Callable + - Give a hook for processing data before fetching + - An example to explain the necessity of the hook: + - A Dataset learned some processors to process data which is related to data segmentation + - It will apply them every time when preparing data. + - The learned processor require the dataframe remains the same format when fitting and applying + - However the data format will change according to the parameters. + - So the processors should be applied to the underlayer data. squeeze : bool whether squeeze columns and index @@ -196,8 +205,15 @@ def fetch( ------- pd.DataFrame. """ + if proc_func is None: + df = self._data + else: + # FIXME: fetching by time first will be more friendly to `proc_func` + # Copy in case of `proc_func` changing the data inplace.... + df = proc_func(fetch_df_by_index(self._data, selector, level, fetch_orig=self.fetch_orig).copy()) + # Fetch column first will be more friendly to SepDataFrame - df = self._fetch_df_by_col(self._data, col_set) + df = self._fetch_df_by_col(df, col_set) df = fetch_df_by_index(df, selector, level, fetch_orig=self.fetch_orig) if squeeze: # squeeze columns @@ -481,6 +497,7 @@ def fetch( level: Union[str, int] = "datetime", col_set=DataHandler.CS_ALL, data_key: str = DK_I, + proc_func: Callable = None, ) -> pd.DataFrame: """ fetch data from underlying data source @@ -495,12 +512,18 @@ def fetch( select a set of meaningful columns.(e.g. features, columns). data_key : str the data to fetch: DK_*. + proc_func: Callable + please refer to the doc of DataHandler.fetch Returns ------- pd.DataFrame: """ df = self._get_df_by_key(data_key) + if proc_func is not None: + # FIXME: fetch by time first will be more friendly to proc_func + # Copy incase of `proc_func` changing the data inplace.... + df = proc_func(fetch_df_by_index(df, selector, level, fetch_orig=self.fetch_orig).copy()) # Fetch column first will be more friendly to SepDataFrame df = self._fetch_df_by_col(df, col_set) return fetch_df_by_index(df, selector, level, fetch_orig=self.fetch_orig) diff --git a/qlib/data/dataset/loader.py b/qlib/data/dataset/loader.py index 58aca1d4f7..2ad110b89d 100644 --- a/qlib/data/dataset/loader.py +++ b/qlib/data/dataset/loader.py @@ -13,6 +13,7 @@ from qlib.data import filter as filter_module from qlib.data.filter import BaseDFilter from qlib.utils import load_dataset, init_instance_by_config +from qlib.log import get_module_logger class DataLoader(abc.ABC): @@ -224,6 +225,10 @@ class DataLoaderDH(DataLoader): DataLoader based on (D)ata (H)andler It is designed to load multiple data from data handler - If you just want to load data from single datahandler, you can write them in single data handler + + TODO: What make this module not that easy to use. + - For online scenario + - The underlayer data handler should be configured. But data loader doesn't provide such interface & hook. """ def __init__(self, handler_config: dict, fetch_kwargs: dict = {}, is_group=False): @@ -265,7 +270,7 @@ def __init__(self, handler_config: dict, fetch_kwargs: dict = {}, is_group=False def load(self, instruments=None, start_time=None, end_time=None) -> pd.DataFrame: if instruments is not None: - LOG.warning(f"instruments[{instruments}] is ignored") + get_module_logger(self.__class__.__name__).warning(f"instruments[{instruments}] is ignored") if self.is_group: df = pd.concat( diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index 6e1ae8b9d3..7ffca20eec 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -6,6 +6,8 @@ from qlib.workflow.recorder import Recorder from qlib.workflow.record_temp import SignalRecord from qlib.workflow.task.manage import TaskManager, run_task +from qlib.data.dataset import Dataset +from qlib.model.base import Model def task_train(task_config: dict, experiment_name: str) -> Recorder: @@ -25,8 +27,8 @@ def task_train(task_config: dict, experiment_name: str) -> Recorder: """ # model initiaiton - model = init_instance_by_config(task_config["model"]) - dataset = init_instance_by_config(task_config["dataset"]) + model: Model = init_instance_by_config(task_config["model"]) + dataset: Dataset = init_instance_by_config(task_config["dataset"]) # start exp with R.start(experiment_name=experiment_name): @@ -37,6 +39,8 @@ def task_train(task_config: dict, experiment_name: str) -> Recorder: recorder = R.get_recorder() R.save_objects(**{"params.pkl": model}) R.save_objects(**{"task": task_config}) # keep the original format and datatype + # This dataset is saved for online inference. So the concrete data should not be dumped + dataset.config(dump_all=False, recursive=True) R.save_objects(**{"dataset": dataset}) # generate records: prediction, backtest, and analysis diff --git a/qlib/utils/__init__.py b/qlib/utils/__init__.py index 2a34035f3b..7e71ba76c3 100644 --- a/qlib/utils/__init__.py +++ b/qlib/utils/__init__.py @@ -6,6 +6,7 @@ from __future__ import print_function import os +import pickle import re import copy import json @@ -26,6 +27,7 @@ from pathlib import Path from typing import Union, Tuple, Any, Text, Optional from types import ModuleType +from urllib.parse import urlparse from ..config import C from ..log import get_module_logger, set_log_with_config @@ -235,7 +237,10 @@ def init_instance_by_config( 'model_path': path, # It is optional if module is given } str example. - "ClassName": getattr(module, config)() will be used. + 1) specify a pickle object + - path like 'file:////obj.pkl' + 2) specify a class name + - "ClassName": getattr(module, config)() will be used. object example: instance of accept_types default_module : Python module @@ -257,6 +262,13 @@ def init_instance_by_config( if isinstance(config, accept_types): return config + if isinstance(config, str): + # path like 'file:////obj.pkl' + pr = urlparse(config) + if pr.scheme == "file": + with open(os.path.join(pr.netloc, pr.path), "rb") as f: + return pickle.load(f) + klass, cls_kwargs = get_cls_kwargs(config, default_module=default_module) return klass(**cls_kwargs, **kwargs) diff --git a/qlib/utils/serial.py b/qlib/utils/serial.py index 4bc57eccd5..b94be464ba 100644 --- a/qlib/utils/serial.py +++ b/qlib/utils/serial.py @@ -33,16 +33,40 @@ def dump_all(self): @property def exclude(self): """ - What attribute will be dumped + What attribute will not be dumped """ return getattr(self, "_exclude", []) - def config(self, dump_all: bool = None, exclude: list = None): - if dump_all is not None: - self._dump_all = dump_all + FLAG_KEY = "_qlib_serial_flag" - if exclude is not None: - self._exclude = exclude + def config(self, dump_all: bool = None, exclude: list = None, recursive=False): + """ + configure the serializable object + + Parameters + ---------- + dump_all : bool + will the object dump all object + exclude : list + What attribute will not be dumped + recursive : bool + will the configuration be recursive + """ + + params = {"dump_all": dump_all, "exclude": exclude} + + for k, v in params.items(): + if v is not None: + attr_name = f"_{k}" + setattr(self, attr_name, v) + + if recursive: + for obj in self.__dict__.values(): + # set flag to prevent endless loop + self.__dict__[self.FLAG_KEY] = True + if isinstance(obj, Serializable) and self.FLAG_KEY not in obj.__dict__: + obj.config(**params, recursive=True) + del self.__dict__[self.FLAG_KEY] def to_pickle(self, path: [Path, str], dump_all: bool = None, exclude: list = None): self.config(dump_all=dump_all, exclude=exclude) diff --git a/qlib/workflow/record_temp.py b/qlib/workflow/record_temp.py index 808aca3024..324b790ac1 100644 --- a/qlib/workflow/record_temp.py +++ b/qlib/workflow/record_temp.py @@ -186,6 +186,9 @@ def generate(self, **kwargs): pred = self.load("pred.pkl") label = self.load("label.pkl") + if label is None or not isinstance(label, pd.DataFrame) or label.empty: + logger.warn(f"Empty label.") + return ic, ric = calc_ic(pred.iloc[:, 0], label.iloc[:, 0]) metrics = { "IC": ic.mean(), From cca43cf102c6c18958d1363e22cc6855aaaeb473 Mon Sep 17 00:00:00 2001 From: Young Date: Sun, 11 Apr 2021 14:39:19 +0000 Subject: [PATCH 38/61] Refactor update & modification when running NN --- qlib/model/ens/ensemble.py | 2 +- qlib/model/ens/group.py | 51 ++++++++----- qlib/model/trainer.py | 6 +- qlib/utils/__init__.py | 4 +- qlib/workflow/online/update.py | 129 +++++++++++++++++++++++++++++++++ qlib/workflow/task/collect.py | 15 ++-- qlib/workflow/task/gen.py | 6 ++ qlib/workflow/task/manage.py | 29 +++++++- 8 files changed, 210 insertions(+), 32 deletions(-) diff --git a/qlib/model/ens/ensemble.py b/qlib/model/ens/ensemble.py index a2333cfeba..942303c18b 100644 --- a/qlib/model/ens/ensemble.py +++ b/qlib/model/ens/ensemble.py @@ -58,7 +58,7 @@ class RollingEnsemble(Ensemble): """Merge the rolling objects in an Ensemble""" - def __call__(self, ensemble_dict: dict, *args, **kwargs): + def __call__(self, ensemble_dict: dict): """Merge a dict of rolling dataframe like `prediction` or `IC` into an ensemble. NOTE: The values of dict must be pd.Dataframe, and have the index "datetime" diff --git a/qlib/model/ens/group.py b/qlib/model/ens/group.py index d138b917c6..f5ab5d8a79 100644 --- a/qlib/model/ens/group.py +++ b/qlib/model/ens/group.py @@ -1,6 +1,7 @@ from qlib.model.ens.ensemble import Ensemble, RollingEnsemble from typing import Callable, Union from qlib.utils.serial import Serializable +from joblib import Parallel, delayed class Group(Serializable): @@ -18,10 +19,23 @@ def __init__(self, group_func=None, ens: Ensemble = None): ens (Ensemble, optional): If not None, do ensemble for grouped value after grouping. """ - self.group = group_func - self.ens = ens + self._group_func = group_func + self._ens_func = ens - def __call__(self, ungrouped_dict: dict, *args, **kwargs): + def group(self, *args, **kwargs): + # TODO: such design is weird when `_group_func` is the only configurable part in the class + if isinstance(getattr(self, "_group_func", None), Callable): + return self._group_func(*args, **kwargs) + else: + raise NotImplementedError(f"Please specify valid `group_func`.") + + def reduce(self, *args, **kwargs): + if isinstance(getattr(self, "_ens_func", None), Callable): + return self._ens_func(*args, **kwargs) + else: + raise NotImplementedError(f"Please specify valid `_ens_func`.") + + def __call__(self, ungrouped_dict: dict, n_jobs=1, verbose=0, *args, **kwargs): """Group the ungrouped_dict into different groups. Args: @@ -30,23 +44,24 @@ def __call__(self, ungrouped_dict: dict, *args, **kwargs): Returns: dict: grouped_dict like {G1: object, G2: object} """ - if isinstance(getattr(self, "group", None), Callable): - grouped_dict = self.group(ungrouped_dict, *args, **kwargs) - if self.ens is not None: - ens_dict = {} - for key, value in grouped_dict.items(): - ens_dict[key] = self.ens(value) - grouped_dict = ens_dict - return grouped_dict - else: - raise NotImplementedError(f"Please specify valid group_func.") + + # FIXME: The multiprocessing will raise the following error + # NotImplementedError: Please specify valid `_ens_func`. + # The problem maybe the state of the function is lost + grouped_dict = self.group(ungrouped_dict, *args, **kwargs) + + key_l = [] + job_l = [] + for key, value in grouped_dict.items(): + key_l.append(key) + job_l.append(delayed(Group.reduce)(self, value)) + return dict(zip(key_l, Parallel(n_jobs=n_jobs, verbose=verbose)(job_l))) class RollingGroup(Group): """group the rolling dict""" - @staticmethod - def rolling_group(rolling_dict: dict): + def group(self, rolling_dict: dict): """Given an rolling dict likes {(A,B,R): things}, return the grouped dict likes {(A,B): {R:things}} NOTE: There is a assumption which is the rolling key is at the end of key tuple, because the rolling results always need to be ensemble firstly. @@ -63,7 +78,5 @@ def rolling_group(rolling_dict: dict): grouped_dict.setdefault(key[:-1], {})[key[-1]] = values return grouped_dict - def __init__(self, group_func=None): - super().__init__(group_func=group_func, ens=RollingEnsemble()) - if group_func is None: - self.group = RollingGroup.rolling_group + def __init__(self): + super().__init__(ens=RollingEnsemble()) diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index 7ffca20eec..5165541558 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -8,6 +8,7 @@ from qlib.workflow.task.manage import TaskManager, run_task from qlib.data.dataset import Dataset from qlib.model.base import Model +import socket def task_train(task_config: dict, experiment_name: str) -> Recorder: @@ -35,16 +36,17 @@ def task_train(task_config: dict, experiment_name: str) -> Recorder: # train model R.log_params(**flatten_dict(task_config)) + R.save_objects(**{"task": task_config}) # keep the original format and datatype + R.set_tags(hostname=socket.gethostname()) model.fit(dataset) - recorder = R.get_recorder() R.save_objects(**{"params.pkl": model}) - R.save_objects(**{"task": task_config}) # keep the original format and datatype # This dataset is saved for online inference. So the concrete data should not be dumped dataset.config(dump_all=False, recursive=True) R.save_objects(**{"dataset": dataset}) # generate records: prediction, backtest, and analysis records = task_config.get("record", []) + recorder = R.get_recorder() if isinstance(records, dict): # prevent only one dict records = [records] for record in records: diff --git a/qlib/utils/__init__.py b/qlib/utils/__init__.py index 7e71ba76c3..3ebc6fc1ca 100644 --- a/qlib/utils/__init__.py +++ b/qlib/utils/__init__.py @@ -522,7 +522,7 @@ def get_date_range(trading_date, left_shift=0, right_shift=0, future=False): return calendar -def get_date_by_shift(trading_date, shift, future=False, clip_shift=True): +def get_date_by_shift(trading_date, shift, future=False, clip_shift=True, freq="day"): """get trading date with shift bias wil cur_date e.g. : shift == 1, return next trading date shift == -1, return previous trading date @@ -535,7 +535,7 @@ def get_date_by_shift(trading_date, shift, future=False, clip_shift=True): """ from qlib.data import D - cal = D.calendar(future=future) + cal = D.calendar(future=future, freq=freq) if pd.to_datetime(trading_date) not in list(cal): raise ValueError("{} is not trading day!".format(str(trading_date))) _index = bisect.bisect_left(cal, trading_date) diff --git a/qlib/workflow/online/update.py b/qlib/workflow/online/update.py index 1a6897d02c..8835fdae2d 100644 --- a/qlib/workflow/online/update.py +++ b/qlib/workflow/online/update.py @@ -1,13 +1,142 @@ from typing import Union, List +from qlib.data.dataset import DatasetH from qlib.workflow import R from qlib.data import D import pandas as pd from qlib import get_module_logger from qlib.workflow import R +from qlib.model import Model from qlib.model.trainer import task_train from qlib.workflow.recorder import Recorder from qlib.workflow.task.utils import list_recorders from qlib.data.dataset.handler import DataHandlerLP +from qlib.data.dataset import DatasetH +from abc import ABCMeta, abstractmethod +from qlib.utils import get_date_by_shift + + +class RMDLoader: + """ + Recorder Model Dataset Loader + """ + + def __init__(self, rec: Recorder): + self.rec = rec + + def get_dataset(self, start_time, end_time, segments=None) -> DatasetH: + """ + load, config and setup dataset. + + This dataset is for inferene + + Parameters + ---------- + start_time : + the start_time of underlying data + end_time : + the end_time of underlying data + segments : dict + the segments config for dataset + Due to the time series dataset (TSDatasetH), the test segments maybe different from start_time and end_time + """ + if segments is None: + segments = {"test": (start_time, end_time)} + dataset: DatasetH = self.rec.load_object("dataset") + dataset.config(handler_kwargs={"start_time": start_time, "end_time": end_time}, segments=segments) + dataset.setup_data(handler_kwargs={"init_type": DataHandlerLP.IT_LS}) + return dataset + + def get_model(self) -> Model: + return self.rec.load_object("params.pkl") + + +class RecordUpdater(metaclass=ABCMeta): + """ + Updata a specific recorders + """ + + def __init__(self, record: Recorder, *args, **kwargs): + self.record = record + + @abstractmethod + def update(self, *args, **kwargs): + """ + Update info for specific recorder + """ + ... + + +class PredUpdater(RecordUpdater): + """ + Update the prediction in the Recorder + """ + + LATEST = "__latest" + + def __init__(self, record: Recorder, to_date=LATEST, hist_ref: int = 0, freq="day"): + """ + Parameters + ---------- + record : Recorder + to_date : + update to prediction to the `to_date` + hist_ref : int + Sometimes, the dataset will have historical depends. + Leave the problem to user to set the length of historical dependancy + NOTE: the start_time is not included in the hist_ref + # TODO: automate this step in the future. + """ + super().__init__(record=record) + + self.to_date = to_date + self.hist_ref = hist_ref + self.freq = freq + self.rmdl = RMDLoader(rec=record) + + if to_date == self.LATEST: + to_date = D.calendar(freq=freq)[-1] + self.to_date = pd.Timestamp(to_date) + self.old_pred = record.load_object("pred.pkl") + self.last_end = self.old_pred.index.get_level_values("datetime").max() + + def prepare_data(self) -> DatasetH: + """ + # Load dataset + + Seperating this function will make it easier to reuse the dataset + """ + start_time_buffer = get_date_by_shift(self.last_end, -self.hist_ref + 1, clip_shift=False, freq=self.freq) + start_time = get_date_by_shift(self.last_end, 1, freq=self.freq) + seg = {"test": (start_time, self.to_date)} + dataset = self.rmdl.get_dataset(start_time=start_time_buffer, end_time=self.to_date, segments=seg) + return dataset + + def update(self, dataset: DatasetH = None): + """ + update the precition in a recorder + """ + # FIXME: the problme below is not solved + # The model dumped on GPU instances can not be loaded on CPU instance. Follow exception will raised + # RuntimeError: Attempting to deserialize object on a CUDA device but torch.cuda.is_available() is False. If you are running on a CPU-only machine, please use torch.load with map_location=torch.device('cpu') to map your storages to the CPU. + + # load dataset + if dataset is None: + # For reusing the dataset + dataset = self.prepare_data() + + # Load model + model = self.rmdl.get_model() + + new_pred = model.predict(dataset) + + cb_pred = pd.concat([self.old_pred, new_pred.to_frame("score")], axis=0) + cb_pred = cb_pred.sort_index() + + self.record.save_objects(**{"pred.pkl": cb_pred}) + + get_module_logger(self.__class__.__name__).info( + f"Finish updating new {new_pred.shape[0]} predictions in {self.record.info['id']}." + ) class ModelUpdater: diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index 6b9418daf9..9bd609670a 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -25,6 +25,8 @@ def collect(self, *args, **kwargs): class RecorderCollector(Collector): + ART_KEY_RAW = "__raw" + def __init__( self, exp_name, @@ -48,9 +50,9 @@ def __init__( rec_key_func = lambda rec: rec.info["id"] if artifacts_key is None: artifacts_key = self.artifacts_path.keys() - self.rec_key = rec_key_func + self._rec_key_func = rec_key_func self.artifacts_key = artifacts_key - self.rec_filter = rec_filter_func + self._rec_filter_func = rec_filter_func def collect(self, artifacts_key=None, rec_filter_func=None): """Collect different artifacts based on recorder after filtering. @@ -65,7 +67,7 @@ def collect(self, artifacts_key=None, rec_filter_func=None): if artifacts_key is None: artifacts_key = self.artifacts_key if rec_filter_func is None: - rec_filter_func = self.rec_filter + rec_filter_func = self._rec_filter_func if isinstance(artifacts_key, str): artifacts_key = [artifacts_key] @@ -74,9 +76,12 @@ def collect(self, artifacts_key=None, rec_filter_func=None): # filter records recs_flt = list_recorders(self.exp_name, rec_filter_func) for _, rec in recs_flt.items(): - rec_key = self.rec_key(rec) + rec_key = self._rec_key_func(rec) for key in artifacts_key: - artifact = rec.load_object(self.artifacts_path[key]) + if self.ART_KEY_RAW == key: + artifact = rec + else: + artifact = rec.load_object(self.artifacts_path[key]) collect_dict.setdefault(key, {})[rec_key] = artifact return collect_dict diff --git a/qlib/workflow/task/gen.py b/qlib/workflow/task/gen.py index a8426d920e..3250891268 100644 --- a/qlib/workflow/task/gen.py +++ b/qlib/workflow/task/gen.py @@ -80,6 +80,12 @@ def generate(self, task: dict) -> typing.List[dict]: """ pass + def __call__(self, *args, **kwargs): + """ + This is just a syntactic sugar for generate + """ + return self.generate(*args, **kwargs) + class RollingGen(TaskGen): ROLL_EX = TimeAdjuster.SHIFT_EX # fixed start date, expanding end date diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 720eeb12f1..815529b66d 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -18,7 +18,8 @@ import pymongo from qlib.config import C from .utils import get_mongodb -from qlib import get_module_logger +from qlib import get_module_logger, auto_init +import fire class TaskManager: @@ -49,7 +50,7 @@ class TaskManager: ENCODE_FIELDS_PREFIX = ["def", "res"] - def __init__(self, task_pool: str): + def __init__(self, task_pool: str = None): """ init Task Manager, remember to make the statement of MongoDB url and database name firstly. @@ -59,7 +60,8 @@ def __init__(self, task_pool: str): the name of Collection in MongoDB """ self.mdb = get_mongodb() - self.task_pool = getattr(self.mdb, task_pool) + if task_pool is not None: + self.task_pool = getattr(self.mdb, task_pool) self.logger = get_module_logger(self.__class__.__name__) def list(self): @@ -287,6 +289,20 @@ def reset_status(self, query, status): query["_id"] = ObjectId(query["_id"]) print(self.task_pool.update_many(query, {"$set": {"status": status}})) + def prioritize(self, task, priority: int): + """ + set priority for task + + Parameters + ---------- + task : dict + The task query from the database + priority : int + the target priority + """ + update_dict = {"$set": {"priority": priority}} + self.task_pool.update_one({"_id": task["_id"]}, update_dict) + def _get_undone_n(self, task_stat): return task_stat.get(self.STATUS_WAITING, 0) + task_stat.get(self.STATUS_RUNNING, 0) @@ -345,3 +361,10 @@ def run_task(task_func, task_pool, force_release=False, *args, **kwargs): ever_run = True return ever_run + + +if __name__ == "__main__": + # This is for using it in cmd + # E.g. : `python -m qlib.workflow.task.manage list` + auto_init() + fire.Fire(TaskManager) From b15e5e33fd0d20e24ac52c9397374af27a5bc987 Mon Sep 17 00:00:00 2001 From: Young Date: Mon, 12 Apr 2021 06:33:31 +0000 Subject: [PATCH 39/61] Fix the multi-processing bug --- qlib/model/ens/group.py | 8 +++----- qlib/utils/serial.py | 6 ++++-- qlib/workflow/task/collect.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/qlib/model/ens/group.py b/qlib/model/ens/group.py index f5ab5d8a79..c80959b0d5 100644 --- a/qlib/model/ens/group.py +++ b/qlib/model/ens/group.py @@ -1,10 +1,9 @@ from qlib.model.ens.ensemble import Ensemble, RollingEnsemble from typing import Callable, Union -from qlib.utils.serial import Serializable from joblib import Parallel, delayed -class Group(Serializable): +class Group: """Group the objects based on dict""" def __init__(self, group_func=None, ens: Ensemble = None): @@ -45,9 +44,8 @@ def __call__(self, ungrouped_dict: dict, n_jobs=1, verbose=0, *args, **kwargs): dict: grouped_dict like {G1: object, G2: object} """ - # FIXME: The multiprocessing will raise the following error - # NotImplementedError: Please specify valid `_ens_func`. - # The problem maybe the state of the function is lost + # NOTE: The multiprocessing will raise error if you use `Serializable` + # Because the `Serializable` will affect the behaviours of pickle grouped_dict = self.group(ungrouped_dict, *args, **kwargs) key_l = [] diff --git a/qlib/utils/serial.py b/qlib/utils/serial.py index b94be464ba..1b775d99a7 100644 --- a/qlib/utils/serial.py +++ b/qlib/utils/serial.py @@ -7,8 +7,10 @@ class Serializable: """ - Serializable behaves like pickle. - But it only saves the state whose name **does not** start with `_` + Serializable will change the behaviours of pickle. + - It only saves the state whose name **does not** start with `_` + It provides a syntactic sugar for distinguish the attributes which user doesn't want. + - For examples, a learnable Datahandler just wants to save the parameters without data when dumping to disk """ def __init__(self): diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index 9bd609670a..f651ef8d8f 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -4,7 +4,7 @@ from qlib.utils.serial import Serializable -class Collector(Serializable): +class Collector: """The collector to collect different results""" def collect(self, *args, **kwargs): From 5095b2a4707657de5edf935a3107a7628f9dde74 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Tue, 13 Apr 2021 09:45:16 +0000 Subject: [PATCH 40/61] simulator & examples --- examples/online_srv/online_simulate.py | 163 ++++++++++++++++++ .../task_manager_rolling_with_updating.py | 9 +- qlib/workflow/online/manager.py | 89 ++++++---- qlib/workflow/online/simulator.py | 80 +++++++++ qlib/workflow/online/update.py | 132 ++------------ qlib/workflow/task/utils.py | 8 +- 6 files changed, 328 insertions(+), 153 deletions(-) create mode 100644 examples/online_srv/online_simulate.py create mode 100644 qlib/workflow/online/simulator.py diff --git a/examples/online_srv/online_simulate.py b/examples/online_srv/online_simulate.py new file mode 100644 index 0000000000..007085c736 --- /dev/null +++ b/examples/online_srv/online_simulate.py @@ -0,0 +1,163 @@ +from abc import abstractmethod +import copy +from pprint import pprint + +import fire +import qlib +from qlib.config import REG_CN +from qlib.model.trainer import task_train +from qlib.workflow import R +from qlib.workflow.task.gen import TaskGen +from qlib.workflow.online.simulator import OnlineSimulator +from qlib.workflow.task.collect import RecorderCollector +from qlib.model.ens.ensemble import RollingEnsemble, ens_workflow +from qlib.workflow.task.gen import RollingGen, task_generator +from qlib.workflow.task.manage import TaskManager, run_task +from qlib.workflow.online.manager import RollingOnlineManager +from qlib.workflow.task.utils import TimeAdjuster, list_recorders +from qlib.model.trainer import TrainerRM +from qlib.model.ens.group import RollingGroup + +data_handler_config = { + "start_time": "2018-01-01", + "end_time": "2018-10-31", + "fit_start_time": "2018-01-01", + "fit_end_time": "2018-03-31", + "instruments": "csi100", +} + +dataset_config = { + "class": "DatasetH", + "module_path": "qlib.data.dataset", + "kwargs": { + "handler": { + "class": "Alpha158", + "module_path": "qlib.contrib.data.handler", + "kwargs": data_handler_config, + }, + "segments": { + "train": ("2018-01-01", "2018-03-31"), + "valid": ("2018-04-01", "2018-05-31"), + "test": ("2018-06-01", "2018-09-10"), + }, + }, +} + +record_config = [ + { + "class": "SignalRecord", + "module_path": "qlib.workflow.record_temp", + }, + { + "class": "SigAnaRecord", + "module_path": "qlib.workflow.record_temp", + }, +] + +# use lgb model +task_lgb_config = { + "model": { + "class": "LGBModel", + "module_path": "qlib.contrib.model.gbdt", + }, + "dataset": dataset_config, + "record": record_config, +} + +# use xgboost model +task_xgboost_config = { + "model": { + "class": "XGBModel", + "module_path": "qlib.contrib.model.xgboost", + }, + "dataset": dataset_config, + "record": record_config, +} + + +class OnlineSimulatorExample: + def __init__( + self, + exp_name="rolling_exp", + task_pool="rolling_task", + provider_uri="~/.qlib/qlib_data/cn_data", + region="cn", + task_url="mongodb://10.0.0.4:27017/", + task_db_name="rolling_db", + rolling_step=80, + ): + self.exp_name = exp_name + self.task_pool = task_pool + mongo_conf = { + "task_url": task_url, # your MongoDB url + "task_db_name": task_db_name, # database name + } + qlib.init(provider_uri=provider_uri, region=region, mongo=mongo_conf) + + self.rolling_gen = RollingGen(step=rolling_step, rtype=RollingGen.ROLL_SD) + self.trainer = TrainerRM(self.exp_name, self.task_pool) + self.task_manager = TaskManager(self.task_pool) + self.rolling_online_manager = RollingOnlineManager( + experiment_name=exp_name, rolling_gen=self.rolling_gen, trainer=self.trainer, need_log=False + ) + + # Reset all things to the first status, be careful to save important data + def reset(self): + print("========== reset ==========") + self.task_manager.remove() + exp = R.get_exp(experiment_name=self.exp_name) + for rid in exp.list_recorders(): + exp.delete_recorder(rid) + + @staticmethod + def rec_key(recorder): + task_config = recorder.load_object("task") + model_key = task_config["model"]["class"] + rolling_key = task_config["dataset"]["kwargs"]["segments"]["test"] + return model_key, rolling_key + + # Run this firstly to see the workflow in Task Management + def first_run(self): + print("========== first_run ==========") + self.reset() + + tasks = task_generator( + tasks=task_xgboost_config, + generators=[self.rolling_gen], # generate different date segment + ) + + pprint(tasks) + + self.trainer.train(tasks) + + print("========== task collecting ==========") + + artifact = ens_workflow(RecorderCollector(exp_name=self.exp_name, rec_key_func=self.rec_key), RollingGroup()) + print(artifact) + + latest_rec, _ = self.rolling_online_manager.list_latest_recorders() + self.rolling_online_manager.set_online_tag(RollingOnlineManager.ONLINE_TAG, list(latest_rec.values())) + + def simulate(self): + + print("========== simulate ==========") + onlinesimulator = OnlineSimulator( + start_time="2018-09-10", + end_time="2018-10-31", + onlinemanager=self.rolling_online_manager, + collector=RecorderCollector(exp_name=self.exp_name, rec_key_func=self.rec_key), + process_list=[RollingGroup()], + ) + results = onlinesimulator.simulate() + print(results) + recs_dict = onlinesimulator.online_models() + for time, recs in recs_dict.items(): + print(f"{str(time[0])} to {str(time[1])}:") + for rec in recs: + print(rec.info["id"]) + + +if __name__ == "__main__": + ose = OnlineSimulatorExample() + ose.first_run() + ose.simulate() diff --git a/examples/online_srv/task_manager_rolling_with_updating.py b/examples/online_srv/task_manager_rolling_with_updating.py index bfdc5f3c03..9195a1de69 100644 --- a/examples/online_srv/task_manager_rolling_with_updating.py +++ b/examples/online_srv/task_manager_rolling_with_updating.py @@ -100,9 +100,9 @@ def __init__( def print_online_model(self): print("========== print_online_model ==========") print("Current 'online' model:") - for rid, rec in list_recorders(self.exp_name).items(): - if self.rolling_online_manager.get_online_tag(rec) == self.rolling_online_manager.ONLINE_TAG: - print(rid) + + for rec in self.rolling_online_manager.online_models(): + print(rec.info["id"]) print("Current 'next online' model:") for rid, rec in list_recorders(self.exp_name).items(): if self.rolling_online_manager.get_online_tag(rec) == self.rolling_online_manager.NEXT_ONLINE_TAG: @@ -161,12 +161,15 @@ def first_run(self): self.reset() tasks = self.task_generating() + pprint(tasks) self.task_training(tasks) self.task_collecting() latest_rec, _ = self.rolling_online_manager.list_latest_recorders() self.rolling_online_manager.reset_online_tag(list(latest_rec.values())) + self.routine() + def routine(self): print("========== routine ==========") self.print_online_model() diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index 66df160cdc..a40512cf3f 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -3,7 +3,7 @@ from qlib.workflow import R from qlib.model.trainer import task_train from qlib.workflow.recorder import MLflowRecorder, Recorder -from qlib.workflow.online.update import ModelUpdater +from qlib.workflow.online.update import PredUpdater, RecordUpdater from qlib.workflow.task.utils import TimeAdjuster from qlib.workflow.task.gen import RollingGen, task_generator from qlib.workflow.task.manage import TaskManager @@ -11,6 +11,7 @@ from qlib.workflow.task.utils import list_recorders from qlib.utils.serial import Serializable from qlib.model.trainer import Trainer, TrainerR +from copy import deepcopy class OnlineManager(Serializable): @@ -20,9 +21,11 @@ class OnlineManager(Serializable): NEXT_ONLINE_TAG = "next_online" # the 'next online' model, which can be 'online' model when call reset_online_model OFFLINE_TAG = "offline" # the 'offline' model, not for online serving - def __init__(self, trainer: Trainer = None): + def __init__(self, trainer: Trainer = None, need_log=True): self._trainer = trainer self.logger = get_module_logger(self.__class__.__name__) + self.need_log = need_log + self.delay_signals = {} def prepare_signals(self, *args, **kwargs): raise NotImplementedError(f"Please implement the `prepare_signals` method.") @@ -31,7 +34,7 @@ def prepare_tasks(self, *args, **kwargs): """return the new tasks waiting for training.""" raise NotImplementedError(f"Please implement the `prepare_tasks` method.") - def prepare_new_models(self, tasks, *args, **kwargs): + def prepare_new_models(self, tasks): """Use trainer to train a list of tasks and set the trained model to next_online. Args: @@ -39,7 +42,7 @@ def prepare_new_models(self, tasks, *args, **kwargs): """ if not (tasks is None or len(tasks) == 0): if self._trainer is not None: - new_models = self._trainer.train(tasks, *args, **kwargs) + new_models = self._trainer.train(tasks) self.set_online_tag(self.NEXT_ONLINE_TAG, new_models) self.logger.info( f"Finished prepare {len(new_models)} new models and set them to `{self.NEXT_ONLINE_TAG}`." @@ -66,15 +69,27 @@ def reset_online_tag(self, *args, **kwargs): """offline all models and set the recorders to 'online'. If no parameter and no 'next online' model, then do nothing.""" raise NotImplementedError(f"Please implement the `reset_online_tag` method.") - def routine(self, *args, **kwargs): + def online_models(self): + """return online models""" + raise NotImplementedError(f"Please implement the `online_models` method.") + + def run_delay_signals(self): + for cur_time, params in self.delay_signals.items(): + self.cur_time = cur_time + self.prepare_signals(*params[0], **params[1]) + self.delay_signals = {} + + def routine(self, cur_time=None, delay_prepare=False, *args, **kwargs): """The typical update process in a routine such as day by day or month by month""" - self.prepare_signals(*args, **kwargs) + self.cur_time = cur_time # None for latest date + if not delay_prepare: + self.prepare_signals(*args, **kwargs) + else: + self.delay_signals[cur_time] = (args, kwargs) tasks = self.prepare_tasks(*args, **kwargs) - self.prepare_new_models(tasks, *args, **kwargs) - self.update_online_pred(*args, **kwargs) - self.reset_online_tag(*args, **kwargs) - - # TODO: first_train? + self.prepare_new_models(tasks) + self.update_online_pred() + return self.reset_online_tag() class OnlineManagerR(OnlineManager): @@ -83,10 +98,9 @@ class OnlineManagerR(OnlineManager): """ - def __init__(self, experiment_name: str, trainer: Trainer = None): + def __init__(self, experiment_name: str, trainer: Trainer = None, need_log=True): trainer = TrainerR(experiment_name) - super().__init__(trainer) - self.logger = get_module_logger(self.__class__.__name__) + super().__init__(trainer, need_log) self.exp_name = experiment_name def set_online_tag(self, tag, recorder: Union[Recorder, List]): @@ -94,7 +108,8 @@ def set_online_tag(self, tag, recorder: Union[Recorder, List]): recorder = [recorder] for rec in recorder: rec.set_tags(**{self.ONLINE_KEY: tag}) - self.logger.info(f"Set {len(recorder)} models to '{tag}'.") + if self.need_log: + self.logger.info(f"Set {len(recorder)} models to '{tag}'.") def get_online_tag(self, recorder: Recorder): tags = recorder.list_tags() @@ -106,6 +121,9 @@ def reset_online_tag(self, recorder: Union[Recorder, List] = None): Args: recorders (Union[List, Dict], optional): the recorders you want to reset to 'online'. If don't give, set 'next online' model to 'online' model. If there isn't any 'next online' model, then maintain existing 'online' model. + + Returns: + list: new online recorder. [] if there is no update. """ if recorder is None: recorder = list( @@ -116,31 +134,35 @@ def reset_online_tag(self, recorder: Union[Recorder, List] = None): if isinstance(recorder, Recorder): recorder = [recorder] if len(recorder) == 0: - self.logger.info("No 'next online' model, just use current 'online' models.") - return + if self.need_log: + self.logger.info("No 'next online' model, just use current 'online' models.") + return [] recs = list_recorders(self.exp_name) self.set_online_tag(OnlineManager.OFFLINE_TAG, list(recs.values())) self.set_online_tag(OnlineManager.ONLINE_TAG, recorder) - self.logger.info(f"Reset {len(recorder)} models to 'online'.") + return recorder - def update_online_pred(self, *args, **kwargs): + def online_models(self): + return list( + list_recorders(self.exp_name, lambda rec: self.get_online_tag(rec) == OnlineManager.ONLINE_TAG).values() + ) + + def update_online_pred(self): """update all online model predictions to the latest day in Calendar""" - mu = ModelUpdater(self.exp_name) - cnt = mu.update_all_pred(lambda rec: self.get_online_tag(rec) == OnlineManager.ONLINE_TAG) - self.logger.info(f"Finish updating {cnt} online model predictions of {self.exp_name}.") + online_models = self.online_models() + for rec in online_models: + PredUpdater(rec, to_date=self.cur_time, need_log=self.need_log).update() + + if self.need_log: + self.logger.info(f"Finish updating {len(online_models)} online model predictions of {self.exp_name}.") class RollingOnlineManager(OnlineManagerR): """An implementation of OnlineManager based on Rolling.""" - def __init__( - self, - experiment_name: str, - rolling_gen: RollingGen, - trainer: Trainer = None, - ): + def __init__(self, experiment_name: str, rolling_gen: RollingGen, trainer: Trainer = None, need_log=True): trainer = TrainerR(experiment_name) - super().__init__(experiment_name, trainer) + super().__init__(experiment_name, trainer, need_log=need_log) self.ta = TimeAdjuster() self.rg = rolling_gen self.logger = get_module_logger(self.__class__.__name__) @@ -154,22 +176,25 @@ def prepare_tasks(self, *args, **kwargs): Returns: list: a list of new tasks. """ + self.ta.set_end_time(self.cur_time) latest_records, max_test = self.list_latest_recorders( lambda rec: self.get_online_tag(rec) == OnlineManager.ONLINE_TAG ) if max_test is None: self.logger.warn(f"No latest online recorders, no new tasks.") return [] - calendar_latest = self.ta.last_date() + calendar_latest = self.ta.last_date() if self.cur_time is None else self.cur_time if self.ta.cal_interval(calendar_latest, max_test[0]) > self.rg.step: old_tasks = [] + tasks_tmp = [] for rid, rec in latest_records.items(): task = rec.load_object("task") + old_tasks.append(deepcopy(task)) test_begin = task["dataset"]["kwargs"]["segments"]["test"][0] # modify the test segment to generate new tasks task["dataset"]["kwargs"]["segments"]["test"] = (test_begin, calendar_latest) - old_tasks.append(task) - new_tasks_tmp = task_generator(old_tasks, self.rg) + tasks_tmp.append(task) + new_tasks_tmp = task_generator(tasks_tmp, self.rg) new_tasks = [task for task in new_tasks_tmp if task not in old_tasks] return new_tasks return [] diff --git a/qlib/workflow/online/simulator.py b/qlib/workflow/online/simulator.py new file mode 100644 index 0000000000..7f08549f01 --- /dev/null +++ b/qlib/workflow/online/simulator.py @@ -0,0 +1,80 @@ +from typing import Callable +import pandas as pd +from qlib.config import C +from qlib.data import D +from qlib import get_module_logger +from qlib.log import set_log_with_config +from qlib.model.ens.ensemble import ens_workflow +from qlib.workflow.online.manager import OnlineManager +from qlib.workflow.task.collect import Collector + + +class OnlineSimulator: + """ + To simulate online serving in the past, like a "online serving backtest". + """ + + def __init__( + self, + start_time, + end_time, + onlinemanager: OnlineManager, + frequency="day", + time_delta="20 hours", + collector: Collector = None, + process_list: list = None, + ): + self.logger = get_module_logger(self.__class__.__name__) + self.cal = D.calendar(start_time=start_time, end_time=end_time, freq=frequency) + self.start_time = self.cal[0] + self.end_time = self.cal[-1] + self.olm = onlinemanager + self.time_delta = time_delta + + if len(self.cal) == 0: + self.logger.warn(f"There is no need to simulate bacause start_time is larger than end_time.") + self.collector = collector + self.process_list = process_list + + def simulate(self, *args, **kwargs): + """ + Starting from start time, this method will simulate every routine in OnlineManager. + NOTE: Considering the parallel training, the signals will be perpared after all routine simulating. + + Returns: + dict: the simulated results collected by collector + """ + self.rec_dict = {} + tmp_begin = self.start_time + tmp_end = None + prev_recorders = self.olm.online_models() + for cur_time in self.cal: + cur_time = cur_time + pd.Timedelta(self.time_delta) + self.logger.info(f"Simulating at {str(cur_time)}......") + recorders = self.olm.routine(cur_time, True, *args, **kwargs) + if len(recorders) == 0: + tmp_end = cur_time + else: + self.rec_dict[(tmp_begin, tmp_end)] = prev_recorders + tmp_begin = cur_time + prev_recorders = recorders + + self.rec_dict[(tmp_begin, self.end_time)] = prev_recorders + # prepare signals again incase there is no trained model when call it + self.olm.run_delay_signals() + self.logger.info(f"Finished preparing signals") + + if self.collector is not None: + return ens_workflow(self.collector, self.process_list) + + def online_models(self): + """ + Return a online models dict likes {(begin_time, end_time):[online models]}. + + Returns: + dict + """ + if hasattr(self, "rec_dict"): + return self.rec_dict + self.logger.warn(f"Please call `simulate` firstly when calling `online_models`") + return {} diff --git a/qlib/workflow/online/update.py b/qlib/workflow/online/update.py index 8835fdae2d..8aa32ff292 100644 --- a/qlib/workflow/online/update.py +++ b/qlib/workflow/online/update.py @@ -27,7 +27,7 @@ def get_dataset(self, start_time, end_time, segments=None) -> DatasetH: """ load, config and setup dataset. - This dataset is for inferene + This dataset is for inference Parameters ---------- @@ -55,8 +55,10 @@ class RecordUpdater(metaclass=ABCMeta): Updata a specific recorders """ - def __init__(self, record: Recorder, *args, **kwargs): + def __init__(self, record: Recorder, need_log=True, *args, **kwargs): self.record = record + self.logger = get_module_logger(self.__class__.__name__) + self.need_log = need_log @abstractmethod def update(self, *args, **kwargs): @@ -73,7 +75,7 @@ class PredUpdater(RecordUpdater): LATEST = "__latest" - def __init__(self, record: Recorder, to_date=LATEST, hist_ref: int = 0, freq="day"): + def __init__(self, record: Recorder, to_date=LATEST, hist_ref: int = 0, freq="day", need_log=True): """ Parameters ---------- @@ -86,14 +88,15 @@ def __init__(self, record: Recorder, to_date=LATEST, hist_ref: int = 0, freq="da NOTE: the start_time is not included in the hist_ref # TODO: automate this step in the future. """ - super().__init__(record=record) + super().__init__(record=record, need_log=need_log) self.to_date = to_date self.hist_ref = hist_ref self.freq = freq self.rmdl = RMDLoader(rec=record) - if to_date == self.LATEST: + # FIXME: why we need LATEST? can we use to_date=None instead? + if to_date == self.LATEST or to_date == None: to_date = D.calendar(freq=freq)[-1] self.to_date = pd.Timestamp(to_date) self.old_pred = record.load_object("pred.pkl") @@ -119,6 +122,12 @@ def update(self, dataset: DatasetH = None): # The model dumped on GPU instances can not be loaded on CPU instance. Follow exception will raised # RuntimeError: Attempting to deserialize object on a CUDA device but torch.cuda.is_available() is False. If you are running on a CPU-only machine, please use torch.load with map_location=torch.device('cpu') to map your storages to the CPU. + start_time = get_date_by_shift(self.last_end, 1, freq=self.freq) + if start_time >= self.to_date: + if self.need_log: + self.logger.info(f"The prediction in {self.record.info['id']} are latest. No need to update.") + return + # load dataset if dataset is None: # For reusing the dataset @@ -134,114 +143,5 @@ def update(self, dataset: DatasetH = None): self.record.save_objects(**{"pred.pkl": cb_pred}) - get_module_logger(self.__class__.__name__).info( - f"Finish updating new {new_pred.shape[0]} predictions in {self.record.info['id']}." - ) - - -class ModelUpdater: - """ - The model updater to update model results in new data. - """ - - def __init__(self, experiment_name: str) -> None: - """ModelUpdater needs experiment name to find the records - - Parameters - ---------- - experiment_name : str - experiment name string - """ - self.exp_name = experiment_name - self.logger = get_module_logger(self.__class__.__name__) - - def _reload_dataset(self, recorder, start_time, end_time): - """reload dataset from pickle file - - Parameters - ---------- - recorder : Recorder - the instance of the Recorder - start_time : Timestamp - the start time you want to load - end_time : Timestamp - the end time you want to load - - Returns - ------- - Dataset - the instance of Dataset - """ - segments = {"test": (start_time, end_time)} - dataset = recorder.load_object("dataset") - dataset.config(handler_kwargs={"start_time": start_time, "end_time": end_time}, segments=segments) - dataset.setup_data(handler_kwargs={"init_type": DataHandlerLP.IT_LS}) - return dataset - - def update_pred(self, recorder: Recorder, frequency="day"): - """update predictions to the latest day in Calendar based on rid - - Parameters - ---------- - recorder: Union[str,Recorder] - the id of a Recorder or the Recorder instance - """ - old_pred = recorder.load_object("pred.pkl") - last_end = old_pred.index.get_level_values("datetime").max() - - # updated to the latest trading day - if frequency == "day": - cal = D.calendar(start_time=last_end + pd.Timedelta(days=1), end_time=None) - else: - raise NotImplementedError("Now `ModelUpdater` only support update daily frequency prediction") - - if len(cal) == 0: - self.logger.info( - f"The prediction in {recorder.info['id']} of {self.exp_name} are latest. No need to update." - ) - return - - start_time, end_time = cal[0], cal[-1] - - dataset = self._reload_dataset(recorder, start_time, end_time) - - model = recorder.load_object("params.pkl") - new_pred = model.predict(dataset) - - cb_pred = pd.concat([old_pred, new_pred.to_frame("score")], axis=0) - cb_pred = cb_pred.sort_index() - - recorder.save_objects(**{"pred.pkl": cb_pred}) - - self.logger.info( - f"Finish updating new {new_pred.shape[0]} predictions in {recorder.info['id']} of {self.exp_name}." - ) - - def update_all_pred(self, rec_filter_func=None): - """update all predictions in this experiment after filter. - - An example of filter function: - - .. code-block:: python - - def record_filter(record): - task_config = record.load_object("task") - if task_config["model"]["class"]=="LGBModel": - return True - return False - - Parameters - ---------- - rec_filter_func : Callable[[Recorder], bool], optional - the filter function to decide whether this record will be updated, by default None - - Returns - ---------- - cnt: int - the count of updated record - - """ - recs = list_recorders(self.exp_name, rec_filter_func=rec_filter_func) - for rid, rec in recs.items(): - self.update_pred(rec) - return len(recs) + if self.need_log: + self.logger.info(f"Finish updating new {new_pred.shape[0]} predictions in {self.record.info['id']}.") diff --git a/qlib/workflow/task/utils.py b/qlib/workflow/task/utils.py index b6287abc2b..87a3a41f3f 100644 --- a/qlib/workflow/task/utils.py +++ b/qlib/workflow/task/utils.py @@ -57,8 +57,12 @@ class TimeAdjuster: find appropriate date and adjust date. """ - def __init__(self, future=False): - self.cals = D.calendar(future=future) + def __init__(self, future=True, end_time=None): + self._future = future + self.cals = D.calendar(future=future, end_time=end_time) + + def set_end_time(self, end_time=None): + self.cals = D.calendar(future=self._future, end_time=end_time) def get(self, idx: int): """ From cec318fbfe5f121c4ac5c080e308c83603d31717 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Fri, 16 Apr 2021 05:37:13 +0000 Subject: [PATCH 41/61] online serving V7 --- .../model_rolling/task_manager_rolling.py | 7 +- .../online_srv/online_management_simulate.py | 198 ++++++++++++++++++ examples/online_srv/online_simulate.py | 163 -------------- .../task_manager_rolling_with_updating.py | 13 +- qlib/data/dataset/__init__.py | 2 +- qlib/model/trainer.py | 3 +- qlib/workflow/online/manager.py | 138 +++++++++--- qlib/workflow/online/simulator.py | 27 +-- qlib/workflow/online/update.py | 3 +- qlib/workflow/task/gen.py | 2 +- qlib/workflow/task/manage.py | 10 +- qlib/workflow/task/utils.py | 29 ++- 12 files changed, 370 insertions(+), 225 deletions(-) create mode 100644 examples/online_srv/online_management_simulate.py delete mode 100644 examples/online_srv/online_simulate.py diff --git a/examples/model_rolling/task_manager_rolling.py b/examples/model_rolling/task_manager_rolling.py index 3e914cc636..4508a87886 100644 --- a/examples/model_rolling/task_manager_rolling.py +++ b/examples/model_rolling/task_manager_rolling.py @@ -1,9 +1,10 @@ from pprint import pprint +import time import fire import qlib from qlib.config import REG_CN -from qlib.model.trainer import task_train +from qlib.model.trainer import TrainerR, task_train from qlib.workflow import R from qlib.workflow.task.gen import RollingGen, task_generator from qlib.workflow.task.manage import TaskManager, run_task @@ -102,7 +103,7 @@ def task_training(tasks, task_pool, exp_name): # This part corresponds to "Task Collecting" in the document -def task_collecting(task_pool, exp_name): +def task_collecting(exp_name): print("========== task_collecting ==========") def rec_key(recorder): @@ -141,7 +142,7 @@ def main( reset(task_pool, experiment_name) tasks = task_generating() task_training(tasks, task_pool, experiment_name) - task_collecting(task_pool, experiment_name) + task_collecting(experiment_name) if __name__ == "__main__": diff --git a/examples/online_srv/online_management_simulate.py b/examples/online_srv/online_management_simulate.py new file mode 100644 index 0000000000..d3a1328799 --- /dev/null +++ b/examples/online_srv/online_management_simulate.py @@ -0,0 +1,198 @@ +import fire +import qlib +from qlib.model.ens.ensemble import ens_workflow +from qlib.model.ens.group import RollingGroup +from qlib.model.trainer import TrainerRM +from qlib.workflow import R +from qlib.workflow.online.manager import RollingOnlineManager +from qlib.workflow.online.simulator import OnlineSimulator +from qlib.workflow.task.collect import RecorderCollector +from qlib.workflow.task.gen import RollingGen, task_generator +from qlib.workflow.task.manage import TaskManager + +""" +This examples is about the OnlineManager and OnlineSimulator based on Rolling tasks. +The OnlineManager will focus on the updating of your online models. +The OnlineSimulator will focus on the simulating real updating routine of your online models. +""" + + +data_handler_config = { + "start_time": "2018-01-01", + "end_time": None, # "2018-10-31", + "fit_start_time": "2018-01-01", + "fit_end_time": "2018-03-31", + "instruments": "csi100", +} + +dataset_config = { + "class": "DatasetH", + "module_path": "qlib.data.dataset", + "kwargs": { + "handler": { + "class": "Alpha158", + "module_path": "qlib.contrib.data.handler", + "kwargs": data_handler_config, + }, + "segments": { + "train": ("2018-01-01", "2018-03-31"), + "valid": ("2018-04-01", "2018-05-31"), + "test": ("2018-06-01", "2018-09-10"), + }, + }, +} + +record_config = [ + { + "class": "SignalRecord", + "module_path": "qlib.workflow.record_temp", + }, + { + "class": "SigAnaRecord", + "module_path": "qlib.workflow.record_temp", + }, +] + +# use lgb model +task_lgb_config = { + "model": { + "class": "LGBModel", + "module_path": "qlib.contrib.model.gbdt", + }, + "dataset": dataset_config, + "record": record_config, +} + +# use xgboost model +task_xgboost_config = { + "model": { + "class": "XGBModel", + "module_path": "qlib.contrib.model.xgboost", + }, + "dataset": dataset_config, + "record": record_config, +} + + +class OnlineManagerExample: + def __init__( + self, + provider_uri="~/.qlib/qlib_data/cn_data", + region="cn", + exp_name="rolling_exp", + task_url="mongodb://10.0.0.4:27017/", + task_db_name="rolling_db", + task_pool="rolling_task", + rolling_step=80, + start_time="2018-09-10", + end_time="2018-10-31", + ): + """ + init OnlineManagerExample. + + Args: + provider_uri (str, optional): the provider uri. Defaults to "~/.qlib/qlib_data/cn_data". + region (str, optional): the stock region. Defaults to "cn". + exp_name (str, optional): the experiment name. Defaults to "rolling_exp". + task_url (str, optional): your MongoDB url. Defaults to "mongodb://10.0.0.4:27017/". + task_db_name (str, optional): database name. Defaults to "rolling_db". + task_pool (str, optional): the task pool name (a task pool is a collection in MongoDB). Defaults to "rolling_task". + rolling_step (int, optional): the step for rolling. Defaults to 80. + start_time (str, optional): the start time of simulating. Defaults to "2018-09-10". + end_time (str, optional): the end time of simulating. Defaults to "2018-10-31". + """ + self.exp_name = exp_name + self.task_pool = task_pool + mongo_conf = { + "task_url": task_url, + "task_db_name": task_db_name, + } + qlib.init(provider_uri=provider_uri, region=region, mongo=mongo_conf) + + self.rolling_gen = RollingGen(step=rolling_step, rtype=RollingGen.ROLL_SD) # The rolling tasks generator + self.trainer = TrainerRM(self.exp_name, self.task_pool) # The trainer based on (R)ecorder and Task(M)anager + self.task_manager = TaskManager(self.task_pool) # A good way to manage all your tasks + self.collector = RecorderCollector(exp_name=self.exp_name, rec_key_func=self.rec_key) # The result collector + self.grouper = RollingGroup() # Divide your results into different rolling group + self.rolling_online_manager = RollingOnlineManager( + experiment_name=exp_name, + rolling_gen=self.rolling_gen, + trainer=self.trainer, + collector=self.collector, + need_log=False, + ) # The OnlineManager based on Rolling + self.onlinesimulator = OnlineSimulator( + start_time=start_time, + end_time=end_time, + onlinemanager=self.rolling_online_manager, + ) + + # Reset all things to the first status, be careful to save important data + def reset(self): + print("========== reset ==========") + self.task_manager.remove() + exp = R.get_exp(experiment_name=self.exp_name) + for rid in exp.list_recorders(): + exp.delete_recorder(rid) + + @staticmethod + def rec_key(recorder): + """ + given a Recorder and return its key to identify it + + Args: + recorder (Recorder): a instance of the Recorder + + Returns: + tuple: (model_key, rolling_key) + """ + task_config = recorder.load_object("task") + model_key = task_config["model"]["class"] + rolling_key = task_config["dataset"]["kwargs"]["segments"]["test"] + return model_key, rolling_key + + def result_collecting(self): + print("========== result collecting ==========") + + # ens_workflow can help collect, group and ensemble results in a easy way + artifact = ens_workflow(self.rolling_online_manager.get_collector(), self.grouper) + print(artifact) + + # Run this firstly to see the workflow in OnlineManager + def first_train(self): + print("========== first train ==========") + self.reset() + + tasks = task_generator( + tasks=[task_xgboost_config, task_lgb_config], + generators=[self.rolling_gen], # generate different date segment + ) + + self.rolling_online_manager.prepare_new_models(tasks=tasks, tag=RollingOnlineManager.ONLINE_TAG) + self.result_collecting() + + # Run this secondly to see the simulating in OnlineSimulator + def simulate(self): + + print("========== simulate ==========") + self.onlinesimulator.simulate() + + self.result_collecting() + + print("========== online models ==========") + recs_dict = self.onlinesimulator.online_models() + for time, recs in recs_dict.items(): + print(f"{str(time[0])} to {str(time[1])}:") + for rec in recs: + print(rec.info["id"]) + + # Run this to run all workflow automaticly + def main(self): + self.first_train() + self.simulate() + + +if __name__ == "__main__": + ## to run all workflow automaticly with your own parameters, use the command below + # python online_management_simulate.py main --experiment_name="your_exp_name" --rolling_step=60 + fire.Fire(OnlineManagerExample) diff --git a/examples/online_srv/online_simulate.py b/examples/online_srv/online_simulate.py deleted file mode 100644 index 007085c736..0000000000 --- a/examples/online_srv/online_simulate.py +++ /dev/null @@ -1,163 +0,0 @@ -from abc import abstractmethod -import copy -from pprint import pprint - -import fire -import qlib -from qlib.config import REG_CN -from qlib.model.trainer import task_train -from qlib.workflow import R -from qlib.workflow.task.gen import TaskGen -from qlib.workflow.online.simulator import OnlineSimulator -from qlib.workflow.task.collect import RecorderCollector -from qlib.model.ens.ensemble import RollingEnsemble, ens_workflow -from qlib.workflow.task.gen import RollingGen, task_generator -from qlib.workflow.task.manage import TaskManager, run_task -from qlib.workflow.online.manager import RollingOnlineManager -from qlib.workflow.task.utils import TimeAdjuster, list_recorders -from qlib.model.trainer import TrainerRM -from qlib.model.ens.group import RollingGroup - -data_handler_config = { - "start_time": "2018-01-01", - "end_time": "2018-10-31", - "fit_start_time": "2018-01-01", - "fit_end_time": "2018-03-31", - "instruments": "csi100", -} - -dataset_config = { - "class": "DatasetH", - "module_path": "qlib.data.dataset", - "kwargs": { - "handler": { - "class": "Alpha158", - "module_path": "qlib.contrib.data.handler", - "kwargs": data_handler_config, - }, - "segments": { - "train": ("2018-01-01", "2018-03-31"), - "valid": ("2018-04-01", "2018-05-31"), - "test": ("2018-06-01", "2018-09-10"), - }, - }, -} - -record_config = [ - { - "class": "SignalRecord", - "module_path": "qlib.workflow.record_temp", - }, - { - "class": "SigAnaRecord", - "module_path": "qlib.workflow.record_temp", - }, -] - -# use lgb model -task_lgb_config = { - "model": { - "class": "LGBModel", - "module_path": "qlib.contrib.model.gbdt", - }, - "dataset": dataset_config, - "record": record_config, -} - -# use xgboost model -task_xgboost_config = { - "model": { - "class": "XGBModel", - "module_path": "qlib.contrib.model.xgboost", - }, - "dataset": dataset_config, - "record": record_config, -} - - -class OnlineSimulatorExample: - def __init__( - self, - exp_name="rolling_exp", - task_pool="rolling_task", - provider_uri="~/.qlib/qlib_data/cn_data", - region="cn", - task_url="mongodb://10.0.0.4:27017/", - task_db_name="rolling_db", - rolling_step=80, - ): - self.exp_name = exp_name - self.task_pool = task_pool - mongo_conf = { - "task_url": task_url, # your MongoDB url - "task_db_name": task_db_name, # database name - } - qlib.init(provider_uri=provider_uri, region=region, mongo=mongo_conf) - - self.rolling_gen = RollingGen(step=rolling_step, rtype=RollingGen.ROLL_SD) - self.trainer = TrainerRM(self.exp_name, self.task_pool) - self.task_manager = TaskManager(self.task_pool) - self.rolling_online_manager = RollingOnlineManager( - experiment_name=exp_name, rolling_gen=self.rolling_gen, trainer=self.trainer, need_log=False - ) - - # Reset all things to the first status, be careful to save important data - def reset(self): - print("========== reset ==========") - self.task_manager.remove() - exp = R.get_exp(experiment_name=self.exp_name) - for rid in exp.list_recorders(): - exp.delete_recorder(rid) - - @staticmethod - def rec_key(recorder): - task_config = recorder.load_object("task") - model_key = task_config["model"]["class"] - rolling_key = task_config["dataset"]["kwargs"]["segments"]["test"] - return model_key, rolling_key - - # Run this firstly to see the workflow in Task Management - def first_run(self): - print("========== first_run ==========") - self.reset() - - tasks = task_generator( - tasks=task_xgboost_config, - generators=[self.rolling_gen], # generate different date segment - ) - - pprint(tasks) - - self.trainer.train(tasks) - - print("========== task collecting ==========") - - artifact = ens_workflow(RecorderCollector(exp_name=self.exp_name, rec_key_func=self.rec_key), RollingGroup()) - print(artifact) - - latest_rec, _ = self.rolling_online_manager.list_latest_recorders() - self.rolling_online_manager.set_online_tag(RollingOnlineManager.ONLINE_TAG, list(latest_rec.values())) - - def simulate(self): - - print("========== simulate ==========") - onlinesimulator = OnlineSimulator( - start_time="2018-09-10", - end_time="2018-10-31", - onlinemanager=self.rolling_online_manager, - collector=RecorderCollector(exp_name=self.exp_name, rec_key_func=self.rec_key), - process_list=[RollingGroup()], - ) - results = onlinesimulator.simulate() - print(results) - recs_dict = onlinesimulator.online_models() - for time, recs in recs_dict.items(): - print(f"{str(time[0])} to {str(time[1])}:") - for rec in recs: - print(rec.info["id"]) - - -if __name__ == "__main__": - ose = OnlineSimulatorExample() - ose.first_run() - ose.simulate() diff --git a/examples/online_srv/task_manager_rolling_with_updating.py b/examples/online_srv/task_manager_rolling_with_updating.py index 9195a1de69..076c1a467c 100644 --- a/examples/online_srv/task_manager_rolling_with_updating.py +++ b/examples/online_srv/task_manager_rolling_with_updating.py @@ -123,7 +123,8 @@ def task_generating(self): return tasks def task_training(self, tasks): - self.trainer.train(tasks) + # self.trainer.train(tasks) + self.rolling_online_manager.prepare_new_models(tasks, tag=RollingOnlineManager.ONLINE_TAG) # This part corresponds to "Task Collecting" in the document def task_collecting(self): @@ -165,10 +166,8 @@ def first_run(self): self.task_training(tasks) self.task_collecting() - latest_rec, _ = self.rolling_online_manager.list_latest_recorders() - self.rolling_online_manager.reset_online_tag(list(latest_rec.values())) - - self.routine() + # latest_rec, _ = self.rolling_online_manager.list_latest_recorders() + # self.rolling_online_manager.reset_online_tag(list(latest_rec.values())) def routine(self): print("========== routine ==========") @@ -177,6 +176,10 @@ def routine(self): self.print_online_model() self.task_collecting() + def main(self): + self.first_run() + self.routine() + if __name__ == "__main__": ####### to train the first version's models, use the command below diff --git a/qlib/data/dataset/__init__.py b/qlib/data/dataset/__init__.py index ef30c634e6..cd15a98c93 100644 --- a/qlib/data/dataset/__init__.py +++ b/qlib/data/dataset/__init__.py @@ -488,7 +488,7 @@ def _prepare_seg(self, slc: slice, **kwargs) -> TSDataSampler: """ split the _prepare_raw_seg is to leave a hook for data preprocessing before creating processing data """ - dtype = kwargs.pop("dtype") + dtype = kwargs.pop("dtype", None) start, end = slc.start, slc.stop data = self._prepare_raw_seg(slc=slc, **kwargs) tsds = TSDataSampler(data=data, start=start, end=end, step_len=self.step_len, dtype=dtype) diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index 5165541558..2182497f5c 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -26,7 +26,6 @@ def task_train(task_config: dict, experiment_name: str) -> Recorder: ---------- Recorder : The instance of the recorder """ - # model initiaiton model: Model = init_instance_by_config(task_config["model"]) dataset: Dataset = init_instance_by_config(task_config["dataset"]) @@ -46,7 +45,7 @@ def task_train(task_config: dict, experiment_name: str) -> Recorder: # generate records: prediction, backtest, and analysis records = task_config.get("record", []) - recorder = R.get_recorder() + recorder: Recorder = R.get_recorder() if isinstance(records, dict): # prevent only one dict records = [records] for record in records: diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index a40512cf3f..fe6f0db6ff 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -4,6 +4,7 @@ from qlib.model.trainer import task_train from qlib.workflow.recorder import MLflowRecorder, Recorder from qlib.workflow.online.update import PredUpdater, RecordUpdater +from qlib.workflow.task.collect import Collector from qlib.workflow.task.utils import TimeAdjuster from qlib.workflow.task.gen import RollingGen, task_generator from qlib.workflow.task.manage import TaskManager @@ -14,78 +15,127 @@ from copy import deepcopy -class OnlineManager(Serializable): +class OnlineManager: ONLINE_KEY = "online_status" # the online status key in recorder ONLINE_TAG = "online" # the 'online' model + # NOTE: The meaning of this tag is that we can not assume the training models can be trained before we need its predition. Whenever finished training, it can be guaranteed that there are some online models. NEXT_ONLINE_TAG = "next_online" # the 'next online' model, which can be 'online' model when call reset_online_model OFFLINE_TAG = "offline" # the 'offline' model, not for online serving - def __init__(self, trainer: Trainer = None, need_log=True): - self._trainer = trainer + def __init__(self, trainer: Trainer = None, collector: Collector = None, need_log=True): + """ + init OnlineManager. + + Args: + trainer (Trainer, optional): a instance of Trainer. Defaults to None. + collector (Collector, optional): a instance of Collector. Defaults to None. + need_log (bool, optional): print log or not. Defaults to True. + """ + self.trainer = trainer self.logger = get_module_logger(self.__class__.__name__) self.need_log = need_log self.delay_signals = {} + self.collector = collector + self.cur_time = None def prepare_signals(self, *args, **kwargs): + """ + After perparing the data of last routine (a box in box-plot) which means the end of the routine, we can prepare trading signals for next routine. + """ raise NotImplementedError(f"Please implement the `prepare_signals` method.") def prepare_tasks(self, *args, **kwargs): - """return the new tasks waiting for training.""" + """ + After the end of a routine, check whether we need to prepare and train some new tasks. + return the new tasks waiting for training. + """ raise NotImplementedError(f"Please implement the `prepare_tasks` method.") - def prepare_new_models(self, tasks): - """Use trainer to train a list of tasks and set the trained model to next_online. + def prepare_new_models(self, tasks, tag=NEXT_ONLINE_TAG): + """ + Use trainer to train a list of tasks and set the trained model to `tag`. Args: tasks (list): a list of tasks. + tag (str): + `ONLINE_TAG` for first train or additional train + `NEXT_ONLINE_TAG` for reset online model when calling `reset_online_tag` + `OFFLINE_TAG` for train but offline those models """ if not (tasks is None or len(tasks) == 0): - if self._trainer is not None: - new_models = self._trainer.train(tasks) - self.set_online_tag(self.NEXT_ONLINE_TAG, new_models) - self.logger.info( - f"Finished prepare {len(new_models)} new models and set them to `{self.NEXT_ONLINE_TAG}`." - ) + if self.trainer is not None: + new_models = self.trainer.train(tasks) + self.set_online_tag(tag, new_models) + if self.need_log: + self.logger.info(f"Finished prepare {len(new_models)} new models and set them to {tag}.") else: self.logger.warn("No trainer to train new tasks.") def update_online_pred(self, *args, **kwargs): + """ + After the end of a routine, update the predictions of online models to latest. + """ raise NotImplementedError(f"Please implement the `update_online_pred` method.") def set_online_tag(self, tag, *args, **kwargs): - """set `tag` to the model to sign whether online + """ + Set `tag` to the model to sign whether online. Args: - tag (str): the tags in ONLINE_TAG, NEXT_ONLINE_TAG, OFFLINE_TAG + tag (str): the tags in `ONLINE_TAG`, `NEXT_ONLINE_TAG`, `OFFLINE_TAG` """ raise NotImplementedError(f"Please implement the `set_online_tag` method.") def get_online_tag(self, *args, **kwargs): - """given a model and return its online tag""" + """ + Given a model and return its online tag. + """ raise NotImplementedError(f"Please implement the `get_online_tag` method.") def reset_online_tag(self, *args, **kwargs): - """offline all models and set the recorders to 'online'. If no parameter and no 'next online' model, then do nothing.""" + """ + Offline all models and set the models to 'online'. + """ raise NotImplementedError(f"Please implement the `reset_online_tag` method.") def online_models(self): - """return online models""" + """ + Return online models. + """ raise NotImplementedError(f"Please implement the `online_models` method.") + def get_collector(self): + """ + Return the collector. + + Returns: + Collector + """ + return self.collector + def run_delay_signals(self): + """ + Prepare all signals if there are some dates waiting for prepare. + """ for cur_time, params in self.delay_signals.items(): self.cur_time = cur_time self.prepare_signals(*params[0], **params[1]) self.delay_signals = {} def routine(self, cur_time=None, delay_prepare=False, *args, **kwargs): - """The typical update process in a routine such as day by day or month by month""" + """ + The typical update process after a routine, such as day by day or month by month. + Prepare signals -> prepare tasks -> prepare new models -> update online prediction -> reset online models + """ self.cur_time = cur_time # None for latest date if not delay_prepare: self.prepare_signals(*args, **kwargs) else: - self.delay_signals[cur_time] = (args, kwargs) + if cur_time is not None: + self.delay_signals[cur_time] = (args, kwargs) + else: + raise ValueError("Can not delay prepare when cur_time is None") tasks = self.prepare_tasks(*args, **kwargs) self.prepare_new_models(tasks) self.update_online_pred() @@ -98,9 +148,18 @@ class OnlineManagerR(OnlineManager): """ - def __init__(self, experiment_name: str, trainer: Trainer = None, need_log=True): + def __init__(self, experiment_name: str, trainer: Trainer = None, collector: Collector = None, need_log=True): + """ + init OnlineManagerR. + + Args: + experiment_name (str): the experiment name. + trainer (Trainer, optional): a instance of Trainer. Defaults to None. + collector (Collector, optional): a instance of Collector. Defaults to None. + need_log (bool, optional): print log or not. Defaults to True. + """ trainer = TrainerR(experiment_name) - super().__init__(trainer, need_log) + super().__init__(trainer=trainer, collector=collector, need_log=need_log) self.exp_name = experiment_name def set_online_tag(self, tag, recorder: Union[Recorder, List]): @@ -148,7 +207,9 @@ def online_models(self): ) def update_online_pred(self): - """update all online model predictions to the latest day in Calendar""" + """ + Update all online model predictions to the latest day in Calendar + """ online_models = self.online_models() for rec in online_models: PredUpdater(rec, to_date=self.cur_time, need_log=self.need_log).update() @@ -160,18 +221,39 @@ def update_online_pred(self): class RollingOnlineManager(OnlineManagerR): """An implementation of OnlineManager based on Rolling.""" - def __init__(self, experiment_name: str, rolling_gen: RollingGen, trainer: Trainer = None, need_log=True): + def __init__( + self, + experiment_name: str, + rolling_gen: RollingGen, + trainer: Trainer = None, + collector: Collector = None, + need_log=True, + ): + """ + init RollingOnlineManager. + + Args: + experiment_name (str): the experiment name. + rolling_gen (RollingGen): a instance of RollingGen + trainer (Trainer, optional): a instance of Trainer. Defaults to None. + collector (Collector, optional): a instance of Collector. Defaults to None. + need_log (bool, optional): print log or not. Defaults to True. + """ trainer = TrainerR(experiment_name) - super().__init__(experiment_name, trainer, need_log=need_log) + super().__init__(experiment_name=experiment_name, trainer=trainer, collector=collector, need_log=need_log) self.ta = TimeAdjuster() self.rg = rolling_gen self.logger = get_module_logger(self.__class__.__name__) def prepare_signals(self, *args, **kwargs): + """ + Must use `pass` even though there is nothing to do. + """ pass def prepare_tasks(self, *args, **kwargs): - """prepare new tasks based on new date. + """ + Prepare new tasks based on new date. Returns: list: a list of new tasks. @@ -184,7 +266,11 @@ def prepare_tasks(self, *args, **kwargs): self.logger.warn(f"No latest online recorders, no new tasks.") return [] calendar_latest = self.ta.last_date() if self.cur_time is None else self.cur_time - if self.ta.cal_interval(calendar_latest, max_test[0]) > self.rg.step: + if self.need_log: + self.logger.info( + f"The interval between current time and last rolling test begin time is {self.ta.cal_interval(calendar_latest, max_test[0])}, the rolling step is {self.rg.step}" + ) + if self.ta.cal_interval(calendar_latest, max_test[0]) >= self.rg.step: old_tasks = [] tasks_tmp = [] for rid, rec in latest_records.items(): diff --git a/qlib/workflow/online/simulator.py b/qlib/workflow/online/simulator.py index 7f08549f01..16628c240c 100644 --- a/qlib/workflow/online/simulator.py +++ b/qlib/workflow/online/simulator.py @@ -1,12 +1,6 @@ -from typing import Callable -import pandas as pd -from qlib.config import C from qlib.data import D from qlib import get_module_logger -from qlib.log import set_log_with_config -from qlib.model.ens.ensemble import ens_workflow from qlib.workflow.online.manager import OnlineManager -from qlib.workflow.task.collect import Collector class OnlineSimulator: @@ -20,21 +14,24 @@ def __init__( end_time, onlinemanager: OnlineManager, frequency="day", - time_delta="20 hours", - collector: Collector = None, - process_list: list = None, ): + """ + init OnlineSimulator. + + Args: + start_time (str or pd.Timestamp): the start time of simulating. + end_time (str or pd.Timestamp): the end time of simulating. If None, then end_time is latest. + onlinemanager (OnlineManager): the instance of OnlineManager + frequency (str, optional): the data frequency. Defaults to "day". + """ self.logger = get_module_logger(self.__class__.__name__) self.cal = D.calendar(start_time=start_time, end_time=end_time, freq=frequency) self.start_time = self.cal[0] self.end_time = self.cal[-1] self.olm = onlinemanager - self.time_delta = time_delta if len(self.cal) == 0: self.logger.warn(f"There is no need to simulate bacause start_time is larger than end_time.") - self.collector = collector - self.process_list = process_list def simulate(self, *args, **kwargs): """ @@ -42,14 +39,13 @@ def simulate(self, *args, **kwargs): NOTE: Considering the parallel training, the signals will be perpared after all routine simulating. Returns: - dict: the simulated results collected by collector + Collector: the OnlineManager's collector """ self.rec_dict = {} tmp_begin = self.start_time tmp_end = None prev_recorders = self.olm.online_models() for cur_time in self.cal: - cur_time = cur_time + pd.Timedelta(self.time_delta) self.logger.info(f"Simulating at {str(cur_time)}......") recorders = self.olm.routine(cur_time, True, *args, **kwargs) if len(recorders) == 0: @@ -64,8 +60,7 @@ def simulate(self, *args, **kwargs): self.olm.run_delay_signals() self.logger.info(f"Finished preparing signals") - if self.collector is not None: - return ens_workflow(self.collector, self.process_list) + return self.olm.get_collector() def online_models(self): """ diff --git a/qlib/workflow/online/update.py b/qlib/workflow/online/update.py index 8aa32ff292..a6f0aeefee 100644 --- a/qlib/workflow/online/update.py +++ b/qlib/workflow/online/update.py @@ -121,6 +121,7 @@ def update(self, dataset: DatasetH = None): # FIXME: the problme below is not solved # The model dumped on GPU instances can not be loaded on CPU instance. Follow exception will raised # RuntimeError: Attempting to deserialize object on a CUDA device but torch.cuda.is_available() is False. If you are running on a CPU-only machine, please use torch.load with map_location=torch.device('cpu') to map your storages to the CPU. + # https://github.com/pytorch/pytorch/issues/16797 start_time = get_date_by_shift(self.last_end, 1, freq=self.freq) if start_time >= self.to_date: @@ -136,7 +137,7 @@ def update(self, dataset: DatasetH = None): # Load model model = self.rmdl.get_model() - new_pred = model.predict(dataset) + new_pred: pd.Series = model.predict(dataset) cb_pred = pd.concat([self.old_pred, new_pred.to_frame("score")], axis=0) cb_pred = cb_pred.sort_index() diff --git a/qlib/workflow/task/gen.py b/qlib/workflow/task/gen.py index 3250891268..542466a5f6 100644 --- a/qlib/workflow/task/gen.py +++ b/qlib/workflow/task/gen.py @@ -168,7 +168,7 @@ def generate(self, task: dict): if prev_seg is None: # First rolling # 1) prepare the end point - segments = copy.deepcopy(self.ta.align_seg(t["dataset"]["kwargs"]["segments"])) + segments: dict = copy.deepcopy(self.ta.align_seg(t["dataset"]["kwargs"]["segments"])) test_end = self.ta.last_date() if segments[self.test_key][1] is None else segments[self.test_key][1] # 2) and init test segments test_start_idx = self.ta.align_idx(segments[self.test_key][0]) diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 815529b66d..b144a8872d 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -12,6 +12,7 @@ from pymongo.errors import InvalidDocument from bson.objectid import ObjectId from contextlib import contextmanager +import qlib from tqdm.cli import tqdm import time import concurrent @@ -65,6 +66,12 @@ def __init__(self, task_pool: str = None): self.logger = get_module_logger(self.__class__.__name__) def list(self): + """ + list the all collection(task_pool) of the db + + Returns: + list + """ return self.mdb.list_collection_names() def _encode_task(self, task): @@ -257,9 +264,6 @@ def remove(self, query={}): query: dict the dict of query - Returns - ------- - """ query = query.copy() if "_id" in query: diff --git a/qlib/workflow/task/utils.py b/qlib/workflow/task/utils.py index 87a3a41f3f..03ba4ed681 100644 --- a/qlib/workflow/task/utils.py +++ b/qlib/workflow/task/utils.py @@ -15,10 +15,21 @@ def get_mongodb(): get database in MongoDB, which means you need to declare the address and the name of database. for example: - C["mongo"] = { - "task_url" : "mongodb://localhost:27017/", - "task_db_name" : "rolling_db" - } + + Using qlib.init(): + + mongo_conf = { + "task_url": task_url, # your MongoDB url + "task_db_name": task_db_name, # database name + } + qlib.init(..., mongo=mongo_conf) + + After qlib.init(): + + C["mongo"] = { + "task_url" : "mongodb://localhost:27017/", + "task_db_name" : "rolling_db" + } """ try: @@ -113,6 +124,16 @@ def align_idx(self, time_point, tp_type="start"): return idx def cal_interval(self, time_point_A, time_point_B): + """ + calculate the trading day interval + + Args: + time_point_A : time_point_A + time_point_B : time_point_B (is the past of time_point_A) + + Returns: + int: the interval between A and B + """ return self.align_idx(time_point_A) - self.align_idx(time_point_B) def align_time(self, time_point, tp_type="start"): From de0a0c083d6e37589e40caede8aca6cfd9178c8e Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Thu, 22 Apr 2021 08:09:15 +0000 Subject: [PATCH 42/61] bug fixed --- qlib/workflow/online/manager.py | 42 +++++++++++++++++++-------------- qlib/workflow/online/update.py | 7 ++---- qlib/workflow/task/gen.py | 2 +- qlib/workflow/task/utils.py | 7 ------ 4 files changed, 27 insertions(+), 31 deletions(-) diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index fe6f0db6ff..f7f9b62d54 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -1,18 +1,13 @@ -from typing import Dict, Union, List +from copy import deepcopy +from typing import Dict, List, Union from qlib import get_module_logger -from qlib.workflow import R -from qlib.model.trainer import task_train -from qlib.workflow.recorder import MLflowRecorder, Recorder -from qlib.workflow.online.update import PredUpdater, RecordUpdater +from qlib.data.data import D +from qlib.model.trainer import Trainer, TrainerR, task_train +from qlib.workflow.online.update import PredUpdater +from qlib.workflow.recorder import Recorder from qlib.workflow.task.collect import Collector -from qlib.workflow.task.utils import TimeAdjuster from qlib.workflow.task.gen import RollingGen, task_generator -from qlib.workflow.task.manage import TaskManager -from qlib.workflow.task.manage import run_task -from qlib.workflow.task.utils import list_recorders -from qlib.utils.serial import Serializable -from qlib.model.trainer import Trainer, TrainerR -from copy import deepcopy +from qlib.workflow.task.utils import TimeAdjuster, list_recorders class OnlineManager: @@ -63,6 +58,7 @@ def prepare_new_models(self, tasks, tag=NEXT_ONLINE_TAG): `NEXT_ONLINE_TAG` for reset online model when calling `reset_online_tag` `OFFLINE_TAG` for train but offline those models """ + # TODO: 回调 if not (tasks is None or len(tasks) == 0): if self.trainer is not None: new_models = self.trainer.train(tasks) @@ -158,7 +154,8 @@ def __init__(self, experiment_name: str, trainer: Trainer = None, collector: Col collector (Collector, optional): a instance of Collector. Defaults to None. need_log (bool, optional): print log or not. Defaults to True. """ - trainer = TrainerR(experiment_name) + if trainer is None: + trainer = TrainerR(experiment_name) super().__init__(trainer=trainer, collector=collector, need_log=need_log) self.exp_name = experiment_name @@ -239,7 +236,8 @@ def __init__( collector (Collector, optional): a instance of Collector. Defaults to None. need_log (bool, optional): print log or not. Defaults to True. """ - trainer = TrainerR(experiment_name) + if trainer is None: + trainer = TrainerR(experiment_name) super().__init__(experiment_name=experiment_name, trainer=trainer, collector=collector, need_log=need_log) self.ta = TimeAdjuster() self.rg = rolling_gen @@ -247,9 +245,17 @@ def __init__( def prepare_signals(self, *args, **kwargs): """ + Average the online models prediction and save them into a recorder + + Must use `pass` even though there is nothing to do. """ - pass + # 检查recorder是否存在，如果不存在就创建一个 + # 检查recorder的上一个信号时间，如果没有那就从上线模型的共同最早时间开始出信号 + # 从recorder的上一个信号时间开始出信号，出到self.cur_time + for model in self.online_models(): + + pass def prepare_tasks(self, *args, **kwargs): """ @@ -258,17 +264,17 @@ def prepare_tasks(self, *args, **kwargs): Returns: list: a list of new tasks. """ - self.ta.set_end_time(self.cur_time) + #TODO: max_test = self.cur_time latest_records, max_test = self.list_latest_recorders( lambda rec: self.get_online_tag(rec) == OnlineManager.ONLINE_TAG ) if max_test is None: self.logger.warn(f"No latest online recorders, no new tasks.") return [] - calendar_latest = self.ta.last_date() if self.cur_time is None else self.cur_time + calendar_latest = D.calendar(end_time=self.cur_time)[-1] if self.cur_time is None else self.cur_time if self.need_log: self.logger.info( - f"The interval between current time and last rolling test begin time is {self.ta.cal_interval(calendar_latest, max_test[0])}, the rolling step is {self.rg.step}" + f"The interval between current time {calendar_latest} and last rolling test begin time {max_test[0]} is {self.ta.cal_interval(calendar_latest, max_test[0])}, the rolling step is {self.rg.step}" ) if self.ta.cal_interval(calendar_latest, max_test[0]) >= self.rg.step: old_tasks = [] diff --git a/qlib/workflow/online/update.py b/qlib/workflow/online/update.py index a6f0aeefee..5b58360d83 100644 --- a/qlib/workflow/online/update.py +++ b/qlib/workflow/online/update.py @@ -73,9 +73,7 @@ class PredUpdater(RecordUpdater): Update the prediction in the Recorder """ - LATEST = "__latest" - - def __init__(self, record: Recorder, to_date=LATEST, hist_ref: int = 0, freq="day", need_log=True): + def __init__(self, record: Recorder, to_date=None, hist_ref: int = 0, freq="day", need_log=True): """ Parameters ---------- @@ -95,8 +93,7 @@ def __init__(self, record: Recorder, to_date=LATEST, hist_ref: int = 0, freq="da self.freq = freq self.rmdl = RMDLoader(rec=record) - # FIXME: why we need LATEST? can we use to_date=None instead? - if to_date == self.LATEST or to_date == None: + if to_date == None: to_date = D.calendar(freq=freq)[-1] self.to_date = pd.Timestamp(to_date) self.old_pred = record.load_object("pred.pkl") diff --git a/qlib/workflow/task/gen.py b/qlib/workflow/task/gen.py index 542466a5f6..ad7a162181 100644 --- a/qlib/workflow/task/gen.py +++ b/qlib/workflow/task/gen.py @@ -169,7 +169,7 @@ def generate(self, task: dict): # First rolling # 1) prepare the end point segments: dict = copy.deepcopy(self.ta.align_seg(t["dataset"]["kwargs"]["segments"])) - test_end = self.ta.last_date() if segments[self.test_key][1] is None else segments[self.test_key][1] + test_end = self.ta.max() if segments[self.test_key][1] is None else segments[self.test_key][1] # 2) and init test segments test_start_idx = self.ta.align_idx(segments[self.test_key][0]) segments[self.test_key] = (self.ta.get(test_start_idx), self.ta.get(test_start_idx + self.step - 1)) diff --git a/qlib/workflow/task/utils.py b/qlib/workflow/task/utils.py index 03ba4ed681..ce8e0dfa35 100644 --- a/qlib/workflow/task/utils.py +++ b/qlib/workflow/task/utils.py @@ -90,17 +90,10 @@ def get(self, idx: int): def max(self): """ - (Deprecated) Return the max calendar datetime """ return max(self.cals) - def last_date(self) -> pd.Timestamp: - """ - Return the last datetime in the calendar - """ - return self.cals[-1] - def align_idx(self, time_point, tp_type="start"): """ align the index of time_point in the calendar From 319396c815c4ac1e5a07d8d2a64623ff0a14a1ba Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Sun, 25 Apr 2021 06:26:45 +0000 Subject: [PATCH 43/61] online serving V8 --- .../model_rolling/task_manager_rolling.py | 153 +++++++++-------- .../online_srv/online_management_simulate.py | 2 +- ...dating.py => rolling_online_management.py} | 103 +++--------- examples/online_srv/update_online_pred.py | 47 +++--- qlib/model/trainer.py | 9 + qlib/workflow/online/manager.py | 158 ++++++++++++++---- qlib/workflow/task/collect.py | 2 +- qlib/workflow/task/gen.py | 9 + 8 files changed, 278 insertions(+), 205 deletions(-) rename examples/online_srv/{task_manager_rolling_with_updating.py => rolling_online_management.py} (52%) diff --git a/examples/model_rolling/task_manager_rolling.py b/examples/model_rolling/task_manager_rolling.py index 4508a87886..9c1cbf8919 100644 --- a/examples/model_rolling/task_manager_rolling.py +++ b/examples/model_rolling/task_manager_rolling.py @@ -15,6 +15,11 @@ from qlib.model.ens.group import RollingGroup from qlib.model.trainer import TrainerRM +""" +This example shows how a Trainer work based on TaskManager with rolling tasks. +After training, how to collect the rolling results will be showed in task_collecting. +""" + data_handler_config = { "start_time": "2008-01-01", "end_time": "2020-08-01", @@ -71,81 +76,83 @@ "record": record_config, } -# Reset all things to the first status, be careful to save important data -def reset(task_pool, exp_name): - print("========== reset ==========") - TaskManager(task_pool=task_pool).remove() - - exp = R.get_exp(experiment_name=exp_name) - - for rid in exp.list_recorders(): - exp.delete_recorder(rid) - - -# This part corresponds to "Task Generating" in the document -def task_generating(): - - print("========== task_generating ==========") - - tasks = task_generator( - tasks=[task_xgboost_config, task_lgb_config], - generators=RollingGen(step=550, rtype=RollingGen.ROLL_SD), # generate different date segment - ) - - pprint(tasks) - - return tasks - - -def task_training(tasks, task_pool, exp_name): - trainer = TrainerRM(exp_name, task_pool) - trainer.train(tasks) - - -# This part corresponds to "Task Collecting" in the document -def task_collecting(exp_name): - print("========== task_collecting ==========") - - def rec_key(recorder): - task_config = recorder.load_object("task") - model_key = task_config["model"]["class"] - rolling_key = task_config["dataset"]["kwargs"]["segments"]["test"] - return model_key, rolling_key - - def my_filter(recorder): - # only choose the results of "LGBModel" - model_key, rolling_key = rec_key(recorder) - if model_key == "LGBModel": - return True - return False - - artifact = ens_workflow( - RecorderCollector(exp_name=exp_name, rec_key_func=rec_key, rec_filter_func=my_filter), - RollingGroup(), - ) - print(artifact) - - -def main( - provider_uri="~/.qlib/qlib_data/cn_data", - task_url="mongodb://10.0.0.4:27017/", - task_db_name="rolling_db", - experiment_name="rolling_exp", - task_pool="rolling_task", -): - mongo_conf = { - "task_url": task_url, - "task_db_name": task_db_name, - } - qlib.init(provider_uri=provider_uri, region=REG_CN, mongo=mongo_conf) - reset(task_pool, experiment_name) - tasks = task_generating() - task_training(tasks, task_pool, experiment_name) - task_collecting(experiment_name) +class RollingTaskExample: + def __init__( + self, + provider_uri="~/.qlib/qlib_data/cn_data", + region=REG_CN, + task_url="mongodb://10.0.0.4:27017/", + task_db_name="rolling_db", + experiment_name="rolling_exp", + task_pool="rolling_task", + task_config=[task_xgboost_config, task_lgb_config], + rolling_step=550, + rolling_type=RollingGen.ROLL_SD, + ): + # TaskManager config + mongo_conf = { + "task_url": task_url, + "task_db_name": task_db_name, + } + qlib.init(provider_uri=provider_uri, region=region, mongo=mongo_conf) + self.experiment_name = experiment_name + self.task_pool = task_pool + self.task_config = task_config + self.rolling_gen = RollingGen(step=rolling_step, rtype=rolling_type) + + # Reset all things to the first status, be careful to save important data + def reset(self): + print("========== reset ==========") + TaskManager(task_pool=self.task_pool).remove() + exp = R.get_exp(experiment_name=self.experiment_name) + for rid in exp.list_recorders(): + exp.delete_recorder(rid) + + def task_generating(self): + print("========== task_generating ==========") + tasks = task_generator( + tasks=self.task_config, + generators=self.rolling_gen, # generate different date segments + ) + pprint(tasks) + return tasks + + def task_training(self, tasks): + print("========== task_training ==========") + trainer = TrainerRM(self.experiment_name, self.task_pool) + trainer.train(tasks) + + def task_collecting(self): + print("========== task_collecting ==========") + + def rec_key(recorder): + task_config = recorder.load_object("task") + model_key = task_config["model"]["class"] + rolling_key = task_config["dataset"]["kwargs"]["segments"]["test"] + return model_key, rolling_key + + def my_filter(recorder): + # only choose the results of "LGBModel" + model_key, rolling_key = rec_key(recorder) + if model_key == "LGBModel": + return True + return False + + artifact = ens_workflow( + RecorderCollector(exp_name=self.experiment_name, rec_key_func=rec_key, rec_filter_func=my_filter), + RollingGroup(), + ) + print(artifact) + + def main(self): + self.reset() + tasks = self.task_generating() + self.task_training(tasks) + self.task_collecting() if __name__ == "__main__": ## to see the whole process with your own parameters, use the command below - # python update_online_pred.py main --experiment_name="your_exp_name" - fire.Fire() + # python task_manager_rolling.py main --experiment_name="your_exp_name" + fire.Fire(RollingTaskExample) diff --git a/examples/online_srv/online_management_simulate.py b/examples/online_srv/online_management_simulate.py index d3a1328799..9b5fbcc037 100644 --- a/examples/online_srv/online_management_simulate.py +++ b/examples/online_srv/online_management_simulate.py @@ -11,7 +11,7 @@ from qlib.workflow.task.manage import TaskManager """ -This examples is about the OnlineManager and OnlineSimulator based on Rolling tasks. +This examples is about the OnlineManager and OnlineSimulator based on rolling tasks. The OnlineManager will focus on the updating of your online models. The OnlineSimulator will focus on the simulating real updating routine of your online models. """ diff --git a/examples/online_srv/task_manager_rolling_with_updating.py b/examples/online_srv/rolling_online_management.py similarity index 52% rename from examples/online_srv/task_manager_rolling_with_updating.py rename to examples/online_srv/rolling_online_management.py index 076c1a467c..6c30f3af3a 100644 --- a/examples/online_srv/task_manager_rolling_with_updating.py +++ b/examples/online_srv/rolling_online_management.py @@ -1,18 +1,21 @@ -from pprint import pprint - +import os +from pathlib import Path +import pickle import fire import qlib -from qlib.config import REG_CN -from qlib.model.trainer import task_train from qlib.workflow import R -from qlib.workflow.task.collect import RecorderCollector -from qlib.model.ens.ensemble import RollingEnsemble, ens_workflow -from qlib.workflow.task.gen import RollingGen, task_generator -from qlib.workflow.task.manage import TaskManager, run_task +from qlib.workflow.task.gen import RollingGen +from qlib.workflow.task.manage import TaskManager from qlib.workflow.online.manager import RollingOnlineManager from qlib.workflow.task.utils import list_recorders from qlib.model.trainer import TrainerRM -from qlib.model.ens.group import RollingGroup + +""" +This example show how RollingOnlineManager works with rolling tasks. +There are two parts including first train and routine. +Firstly, the RollingOnlineManager will finish the first training and set trained models to `online` models. +Next, the RollingOnlineManager will finish a routine process, including update online prediction -> prepare signals -> prepare tasks -> prepare new models -> reset online models +""" data_handler_config = { "start_time": "2013-01-01", @@ -89,92 +92,38 @@ def __init__( "task_db_name": task_db_name, # database name } qlib.init(provider_uri=provider_uri, region=region, mongo=mongo_conf) - - self.rolling_gen = RollingGen(step=rolling_step, rtype=RollingGen.ROLL_SD) - self.trainer = TrainerRM(self.exp_name, self.task_pool) - self.task_manager = TaskManager(self.task_pool) self.rolling_online_manager = RollingOnlineManager( - experiment_name=exp_name, rolling_gen=self.rolling_gen, trainer=self.trainer - ) - - def print_online_model(self): - print("========== print_online_model ==========") - print("Current 'online' model:") - - for rec in self.rolling_online_manager.online_models(): - print(rec.info["id"]) - print("Current 'next online' model:") - for rid, rec in list_recorders(self.exp_name).items(): - if self.rolling_online_manager.get_online_tag(rec) == self.rolling_online_manager.NEXT_ONLINE_TAG: - print(rid) - - # This part corresponds to "Task Generating" in the document - def task_generating(self): - - print("========== task_generating ==========") - - tasks = task_generator( - tasks=[task_xgboost_config, task_lgb_config], - generators=self.rolling_gen, # generate different date segment + experiment_name=exp_name, + rolling_gen=RollingGen(step=rolling_step, rtype=RollingGen.ROLL_SD), + trainer=TrainerRM(self.exp_name, self.task_pool), ) - pprint(tasks) - - return tasks - - def task_training(self, tasks): - # self.trainer.train(tasks) - self.rolling_online_manager.prepare_new_models(tasks, tag=RollingOnlineManager.ONLINE_TAG) - - # This part corresponds to "Task Collecting" in the document - def task_collecting(self): - print("========== task_collecting ==========") - - def rec_key(recorder): - task_config = recorder.load_object("task") - model_key = task_config["model"]["class"] - rolling_key = task_config["dataset"]["kwargs"]["segments"]["test"] - return model_key, rolling_key - - def my_filter(recorder): - # only choose the results of "LGBModel" - model_key, rolling_key = rec_key(recorder) - if model_key == "LGBModel": - return True - return False - - artifact = ens_workflow( - RecorderCollector(exp_name=self.exp_name, rec_key_func=rec_key, rec_filter_func=my_filter), RollingGroup() - ) - print(artifact) + _ROLLING_MANAGER_PATH = ".rolling_manager" # the RollingOnlineManager will dump to this file, for it will be loaded when calling routine. # Reset all things to the first status, be careful to save important data def reset(self): print("========== reset ==========") - self.task_manager.remove() + TaskManager(self.task_pool).remove() exp = R.get_exp(experiment_name=self.exp_name) for rid in exp.list_recorders(): exp.delete_recorder(rid) - # Run this firstly to see the workflow in Task Management + if os.path.exists(self._ROLLING_MANAGER_PATH): + os.remove(self._ROLLING_MANAGER_PATH) + def first_run(self): print("========== first_run ==========") self.reset() - - tasks = self.task_generating() - pprint(tasks) - self.task_training(tasks) - self.task_collecting() - - # latest_rec, _ = self.rolling_online_manager.list_latest_recorders() - # self.rolling_online_manager.reset_online_tag(list(latest_rec.values())) + self.rolling_online_manager.first_train([task_xgboost_config, task_lgb_config]) + self.rolling_online_manager.to_pickle(self._ROLLING_MANAGER_PATH) + print(self.rolling_online_manager.collect_artifact()) def routine(self): print("========== routine ==========") - self.print_online_model() + with Path(self._ROLLING_MANAGER_PATH).open("rb") as f: + self.rolling_online_manager = pickle.load(f) self.rolling_online_manager.routine() - self.print_online_model() - self.task_collecting() + print(self.rolling_online_manager.collect_artifact()) def main(self): self.first_run() diff --git a/examples/online_srv/update_online_pred.py b/examples/online_srv/update_online_pred.py index 0f075abcd0..ed2ad6997e 100644 --- a/examples/online_srv/update_online_pred.py +++ b/examples/online_srv/update_online_pred.py @@ -5,6 +5,13 @@ from qlib.workflow.online.manager import OnlineManagerR from qlib.workflow.task.utils import list_recorders +""" +This example show how OnlineManager works when we need update prediction. +There are two parts including first_train and update_online_pred. +Firstly, the RollingOnlineManager will finish the first training and set the trained model to `online` model. +Next, the RollingOnlineManager will finish updating online prediction +""" + data_handler_config = { "start_time": "2008-01-01", "end_time": "2020-08-01", @@ -52,31 +59,25 @@ } -def first_train(experiment_name="online_srv"): - - rec = task_train(task_config=task, experiment_name=experiment_name) - - online_manager = OnlineManagerR(experiment_name) - online_manager.reset_online_tag(rec) - - -def update_online_pred(experiment_name="online_srv"): - - online_manager = OnlineManagerR(experiment_name) - - print("Here are the online models waiting for update:") - for rid, rec in list_recorders(experiment_name).items(): - if online_manager.get_online_tag(rec) == OnlineManagerR.ONLINE_TAG: - print(rid) +class UpdatePredExample: + def __init__( + self, provider_uri="~/.qlib/qlib_data/cn_data", region=REG_CN, experiment_name="online_srv", task_config=task + ): + qlib.init(provider_uri=provider_uri, region=region) + self.experiment_name = experiment_name + self.online_manager = OnlineManagerR(self.experiment_name) + self.task_config = task_config - online_manager.update_online_pred() + def first_train(self): + rec = task_train(self.task_config, experiment_name=self.experiment_name) + self.online_manager.reset_online_tag(rec) # set to online model + def update_online_pred(self): + self.online_manager.update_online_pred() -def main(provider_uri="~/.qlib/qlib_data/cn_data", region=REG_CN, experiment_name="online_srv"): - provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir - qlib.init(provider_uri=provider_uri, region=region) - first_train(experiment_name) - update_online_pred(experiment_name) + def main(self): + self.first_train() + self.update_online_pred() if __name__ == "__main__": @@ -86,4 +87,4 @@ def main(provider_uri="~/.qlib/qlib_data/cn_data", region=REG_CN, experiment_nam # python update_online_pred.py update_online_pred ## to see the whole process with your own parameters, use the command below # python update_online_pred.py main --experiment_name="your_exp_name" - fire.Fire() + fire.Fire(UpdatePredExample) diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index 2182497f5c..348f6b5218 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -135,3 +135,12 @@ def train(self, tasks: list, train_func=None, *args, **kwargs): for _id in _id_list: recs.append(tm.re_query(_id)["res"]) return recs + + +class DelayTrainer(Trainer): + def fake_train(self): + self.fake_trained = [] + + def train(self): + for rec in self.fake_trained: + pass diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index f7f9b62d54..e744880402 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -1,16 +1,29 @@ from copy import deepcopy +from operator import index +import pandas as pd +from qlib.model.ens.ensemble import ens_workflow +from qlib.model.ens.group import RollingGroup +from qlib.utils.serial import Serializable from typing import Dict, List, Union from qlib import get_module_logger from qlib.data.data import D from qlib.model.trainer import Trainer, TrainerR, task_train +from qlib.workflow import R from qlib.workflow.online.update import PredUpdater from qlib.workflow.recorder import Recorder -from qlib.workflow.task.collect import Collector +from qlib.workflow.task.collect import Collector, RecorderCollector from qlib.workflow.task.gen import RollingGen, task_generator from qlib.workflow.task.utils import TimeAdjuster, list_recorders +""" +This class is a component of online serving, it can manage a series of models dynamically. +With the change of time, the decisive models will be also changed. In this module, we called those contributing models as `online` models. +In every routine(such as everyday or every minutes), the `online` models maybe changed and the prediction of them need to be updated. +So this module provide a series methods to control this process. +""" -class OnlineManager: + +class OnlineManager(Serializable): ONLINE_KEY = "online_status" # the online status key in recorder ONLINE_TAG = "online" # the 'online' model @@ -18,26 +31,28 @@ class OnlineManager: NEXT_ONLINE_TAG = "next_online" # the 'next online' model, which can be 'online' model when call reset_online_model OFFLINE_TAG = "offline" # the 'offline' model, not for online serving - def __init__(self, trainer: Trainer = None, collector: Collector = None, need_log=True): + SIGNAL_EXP = "OnlineManagerSignals" # a specific experiment to save signals of different experiment. + + def __init__(self, trainer: Trainer = None, need_log=True): """ init OnlineManager. Args: trainer (Trainer, optional): a instance of Trainer. Defaults to None. - collector (Collector, optional): a instance of Collector. Defaults to None. need_log (bool, optional): print log or not. Defaults to True. """ self.trainer = trainer self.logger = get_module_logger(self.__class__.__name__) self.need_log = need_log self.delay_signals = {} - self.collector = collector self.cur_time = None def prepare_signals(self, *args, **kwargs): """ After perparing the data of last routine (a box in box-plot) which means the end of the routine, we can prepare trading signals for next routine. + Must use `pass` even though there is nothing to do. """ + raise NotImplementedError(f"Please implement the `prepare_signals` method.") def prepare_tasks(self, *args, **kwargs): @@ -47,7 +62,7 @@ def prepare_tasks(self, *args, **kwargs): """ raise NotImplementedError(f"Please implement the `prepare_tasks` method.") - def prepare_new_models(self, tasks, tag=NEXT_ONLINE_TAG): + def prepare_new_models(self, tasks, tag=NEXT_ONLINE_TAG, check_func=None): """ Use trainer to train a list of tasks and set the trained model to `tag`. @@ -57,14 +72,20 @@ def prepare_new_models(self, tasks, tag=NEXT_ONLINE_TAG): `ONLINE_TAG` for first train or additional train `NEXT_ONLINE_TAG` for reset online model when calling `reset_online_tag` `OFFLINE_TAG` for train but offline those models + check_func: the method to judge if a model can be online. + The parameter is the model record and return True for online. + None for online every models. + """ - # TODO: 回调 - if not (tasks is None or len(tasks) == 0): + if check_func is None: + check_func = lambda x: True + if len(tasks) > 0: if self.trainer is not None: new_models = self.trainer.train(tasks) - self.set_online_tag(tag, new_models) - if self.need_log: - self.logger.info(f"Finished prepare {len(new_models)} new models and set them to {tag}.") + if check_func(new_models): + self.set_online_tag(tag, new_models) + if self.need_log: + self.logger.info(f"Finished preparing {len(new_models)} new models and set them to {tag}.") else: self.logger.warn("No trainer to train new tasks.") @@ -101,6 +122,12 @@ def online_models(self): """ raise NotImplementedError(f"Please implement the `online_models` method.") + def first_train(self): + """ + Train a series of models firstly and set some of them into online models. + """ + raise NotImplementedError(f"Please implement the `first_train` method.") + def get_collector(self): """ Return the collector. @@ -108,7 +135,7 @@ def get_collector(self): Returns: Collector """ - return self.collector + raise NotImplementedError(f"Please implement the `get_collector` method.") def run_delay_signals(self): """ @@ -122,9 +149,10 @@ def run_delay_signals(self): def routine(self, cur_time=None, delay_prepare=False, *args, **kwargs): """ The typical update process after a routine, such as day by day or month by month. - Prepare signals -> prepare tasks -> prepare new models -> update online prediction -> reset online models + update online prediction -> prepare signals -> prepare tasks -> prepare new models -> reset online models """ self.cur_time = cur_time # None for latest date + self.update_online_pred() if not delay_prepare: self.prepare_signals(*args, **kwargs) else: @@ -134,7 +162,7 @@ def routine(self, cur_time=None, delay_prepare=False, *args, **kwargs): raise ValueError("Can not delay prepare when cur_time is None") tasks = self.prepare_tasks(*args, **kwargs) self.prepare_new_models(tasks) - self.update_online_pred() + return self.reset_online_tag() @@ -144,19 +172,18 @@ class OnlineManagerR(OnlineManager): """ - def __init__(self, experiment_name: str, trainer: Trainer = None, collector: Collector = None, need_log=True): + def __init__(self, experiment_name: str, trainer: Trainer = None, need_log=True): """ init OnlineManagerR. Args: experiment_name (str): the experiment name. trainer (Trainer, optional): a instance of Trainer. Defaults to None. - collector (Collector, optional): a instance of Collector. Defaults to None. need_log (bool, optional): print log or not. Defaults to True. """ if trainer is None: trainer = TrainerR(experiment_name) - super().__init__(trainer=trainer, collector=collector, need_log=need_log) + super().__init__(trainer=trainer, need_log=need_log) self.exp_name = experiment_name def set_online_tag(self, tag, recorder: Union[Recorder, List]): @@ -212,7 +239,40 @@ def update_online_pred(self): PredUpdater(rec, to_date=self.cur_time, need_log=self.need_log).update() if self.need_log: - self.logger.info(f"Finish updating {len(online_models)} online model predictions of {self.exp_name}.") + self.logger.info(f"Finished updating {len(online_models)} online model predictions of {self.exp_name}.") + + def prepare_signals(self, over_write=False): + """ + Average the predictions of online models and offer a trading signals every routine. + The signals will be saved to `signal` file of a recorder named self.exp_name of a experiment using the name of `SIGNAL_EXP` + + Args: + over_write (bool, optional): If True, the new signals will overwrite the file. If False, the new signals will append to the end of signals. Defaults to False. + """ + + with R.start(experiment_name=self.SIGNAL_EXP, recorder_name=self.exp_name, resume=True): + recorder = R.get_recorder() + pred = [] + + try: + old_signals = recorder.load_object("signals") + except OSError: + old_signals = None + + for rec in self.online_models(): + pred.append(rec.load_object("pred.pkl")) + + signals = pd.concat(pred, axis=1).mean(axis=1).to_frame("score") + signals = signals.sort_index() + if old_signals is not None and not over_write: + # signals = old_signals.reindex(signals.index).combine_first(signals) + old_max = old_signals.index.get_level_values("datetime").max() + new_signals = signals.loc[old_max:] + signals = pd.concat([old_signals, new_signals], axis=0) + else: + new_signals = signals + self.logger.info(f"Finished preparing new {len(new_signals)} signals to {self.SIGNAL_EXP}/{self.exp_name}.") + recorder.save_objects(**{"signals": signals}) class RollingOnlineManager(OnlineManagerR): @@ -223,7 +283,6 @@ def __init__( experiment_name: str, rolling_gen: RollingGen, trainer: Trainer = None, - collector: Collector = None, need_log=True, ): """ @@ -238,24 +297,64 @@ def __init__( """ if trainer is None: trainer = TrainerR(experiment_name) - super().__init__(experiment_name=experiment_name, trainer=trainer, collector=collector, need_log=need_log) + super().__init__(experiment_name=experiment_name, trainer=trainer, need_log=need_log) self.ta = TimeAdjuster() self.rg = rolling_gen self.logger = get_module_logger(self.__class__.__name__) - def prepare_signals(self, *args, **kwargs): + def get_collector(self, rec_key_func=None, rec_filter_func=None): """ - Average the online models prediction and save them into a recorder - + get the instance of collector to collect results - Must use `pass` even though there is nothing to do. + Args: + rec_key_func (Callable): a function to get the key of a recorder. If None, use recorder id. + rec_filter_func (Callable, optional): filter the recorder by return True or False. Defaults to None. + """ + + def rec_key(recorder): + task_config = recorder.load_object("task") + model_key = task_config["model"]["class"] + rolling_key = task_config["dataset"]["kwargs"]["segments"]["test"] + return model_key, rolling_key + + if rec_key_func is None: + rec_key_func = rec_key + + return RecorderCollector(exp_name=self.exp_name, rec_key_func=rec_key_func, rec_filter_func=rec_filter_func) + + def collect_artifact(self, rec_key_func=None, rec_filter_func=None): + """ + collecting artifact based on the collector and RollingGroup. + + Args: + rec_key_func (Callable): a function to get the key of a recorder. If None, use recorder id. + rec_filter_func (Callable, optional): filter the recorder by return True or False. Defaults to None. + + Returns: + dict: the artifact dict after rolling ensemble + """ + artifact = ens_workflow( + self.get_collector(rec_key_func=rec_key_func, rec_filter_func=rec_filter_func), RollingGroup() + ) + return artifact + + def first_train(self, task_configs: list): """ - # 检查recorder是否存在，如果不存在就创建一个 - # 检查recorder的上一个信号时间，如果没有那就从上线模型的共同最早时间开始出信号 - # 从recorder的上一个信号时间开始出信号，出到self.cur_time - for model in self.online_models(): + Use rolling_gen to generate different tasks based on task_configs and trained them. - pass + Args: + task_configs (list or dict): a list of task configs or a task config + + Returns: + Collector: a instance of a Collector. + """ + tasks = task_generator( + tasks=task_configs, + generators=self.rg, # generate different date segment + ) + self.prepare_new_models(tasks, tag=self.ONLINE_TAG) + self.prepare_signals(over_write=True) + return self.get_collector() def prepare_tasks(self, *args, **kwargs): """ @@ -264,7 +363,6 @@ def prepare_tasks(self, *args, **kwargs): Returns: list: a list of new tasks. """ - #TODO: max_test = self.cur_time latest_records, max_test = self.list_latest_recorders( lambda rec: self.get_online_tag(rec) == OnlineManager.ONLINE_TAG ) diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index f651ef8d8f..ef6a7a7d4f 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -49,7 +49,7 @@ def __init__( if rec_key_func is None: rec_key_func = lambda rec: rec.info["id"] if artifacts_key is None: - artifacts_key = self.artifacts_path.keys() + artifacts_key = list(self.artifacts_path.keys()) self._rec_key_func = rec_key_func self.artifacts_key = artifacts_key self._rec_filter_func = rec_filter_func diff --git a/qlib/workflow/task/gen.py b/qlib/workflow/task/gen.py index ad7a162181..9e273b74f3 100644 --- a/qlib/workflow/task/gen.py +++ b/qlib/workflow/task/gen.py @@ -194,6 +194,15 @@ def generate(self, task: dict): # update segments of this task t["dataset"]["kwargs"]["segments"] = copy.deepcopy(segments) + # if end_time < the end of test_segments, then change end_time to allow load more data + if ( + self.ta.cal_interval( + t["dataset"]["kwargs"]["handler"]["kwargs"]["end_time"], + t["dataset"]["kwargs"]["segments"][self.test_key][1], + ) + < 0 + ): + t["dataset"]["kwargs"]["handler"]["kwargs"]["end_time"] = copy.deepcopy(segments[self.test_key][1]) prev_seg = segments res.append(t) return res From 0058f7d0dcf29106f245ac4d69ec8e84ac2dcfa5 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Mon, 26 Apr 2021 09:31:47 +0000 Subject: [PATCH 44/61] Online Serving V8 --- .../online_srv/online_management_simulate.py | 68 ++---- .../online_srv/rolling_online_management.py | 5 + qlib/model/trainer.py | 212 +++++++++++++++--- qlib/workflow/__init__.py | 2 +- qlib/workflow/online/manager.py | 176 ++++++++++----- qlib/workflow/online/simulator.py | 13 +- qlib/workflow/task/gen.py | 9 +- qlib/workflow/task/manage.py | 36 ++- 8 files changed, 365 insertions(+), 156 deletions(-) diff --git a/examples/online_srv/online_management_simulate.py b/examples/online_srv/online_management_simulate.py index 9b5fbcc037..1b1fed6603 100644 --- a/examples/online_srv/online_management_simulate.py +++ b/examples/online_srv/online_management_simulate.py @@ -1,14 +1,14 @@ import fire import qlib from qlib.model.ens.ensemble import ens_workflow -from qlib.model.ens.group import RollingGroup -from qlib.model.trainer import TrainerRM +from qlib.model.trainer import DelayTrainerR, DelayTrainerRM, TrainerRM from qlib.workflow import R from qlib.workflow.online.manager import RollingOnlineManager from qlib.workflow.online.simulator import OnlineSimulator from qlib.workflow.task.collect import RecorderCollector from qlib.workflow.task.gen import RollingGen, task_generator from qlib.workflow.task.manage import TaskManager +from qlib.workflow.task.utils import list_recorders """ This examples is about the OnlineManager and OnlineSimulator based on rolling tasks. @@ -19,7 +19,7 @@ data_handler_config = { "start_time": "2018-01-01", - "end_time": None, # "2018-10-31", + "end_time": "2018-10-31", "fit_start_time": "2018-01-01", "fit_end_time": "2018-03-31", "instruments": "csi100", @@ -74,7 +74,7 @@ } -class OnlineManagerExample: +class OnlineSimulationExample: def __init__( self, provider_uri="~/.qlib/qlib_data/cn_data", @@ -86,6 +86,7 @@ def __init__( rolling_step=80, start_time="2018-09-10", end_time="2018-10-31", + tasks=[task_xgboost_config], # , task_lgb_config] ): """ init OnlineManagerExample. @@ -100,6 +101,7 @@ def __init__( rolling_step (int, optional): the step for rolling. Defaults to 80. start_time (str, optional): the start time of simulating. Defaults to "2018-09-10". end_time (str, optional): the end time of simulating. Defaults to "2018-10-31". + tasks (dict or list[dict]): a set of the task config waiting for rolling and training """ self.exp_name = exp_name self.task_pool = task_pool @@ -108,76 +110,49 @@ def __init__( "task_db_name": task_db_name, } qlib.init(provider_uri=provider_uri, region=region, mongo=mongo_conf) - - self.rolling_gen = RollingGen(step=rolling_step, rtype=RollingGen.ROLL_SD) # The rolling tasks generator - self.trainer = TrainerRM(self.exp_name, self.task_pool) # The trainer based on (R)ecorder and Task(M)anager + self.rolling_gen = RollingGen( + step=rolling_step, rtype=RollingGen.ROLL_SD, modify_end_time=False + ) # The rolling tasks generator, modify_end_time is false because we just need simulate to 2018-10-31. + self.trainer = DelayTrainerRM(self.exp_name, self.task_pool) self.task_manager = TaskManager(self.task_pool) # A good way to manage all your tasks - self.collector = RecorderCollector(exp_name=self.exp_name, rec_key_func=self.rec_key) # The result collector - self.grouper = RollingGroup() # Divide your results into different rolling group self.rolling_online_manager = RollingOnlineManager( experiment_name=exp_name, rolling_gen=self.rolling_gen, trainer=self.trainer, - collector=self.collector, need_log=False, ) # The OnlineManager based on Rolling self.onlinesimulator = OnlineSimulator( start_time=start_time, end_time=end_time, - onlinemanager=self.rolling_online_manager, + online_manager=self.rolling_online_manager, ) + self.tasks = tasks # Reset all things to the first status, be careful to save important data def reset(self): print("========== reset ==========") self.task_manager.remove() + exp = R.get_exp(experiment_name=self.exp_name) for rid in exp.list_recorders(): exp.delete_recorder(rid) - @staticmethod - def rec_key(recorder): - """ - given a Recorder and return its key to identify it - - Args: - recorder (Recorder): a instance of the Recorder - - Returns: - tuple: (model_key, rolling_key) - """ - task_config = recorder.load_object("task") - model_key = task_config["model"]["class"] - rolling_key = task_config["dataset"]["kwargs"]["segments"]["test"] - return model_key, rolling_key - - def result_collecting(self): - print("========== result collecting ==========") - - # ens_workflow can help collect, group and ensemble results in a easy way - artifact = ens_workflow(self.rolling_online_manager.get_collector(), self.grouper) - print(artifact) + for rid in list_recorders( + RollingOnlineManager.SIGNAL_EXP, lambda x: True if x.info["name"] == self.exp_name else False + ): + exp.delete_recorder(rid) # Run this firstly to see the workflow in OnlineManager def first_train(self): print("========== first train ==========") self.reset() - - tasks = task_generator( - tasks=[task_xgboost_config, task_lgb_config], - generators=[self.rolling_gen], # generate different date segment - ) - - self.rolling_online_manager.prepare_new_models(tasks=tasks, tag=RollingOnlineManager.ONLINE_TAG) - self.result_collecting() + self.rolling_online_manager.first_train(self.tasks) # Run this secondly to see the simulating in OnlineSimulator def simulate(self): - print("========== simulate ==========") self.onlinesimulator.simulate() - - self.result_collecting() + print(self.rolling_online_manager.collect_artifact()) print("========== online models ==========") recs_dict = self.onlinesimulator.online_models() @@ -186,6 +161,9 @@ def simulate(self): for rec in recs: print(rec.info["id"]) + print("========== online signals ==========") + print(self.rolling_online_manager.get_signals()) + # Run this to run all workflow automaticly def main(self): self.first_train() @@ -195,4 +173,4 @@ def main(self): if __name__ == "__main__": ## to run all workflow automaticly with your own parameters, use the command below # python online_management_simulate.py main --experiment_name="your_exp_name" --rolling_step=60 - fire.Fire(OnlineManagerExample) + fire.Fire(OnlineSimulationExample) diff --git a/examples/online_srv/rolling_online_management.py b/examples/online_srv/rolling_online_management.py index 6c30f3af3a..d118afe759 100644 --- a/examples/online_srv/rolling_online_management.py +++ b/examples/online_srv/rolling_online_management.py @@ -111,6 +111,11 @@ def reset(self): if os.path.exists(self._ROLLING_MANAGER_PATH): os.remove(self._ROLLING_MANAGER_PATH) + for rid in list_recorders( + RollingOnlineManager.SIGNAL_EXP, lambda x: True if x.info["name"] == self.exp_name else False + ): + exp.delete_recorder(rid) + def first_run(self): print("========== first_run ==========") self.reset() diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index 348f6b5218..af65c58863 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -1,6 +1,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +import copy +import time +from xxlimited import Str from qlib.utils import init_instance_by_config, flatten_dict, get_cls_kwargs from qlib.workflow import R from qlib.workflow.recorder import Recorder @@ -11,51 +14,80 @@ import socket -def task_train(task_config: dict, experiment_name: str) -> Recorder: +def begin_task_train(task_config: dict, experiment_name: str, *args, **kwargs) -> Recorder: """ - task based training + Begin a task training with starting a recorder and saving the task config. - Parameters - ---------- - task_config : dict - A dict describes a task setting. - experiment_name: str - The name of experiment + Args: + task_config (dict) + experiment_name (str) - Returns - ---------- - Recorder : The instance of the recorder + Returns: + Recorder """ - # model initiaiton - model: Model = init_instance_by_config(task_config["model"]) - dataset: Dataset = init_instance_by_config(task_config["dataset"]) - - # start exp - with R.start(experiment_name=experiment_name): - - # train model + with R.start(experiment_name=experiment_name, recorder_name=str(time.time())): R.log_params(**flatten_dict(task_config)) R.save_objects(**{"task": task_config}) # keep the original format and datatype - R.set_tags(hostname=socket.gethostname()) + R.set_tags(**{"hostname": socket.gethostname(), "train_status": "begin_task_train"}) + recorder: Recorder = R.get_recorder() + return recorder + + +def end_task_train(rec: Recorder, experiment_name: str, *args, **kwargs): + """ + Finished task training with real model fitting and saving. + + Args: + rec (Recorder): This recorder will be resumed + experiment_name (str) + + Returns: + Recorder + """ + with R.start(experiment_name=experiment_name, recorder_name=rec.info["name"], resume=True): + task_config = R.load_object("task") + # model & dataset initiaiton + model: Model = init_instance_by_config(task_config["model"]) + dataset: Dataset = init_instance_by_config(task_config["dataset"]) + # model training model.fit(dataset) R.save_objects(**{"params.pkl": model}) # This dataset is saved for online inference. So the concrete data should not be dumped dataset.config(dump_all=False, recursive=True) R.save_objects(**{"dataset": dataset}) - # generate records: prediction, backtest, and analysis records = task_config.get("record", []) - recorder: Recorder = R.get_recorder() if isinstance(records, dict): # prevent only one dict records = [records] for record in records: cls, kwargs = get_cls_kwargs(record, default_module="qlib.workflow.record_temp") if cls is SignalRecord: - rconf = {"model": model, "dataset": dataset, "recorder": recorder} + rconf = {"model": model, "dataset": dataset, "recorder": rec} else: - rconf = {"recorder": recorder} + rconf = {"recorder": rec} r = cls(**kwargs, **rconf) r.generate() + R.set_tags(**{"train_status": "end_task_train"}) + return rec + + +def task_train(task_config: dict, experiment_name: str) -> Recorder: + """ + task based training + + Parameters + ---------- + task_config : dict + A dict describes a task setting. + experiment_name: str + The name of experiment + + Returns + ---------- + Recorder : The instance of the recorder + """ + recorder = begin_task_train(task_config, experiment_name) + recorder = end_task_train(recorder, experiment_name) return recorder @@ -64,14 +96,22 @@ class Trainer: The trainer which can train a list of model """ - def train(self, *args, **kwargs): - """Given a list of model definition, finished training and return the results of them. + def train(self, tasks: list, *args, **kwargs): + """Given a list of model definition, begin a training and return the models. Returns: - list: a list of trained results + list: a list of models """ raise NotImplementedError(f"Please implement the `train` method.") + def end_train(self, models, *args, **kwargs): + """Given a list of models, finished something in the end of training if you need. + + Returns: + list: a list of models + """ + pass + class TrainerR(Trainer): """Trainer based on (R)ecorder. @@ -112,7 +152,15 @@ def __init__(self, experiment_name: str, task_pool: str, train_func=task_train): self.task_pool = task_pool self.train_func = train_func - def train(self, tasks: list, train_func=None, *args, **kwargs): + def train( + self, + tasks: list, + train_func=None, + before_status=TaskManager.STATUS_WAITING, + after_status=TaskManager.STATUS_DONE, + *args, + **kwargs, + ): """Given a list of `task`s and return a list of trained Recorder. The order can be guaranteed. This method defaults to a single process, but TaskManager offered a great way to parallel training. @@ -129,7 +177,15 @@ def train(self, tasks: list, train_func=None, *args, **kwargs): train_func = self.train_func tm = TaskManager(task_pool=self.task_pool) _id_list = tm.create_task(tasks) # all tasks will be saved to MongoDB - run_task(train_func, self.task_pool, experiment_name=self.experiment_name, *args, **kwargs) + run_task( + train_func, + self.task_pool, + experiment_name=self.experiment_name, + before_status=before_status, + after_status=after_status, + *args, + **kwargs, + ) recs = [] for _id in _id_list: @@ -137,10 +193,96 @@ def train(self, tasks: list, train_func=None, *args, **kwargs): return recs -class DelayTrainer(Trainer): - def fake_train(self): - self.fake_trained = [] +class DelayTrainerR(TrainerR): + """ + A delayed implementation based on TrainerR, which means `train` method may only do some preparation and `end_train` method can do the real model fitting. + + """ + + def __init__(self, experiment_name, train_func=begin_task_train, end_train_func=end_task_train): + super().__init__(experiment_name, train_func) + self.end_train_func = end_train_func + self.recs = [] + + def train(self, tasks: list, train_func, *args, **kwargs): + """ + Same as `train` of TrainerR, the results will be recorded in self.recs + + Args: + tasks (list): a list of definition based on `task` dict + train_func (Callable): the train method which need at least `task` and `experiment_name`. None for default. + + Returns: + list: a list of Recorders + """ + self.recs = super().train(tasks, train_func=train_func, *args, **kwargs) + return self.recs + + def end_train(self, recs=None, end_train_func=None): + """ + Given a list of Recorder and return a list of trained Recorder. + This class will finished real data loading and model fitting. + + Args: + recs (list, optional): a list of Recorder, the tasks have been saved to them. Defaults to None for using self.recs. + end_train_func (Callable, optional): the end_train method which need at least `rec` and `experiment_name`. Defaults to None for using self.end_train_func. + + Returns: + list: a list of Recorders + """ + if recs is None: + recs = copy.deepcopy(self.recs) + # the models will be only trained once + self.recs = [] + if end_train_func is None: + end_train_func = self.end_train_func + for rec in recs: + end_train_func(rec) + return recs + + +class DelayTrainerRM(TrainerRM): + """ + A delayed implementation based on TrainerRM, which means `train` method may only do some preparation and `end_train` method can do the real model fitting. - def train(self): - for rec in self.fake_trained: - pass + """ + + def __init__(self, experiment_name, task_pool: str, train_func=begin_task_train, end_train_func=end_task_train): + super().__init__(experiment_name, task_pool, train_func) + self.end_train_func = end_train_func + + def train(self, tasks: list, train_func=None, *args, **kwargs): + """ + Same as `train` of TrainerRM, the results will be recorded in self.recs + + Args: + tasks (list): a list of definition based on `task` dict + train_func (Callable): the train method which need at least `task` and `experiment_name`. None for default. + + Returns: + list: a list of Recorders + """ + return super().train(tasks, train_func=train_func, after_status=TaskManager.STATUS_PART_DONE, *args, **kwargs) + + def end_train(self, recs, end_train_func=None): + """ + Given a list of Recorder and return a list of trained Recorder. + This class will finished real data loading and model fitting. + + Args: + recs (list, optional): a list of Recorder, the tasks have been saved to them. Defaults to None for using self.recs.. + end_train_func (Callable, optional): the end_train method which need at least `rec` and `experiment_name`. Defaults to None for using self.end_train_func. + + Returns: + list: a list of Recorders + """ + + if end_train_func is None: + end_train_func = self.end_train_func + run_task( + end_train_func, + self.task_pool, + experiment_name=self.experiment_name, + before_status=TaskManager.STATUS_PART_DONE, + ) + return recs diff --git a/qlib/workflow/__init__.py b/qlib/workflow/__init__.py index a036656262..46f9c563f5 100644 --- a/qlib/workflow/__init__.py +++ b/qlib/workflow/__init__.py @@ -304,7 +304,7 @@ def set_uri(self, uri: Optional[Text]): """ self.exp_manager.set_uri(uri) - def get_recorder(self, recorder_id=None, recorder_name=None, experiment_name=None): + def get_recorder(self, recorder_id=None, recorder_name=None, experiment_name=None) -> Recorder: """ Method for retrieving a recorder. diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index e744880402..c94cf24557 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -44,17 +44,21 @@ def __init__(self, trainer: Trainer = None, need_log=True): self.trainer = trainer self.logger = get_module_logger(self.__class__.__name__) self.need_log = need_log - self.delay_signals = {} self.cur_time = None - def prepare_signals(self, *args, **kwargs): + def prepare_signals(self): """ After perparing the data of last routine (a box in box-plot) which means the end of the routine, we can prepare trading signals for next routine. Must use `pass` even though there is nothing to do. """ - raise NotImplementedError(f"Please implement the `prepare_signals` method.") + def get_signals(self): + """ + After preparing signals, here is the method to get them. + """ + raise NotImplementedError(f"Please implement the `get_signals` method.") + def prepare_tasks(self, *args, **kwargs): """ After the end of a routine, check whether we need to prepare and train some new tasks. @@ -62,7 +66,7 @@ def prepare_tasks(self, *args, **kwargs): """ raise NotImplementedError(f"Please implement the `prepare_tasks` method.") - def prepare_new_models(self, tasks, tag=NEXT_ONLINE_TAG, check_func=None): + def prepare_new_models(self, tasks, tag=NEXT_ONLINE_TAG, check_func=None, *args, **kwargs): """ Use trainer to train a list of tasks and set the trained model to `tag`. @@ -75,13 +79,14 @@ def prepare_new_models(self, tasks, tag=NEXT_ONLINE_TAG, check_func=None): check_func: the method to judge if a model can be online. The parameter is the model record and return True for online. None for online every models. + *args, **kwargs: will be passed to end_train which means will be passed to customized train method. """ if check_func is None: check_func = lambda x: True if len(tasks) > 0: if self.trainer is not None: - new_models = self.trainer.train(tasks) + new_models = self.trainer.train(tasks, *args, **kwargs) if check_func(new_models): self.set_online_tag(tag, new_models) if self.need_log: @@ -89,13 +94,13 @@ def prepare_new_models(self, tasks, tag=NEXT_ONLINE_TAG, check_func=None): else: self.logger.warn("No trainer to train new tasks.") - def update_online_pred(self, *args, **kwargs): + def update_online_pred(self): """ After the end of a routine, update the predictions of online models to latest. """ raise NotImplementedError(f"Please implement the `update_online_pred` method.") - def set_online_tag(self, tag, *args, **kwargs): + def set_online_tag(self, tag, recorder): """ Set `tag` to the model to sign whether online. @@ -104,15 +109,21 @@ def set_online_tag(self, tag, *args, **kwargs): """ raise NotImplementedError(f"Please implement the `set_online_tag` method.") - def get_online_tag(self, *args, **kwargs): + def get_online_tag(self): """ Given a model and return its online tag. """ raise NotImplementedError(f"Please implement the `get_online_tag` method.") - def reset_online_tag(self, *args, **kwargs): - """ - Offline all models and set the models to 'online'. + def reset_online_tag(self, recorders=None): + """offline all models and set the recorders to 'online'. If no parameter and no 'next online' model, then do nothing. + + Args: + recorders (List, optional): + the recorders you want to reset to 'online'. If don't give, set 'next online' model to 'online' model. If there isn't any 'next online' model, then maintain existing 'online' model. + + Returns: + list: new online recorder. [] if there is no update. """ raise NotImplementedError(f"Please implement the `reset_online_tag` method.") @@ -137,31 +148,46 @@ def get_collector(self): """ raise NotImplementedError(f"Please implement the `get_collector` method.") - def run_delay_signals(self): - """ - Prepare all signals if there are some dates waiting for prepare. + def delay_prepare(self, rec_dict, *args, **kwargs): """ - for cur_time, params in self.delay_signals.items(): - self.cur_time = cur_time - self.prepare_signals(*params[0], **params[1]) - self.delay_signals = {} + Prepare all models and signals if there are something waiting for prepare. + NOTE: Assumption: the predictions of online models are between `time_segment`, or this method will work in a wrong way. + + Args: + rec_dict (str): an online models dict likes {(begin_time, end_time):[online models]}. + *args, **kwargs: will be passed to end_train which means will be passed to customized train method. + """ + for time_segment, recs_list in rec_dict.items(): + self.trainer.end_train(recs_list, *args, **kwargs) + self.reset_online_tag(recs_list) + self.prepare_signals() + signal_max = self.get_signals().index.get_level_values("datetime").max() + if time_segment[1] is not None and signal_max > time_segment[1]: + raise ValueError( + f"The max time of signals prepared by online models is {signal_max}, but those models only online in {time_segment}" + ) def routine(self, cur_time=None, delay_prepare=False, *args, **kwargs): """ The typical update process after a routine, such as day by day or month by month. update online prediction -> prepare signals -> prepare tasks -> prepare new models -> reset online models + + NOTE: Assumption: if using simulator (delay_prepare is True), the prediction will be prepared well after every training, so there is no need to update predictions. + + Args: + cur_time ([type], optional): [description]. Defaults to None. + delay_prepare (bool, optional): [description]. Defaults to False. + *args, **kwargs: will be passed to `prepare_tasks` and `prepare_new_models`. It can be some hyper parameter or training config. + + Returns: + [type]: [description] """ self.cur_time = cur_time # None for latest date - self.update_online_pred() if not delay_prepare: - self.prepare_signals(*args, **kwargs) - else: - if cur_time is not None: - self.delay_signals[cur_time] = (args, kwargs) - else: - raise ValueError("Can not delay prepare when cur_time is None") + self.update_online_pred() + self.prepare_signals() tasks = self.prepare_tasks(*args, **kwargs) - self.prepare_new_models(tasks) + self.prepare_new_models(tasks, *args, **kwargs) return self.reset_online_tag() @@ -185,8 +211,16 @@ def __init__(self, experiment_name: str, trainer: Trainer = None, need_log=True) trainer = TrainerR(experiment_name) super().__init__(trainer=trainer, need_log=need_log) self.exp_name = experiment_name + self.signal_rec = None def set_online_tag(self, tag, recorder: Union[Recorder, List]): + """ + Set `tag` to the model to sign whether online. + + Args: + tag (str): the tags in `ONLINE_TAG`, `NEXT_ONLINE_TAG`, `OFFLINE_TAG` + recorder (Union[Recorder, List]) + """ if isinstance(recorder, Recorder): recorder = [recorder] for rec in recorder: @@ -195,6 +229,15 @@ def set_online_tag(self, tag, recorder: Union[Recorder, List]): self.logger.info(f"Set {len(recorder)} models to '{tag}'.") def get_online_tag(self, recorder: Recorder): + """ + Given a model and return its online tag. + + Args: + recorder (Recorder): a instance of recorder + + Returns: + str: the tag + """ tags = recorder.list_tags() return tags.get(OnlineManager.ONLINE_KEY, OnlineManager.OFFLINE_TAG) @@ -202,7 +245,7 @@ def reset_online_tag(self, recorder: Union[Recorder, List] = None): """offline all models and set the recorders to 'online'. If no parameter and no 'next online' model, then do nothing. Args: - recorders (Union[List, Dict], optional): + recorders (Union[Recorder, List], optional): the recorders you want to reset to 'online'. If don't give, set 'next online' model to 'online' model. If there isn't any 'next online' model, then maintain existing 'online' model. Returns: @@ -225,7 +268,30 @@ def reset_online_tag(self, recorder: Union[Recorder, List] = None): self.set_online_tag(OnlineManager.ONLINE_TAG, recorder) return recorder + def get_signals(self): + """ + get signals from the recorder(named self.exp_name) of the experiment(named self.SIGNAL_EXP) + + Returns: + signals + """ + if self.signal_rec is None: + with R.start(experiment_name=self.SIGNAL_EXP, recorder_name=self.exp_name, resume=True): + self.signal_rec = R.get_recorder() + signals = None + try: + signals = self.signal_rec.load_object("signals") + except OSError: + self.logger.warn("Can not find `signals`, have you called `prepare_signals` before?") + return signals + def online_models(self): + """ + Return online models. + + Returns: + list: the list of online models + """ return list( list_recorders(self.exp_name, lambda rec: self.get_online_tag(rec) == OnlineManager.ONLINE_TAG).values() ) @@ -245,34 +311,35 @@ def prepare_signals(self, over_write=False): """ Average the predictions of online models and offer a trading signals every routine. The signals will be saved to `signal` file of a recorder named self.exp_name of a experiment using the name of `SIGNAL_EXP` - + Even if the latest signal already exists, the latest calculation result will be overwritten. + NOTE: Given a prediction of a certain time, all signals before this time will be prepared well. Args: over_write (bool, optional): If True, the new signals will overwrite the file. If False, the new signals will append to the end of signals. Defaults to False. """ - - with R.start(experiment_name=self.SIGNAL_EXP, recorder_name=self.exp_name, resume=True): - recorder = R.get_recorder() - pred = [] - - try: - old_signals = recorder.load_object("signals") - except OSError: - old_signals = None - - for rec in self.online_models(): - pred.append(rec.load_object("pred.pkl")) - - signals = pd.concat(pred, axis=1).mean(axis=1).to_frame("score") - signals = signals.sort_index() - if old_signals is not None and not over_write: - # signals = old_signals.reindex(signals.index).combine_first(signals) - old_max = old_signals.index.get_level_values("datetime").max() - new_signals = signals.loc[old_max:] - signals = pd.concat([old_signals, new_signals], axis=0) - else: - new_signals = signals + if self.signal_rec is None: + with R.start(experiment_name=self.SIGNAL_EXP, recorder_name=self.exp_name, resume=True): + self.signal_rec = R.get_recorder() + + pred = [] + try: + old_signals = self.signal_rec.load_object("signals") + except OSError: + old_signals = None + + for rec in self.online_models(): + pred.append(rec.load_object("pred.pkl")) + + signals = pd.concat(pred, axis=1).mean(axis=1).to_frame("score") + signals = signals.sort_index() + if old_signals is not None and not over_write: + old_max = old_signals.index.get_level_values("datetime").max() + new_signals = signals.loc[old_max:] + signals = pd.concat([old_signals, new_signals], axis=0) + else: + new_signals = signals + if self.need_log: self.logger.info(f"Finished preparing new {len(new_signals)} signals to {self.SIGNAL_EXP}/{self.exp_name}.") - recorder.save_objects(**{"signals": signals}) + self.signal_rec.save_objects(**{"signals": signals}) class RollingOnlineManager(OnlineManagerR): @@ -304,7 +371,9 @@ def __init__( def get_collector(self, rec_key_func=None, rec_filter_func=None): """ - get the instance of collector to collect results + Get the instance of collector to collect results. The returned collector must can distinguish results in different models. + Assumption: the models can be distinguished based on model name and rolling test segments. + If you do not want this assumption, please implement your own method or use another rec_key_func. Args: rec_key_func (Callable): a function to get the key of a recorder. If None, use recorder id. @@ -353,10 +422,9 @@ def first_train(self, task_configs: list): generators=self.rg, # generate different date segment ) self.prepare_new_models(tasks, tag=self.ONLINE_TAG) - self.prepare_signals(over_write=True) return self.get_collector() - def prepare_tasks(self, *args, **kwargs): + def prepare_tasks(self): """ Prepare new tasks based on new date. diff --git a/qlib/workflow/online/simulator.py b/qlib/workflow/online/simulator.py index 16628c240c..d45b7d99da 100644 --- a/qlib/workflow/online/simulator.py +++ b/qlib/workflow/online/simulator.py @@ -12,7 +12,7 @@ def __init__( self, start_time, end_time, - onlinemanager: OnlineManager, + online_manager: OnlineManager, frequency="day", ): """ @@ -28,15 +28,14 @@ def __init__( self.cal = D.calendar(start_time=start_time, end_time=end_time, freq=frequency) self.start_time = self.cal[0] self.end_time = self.cal[-1] - self.olm = onlinemanager - + self.olm = online_manager if len(self.cal) == 0: self.logger.warn(f"There is no need to simulate bacause start_time is larger than end_time.") def simulate(self, *args, **kwargs): """ Starting from start time, this method will simulate every routine in OnlineManager. - NOTE: Considering the parallel training, the signals will be perpared after all routine simulating. + NOTE: Considering the parallel training, the models and signals can be perpared after all routine simulating. Returns: Collector: the OnlineManager's collector @@ -54,12 +53,10 @@ def simulate(self, *args, **kwargs): self.rec_dict[(tmp_begin, tmp_end)] = prev_recorders tmp_begin = cur_time prev_recorders = recorders - self.rec_dict[(tmp_begin, self.end_time)] = prev_recorders - # prepare signals again incase there is no trained model when call it - self.olm.run_delay_signals() + # finished perparing models (and pred) and signals + self.olm.delay_prepare(self.rec_dict) self.logger.info(f"Finished preparing signals") - return self.olm.get_collector() def online_models(self): diff --git a/qlib/workflow/task/gen.py b/qlib/workflow/task/gen.py index 9e273b74f3..158bc99168 100644 --- a/qlib/workflow/task/gen.py +++ b/qlib/workflow/task/gen.py @@ -91,7 +91,7 @@ class RollingGen(TaskGen): ROLL_EX = TimeAdjuster.SHIFT_EX # fixed start date, expanding end date ROLL_SD = TimeAdjuster.SHIFT_SD # fixed segments size, slide it from start date - def __init__(self, step: int = 40, rtype: str = ROLL_EX): + def __init__(self, step: int = 40, rtype: str = ROLL_EX, modify_end_time=True): """ Generate tasks for rolling @@ -101,9 +101,12 @@ def __init__(self, step: int = 40, rtype: str = ROLL_EX): step to rolling rtype : str rolling type (expanding, sliding) + modify_end_time: bool + Whether the data set configuration needs to be modified when the required scope exceeds the original data set scope """ self.step = step self.rtype = rtype + self.modify_end_time = modify_end_time # TODO: Ask pengrong to update future date in dataset self.ta = TimeAdjuster(future=True) @@ -113,7 +116,6 @@ def __init__(self, step: int = 40, rtype: str = ROLL_EX): def generate(self, task: dict): """ Converting the task into a rolling task. - # FIXME: only modify dataset layer, user need to change datahandler firstly. Parameters ---------- @@ -196,7 +198,8 @@ def generate(self, task: dict): t["dataset"]["kwargs"]["segments"] = copy.deepcopy(segments) # if end_time < the end of test_segments, then change end_time to allow load more data if ( - self.ta.cal_interval( + self.modify_end_time + and self.ta.cal_interval( t["dataset"]["kwargs"]["handler"]["kwargs"]["end_time"], t["dataset"]["kwargs"]["segments"][self.test_key][1], ) diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index b144a8872d..9d50d85638 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -174,11 +174,11 @@ def create_task(self, task_def_l, dry_run=False, print_nt=False): return _id_list - def fetch_task(self, query={}): + def fetch_task(self, query={}, status=STATUS_WAITING): query = query.copy() if "_id" in query: query["_id"] = ObjectId(query["_id"]) - query.update({"status": self.STATUS_WAITING}) + query.update({"status": status}) task = self.task_pool.find_one_and_update( query, {"$set": {"status": self.STATUS_RUNNING}}, sort=[("priority", pymongo.DESCENDING)] ) @@ -189,7 +189,7 @@ def fetch_task(self, query={}): return self._decode_task(task) @contextmanager - def safe_fetch_task(self, query={}): + def safe_fetch_task(self, query={}, status=STATUS_WAITING): """ fetch task from task_pool using query with contextmanager @@ -202,7 +202,7 @@ def safe_fetch_task(self, query={}): ------- """ - task = self.fetch_task(query=query) + task = self.fetch_task(query=query, status=status) try: yield task except Exception: @@ -330,7 +330,15 @@ def __str__(self): return f"TaskManager({self.task_pool})" -def run_task(task_func, task_pool, force_release=False, *args, **kwargs): +def run_task( + task_func, + task_pool, + force_release=False, + before_status=TaskManager.STATUS_WAITING, + after_status=TaskManager.STATUS_DONE, + *args, + **kwargs, +): """ While task pool is not empty (has WAITING tasks), use task_func to fetch and run tasks in task_pool @@ -352,16 +360,24 @@ def run_task(task_func, task_pool, force_release=False, *args, **kwargs): ever_run = False while True: - with tm.safe_fetch_task() as task: + with tm.safe_fetch_task(status=before_status) as task: if task is None: break get_module_logger("run_task").info(task["def"]) + # when fetching `WAITING` task, use task_def to train + if before_status == TaskManager.STATUS_WAITING: + param = task["def"] + # when fetching `PART_DONE` task, use task_res to train for the result has been saved + elif before_status == TaskManager.STATUS_PART_DONE: + param = task["res"] + else: + raise ValueError("The fetched task must be `STATUS_WAITING` or `STATUS_PART_DONE`!") if force_release: - with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: # what this means? - res = executor.submit(task_func, task["def"], *args, **kwargs).result() + with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: + res = executor.submit(task_func, param, *args, **kwargs).result() else: - res = task_func(task["def"], *args, **kwargs) - tm.commit_task_res(task, res) + res = task_func(param, *args, **kwargs) + tm.commit_task_res(task, res, status=after_status) ever_run = True return ever_run From 42f510024cfbff7ce412a95e1ad7c05c85f59ec1 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Tue, 27 Apr 2021 04:12:08 +0000 Subject: [PATCH 45/61] update collector --- .../model_rolling/task_manager_rolling.py | 2 +- qlib/model/ens/ensemble.py | 9 +- qlib/workflow/online/manager.py | 2 +- qlib/workflow/task/collect.py | 107 +++++++++++++++++- setup.py | 1 + 5 files changed, 106 insertions(+), 15 deletions(-) diff --git a/examples/model_rolling/task_manager_rolling.py b/examples/model_rolling/task_manager_rolling.py index 9c1cbf8919..ab3a4eee52 100644 --- a/examples/model_rolling/task_manager_rolling.py +++ b/examples/model_rolling/task_manager_rolling.py @@ -140,7 +140,7 @@ def my_filter(recorder): return False artifact = ens_workflow( - RecorderCollector(exp_name=self.experiment_name, rec_key_func=rec_key, rec_filter_func=my_filter), + RecorderCollector(experiment=self.experiment_name, rec_key_func=rec_key, rec_filter_func=my_filter), RollingGroup(), ) print(artifact) diff --git a/qlib/model/ens/ensemble.py b/qlib/model/ens/ensemble.py index 942303c18b..63f6438c27 100644 --- a/qlib/model/ens/ensemble.py +++ b/qlib/model/ens/ensemble.py @@ -13,12 +13,7 @@ def ens_workflow(collector: Collector, process_list, *args, **kwargs): collector (Collector): the collector to collect the result into {result_key: things} process_list (list or Callable): the list of processors or the instance of processor to process dict. The processor order is same as the list order. - - For example: [Group1(..., Ensemble1()), Group2(..., Ensemble2())] - - artifacts_key (list, optional): the artifacts key you want to get. If None, get all artifacts. - rec_filter_func (Callable, optional): filter the recorder by return True or False. Defaults to None. - + For example: [Group1(..., Ensemble1()), Group2(..., Ensemble2())] Returns: dict: the ensemble dict """ @@ -38,7 +33,7 @@ def ens_workflow(collector: Collector, process_list, *args, **kwargs): return ensemble -class Ensemble(Serializable): +class Ensemble: """Merge the objects in an Ensemble.""" def __call__(self, ensemble_dict: dict, *args, **kwargs): diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index c94cf24557..e107271d0a 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -389,7 +389,7 @@ def rec_key(recorder): if rec_key_func is None: rec_key_func = rec_key - return RecorderCollector(exp_name=self.exp_name, rec_key_func=rec_key_func, rec_filter_func=rec_filter_func) + return RecorderCollector(experiment=self.exp_name, rec_key_func=rec_key_func, rec_filter_func=rec_filter_func) def collect_artifact(self, rec_key_func=None, rec_filter_func=None): """ diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index ef6a7a7d4f..b4c81122d2 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -1,17 +1,28 @@ from abc import abstractmethod from typing import Callable, Union +from qlib.workflow import R from qlib.workflow.task.utils import list_recorders from qlib.utils.serial import Serializable +import dill as pickle class Collector: """The collector to collect different results""" - def collect(self, *args, **kwargs): + def __init__(self, process_list=[]): + """ + Args: + process_list (list, optional): process_list (list or Callable): the list of processors or the instance of processor to process dict. + """ + if not isinstance(process_list, list): + process_list = [process_list] + self.process_list = process_list + + def collect(self): """Collect the results and return a dict like {key: things} Returns: - dict: the dict after collected. + dict: the dict after collecting. For example: @@ -23,13 +34,88 @@ def collect(self, *args, **kwargs): """ raise NotImplementedError(f"Please implement the `collect` method.") + @staticmethod + def process_collect(collected_dict, process_list=[], *args, **kwargs): + """do a series of processing to the dict returned by collect and return a dict like {key: things} + For example: you can group and ensemble. + + Args: + collected_dict (dict): the dict return by `collect` + process_list (list or Callable): the list of processors or the instance of processor to process dict. + The processor order is same as the list order. + For example: [Group1(..., Ensemble1()), Group2(..., Ensemble2())] + + Returns: + dict: the dict after processing. + """ + if not isinstance(process_list, list): + process_list = [process_list] + result = {} + for artifact in collected_dict: + value = collected_dict[artifact] + for process in process_list: + if not callable(process): + raise NotImplementedError(f"{type(process)} is not supported in `process_collect`.") + value = process(value, *args, **kwargs) + result[artifact] = value + return result + + def __call__(self, *args, **kwargs): + """ + do the workflow including collect and process_collect + + Returns: + dict: the dict after collecting and processing. + """ + collected = self.collect() + return self.process_collect(collected, self.process_list, *args, **kwargs) + + def save(self, filepath): + """ + save the collector into a file + + Args: + filepath (str): the path of file + + Returns: + bool: if successed + """ + try: + with open(filepath, "wb") as f: + pickle.dump(self, f) + except Exception: + return False + return True + + @staticmethod + def load(filepath): + """ + load the collector from a file + + Args: + filepath (str): the path of file + + Raises: + TypeError: the pickled file must be `Collector` + + Returns: + Collector: the instance of Collector + """ + with open(filepath, "rb") as f: + collector = pickle.load(f) + if isinstance(collector, Collector): + return collector + else: + raise TypeError(f"The instance of {type(collector)} is not a valid `Collector`!") + class RecorderCollector(Collector): ART_KEY_RAW = "__raw" def __init__( self, - exp_name, + experiment, + process_list=[], rec_key_func=None, rec_filter_func=None, artifacts_path={"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"}, @@ -38,13 +124,17 @@ def __init__( """init RecorderCollector Args: - exp_name (str): the name of Experiment + experiment (Experiment or str): an instance of a Experiment or the name of a Experiment + process_list (list or Callable): the list of processors or the instance of processor to process dict. rec_key_func (Callable): a function to get the key of a recorder. If None, use recorder id. rec_filter_func (Callable, optional): filter the recorder by return True or False. Defaults to None. artifacts_path (dict, optional): The artifacts name and its path in Recorder. Defaults to {"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"}. artifacts_key (str or List, optional): the artifacts key you want to get. If None, get all artifacts. """ - self.exp_name = exp_name + if isinstance(experiment, str): + experiment = R.get_exp(experiment_name=experiment) + self.experiment = experiment + self.process_list = process_list self.artifacts_path = artifacts_path if rec_key_func is None: rec_key_func = lambda rec: rec.info["id"] @@ -74,7 +164,12 @@ def collect(self, artifacts_key=None, rec_filter_func=None): collect_dict = {} # filter records - recs_flt = list_recorders(self.exp_name, rec_filter_func) + recs = self.experiment.list_recorders() + recs_flt = {} + for rid, rec in recs.items(): + if rec_filter_func is None or rec_filter_func(rec): + recs_flt[rid] = rec + for _, rec in recs_flt.items(): rec_key = self._rec_key_func(rec) for key in artifacts_key: diff --git a/setup.py b/setup.py index 699fdf75d8..c90d7d1c36 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ "ruamel.yaml>=0.16.12", "pymongo==3.7.2", # For task management "scikit-learn>=0.22", + "dill", ] # Numpy include From 36ab078fbdbdd69f1ac93b0be75ab29253b357d3 Mon Sep 17 00:00:00 2001 From: blin Date: Wed, 28 Apr 2021 07:15:59 +0000 Subject: [PATCH 46/61] filter --- qlib/data/dataset/__init__.py | 44 ++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/qlib/data/dataset/__init__.py b/qlib/data/dataset/__init__.py index cd15a98c93..5485796efb 100644 --- a/qlib/data/dataset/__init__.py +++ b/qlib/data/dataset/__init__.py @@ -114,6 +114,7 @@ def __init__(self, handler: Union[Dict, DataHandler], segments: Dict[Text, Tuple """ self.handler: DataHandler = init_instance_by_config(handler, accept_types=DataHandler) self.segments = segments.copy() + self.fetch_kwargs = {} super().__init__(**kwargs) def config(self, handler_kwargs: dict = None, **kwargs): @@ -171,7 +172,7 @@ def _prepare_seg(self, slc: slice, **kwargs): ---------- slc : slice """ - return self.handler.fetch(slc, **kwargs) + return self.handler.fetch(slc, **kwargs, **self.fetch_kwargs) def prepare( self, @@ -288,13 +289,29 @@ def __init__(self, data: pd.DataFrame, start, end, step_len: int, fillna_type: s # the data type will be changed # The index of usable data is between start_idx and end_idx - self.start_idx, self.end_idx = self.data.index.slice_locs(start=pd.Timestamp(start), end=pd.Timestamp(end)) self.idx_df, self.idx_map = self.build_index(self.data) - self.idx_arr = np.array(self.idx_df.values, dtype=np.float64) # for better performance - self.data_idx = deepcopy(self.data.index) + self.data_index = deepcopy(self.data.index) + + if flt_data is not None: + self.flt_data = np.array(flt_data).reshape(-1) + self.idx_map = self.flt_idx_map(self.flt_data, self.idx_map) + self.data_index = self.data_index[np.where(self.flt_data == True)[0]] + self.start_idx, self.end_idx = self.data_index.slice_locs(start=pd.Timestamp(start), end=pd.Timestamp(end)) + self.idx_arr = np.array(self.idx_df.values, dtype=np.float64) # for better performance + del self.data # save memory + @staticmethod + def flt_idx_map(flt_data, idx_map): + idx = 0 + new_idx_map = {} + for i, exist in enumerate(flt_data): + if exist: + new_idx_map[idx] = idx_map[i] + idx += 1 + return new_idx_map + def get_index(self): """ Get the pandas index of the data, it will be useful in following scenarios @@ -488,8 +505,19 @@ def _prepare_seg(self, slc: slice, **kwargs) -> TSDataSampler: """ split the _prepare_raw_seg is to leave a hook for data preprocessing before creating processing data """ - dtype = kwargs.pop("dtype", None) + dtype = kwargs.pop("dtype") start, end = slc.start, slc.stop - data = self._prepare_raw_seg(slc=slc, **kwargs) - tsds = TSDataSampler(data=data, start=start, end=end, step_len=self.step_len, dtype=dtype) - return tsds + flt_col = kwargs.pop('flt_col', None) + # TSDatasetH will retrieve more data for complete + data = self._prepare_raw_seg(slc, **kwargs) + + flt_kwargs = deepcopy(kwargs) + if flt_col is not None: + flt_kwargs['col_set'] = flt_col + flt_data = self._prepare_raw_seg(slc, **flt_kwargs) + assert len(flt_data.columns) == 1 + else: + flt_data = None + + tsds = TSDataSampler(data=data, start=start, end=end, step_len=self.step_len, dtype=dtype, flt_data=flt_data) + return tsds \ No newline at end of file From 45c6dfc5daed1c8e2678fa9ca68c35a514748d3f Mon Sep 17 00:00:00 2001 From: blin Date: Wed, 28 Apr 2021 07:25:19 +0000 Subject: [PATCH 47/61] filter --- qlib/data/dataset/__init__.py | 44 ++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/qlib/data/dataset/__init__.py b/qlib/data/dataset/__init__.py index cd15a98c93..5485796efb 100644 --- a/qlib/data/dataset/__init__.py +++ b/qlib/data/dataset/__init__.py @@ -114,6 +114,7 @@ def __init__(self, handler: Union[Dict, DataHandler], segments: Dict[Text, Tuple """ self.handler: DataHandler = init_instance_by_config(handler, accept_types=DataHandler) self.segments = segments.copy() + self.fetch_kwargs = {} super().__init__(**kwargs) def config(self, handler_kwargs: dict = None, **kwargs): @@ -171,7 +172,7 @@ def _prepare_seg(self, slc: slice, **kwargs): ---------- slc : slice """ - return self.handler.fetch(slc, **kwargs) + return self.handler.fetch(slc, **kwargs, **self.fetch_kwargs) def prepare( self, @@ -288,13 +289,29 @@ def __init__(self, data: pd.DataFrame, start, end, step_len: int, fillna_type: s # the data type will be changed # The index of usable data is between start_idx and end_idx - self.start_idx, self.end_idx = self.data.index.slice_locs(start=pd.Timestamp(start), end=pd.Timestamp(end)) self.idx_df, self.idx_map = self.build_index(self.data) - self.idx_arr = np.array(self.idx_df.values, dtype=np.float64) # for better performance - self.data_idx = deepcopy(self.data.index) + self.data_index = deepcopy(self.data.index) + + if flt_data is not None: + self.flt_data = np.array(flt_data).reshape(-1) + self.idx_map = self.flt_idx_map(self.flt_data, self.idx_map) + self.data_index = self.data_index[np.where(self.flt_data == True)[0]] + self.start_idx, self.end_idx = self.data_index.slice_locs(start=pd.Timestamp(start), end=pd.Timestamp(end)) + self.idx_arr = np.array(self.idx_df.values, dtype=np.float64) # for better performance + del self.data # save memory + @staticmethod + def flt_idx_map(flt_data, idx_map): + idx = 0 + new_idx_map = {} + for i, exist in enumerate(flt_data): + if exist: + new_idx_map[idx] = idx_map[i] + idx += 1 + return new_idx_map + def get_index(self): """ Get the pandas index of the data, it will be useful in following scenarios @@ -488,8 +505,19 @@ def _prepare_seg(self, slc: slice, **kwargs) -> TSDataSampler: """ split the _prepare_raw_seg is to leave a hook for data preprocessing before creating processing data """ - dtype = kwargs.pop("dtype", None) + dtype = kwargs.pop("dtype") start, end = slc.start, slc.stop - data = self._prepare_raw_seg(slc=slc, **kwargs) - tsds = TSDataSampler(data=data, start=start, end=end, step_len=self.step_len, dtype=dtype) - return tsds + flt_col = kwargs.pop('flt_col', None) + # TSDatasetH will retrieve more data for complete + data = self._prepare_raw_seg(slc, **kwargs) + + flt_kwargs = deepcopy(kwargs) + if flt_col is not None: + flt_kwargs['col_set'] = flt_col + flt_data = self._prepare_raw_seg(slc, **flt_kwargs) + assert len(flt_data.columns) == 1 + else: + flt_data = None + + tsds = TSDataSampler(data=data, start=start, end=end, step_len=self.step_len, dtype=dtype, flt_data=flt_data) + return tsds \ No newline at end of file From fa4511cb0a82f994b094df25f226e18bb8deb543 Mon Sep 17 00:00:00 2001 From: blin Date: Wed, 28 Apr 2021 07:30:22 +0000 Subject: [PATCH 48/61] filter --- qlib/data/dataset/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qlib/data/dataset/__init__.py b/qlib/data/dataset/__init__.py index 5485796efb..0bdb5018b4 100644 --- a/qlib/data/dataset/__init__.py +++ b/qlib/data/dataset/__init__.py @@ -505,7 +505,7 @@ def _prepare_seg(self, slc: slice, **kwargs) -> TSDataSampler: """ split the _prepare_raw_seg is to leave a hook for data preprocessing before creating processing data """ - dtype = kwargs.pop("dtype") + dtype = kwargs.pop("dtype", None) start, end = slc.start, slc.stop flt_col = kwargs.pop('flt_col', None) # TSDatasetH will retrieve more data for complete From 40cf83e5572c141fd837c1c2e923499a6a88a31b Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Wed, 28 Apr 2021 09:23:07 +0000 Subject: [PATCH 49/61] online serving V9 middle status --- .../online_srv/online_management_simulate.py | 71 ++--- .../online_srv/rolling_online_management.py | 68 ++-- examples/online_srv/update_online_pred.py | 22 +- qlib/model/trainer.py | 10 + qlib/workflow/online/manager.py | 123 +++++++- qlib/workflow/online/simulator.py | 61 ++-- qlib/workflow/online/strategy.py | 293 ++++++++++++++++++ qlib/workflow/online/utils.py | 165 ++++++++++ qlib/workflow/task/collect.py | 25 ++ 9 files changed, 712 insertions(+), 126 deletions(-) create mode 100644 qlib/workflow/online/strategy.py create mode 100644 qlib/workflow/online/utils.py diff --git a/examples/online_srv/online_management_simulate.py b/examples/online_srv/online_management_simulate.py index 1b1fed6603..6a1d233ae6 100644 --- a/examples/online_srv/online_management_simulate.py +++ b/examples/online_srv/online_management_simulate.py @@ -1,20 +1,23 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +""" +This examples is about the OnlineManager and OnlineSimulator based on rolling tasks. +The OnlineManager will focus on the updating of your online models. +The OnlineSimulator will focus on the simulating real updating routine of your online models. +""" import fire import qlib from qlib.model.ens.ensemble import ens_workflow from qlib.model.trainer import DelayTrainerR, DelayTrainerRM, TrainerRM from qlib.workflow import R -from qlib.workflow.online.manager import RollingOnlineManager -from qlib.workflow.online.simulator import OnlineSimulator +from qlib.workflow.online.manager import OnlineM # RollingOnlineManager +from qlib.workflow.online.strategy import OnlineStrategy, RollingAverageStrategy from qlib.workflow.task.collect import RecorderCollector from qlib.workflow.task.gen import RollingGen, task_generator from qlib.workflow.task.manage import TaskManager from qlib.workflow.task.utils import list_recorders -""" -This examples is about the OnlineManager and OnlineSimulator based on rolling tasks. -The OnlineManager will focus on the updating of your online models. -The OnlineSimulator will focus on the simulating real updating routine of your online models. -""" + data_handler_config = { @@ -105,6 +108,8 @@ def __init__( """ self.exp_name = exp_name self.task_pool = task_pool + self.start_time = start_time + self.end_time = end_time mongo_conf = { "task_url": task_url, "task_db_name": task_db_name, @@ -115,17 +120,18 @@ def __init__( ) # The rolling tasks generator, modify_end_time is false because we just need simulate to 2018-10-31. self.trainer = DelayTrainerRM(self.exp_name, self.task_pool) self.task_manager = TaskManager(self.task_pool) # A good way to manage all your tasks - self.rolling_online_manager = RollingOnlineManager( - experiment_name=exp_name, - rolling_gen=self.rolling_gen, - trainer=self.trainer, + self.rolling_online_manager = OnlineM( + RollingAverageStrategy( + exp_name, task_template=tasks, rolling_gen=self.rolling_gen, trainer=self.trainer, need_log=False + ), + begin_time=self.start_time, need_log=False, ) # The OnlineManager based on Rolling - self.onlinesimulator = OnlineSimulator( - start_time=start_time, - end_time=end_time, - online_manager=self.rolling_online_manager, - ) + # self.onlinesimulator = OnlineSimulator( + # start_time=start_time, + # end_time=end_time, + # online_manager=self.rolling_online_manager, + # ) self.tasks = tasks # Reset all things to the first status, be careful to save important data @@ -137,37 +143,16 @@ def reset(self): for rid in exp.list_recorders(): exp.delete_recorder(rid) - for rid in list_recorders( - RollingOnlineManager.SIGNAL_EXP, lambda x: True if x.info["name"] == self.exp_name else False - ): + for rid in list_recorders("OnlineManagerSignals", lambda x: True if x.info["name"] == self.exp_name else False): exp.delete_recorder(rid) - # Run this firstly to see the workflow in OnlineManager - def first_train(self): - print("========== first train ==========") - self.reset() - self.rolling_online_manager.first_train(self.tasks) - - # Run this secondly to see the simulating in OnlineSimulator - def simulate(self): - print("========== simulate ==========") - self.onlinesimulator.simulate() - print(self.rolling_online_manager.collect_artifact()) - - print("========== online models ==========") - recs_dict = self.onlinesimulator.online_models() - for time, recs in recs_dict.items(): - print(f"{str(time[0])} to {str(time[1])}:") - for rec in recs: - print(rec.info["id"]) - - print("========== online signals ==========") - print(self.rolling_online_manager.get_signals()) - # Run this to run all workflow automaticly def main(self): - self.first_train() - self.simulate() + self.reset() + print("========== simulate ==========") + self.rolling_online_manager.simulate(end_time=self.end_time) + print(self.rolling_online_manager.get_collector()()) + print(self.rolling_online_manager.get_online_history(self.exp_name)) if __name__ == "__main__": diff --git a/examples/online_srv/rolling_online_management.py b/examples/online_srv/rolling_online_management.py index d118afe759..7b2f58909a 100644 --- a/examples/online_srv/rolling_online_management.py +++ b/examples/online_srv/rolling_online_management.py @@ -1,21 +1,22 @@ +""" +This example show how RollingOnlineManager works with rolling tasks. +There are two parts including first train and routine. +Firstly, the RollingOnlineManager will finish the first training and set trained models to `online` models. +Next, the RollingOnlineManager will finish a routine process, including update online prediction -> prepare signals -> prepare tasks -> prepare new models -> reset online models +""" import os from pathlib import Path import pickle import fire import qlib from qlib.workflow import R +from qlib.workflow.online.strategy import OnlineStrategy, RollingAverageStrategy from qlib.workflow.task.gen import RollingGen from qlib.workflow.task.manage import TaskManager -from qlib.workflow.online.manager import RollingOnlineManager +from qlib.workflow.online.manager import OnlineM from qlib.workflow.task.utils import list_recorders from qlib.model.trainer import TrainerRM - -""" -This example show how RollingOnlineManager works with rolling tasks. -There are two parts including first train and routine. -Firstly, the RollingOnlineManager will finish the first training and set trained models to `online` models. -Next, the RollingOnlineManager will finish a routine process, including update online prediction -> prepare signals -> prepare tasks -> prepare new models -> reset online models -""" +from pprint import pprint data_handler_config = { "start_time": "2013-01-01", @@ -77,58 +78,65 @@ class RollingOnlineExample: def __init__( self, - exp_name="rolling_exp", - task_pool="rolling_task", provider_uri="~/.qlib/qlib_data/cn_data", region="cn", task_url="mongodb://10.0.0.4:27017/", task_db_name="rolling_db", rolling_step=550, + tasks=[task_xgboost_config, task_lgb_config], ): - self.exp_name = exp_name - self.task_pool = task_pool mongo_conf = { "task_url": task_url, # your MongoDB url "task_db_name": task_db_name, # database name } qlib.init(provider_uri=provider_uri, region=region, mongo=mongo_conf) - self.rolling_online_manager = RollingOnlineManager( - experiment_name=exp_name, - rolling_gen=RollingGen(step=rolling_step, rtype=RollingGen.ROLL_SD), - trainer=TrainerRM(self.exp_name, self.task_pool), - ) + self.tasks = tasks + self.rolling_step = rolling_step + strategy = [] + for task in tasks: + name_id = task["model"]["class"] + "_" + str(self.rolling_step) + strategy.append( + RollingAverageStrategy( + name_id, + task, + RollingGen(step=rolling_step, rtype=RollingGen.ROLL_SD), + TrainerRM(experiment_name=name_id, task_pool=name_id), + ) + ) + + self.rolling_online_manager = OnlineM(strategy) _ROLLING_MANAGER_PATH = ".rolling_manager" # the RollingOnlineManager will dump to this file, for it will be loaded when calling routine. # Reset all things to the first status, be careful to save important data def reset(self): print("========== reset ==========") - TaskManager(self.task_pool).remove() - exp = R.get_exp(experiment_name=self.exp_name) - for rid in exp.list_recorders(): - exp.delete_recorder(rid) + for task in self.tasks: + name_id = task["model"]["class"] + "_" + str(self.rolling_step) + TaskManager(name_id).remove() + exp = R.get_exp(experiment_name=name_id) + for rid in exp.list_recorders(): + exp.delete_recorder(rid) - if os.path.exists(self._ROLLING_MANAGER_PATH): - os.remove(self._ROLLING_MANAGER_PATH) + if os.path.exists(self._ROLLING_MANAGER_PATH): + os.remove(self._ROLLING_MANAGER_PATH) - for rid in list_recorders( - RollingOnlineManager.SIGNAL_EXP, lambda x: True if x.info["name"] == self.exp_name else False - ): - exp.delete_recorder(rid) + for rid in list_recorders("OnlineManagerSignals", lambda x: True if x.info["name"] == name_id else False): + exp.delete_recorder(rid) def first_run(self): print("========== first_run ==========") self.reset() - self.rolling_online_manager.first_train([task_xgboost_config, task_lgb_config]) + self.rolling_online_manager.first_train() self.rolling_online_manager.to_pickle(self._ROLLING_MANAGER_PATH) - print(self.rolling_online_manager.collect_artifact()) + print(self.rolling_online_manager.get_collector()()) def routine(self): print("========== routine ==========") with Path(self._ROLLING_MANAGER_PATH).open("rb") as f: self.rolling_online_manager = pickle.load(f) self.rolling_online_manager.routine() - print(self.rolling_online_manager.collect_artifact()) + print(self.rolling_online_manager.get_collector()()) def main(self): self.first_run() diff --git a/examples/online_srv/update_online_pred.py b/examples/online_srv/update_online_pred.py index ed2ad6997e..a02b209bde 100644 --- a/examples/online_srv/update_online_pred.py +++ b/examples/online_srv/update_online_pred.py @@ -1,16 +1,14 @@ +""" +This example show how OnlineTool works when we need update prediction. +There are two parts including first_train and update_online_pred. +Firstly, we will finish the training and set the trained model to `online` model. +Next, we will finish updating online prediction. +""" import fire import qlib from qlib.config import REG_CN from qlib.model.trainer import task_train -from qlib.workflow.online.manager import OnlineManagerR -from qlib.workflow.task.utils import list_recorders - -""" -This example show how OnlineManager works when we need update prediction. -There are two parts including first_train and update_online_pred. -Firstly, the RollingOnlineManager will finish the first training and set the trained model to `online` model. -Next, the RollingOnlineManager will finish updating online prediction -""" +from qlib.workflow.online.utils import OnlineToolR data_handler_config = { "start_time": "2008-01-01", @@ -65,15 +63,15 @@ def __init__( ): qlib.init(provider_uri=provider_uri, region=region) self.experiment_name = experiment_name - self.online_manager = OnlineManagerR(self.experiment_name) + self.online_tool = OnlineToolR(self.experiment_name) self.task_config = task_config def first_train(self): rec = task_train(self.task_config, experiment_name=self.experiment_name) - self.online_manager.reset_online_tag(rec) # set to online model + self.online_tool.reset_online_tag(rec) # set to online model def update_online_pred(self): - self.online_manager.update_online_pred() + self.online_tool.update_online_pred() def main(self): self.first_train() diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index af65c58863..0dcc1d67a4 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -25,6 +25,7 @@ def begin_task_train(task_config: dict, experiment_name: str, *args, **kwargs) - Returns: Recorder """ + # FIXME: recorder_id with R.start(experiment_name=experiment_name, recorder_name=str(time.time())): R.log_params(**flatten_dict(task_config)) R.save_objects(**{"task": task_config}) # keep the original format and datatype @@ -112,6 +113,9 @@ def end_train(self, models, *args, **kwargs): """ pass + def is_delay(self): + return False + class TrainerR(Trainer): """Trainer based on (R)ecorder. @@ -240,6 +244,9 @@ def end_train(self, recs=None, end_train_func=None): end_train_func(rec) return recs + def is_delay(self): + return True + class DelayTrainerRM(TrainerRM): """ @@ -286,3 +293,6 @@ def end_train(self, recs, end_train_func=None): before_status=TaskManager.STATUS_PART_DONE, ) return recs + + def is_delay(self): + return True diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index e107271d0a..f8266577b0 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -1,5 +1,14 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +""" +This class is a component of online serving, it can manage a series of models dynamically. +With the change of time, the decisive models will be also changed. In this module, we called those contributing models as `online` models. +In every routine(such as everyday or every minutes), the `online` models maybe changed and the prediction of them need to be updated. +So this module provide a series methods to control this process. +""" from copy import deepcopy -from operator import index +from pprint import pprint import pandas as pd from qlib.model.ens.ensemble import ens_workflow from qlib.model.ens.group import RollingGroup @@ -9,20 +18,13 @@ from qlib.data.data import D from qlib.model.trainer import Trainer, TrainerR, task_train from qlib.workflow import R +from qlib.workflow.online.strategy import OnlineStrategy from qlib.workflow.online.update import PredUpdater from qlib.workflow.recorder import Recorder -from qlib.workflow.task.collect import Collector, RecorderCollector +from qlib.workflow.task.collect import Collector, HyperCollector, RecorderCollector from qlib.workflow.task.gen import RollingGen, task_generator from qlib.workflow.task.utils import TimeAdjuster, list_recorders -""" -This class is a component of online serving, it can manage a series of models dynamically. -With the change of time, the decisive models will be also changed. In this module, we called those contributing models as `online` models. -In every routine(such as everyday or every minutes), the `online` models maybe changed and the prediction of them need to be updated. -So this module provide a series methods to control this process. -""" - - class OnlineManager(Serializable): ONLINE_KEY = "online_status" # the online status key in recorder @@ -357,9 +359,9 @@ def __init__( Args: experiment_name (str): the experiment name. - rolling_gen (RollingGen): a instance of RollingGen - trainer (Trainer, optional): a instance of Trainer. Defaults to None. - collector (Collector, optional): a instance of Collector. Defaults to None. + rolling_gen (RollingGen): an instance of RollingGen + trainer (Trainer, optional): an instance of Trainer. Defaults to None. + collector (Collector, optional): an instance of Collector. Defaults to None. need_log (bool, optional): print log or not. Defaults to True. """ if trainer is None: @@ -475,3 +477,98 @@ def list_latest_recorders(self, rec_filter_func=None): if rec.load_object("task")["dataset"]["kwargs"]["segments"]["test"] == max_test: latest_rec[rid] = rec return latest_rec, max_test + + +class OnlineM(Serializable): + def __init__( + self, strategy: Union[OnlineStrategy, List[OnlineStrategy]], begin_time=None, freq="day", need_log=True + ): + self.logger = get_module_logger(self.__class__.__name__) + self.need_log = need_log + if not isinstance(strategy, list): + strategy = [strategy] + self.strategy = strategy + self.freq = freq + if begin_time is None: + begin_time = D.calendar(freq=self.freq).max() + self.cur_time = pd.Timestamp(begin_time) + self.history = {} + + def first_train(self): + """ + Train a series of models firstly and set some of them into online models. + """ + for strategy in self.strategy: + self.logger.info(f"Strategy `{strategy.name_id}` begins first training...") + online_models = strategy.first_train() + self.history.setdefault(strategy.name_id, {})[self.cur_time] = online_models + + def routine(self, cur_time=None, task_kwargs={}, model_kwargs={}): + """ + The typical update process after a routine, such as day by day or month by month. + update online prediction -> prepare signals -> prepare tasks -> prepare new models -> reset online models + + NOTE: Assumption: if using simulator (delay_prepare is True), the prediction will be prepared well after every training, so there is no need to update predictions. + + Args: + cur_time ([type], optional): [description]. Defaults to None. + delay_prepare (bool, optional): [description]. Defaults to False. + *args, **kwargs: will be passed to `prepare_tasks` and `prepare_new_models`. It can be some hyper parameter or training config. + + Returns: + [type]: [description] + """ + if cur_time is None: + cur_time = D.calendar(freq=self.freq).max() + self.cur_time = pd.Timestamp(cur_time) # None for latest date + for strategy in self.strategy: + self.logger.info(f"Strategy `{strategy.name_id}` begins routine...") + if not strategy.trainer.is_delay(): + strategy.prepare_signals() + tasks = strategy.prepare_tasks(self.cur_time, **task_kwargs) + online_models = strategy.prepare_online_models(tasks, **model_kwargs) + if len(online_models) > 0: + self.history.setdefault(strategy.name_id, {})[self.cur_time] = online_models + + def get_collector(self): + collector_dict = {} + for strategy in self.strategy: + collector_dict[strategy.name_id] = strategy.get_collector() + return HyperCollector(collector_dict) + + def get_online_history(self, strategy_name_id): + history_dict = self.history[strategy_name_id] + history = [] + for time in sorted(history_dict): + models = history_dict[time] + history.append((time, models)) + return history + + def delay_prepare(self, delay_kwargs={}): + """ + Prepare all models and signals if there are something waiting for prepare. + NOTE: Assumption: the predictions of online models are between `time_segment`, or this method will work in a wrong way. + + Args: + rec_dict (str): an online models dict likes {(begin_time, end_time):[online models]}. + *args, **kwargs: will be passed to end_train which means will be passed to customized train method. + """ + for strategy in self.strategy: + strategy.delay_prepare(self.get_online_history(strategy.name_id), **delay_kwargs) + + def simulate(self, end_time, frequency="day", task_kwargs={}, model_kwargs={}, delay_kwargs={}): + """ + Starting from start time, this method will simulate every routine in OnlineManager. + NOTE: Considering the parallel training, the models and signals can be perpared after all routine simulating. + + Returns: + Collector: the OnlineManager's collector + """ + cal = D.calendar(start_time=self.cur_time, end_time=end_time, freq=frequency) + self.first_train() + for cur_time in cal: + self.logger.info(f"Simulating at {str(cur_time)}......") + self.routine(cur_time, task_kwargs=task_kwargs, model_kwargs=model_kwargs) + self.delay_prepare(delay_kwargs=delay_kwargs) + self.logger.info(f"Finished preparing signals") + return self.get_collector() diff --git a/qlib/workflow/online/simulator.py b/qlib/workflow/online/simulator.py index d45b7d99da..ddaf2471c1 100644 --- a/qlib/workflow/online/simulator.py +++ b/qlib/workflow/online/simulator.py @@ -1,6 +1,6 @@ from qlib.data import D from qlib import get_module_logger -from qlib.workflow.online.manager import OnlineManager +from qlib.workflow.online.manager import OnlineM class OnlineSimulator: @@ -32,7 +32,35 @@ def __init__( if len(self.cal) == 0: self.logger.warn(f"There is no need to simulate bacause start_time is larger than end_time.") - def simulate(self, *args, **kwargs): + # def simulate(self, *args, **kwargs): + # """ + # Starting from start time, this method will simulate every routine in OnlineManager. + # NOTE: Considering the parallel training, the models and signals can be perpared after all routine simulating. + + # Returns: + # Collector: the OnlineManager's collector + # """ + # self.rec_dict = {} + # tmp_begin = self.start_time + # tmp_end = None + # self.olm.first_train() + # prev_recorders = self.olm.online_models() + # for cur_time in self.cal: + # self.logger.info(f"Simulating at {str(cur_time)}......") + # recorders = self.olm.routine(cur_time, True, *args, **kwargs) + # if len(recorders) == 0: + # tmp_end = cur_time + # else: + # self.rec_dict[(tmp_begin, tmp_end)] = prev_recorders + # tmp_begin = cur_time + # prev_recorders = recorders + # self.rec_dict[(tmp_begin, self.end_time)] = prev_recorders + # # finished perparing models (and pred) and signals + # self.olm.delay_prepare(self.rec_dict) + # self.logger.info(f"Finished preparing signals") + # return self.olm.get_collector() + + def simulate(self, task_kwargs={}, model_kwargs={}): """ Starting from start time, this method will simulate every routine in OnlineManager. NOTE: Considering the parallel training, the models and signals can be perpared after all routine simulating. @@ -40,33 +68,10 @@ def simulate(self, *args, **kwargs): Returns: Collector: the OnlineManager's collector """ - self.rec_dict = {} - tmp_begin = self.start_time - tmp_end = None - prev_recorders = self.olm.online_models() + self.olm.first_train() for cur_time in self.cal: self.logger.info(f"Simulating at {str(cur_time)}......") - recorders = self.olm.routine(cur_time, True, *args, **kwargs) - if len(recorders) == 0: - tmp_end = cur_time - else: - self.rec_dict[(tmp_begin, tmp_end)] = prev_recorders - tmp_begin = cur_time - prev_recorders = recorders - self.rec_dict[(tmp_begin, self.end_time)] = prev_recorders - # finished perparing models (and pred) and signals - self.olm.delay_prepare(self.rec_dict) + self.olm.routine(cur_time, task_kwargs={}, model_kwargs={}) + self.olm.delay_prepare() self.logger.info(f"Finished preparing signals") return self.olm.get_collector() - - def online_models(self): - """ - Return a online models dict likes {(begin_time, end_time):[online models]}. - - Returns: - dict - """ - if hasattr(self, "rec_dict"): - return self.rec_dict - self.logger.warn(f"Please call `simulate` firstly when calling `online_models`") - return {} diff --git a/qlib/workflow/online/strategy.py b/qlib/workflow/online/strategy.py new file mode 100644 index 0000000000..5e4dcc024b --- /dev/null +++ b/qlib/workflow/online/strategy.py @@ -0,0 +1,293 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +""" +This module is working with OnlineManager, responsing for a set of strategy about how the models are updated and signals are perpared. +""" + +from copy import deepcopy +from typing import List, Union +import pandas as pd +from qlib.data.data import D +from qlib.log import get_module_logger +from qlib.model.ens.group import RollingGroup +from qlib.model.trainer import Trainer, TrainerR +from qlib.workflow import R +from qlib.workflow.online.utils import OnlineTool, OnlineToolR +from qlib.workflow.task.collect import HyperCollector, RecorderCollector +from qlib.workflow.task.gen import RollingGen, task_generator +from qlib.workflow.task.utils import TimeAdjuster, list_recorders + + +class OnlineStrategy: + def __init__(self, name_id: str, trainer: Trainer = None, need_log=True): + """ + init OnlineManager. + + Args: + name_id (str): a unique name or id + trainer (Trainer, optional): a instance of Trainer. Defaults to None. + need_log (bool, optional): print log or not. Defaults to True. + """ + self.name_id = name_id + self.trainer = trainer + self.logger = get_module_logger(self.__class__.__name__) + self.need_log = need_log + self.tool = OnlineTool() + self.history = {} + + def prepare_signals(self, delay=False): + """ + After perparing the data of last routine (a box in box-plot) which means the end of the routine, we can prepare trading signals for next routine. + Must use `pass` even though there is nothing to do. + """ + raise NotImplementedError(f"Please implement the `prepare_signals` method.") + + def prepare_tasks(self, *args, **kwargs): + """ + After the end of a routine, check whether we need to prepare and train some new tasks. + return the new tasks waiting for training. + """ + raise NotImplementedError(f"Please implement the `prepare_tasks` method.") + + def prepare_online_models(self, tasks, check_func=None, **kwargs): + """ + Use trainer to train a list of tasks and set the trained model to `online`. + + Args: + tasks (list): a list of tasks. + tag (str): + `ONLINE_TAG` for first train or additional train + `NEXT_ONLINE_TAG` for reset online model when calling `reset_online_tag` + `OFFLINE_TAG` for train but offline those models + check_func: the method to judge if a model can be online. + The parameter is the model record and return True for online. + None for online every models. + **kwargs: will be passed to end_train which means will be passed to customized train method. + + """ + if check_func is None: + check_func = lambda x: True + online_models = [] + if len(tasks) > 0: + new_models = self.trainer.train(tasks, **kwargs) + for model in new_models: + if check_func(model): + online_models.append(model) + self.tool.reset_online_tag(online_models) + return online_models + + def first_train(self): + """ + Train a series of models firstly and set some of them into online models. + """ + raise NotImplementedError(f"Please implement the `first_train` method.") + + def get_collector(self): + """ + Return the collector. + + Returns: + Collector + """ + raise NotImplementedError(f"Please implement the `get_collector` method.") + + def delay_prepare(self, history, **kwargs): + """ + Prepare all models and signals if there are something waiting for prepare. + NOTE: Assumption: the predictions of online models are between `time_segment`, or this method will work in a wrong way. + + Args: + rec_dict (str): an online models dict likes {(begin_time, end_time):[online models]}. + *args, **kwargs: will be passed to end_train which means will be passed to customized train method. + """ + for time_begin, recs_list in history: + self.trainer.end_train(recs_list, **kwargs) + self.tool.reset_online_tag(recs_list) + self.prepare_signals(delay=True) + + +class RollingAverageStrategy(OnlineStrategy): + + """ + This example strategy always use latest rolling model as online model and prepare trading signals using the average prediction of online models + """ + + def __init__( + self, + name_id: str, + task_template: Union[dict, List[dict]], + rolling_gen: RollingGen, + trainer: Trainer = None, + need_log=True, + signal_exp_name="OnlineManagerSignals", + ): + """ + init OnlineManagerR. + + Assumption: the str of name_id, the experiment name and the trainer's experiment name are same one. + + Args: + name_id (str): a unique name or id. Will be also the name of Experiment. + task_template (Union[dict,List[dict]]): a list of task_template or a single template, which will be used to generate many tasks using rolling_gen. + rolling_gen (RollingGen): an instance of RollingGen + trainer (Trainer, optional): a instance of Trainer. Defaults to None. + need_log (bool, optional): print log or not. Defaults to True. + signal_exp_path (str): a specific experiment to save signals of different experiment. + """ + super().__init__(name_id=name_id, trainer=trainer, need_log=need_log) + self.exp_name = self.name_id + if not isinstance(task_template, list): + task_template = [task_template] + self.task_template = task_template + self.signal_rec = None + self.signal_exp_name = signal_exp_name + self.ta = TimeAdjuster() + self.rg = rolling_gen + self.tool = OnlineToolR(self.exp_name) + + def get_collector(self, rec_key_func=None, rec_filter_func=None): + """ + Get the instance of collector to collect results. The returned collector must can distinguish results in different models. + Assumption: the models can be distinguished based on model name and rolling test segments. + If you do not want this assumption, please implement your own method or use another rec_key_func. + + Args: + rec_key_func (Callable): a function to get the key of a recorder. If None, use recorder id. + rec_filter_func (Callable, optional): filter the recorder by return True or False. Defaults to None. + """ + + def rec_key(recorder): + task_config = recorder.load_object("task") + model_key = task_config["model"]["class"] + rolling_key = task_config["dataset"]["kwargs"]["segments"]["test"] + return model_key, rolling_key + + if rec_key_func is None: + rec_key_func = rec_key + + artifacts_collector = RecorderCollector( + experiment=self.exp_name, + process_list=RollingGroup(), + rec_key_func=rec_key_func, + rec_filter_func=rec_filter_func, + ) + + signals_collector = RecorderCollector( + experiment=self.signal_exp_name, + rec_key_func=lambda rec: rec.info["name"], + rec_filter_func=lambda rec: rec.info["name"] == self.exp_name, + artifacts_path={"signals": "signals"}, + ) + return HyperCollector({"artifacts": artifacts_collector, "signals": signals_collector}) + + def first_train(self): + """ + Use rolling_gen to generate different tasks based on task_template and trained them. + + Returns: + Collector: a instance of a Collector. + """ + tasks = task_generator( + tasks=self.task_template, + generators=self.rg, # generate different date segment + ) + return self.prepare_online_models(tasks) + + def prepare_tasks(self, cur_time): + """ + Prepare new tasks based on cur_time (None for latest). + + Returns: + list: a list of new tasks. + """ + latest_records, max_test = self._list_latest(self.tool.online_models()) + if max_test is None: + self.logger.warn(f"No latest online recorders, no new tasks.") + return [] + calendar_latest = D.calendar(end_time=cur_time)[-1] if cur_time is None else cur_time + if self.need_log: + self.logger.info( + f"The interval between current time {calendar_latest} and last rolling test begin time {max_test[0]} is {self.ta.cal_interval(calendar_latest, max_test[0])}, the rolling step is {self.rg.step}" + ) + if self.ta.cal_interval(calendar_latest, max_test[0]) >= self.rg.step: + old_tasks = [] + tasks_tmp = [] + for rec in latest_records: + task = rec.load_object("task") + old_tasks.append(deepcopy(task)) + test_begin = task["dataset"]["kwargs"]["segments"]["test"][0] + # modify the test segment to generate new tasks + task["dataset"]["kwargs"]["segments"]["test"] = (test_begin, calendar_latest) + tasks_tmp.append(task) + new_tasks_tmp = task_generator(tasks_tmp, self.rg) + new_tasks = [task for task in new_tasks_tmp if task not in old_tasks] + return new_tasks + return [] + + def prepare_signals(self, delay=False, over_write=False): + """ + Average the predictions of online models and offer a trading signals every routine. + The signals will be saved to `signal` file of a recorder named self.exp_name of a experiment using the name of `SIGNAL_EXP` + Even if the latest signal already exists, the latest calculation result will be overwritten. + NOTE: Given a prediction of a certain time, all signals before this time will be prepared well. + Args: + over_write (bool, optional): If True, the new signals will overwrite the file. If False, the new signals will append to the end of signals. Defaults to False. + Returns: + object: the signals. + """ + if not delay: + self.tool.update_online_pred() + if self.signal_rec is None: + with R.start(experiment_name=self.signal_exp_name, recorder_name=self.exp_name, resume=True): + self.signal_rec = R.get_recorder() + + pred = [] + try: + old_signals = self.signal_rec.load_object("signals") + except OSError: + old_signals = None + + for rec in self.tool.online_models(): + pred.append(rec.load_object("pred.pkl")) + + signals = pd.concat(pred, axis=1).mean(axis=1).to_frame("score") + signals = signals.sort_index() + if old_signals is not None and not over_write: + old_max = old_signals.index.get_level_values("datetime").max() + new_signals = signals.loc[old_max:] + signals = pd.concat([old_signals, new_signals], axis=0) + else: + new_signals = signals + if self.need_log: + self.logger.info( + f"Finished preparing new {len(new_signals)} signals to {self.signal_exp_name}/{self.exp_name}." + ) + self.signal_rec.save_objects(**{"signals": signals}) + return signals + + # def get_signals(self): + # """ + # get signals from the recorder(named self.exp_name) of the experiment(named self.SIGNAL_EXP) + + # Returns: + # signals + # """ + # if self.signal_rec is None: + # with R.start(experiment_name=self.signal_exp_name, recorder_name=self.exp_name, resume=True): + # self.signal_rec = R.get_recorder() + # signals = None + # try: + # signals = self.signal_rec.load_object("signals") + # except OSError: + # self.logger.warn("Can not find `signals`, have you called `prepare_signals` before?") + # return signals + + def _list_latest(self, rec_list): + if len(rec_list) == 0: + return rec_list, None + max_test = max(rec.load_object("task")["dataset"]["kwargs"]["segments"]["test"] for rec in rec_list) + latest_rec = [] + for rec in rec_list: + if rec.load_object("task")["dataset"]["kwargs"]["segments"]["test"] == max_test: + latest_rec.append(rec) + return latest_rec, max_test diff --git a/qlib/workflow/online/utils.py b/qlib/workflow/online/utils.py new file mode 100644 index 0000000000..1cd89d6680 --- /dev/null +++ b/qlib/workflow/online/utils.py @@ -0,0 +1,165 @@ +""" +This module is like a online backend, deciding which models are `online` models and how can change them +""" +from typing import List, Union +from qlib.log import get_module_logger +from qlib.workflow.online.update import PredUpdater +from qlib.workflow.recorder import Recorder +from qlib.workflow.task.utils import list_recorders + + +class OnlineTool: + + ONLINE_KEY = "online_status" # the online status key in recorder + ONLINE_TAG = "online" # the 'online' model + # NOTE: The meaning of this tag is that we can not assume the training models can be trained before we need its predition. Whenever finished training, it can be guaranteed that there are some online models. + NEXT_ONLINE_TAG = "next_online" # the 'next online' model, which can be 'online' model when call reset_online_model + OFFLINE_TAG = "offline" # the 'offline' model, not for online serving + + def __init__(self, need_log=True): + """ + init OnlineTool. + + Args: + need_log (bool, optional): print log or not. Defaults to True. + """ + self.logger = get_module_logger(self.__class__.__name__) + self.need_log = need_log + self.cur_time = None + + def set_online_tag(self, tag, recorder): + """ + Set `tag` to the model to sign whether online. + + Args: + tag (str): the tags in `ONLINE_TAG`, `NEXT_ONLINE_TAG`, `OFFLINE_TAG` + """ + raise NotImplementedError(f"Please implement the `set_online_tag` method.") + + def get_online_tag(self): + """ + Given a model and return its online tag. + """ + raise NotImplementedError(f"Please implement the `get_online_tag` method.") + + def reset_online_tag(self, recorders=None): + """offline all models and set the recorders to 'online'. If no parameter and no 'next online' model, then do nothing. + + Args: + recorders (List, optional): + the recorders you want to reset to 'online'. If don't give, set 'next online' model to 'online' model. If there isn't any 'next online' model, then maintain existing 'online' model. + + Returns: + list: new online recorder. [] if there is no update. + """ + raise NotImplementedError(f"Please implement the `reset_online_tag` method.") + + def online_models(self): + """ + Return `online` models. + """ + raise NotImplementedError(f"Please implement the `online_models` method.") + + def update_online_pred(self, to_date=None): + """ + Update the predictions of online models to a date. + + Args: + to_date (pd.Timestamp): the pred before this date will be updated. None for latest. + + """ + raise NotImplementedError(f"Please implement the `update_online_pred` method.") + + +class OnlineToolR(OnlineTool): + """ + The implementation of OnlineTool based on (R)ecorder. + + """ + + def __init__(self, experiment_name: str, need_log=True): + """ + init OnlineToolR. + + Args: + experiment_name (str): the experiment name. + need_log (bool, optional): print log or not. Defaults to True. + """ + super().__init__(need_log=need_log) + self.exp_name = experiment_name + + def set_online_tag(self, tag, recorder: Union[Recorder, List]): + """ + Set `tag` to the model to sign whether online. + + Args: + tag (str): the tags in `ONLINE_TAG`, `NEXT_ONLINE_TAG`, `OFFLINE_TAG` + recorder (Union[Recorder, List]) + """ + if isinstance(recorder, Recorder): + recorder = [recorder] + for rec in recorder: + rec.set_tags(**{self.ONLINE_KEY: tag}) + if self.need_log: + self.logger.info(f"Set {len(recorder)} models to '{tag}'.") + + def get_online_tag(self, recorder: Recorder): + """ + Given a model and return its online tag. + + Args: + recorder (Recorder): a instance of recorder + + Returns: + str: the tag + """ + tags = recorder.list_tags() + return tags.get(self.ONLINE_KEY, self.OFFLINE_TAG) + + def reset_online_tag(self, recorder: Union[Recorder, List] = None): + """offline all models and set the recorders to 'online'. If no parameter and no 'next online' model, then do nothing. + + Args: + recorders (Union[Recorder, List], optional): + the recorders you want to reset to 'online'. If don't give, set 'next online' model to 'online' model. If there isn't any 'next online' model, then maintain existing 'online' model. + + Returns: + list: new online recorder. [] if there is no update. + """ + if recorder is None: + recorder = list( + list_recorders(self.exp_name, lambda rec: self.get_online_tag(rec) == self.NEXT_ONLINE_TAG).values() + ) + if isinstance(recorder, Recorder): + recorder = [recorder] + if len(recorder) == 0: + if self.need_log: + self.logger.info("No 'next online' model, just use current 'online' models.") + return [] + recs = list_recorders(self.exp_name) + self.set_online_tag(self.OFFLINE_TAG, list(recs.values())) + self.set_online_tag(self.ONLINE_TAG, recorder) + return recorder + + def online_models(self): + """ + Return online models. + + Returns: + list: the list of online models + """ + return list(list_recorders(self.exp_name, lambda rec: self.get_online_tag(rec) == self.ONLINE_TAG).values()) + + def update_online_pred(self, to_date=None): + """ + Update the predictions of online models to a date. + + Args: + to_date (pd.Timestamp): the pred before this date will be updated. None for latest in Calendar. + """ + online_models = self.online_models() + for rec in online_models: + PredUpdater(rec, to_date=to_date, need_log=self.need_log).update() + + if self.need_log: + self.logger.info(f"Finished updating {len(online_models)} online model predictions of {self.exp_name}.") diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index b4c81122d2..eb0a20029c 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -1,5 +1,6 @@ from abc import abstractmethod from typing import Callable, Union +from qlib import init from qlib.workflow import R from qlib.workflow.task.utils import list_recorders from qlib.utils.serial import Serializable @@ -109,6 +110,27 @@ def load(filepath): raise TypeError(f"The instance of {type(collector)} is not a valid `Collector`!") +class HyperCollector(Collector): + """ + A collector to collect the results of other Collectors + """ + + def __init__(self, collector_dict, process_list=[]): + """ + Args: + collector_dict (dict): the dict like {collector_key, Collector} + process_list (list or Callable): the list of processors or the instance of processor to process dict. + """ + super().__init__(process_list=process_list) + self.collector_dict = collector_dict + + def collect(self): + collect_dict = {} + for key, collector in self.collector_dict.items(): + collect_dict[key] = collector() + return collect_dict + + class RecorderCollector(Collector): ART_KEY_RAW = "__raw" @@ -180,3 +202,6 @@ def collect(self, artifacts_key=None, rec_filter_func=None): collect_dict.setdefault(key, {})[rec_key] = artifact return collect_dict + + def get_exp_name(self): + return self.experiment.name From 67c5740c83b428519427854efb214e58c28eb9ab Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Thu, 29 Apr 2021 04:30:09 +0000 Subject: [PATCH 50/61] OnlineServing V9 --- .../model_rolling/task_manager_rolling.py | 29 +- .../online_srv/online_management_simulate.py | 54 +- .../online_srv/rolling_online_management.py | 43 +- examples/online_srv/update_online_pred.py | 3 + qlib/data/dataset/__init__.py | 8 +- qlib/model/ens/ensemble.py | 45 +- qlib/model/ens/group.py | 15 +- qlib/model/task.py | 27 - qlib/model/trainer.py | 286 ++++++---- qlib/utils/serial.py | 5 +- qlib/workflow/online/manager.py | 540 ++---------------- qlib/workflow/online/simulator.py | 77 --- qlib/workflow/online/strategy.py | 103 +++- qlib/workflow/online/update.py | 88 +-- qlib/workflow/online/utils.py | 94 +-- qlib/workflow/task/collect.py | 32 +- qlib/workflow/task/gen.py | 24 +- qlib/workflow/task/manage.py | 151 +++-- qlib/workflow/task/utils.py | 61 +- 19 files changed, 676 insertions(+), 1009 deletions(-) delete mode 100644 qlib/model/task.py delete mode 100644 qlib/workflow/online/simulator.py diff --git a/examples/model_rolling/task_manager_rolling.py b/examples/model_rolling/task_manager_rolling.py index ab3a4eee52..1753198858 100644 --- a/examples/model_rolling/task_manager_rolling.py +++ b/examples/model_rolling/task_manager_rolling.py @@ -1,24 +1,23 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +""" +This example shows how a TrainerRM work based on TaskManager with rolling tasks. +After training, how to collect the rolling results will be showed in task_collecting. +""" + from pprint import pprint -import time import fire import qlib from qlib.config import REG_CN -from qlib.model.trainer import TrainerR, task_train from qlib.workflow import R from qlib.workflow.task.gen import RollingGen, task_generator -from qlib.workflow.task.manage import TaskManager, run_task +from qlib.workflow.task.manage import TaskManager from qlib.workflow.task.collect import RecorderCollector -from qlib.model.ens.ensemble import RollingEnsemble, ens_workflow -import pandas as pd -from qlib.workflow.task.utils import list_recorders from qlib.model.ens.group import RollingGroup from qlib.model.trainer import TrainerRM -""" -This example shows how a Trainer work based on TaskManager with rolling tasks. -After training, how to collect the rolling results will be showed in task_collecting. -""" data_handler_config = { "start_time": "2008-01-01", @@ -139,11 +138,13 @@ def my_filter(recorder): return True return False - artifact = ens_workflow( - RecorderCollector(experiment=self.experiment_name, rec_key_func=rec_key, rec_filter_func=my_filter), - RollingGroup(), + collector = RecorderCollector( + experiment=self.experiment_name, + process_list=RollingGroup(), + rec_key_func=rec_key, + rec_filter_func=my_filter, ) - print(artifact) + print(collector()) def main(self): self.reset() diff --git a/examples/online_srv/online_management_simulate.py b/examples/online_srv/online_management_simulate.py index 6a1d233ae6..16e985ccd3 100644 --- a/examples/online_srv/online_management_simulate.py +++ b/examples/online_srv/online_management_simulate.py @@ -1,23 +1,17 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. + """ -This examples is about the OnlineManager and OnlineSimulator based on rolling tasks. -The OnlineManager will focus on the updating of your online models. -The OnlineSimulator will focus on the simulating real updating routine of your online models. +This examples is about how can simulate the OnlineManager based on rolling tasks. """ + import fire import qlib -from qlib.model.ens.ensemble import ens_workflow -from qlib.model.trainer import DelayTrainerR, DelayTrainerRM, TrainerRM -from qlib.workflow import R -from qlib.workflow.online.manager import OnlineM # RollingOnlineManager -from qlib.workflow.online.strategy import OnlineStrategy, RollingAverageStrategy -from qlib.workflow.task.collect import RecorderCollector -from qlib.workflow.task.gen import RollingGen, task_generator +from qlib.model.trainer import DelayTrainerRM +from qlib.workflow.online.manager import OnlineManager +from qlib.workflow.online.strategy import RollingAverageStrategy +from qlib.workflow.task.gen import RollingGen from qlib.workflow.task.manage import TaskManager -from qlib.workflow.task.utils import list_recorders - - data_handler_config = { @@ -89,10 +83,10 @@ def __init__( rolling_step=80, start_time="2018-09-10", end_time="2018-10-31", - tasks=[task_xgboost_config], # , task_lgb_config] + tasks=[task_xgboost_config, task_lgb_config], ): """ - init OnlineManagerExample. + Init OnlineManagerExample. Args: provider_uri (str, optional): the provider uri. Defaults to "~/.qlib/qlib_data/cn_data". @@ -120,42 +114,28 @@ def __init__( ) # The rolling tasks generator, modify_end_time is false because we just need simulate to 2018-10-31. self.trainer = DelayTrainerRM(self.exp_name, self.task_pool) self.task_manager = TaskManager(self.task_pool) # A good way to manage all your tasks - self.rolling_online_manager = OnlineM( + self.rolling_online_manager = OnlineManager( RollingAverageStrategy( exp_name, task_template=tasks, rolling_gen=self.rolling_gen, trainer=self.trainer, need_log=False ), begin_time=self.start_time, need_log=False, - ) # The OnlineManager based on Rolling - # self.onlinesimulator = OnlineSimulator( - # start_time=start_time, - # end_time=end_time, - # online_manager=self.rolling_online_manager, - # ) + ) self.tasks = tasks - # Reset all things to the first status, be careful to save important data - def reset(self): - print("========== reset ==========") - self.task_manager.remove() - - exp = R.get_exp(experiment_name=self.exp_name) - for rid in exp.list_recorders(): - exp.delete_recorder(rid) - - for rid in list_recorders("OnlineManagerSignals", lambda x: True if x.info["name"] == self.exp_name else False): - exp.delete_recorder(rid) - - # Run this to run all workflow automaticly + # Run this to run all workflow automatically def main(self): - self.reset() + print("========== reset ==========") + self.rolling_online_manager.reset() print("========== simulate ==========") self.rolling_online_manager.simulate(end_time=self.end_time) + print("========== collect results ==========") print(self.rolling_online_manager.get_collector()()) + print("========== online history ==========") print(self.rolling_online_manager.get_online_history(self.exp_name)) if __name__ == "__main__": - ## to run all workflow automaticly with your own parameters, use the command below + ## to run all workflow automatically with your own parameters, use the command below # python online_management_simulate.py main --experiment_name="your_exp_name" --rolling_step=60 fire.Fire(OnlineSimulationExample) diff --git a/examples/online_srv/rolling_online_management.py b/examples/online_srv/rolling_online_management.py index 7b2f58909a..950c9684d4 100644 --- a/examples/online_srv/rolling_online_management.py +++ b/examples/online_srv/rolling_online_management.py @@ -1,22 +1,25 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + """ -This example show how RollingOnlineManager works with rolling tasks. +This example show how OnlineManager works with rolling tasks. There are two parts including first train and routine. -Firstly, the RollingOnlineManager will finish the first training and set trained models to `online` models. -Next, the RollingOnlineManager will finish a routine process, including update online prediction -> prepare signals -> prepare tasks -> prepare new models -> reset online models +Firstly, the OnlineManager will finish the first training and set trained models to `online` models. +Next, the OnlineManager will finish a routine process, including update online prediction -> prepare signals -> prepare tasks -> prepare new models -> reset online models """ + import os from pathlib import Path import pickle import fire import qlib from qlib.workflow import R -from qlib.workflow.online.strategy import OnlineStrategy, RollingAverageStrategy +from qlib.workflow.online.strategy import RollingAverageStrategy from qlib.workflow.task.gen import RollingGen from qlib.workflow.task.manage import TaskManager -from qlib.workflow.online.manager import OnlineM +from qlib.workflow.online.manager import OnlineManager from qlib.workflow.task.utils import list_recorders from qlib.model.trainer import TrainerRM -from pprint import pprint data_handler_config = { "start_time": "2013-01-01", @@ -94,7 +97,7 @@ def __init__( self.rolling_step = rolling_step strategy = [] for task in tasks: - name_id = task["model"]["class"] + "_" + str(self.rolling_step) + name_id = task["model"]["class"] # NOTE: Assumption: The model class can specify only one strategy strategy.append( RollingAverageStrategy( name_id, @@ -104,9 +107,12 @@ def __init__( ) ) - self.rolling_online_manager = OnlineM(strategy) + self.rolling_online_manager = OnlineManager(strategy) + self.collector = self.rolling_online_manager.get_collector() - _ROLLING_MANAGER_PATH = ".rolling_manager" # the RollingOnlineManager will dump to this file, for it will be loaded when calling routine. + _ROLLING_MANAGER_PATH = ( + ".RollingOnlineExample" # the OnlineManager will dump to this file, for it can be loaded when calling routine. + ) # Reset all things to the first status, be careful to save important data def reset(self): @@ -125,18 +131,23 @@ def reset(self): exp.delete_recorder(rid) def first_run(self): + print("========== reset ==========") + self.rolling_online_manager.reset() print("========== first_run ==========") - self.reset() self.rolling_online_manager.first_train() + print("========== dump ==========") self.rolling_online_manager.to_pickle(self._ROLLING_MANAGER_PATH) - print(self.rolling_online_manager.get_collector()()) + print("========== collect results ==========") + print(self.collector()) def routine(self): - print("========== routine ==========") + print("========== load ==========") with Path(self._ROLLING_MANAGER_PATH).open("rb") as f: self.rolling_online_manager = pickle.load(f) + print("========== routine ==========") self.rolling_online_manager.routine() - print(self.rolling_online_manager.get_collector()()) + print("========== collect results ==========") + print(self.collector()) def main(self): self.first_run() @@ -145,11 +156,11 @@ def main(self): if __name__ == "__main__": ####### to train the first version's models, use the command below - # python task_manager_rolling_with_updating.py first_run + # python rolling_online_management.py first_run ####### to update the models and predictions after the trading time, use the command below - # python task_manager_rolling_with_updating.py after_day + # python rolling_online_management.py after_day ####### to define your own parameters, use `--` - # python task_manager_rolling_with_updating.py first_run --exp_name='your_exp_name' --rolling_step=40 + # python rolling_online_management.py first_run --exp_name='your_exp_name' --rolling_step=40 fire.Fire(RollingOnlineExample) diff --git a/examples/online_srv/update_online_pred.py b/examples/online_srv/update_online_pred.py index a02b209bde..6e2725c7a2 100644 --- a/examples/online_srv/update_online_pred.py +++ b/examples/online_srv/update_online_pred.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + """ This example show how OnlineTool works when we need update prediction. There are two parts including first_train and update_online_pred. diff --git a/qlib/data/dataset/__init__.py b/qlib/data/dataset/__init__.py index 5485796efb..4457dda5fa 100644 --- a/qlib/data/dataset/__init__.py +++ b/qlib/data/dataset/__init__.py @@ -299,7 +299,7 @@ def __init__(self, data: pd.DataFrame, start, end, step_len: int, fillna_type: s self.start_idx, self.end_idx = self.data_index.slice_locs(start=pd.Timestamp(start), end=pd.Timestamp(end)) self.idx_arr = np.array(self.idx_df.values, dtype=np.float64) # for better performance - + del self.data # save memory @staticmethod @@ -507,17 +507,17 @@ def _prepare_seg(self, slc: slice, **kwargs) -> TSDataSampler: """ dtype = kwargs.pop("dtype") start, end = slc.start, slc.stop - flt_col = kwargs.pop('flt_col', None) + flt_col = kwargs.pop("flt_col", None) # TSDatasetH will retrieve more data for complete data = self._prepare_raw_seg(slc, **kwargs) flt_kwargs = deepcopy(kwargs) if flt_col is not None: - flt_kwargs['col_set'] = flt_col + flt_kwargs["col_set"] = flt_col flt_data = self._prepare_raw_seg(slc, **flt_kwargs) assert len(flt_data.columns) == 1 else: flt_data = None tsds = TSDataSampler(data=data, start=start, end=end, step_len=self.step_len, dtype=dtype, flt_data=flt_data) - return tsds \ No newline at end of file + return tsds diff --git a/qlib/model/ens/ensemble.py b/qlib/model/ens/ensemble.py index 63f6438c27..7ccf98ab2e 100644 --- a/qlib/model/ens/ensemble.py +++ b/qlib/model/ens/ensemble.py @@ -1,36 +1,11 @@ -from abc import abstractmethod -from typing import Callable, Union +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. -import pandas as pd -from qlib.workflow.task.collect import Collector -from qlib.utils.serial import Serializable - - -def ens_workflow(collector: Collector, process_list, *args, **kwargs): - """the ensemble workflow based on collector and different dict processors. +""" +Ensemble can merge the objects in an Ensemble. For example, if there are many submodels predictions, we may need to merge them in an ensemble predictions. +""" - Args: - collector (Collector): the collector to collect the result into {result_key: things} - process_list (list or Callable): the list of processors or the instance of processor to process dict. - The processor order is same as the list order. - For example: [Group1(..., Ensemble1()), Group2(..., Ensemble2())] - Returns: - dict: the ensemble dict - """ - collect_dict = collector.collect() - if not isinstance(process_list, list): - process_list = [process_list] - - ensemble = {} - for artifact in collect_dict: - value = collect_dict[artifact] - for process in process_list: - if not callable(process): - raise NotImplementedError(f"{type(process)} is not supported in `ens_workflow`.") - value = process(value, *args, **kwargs) - ensemble[artifact] = value - - return ensemble +import pandas as pd class Ensemble: @@ -53,17 +28,17 @@ class RollingEnsemble(Ensemble): """Merge the rolling objects in an Ensemble""" - def __call__(self, ensemble_dict: dict): + def __call__(self, ensemble_dict: dict) -> pd.DataFrame: """Merge a dict of rolling dataframe like `prediction` or `IC` into an ensemble. - NOTE: The values of dict must be pd.Dataframe, and have the index "datetime" + NOTE: The values of dict must be pd.DataFrame, and have the index "datetime" Args: - ensemble_dict (dict): a dict like {"A": pd.Dataframe, "B": pd.Dataframe}. + ensemble_dict (dict): a dict like {"A": pd.DataFrame, "B": pd.DataFrame}. The key of the dict will be ignored. Returns: - pd.Dataframe: the complete result of rolling. + pd.DataFrame: the complete result of rolling. """ artifact_list = list(ensemble_dict.values()) artifact_list.sort(key=lambda x: x.index.get_level_values("datetime").min()) diff --git a/qlib/model/ens/group.py b/qlib/model/ens/group.py index c80959b0d5..d53a55f4c8 100644 --- a/qlib/model/ens/group.py +++ b/qlib/model/ens/group.py @@ -1,3 +1,10 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +""" +Group can group a set of object based on `group_func` and change them to a dict. +""" + from qlib.model.ens.ensemble import Ensemble, RollingEnsemble from typing import Callable, Union from joblib import Parallel, delayed @@ -21,20 +28,20 @@ def __init__(self, group_func=None, ens: Ensemble = None): self._group_func = group_func self._ens_func = ens - def group(self, *args, **kwargs): + def group(self, *args, **kwargs) -> dict: # TODO: such design is weird when `_group_func` is the only configurable part in the class if isinstance(getattr(self, "_group_func", None), Callable): return self._group_func(*args, **kwargs) else: raise NotImplementedError(f"Please specify valid `group_func`.") - def reduce(self, *args, **kwargs): + def reduce(self, *args, **kwargs) -> dict: if isinstance(getattr(self, "_ens_func", None), Callable): return self._ens_func(*args, **kwargs) else: raise NotImplementedError(f"Please specify valid `_ens_func`.") - def __call__(self, ungrouped_dict: dict, n_jobs=1, verbose=0, *args, **kwargs): + def __call__(self, ungrouped_dict: dict, n_jobs=1, verbose=0, *args, **kwargs) -> dict: """Group the ungrouped_dict into different groups. Args: @@ -59,7 +66,7 @@ def __call__(self, ungrouped_dict: dict, n_jobs=1, verbose=0, *args, **kwargs): class RollingGroup(Group): """group the rolling dict""" - def group(self, rolling_dict: dict): + def group(self, rolling_dict: dict) -> dict: """Given an rolling dict likes {(A,B,R): things}, return the grouped dict likes {(A,B): {R:things}} NOTE: There is a assumption which is the rolling key is at the end of key tuple, because the rolling results always need to be ensemble firstly. diff --git a/qlib/model/task.py b/qlib/model/task.py deleted file mode 100644 index f29f513a4e..0000000000 --- a/qlib/model/task.py +++ /dev/null @@ -1,27 +0,0 @@ -import abc -import typing - - -class TaskGen(metaclass=abc.ABCMeta): - @abc.abstractmethod - def __call__(self, *args, **kwargs) -> typing.List[dict]: - """ - generate - - Parameters - ---------- - args, kwargs: - The info for generating tasks - Example 1): - input: a specific task template - output: rolling version of the tasks - Example 2): - input: a specific task template - output: a set of tasks with different losses - - Returns - ------- - typing.List[dict]: - A list of tasks - """ - pass diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index 0dcc1d67a4..a0d252ab44 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -1,59 +1,72 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -import copy +""" +The Trainer will train a list of tasks and return a list of model recorder. +There are two steps in each Trainer including `train`(make model recorder) and `end_train`(modify model recorder). + +This is concept called "DelayTrainer", which can be used in online simulating to parallel training. +In "DelayTrainer", the first step is only to save some necessary info to model recorder, and the second step which will be finished in the end can do some concurrent and time-consuming operations such as model fitting. + +`Qlib` offer two kind of Trainer, TrainerR is simplest and TrainerRM is based on TaskManager to help manager tasks lifecycle automatically. +""" + +import socket import time -from xxlimited import Str -from qlib.utils import init_instance_by_config, flatten_dict, get_cls_kwargs +from typing import Callable, List + +from qlib.data.dataset import Dataset +from qlib.model.base import Model +from qlib.utils import flatten_dict, get_cls_kwargs, init_instance_by_config from qlib.workflow import R -from qlib.workflow.recorder import Recorder from qlib.workflow.record_temp import SignalRecord +from qlib.workflow.recorder import Recorder from qlib.workflow.task.manage import TaskManager, run_task -from qlib.data.dataset import Dataset -from qlib.model.base import Model -import socket -def begin_task_train(task_config: dict, experiment_name: str, *args, **kwargs) -> Recorder: +def begin_task_train(task_config: dict, experiment_name: str, recorder_name: str = None) -> Recorder: """ - Begin a task training with starting a recorder and saving the task config. + Begin a task training to start a recorder and save the task config. Args: - task_config (dict) - experiment_name (str) + task_config (dict): the config of a task + experiment_name (str): the name of experiment + recorder_name (str): the given name will be the recorder name. None for using rid. Returns: - Recorder + Recorder: the model recorder """ # FIXME: recorder_id - with R.start(experiment_name=experiment_name, recorder_name=str(time.time())): + if recorder_name is None: + recorder_name = str(time.time()) + with R.start(experiment_name=experiment_name, recorder_name=recorder_name): R.log_params(**flatten_dict(task_config)) R.save_objects(**{"task": task_config}) # keep the original format and datatype - R.set_tags(**{"hostname": socket.gethostname(), "train_status": "begin_task_train"}) + R.set_tags(**{"hostname": socket.gethostname()}) recorder: Recorder = R.get_recorder() return recorder -def end_task_train(rec: Recorder, experiment_name: str, *args, **kwargs): +def end_task_train(rec: Recorder, experiment_name: str) -> Recorder: """ - Finished task training with real model fitting and saving. + Finish task training with real model fitting and saving. Args: - rec (Recorder): This recorder will be resumed - experiment_name (str) + rec (Recorder): the recorder will be resumed + experiment_name (str): the name of experiment Returns: - Recorder + Recorder: the model recorder """ with R.start(experiment_name=experiment_name, recorder_name=rec.info["name"], resume=True): task_config = R.load_object("task") - # model & dataset initiaiton + # model & dataset initiation model: Model = init_instance_by_config(task_config["model"]) dataset: Dataset = init_instance_by_config(task_config["dataset"]) # model training model.fit(dataset) R.save_objects(**{"params.pkl": model}) - # This dataset is saved for online inference. So the concrete data should not be dumped + # this dataset is saved for online inference. So the concrete data should not be dumped dataset.config(dump_all=False, recursive=True) R.save_objects(**{"dataset": dataset}) # generate records: prediction, backtest, and analysis @@ -68,18 +81,18 @@ def end_task_train(rec: Recorder, experiment_name: str, *args, **kwargs): rconf = {"recorder": rec} r = cls(**kwargs, **rconf) r.generate() - R.set_tags(**{"train_status": "end_task_train"}) + return rec def task_train(task_config: dict, experiment_name: str) -> Recorder: """ - task based training + Task based training, will be divided into two steps. Parameters ---------- task_config : dict - A dict describes a task setting. + The config of a task. experiment_name: str The name of experiment @@ -97,42 +110,79 @@ class Trainer: The trainer which can train a list of model """ - def train(self, tasks: list, *args, **kwargs): - """Given a list of model definition, begin a training and return the models. + def __init__(self): + self.delay = False + + def train(self, tasks: list, *args, **kwargs) -> list: + """ + Given a list of model definition, begin a training and return the models. + + Args: + tasks: a list of tasks Returns: list: a list of models """ raise NotImplementedError(f"Please implement the `train` method.") - def end_train(self, models, *args, **kwargs): - """Given a list of models, finished something in the end of training if you need. + def end_train(self, models: list, *args, **kwargs) -> list: + """ + Given a list of models, finished something in the end of training if you need. + The models maybe Recorder, txt file, database and so on. + + Args: + models: a list of models Returns: list: a list of models """ - pass + # do nothing if you finished all work in `train` method + return models - def is_delay(self): - return False + def is_delay(self) -> bool: + """ + If Trainer will delay finishing `end_train`. + + Returns: + bool: if DelayTrainer + """ + return self.delay + + def reset(self): + """ + Reset the Trainer status. + """ + pass class TrainerR(Trainer): - """Trainer based on (R)ecorder. + """ + Trainer based on (R)ecorder. + It will train a list of tasks and return a list of model recorder in a linear way. Assumption: models were defined by `task` and the results will saved to `Recorder` """ - def __init__(self, experiment_name, train_func=task_train): + def __init__(self, experiment_name: str, train_func: Callable = task_train): + """ + Init TrainerR. + + Args: + experiment_name (str): the name of experiment. + train_func (Callable, optional): default training method. Defaults to `task_train`. + """ + super().__init__() self.experiment_name = experiment_name self.train_func = train_func - def train(self, tasks: list, train_func=None, *args, **kwargs): - """Given a list of `task`s and return a list of trained Recorder. The order can be guaranteed. + def train(self, tasks: list, train_func: Callable = None, **kwargs) -> List[Recorder]: + """ + Given a list of `task`s and return a list of trained Recorder. The order can be guaranteed. Args: tasks (list): a list of definition based on `task` dict - train_func (Callable): the train method which need at least `task` and `experiment_name`. None for default. + train_func (Callable): the train method which need at least `task`s and `experiment_name`. None for default training method. + kwargs: the params for train_func. Returns: list: a list of Recorders @@ -141,17 +191,74 @@ def train(self, tasks: list, train_func=None, *args, **kwargs): train_func = self.train_func recs = [] for task in tasks: - recs.append(train_func(task, self.experiment_name, *args, **kwargs)) + rec = train_func(task, self.experiment_name, **kwargs) + rec.set_tags(**{"train_status": "begin_task_train"}) + recs.append(rec) + return recs + + def end_train(self, recs: list, **kwargs) -> list: + for rec in recs: + rec.set_tags(**{"train_status": "end_task_train"}) + return recs + + +class DelayTrainerR(TrainerR): + """ + A delayed implementation based on TrainerR, which means `train` method may only do some preparation and `end_train` method can do the real model fitting. + """ + + def __init__(self, experiment_name, train_func=begin_task_train, end_train_func=end_task_train): + """ + Init TrainerRM. + + Args: + experiment_name (str): the name of experiment. + train_func (Callable, optional): default train method. Defaults to `begin_task_train`. + end_train_func (Callable, optional): default end_train method. Defaults to `end_task_train`. + """ + super().__init__(experiment_name, train_func) + self.end_train_func = end_train_func + self.delay = True + + def end_train(self, recs, end_train_func=None, **kwargs) -> List[Recorder]: + """ + Given a list of Recorder and return a list of trained Recorder. + This class will finish real data loading and model fitting. + + Args: + recs (list): a list of Recorder, the tasks have been saved to them + end_train_func (Callable, optional): the end_train method which need at least `recorder`s and `experiment_name`. Defaults to None for using self.end_train_func. + kwargs: the params for end_train_func. + + Returns: + list: a list of Recorders + """ + if end_train_func is None: + end_train_func = self.end_train_func + for rec in recs: + end_train_func(rec, **kwargs) + rec.set_tags(**{"train_status": "end_task_train"}) return recs class TrainerRM(Trainer): - """Trainer based on (R)ecorder and Task(M)anager + """ + Trainer based on (R)ecorder and Task(M)anager. + It can train a list of tasks and return a list of model recorder in a multiprocessing way. Assumption: `task` will be saved to TaskManager and `task` will be fetched and trained from TaskManager """ def __init__(self, experiment_name: str, task_pool: str, train_func=task_train): + """ + Init TrainerR. + + Args: + experiment_name (str): the name of experiment. + task_pool (str): task pool name in TaskManager. + train_func (Callable, optional): default training method. Defaults to `task_train`. + """ + super().__init__() self.experiment_name = experiment_name self.task_pool = task_pool self.train_func = train_func @@ -159,20 +266,23 @@ def __init__(self, experiment_name: str, task_pool: str, train_func=task_train): def train( self, tasks: list, - train_func=None, - before_status=TaskManager.STATUS_WAITING, - after_status=TaskManager.STATUS_DONE, - *args, + train_func: Callable = None, + before_status: str = TaskManager.STATUS_WAITING, + after_status: str = TaskManager.STATUS_DONE, **kwargs, - ): - """Given a list of `task`s and return a list of trained Recorder. The order can be guaranteed. + ) -> List[Recorder]: + """ + Given a list of `task`s and return a list of trained Recorder. The order can be guaranteed. This method defaults to a single process, but TaskManager offered a great way to parallel training. Users can customize their train_func to realize multiple processes or even multiple machines. Args: tasks (list): a list of definition based on `task` dict - train_func (Callable): the train method which need at least `task` and `experiment_name`. None for default. + train_func (Callable): the train method which need at least `task`s and `experiment_name`. None for default training method. + before_status (str): the tasks in before_status will be fetched and trained. Can be STATUS_WAITING, STATUS_PART_DONE. + after_status (str): the tasks after trained will become after_status. Can be STATUS_WAITING, STATUS_PART_DONE. + kwargs: the params for train_func. Returns: list: a list of Recorders @@ -187,65 +297,27 @@ def train( experiment_name=self.experiment_name, before_status=before_status, after_status=after_status, - *args, **kwargs, ) recs = [] for _id in _id_list: - recs.append(tm.re_query(_id)["res"]) + rec = tm.re_query(_id)["res"] + rec.set_tags(**{"train_status": "begin_task_train"}) + recs.append(rec) return recs - -class DelayTrainerR(TrainerR): - """ - A delayed implementation based on TrainerR, which means `train` method may only do some preparation and `end_train` method can do the real model fitting. - - """ - - def __init__(self, experiment_name, train_func=begin_task_train, end_train_func=end_task_train): - super().__init__(experiment_name, train_func) - self.end_train_func = end_train_func - self.recs = [] - - def train(self, tasks: list, train_func, *args, **kwargs): - """ - Same as `train` of TrainerR, the results will be recorded in self.recs - - Args: - tasks (list): a list of definition based on `task` dict - train_func (Callable): the train method which need at least `task` and `experiment_name`. None for default. - - Returns: - list: a list of Recorders - """ - self.recs = super().train(tasks, train_func=train_func, *args, **kwargs) - return self.recs - - def end_train(self, recs=None, end_train_func=None): - """ - Given a list of Recorder and return a list of trained Recorder. - This class will finished real data loading and model fitting. - - Args: - recs (list, optional): a list of Recorder, the tasks have been saved to them. Defaults to None for using self.recs. - end_train_func (Callable, optional): the end_train method which need at least `rec` and `experiment_name`. Defaults to None for using self.end_train_func. - - Returns: - list: a list of Recorders - """ - if recs is None: - recs = copy.deepcopy(self.recs) - # the models will be only trained once - self.recs = [] - if end_train_func is None: - end_train_func = self.end_train_func + def end_train(self, recs: list, **kwargs) -> list: for rec in recs: - end_train_func(rec) + rec.set_tags(**{"train_status": "end_task_train"}) return recs - def is_delay(self): - return True + def reset(self): + """ + NOTE: this method will delete all task in this task_pool! + """ + tm = TaskManager(task_pool=self.task_pool) + tm.remove() class DelayTrainerRM(TrainerRM): @@ -257,28 +329,28 @@ class DelayTrainerRM(TrainerRM): def __init__(self, experiment_name, task_pool: str, train_func=begin_task_train, end_train_func=end_task_train): super().__init__(experiment_name, task_pool, train_func) self.end_train_func = end_train_func + self.delay = True - def train(self, tasks: list, train_func=None, *args, **kwargs): + def train(self, tasks: list, train_func=None, **kwargs): """ - Same as `train` of TrainerRM, the results will be recorded in self.recs - + Same as `train` of TrainerRM, after_status will be STATUS_PART_DONE. Args: tasks (list): a list of definition based on `task` dict - train_func (Callable): the train method which need at least `task` and `experiment_name`. None for default. - + train_func (Callable): the train method which need at least `task`s and `experiment_name`. Defaults to None for using self.train_func. Returns: list: a list of Recorders """ - return super().train(tasks, train_func=train_func, after_status=TaskManager.STATUS_PART_DONE, *args, **kwargs) + return super().train(tasks, train_func=train_func, after_status=TaskManager.STATUS_PART_DONE, **kwargs) - def end_train(self, recs, end_train_func=None): + def end_train(self, recs, end_train_func=None, **kwargs): """ Given a list of Recorder and return a list of trained Recorder. - This class will finished real data loading and model fitting. + This class will finish real data loading and model fitting. Args: - recs (list, optional): a list of Recorder, the tasks have been saved to them. Defaults to None for using self.recs.. - end_train_func (Callable, optional): the end_train method which need at least `rec` and `experiment_name`. Defaults to None for using self.end_train_func. + recs (list): a list of Recorder, the tasks have been saved to them. + end_train_func (Callable, optional): the end_train method which need at least `recorder`s and `experiment_name`. Defaults to None for using self.end_train_func. + kwargs: the params for end_train_func. Returns: list: a list of Recorders @@ -291,8 +363,8 @@ def end_train(self, recs, end_train_func=None): self.task_pool, experiment_name=self.experiment_name, before_status=TaskManager.STATUS_PART_DONE, + **kwargs, ) + for rec in recs: + rec.set_tags(**{"train_status": "end_task_train"}) return recs - - def is_delay(self): - return True diff --git a/qlib/utils/serial.py b/qlib/utils/serial.py index 1b775d99a7..52d326c2ab 100644 --- a/qlib/utils/serial.py +++ b/qlib/utils/serial.py @@ -3,11 +3,12 @@ from pathlib import Path import pickle +from typing import Union class Serializable: """ - Serializable will change the behaviours of pickle. + Serializable will change the behaviors of pickle. - It only saves the state whose name **does not** start with `_` It provides a syntactic sugar for distinguish the attributes which user doesn't want. - For examples, a learnable Datahandler just wants to save the parameters without data when dumping to disk @@ -70,7 +71,7 @@ def config(self, dump_all: bool = None, exclude: list = None, recursive=False): obj.config(**params, recursive=True) del self.__dict__[self.FLAG_KEY] - def to_pickle(self, path: [Path, str], dump_all: bool = None, exclude: list = None): + def to_pickle(self, path: Union[Path, str], dump_all: bool = None, exclude: list = None): self.config(dump_all=dump_all, exclude=exclude) with Path(path).open("wb") as f: pickle.dump(self, f) diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index f8266577b0..4e92900960 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -2,487 +2,40 @@ # Licensed under the MIT License. """ -This class is a component of online serving, it can manage a series of models dynamically. -With the change of time, the decisive models will be also changed. In this module, we called those contributing models as `online` models. +OnlineManager can manage a set of OnlineStrategy and run them dynamically. + +With the change of time, the decisive models will be also changed. In this module, we call those contributing models as `online` models. In every routine(such as everyday or every minutes), the `online` models maybe changed and the prediction of them need to be updated. So this module provide a series methods to control this process. """ -from copy import deepcopy -from pprint import pprint -import pandas as pd -from qlib.model.ens.ensemble import ens_workflow -from qlib.model.ens.group import RollingGroup -from qlib.utils.serial import Serializable + from typing import Dict, List, Union + +import pandas as pd from qlib import get_module_logger from qlib.data.data import D -from qlib.model.trainer import Trainer, TrainerR, task_train -from qlib.workflow import R +from qlib.utils.serial import Serializable from qlib.workflow.online.strategy import OnlineStrategy -from qlib.workflow.online.update import PredUpdater -from qlib.workflow.recorder import Recorder -from qlib.workflow.task.collect import Collector, HyperCollector, RecorderCollector -from qlib.workflow.task.gen import RollingGen, task_generator -from qlib.workflow.task.utils import TimeAdjuster, list_recorders - -class OnlineManager(Serializable): - - ONLINE_KEY = "online_status" # the online status key in recorder - ONLINE_TAG = "online" # the 'online' model - # NOTE: The meaning of this tag is that we can not assume the training models can be trained before we need its predition. Whenever finished training, it can be guaranteed that there are some online models. - NEXT_ONLINE_TAG = "next_online" # the 'next online' model, which can be 'online' model when call reset_online_model - OFFLINE_TAG = "offline" # the 'offline' model, not for online serving - - SIGNAL_EXP = "OnlineManagerSignals" # a specific experiment to save signals of different experiment. - - def __init__(self, trainer: Trainer = None, need_log=True): - """ - init OnlineManager. - - Args: - trainer (Trainer, optional): a instance of Trainer. Defaults to None. - need_log (bool, optional): print log or not. Defaults to True. - """ - self.trainer = trainer - self.logger = get_module_logger(self.__class__.__name__) - self.need_log = need_log - self.cur_time = None - - def prepare_signals(self): - """ - After perparing the data of last routine (a box in box-plot) which means the end of the routine, we can prepare trading signals for next routine. - Must use `pass` even though there is nothing to do. - """ - raise NotImplementedError(f"Please implement the `prepare_signals` method.") - - def get_signals(self): - """ - After preparing signals, here is the method to get them. - """ - raise NotImplementedError(f"Please implement the `get_signals` method.") - - def prepare_tasks(self, *args, **kwargs): - """ - After the end of a routine, check whether we need to prepare and train some new tasks. - return the new tasks waiting for training. - """ - raise NotImplementedError(f"Please implement the `prepare_tasks` method.") - - def prepare_new_models(self, tasks, tag=NEXT_ONLINE_TAG, check_func=None, *args, **kwargs): - """ - Use trainer to train a list of tasks and set the trained model to `tag`. - - Args: - tasks (list): a list of tasks. - tag (str): - `ONLINE_TAG` for first train or additional train - `NEXT_ONLINE_TAG` for reset online model when calling `reset_online_tag` - `OFFLINE_TAG` for train but offline those models - check_func: the method to judge if a model can be online. - The parameter is the model record and return True for online. - None for online every models. - *args, **kwargs: will be passed to end_train which means will be passed to customized train method. - - """ - if check_func is None: - check_func = lambda x: True - if len(tasks) > 0: - if self.trainer is not None: - new_models = self.trainer.train(tasks, *args, **kwargs) - if check_func(new_models): - self.set_online_tag(tag, new_models) - if self.need_log: - self.logger.info(f"Finished preparing {len(new_models)} new models and set them to {tag}.") - else: - self.logger.warn("No trainer to train new tasks.") - - def update_online_pred(self): - """ - After the end of a routine, update the predictions of online models to latest. - """ - raise NotImplementedError(f"Please implement the `update_online_pred` method.") - - def set_online_tag(self, tag, recorder): - """ - Set `tag` to the model to sign whether online. - - Args: - tag (str): the tags in `ONLINE_TAG`, `NEXT_ONLINE_TAG`, `OFFLINE_TAG` - """ - raise NotImplementedError(f"Please implement the `set_online_tag` method.") - - def get_online_tag(self): - """ - Given a model and return its online tag. - """ - raise NotImplementedError(f"Please implement the `get_online_tag` method.") - - def reset_online_tag(self, recorders=None): - """offline all models and set the recorders to 'online'. If no parameter and no 'next online' model, then do nothing. - - Args: - recorders (List, optional): - the recorders you want to reset to 'online'. If don't give, set 'next online' model to 'online' model. If there isn't any 'next online' model, then maintain existing 'online' model. - - Returns: - list: new online recorder. [] if there is no update. - """ - raise NotImplementedError(f"Please implement the `reset_online_tag` method.") - - def online_models(self): - """ - Return online models. - """ - raise NotImplementedError(f"Please implement the `online_models` method.") - - def first_train(self): - """ - Train a series of models firstly and set some of them into online models. - """ - raise NotImplementedError(f"Please implement the `first_train` method.") - - def get_collector(self): - """ - Return the collector. - - Returns: - Collector - """ - raise NotImplementedError(f"Please implement the `get_collector` method.") - - def delay_prepare(self, rec_dict, *args, **kwargs): - """ - Prepare all models and signals if there are something waiting for prepare. - NOTE: Assumption: the predictions of online models are between `time_segment`, or this method will work in a wrong way. - - Args: - rec_dict (str): an online models dict likes {(begin_time, end_time):[online models]}. - *args, **kwargs: will be passed to end_train which means will be passed to customized train method. - """ - for time_segment, recs_list in rec_dict.items(): - self.trainer.end_train(recs_list, *args, **kwargs) - self.reset_online_tag(recs_list) - self.prepare_signals() - signal_max = self.get_signals().index.get_level_values("datetime").max() - if time_segment[1] is not None and signal_max > time_segment[1]: - raise ValueError( - f"The max time of signals prepared by online models is {signal_max}, but those models only online in {time_segment}" - ) - - def routine(self, cur_time=None, delay_prepare=False, *args, **kwargs): - """ - The typical update process after a routine, such as day by day or month by month. - update online prediction -> prepare signals -> prepare tasks -> prepare new models -> reset online models - - NOTE: Assumption: if using simulator (delay_prepare is True), the prediction will be prepared well after every training, so there is no need to update predictions. - - Args: - cur_time ([type], optional): [description]. Defaults to None. - delay_prepare (bool, optional): [description]. Defaults to False. - *args, **kwargs: will be passed to `prepare_tasks` and `prepare_new_models`. It can be some hyper parameter or training config. - - Returns: - [type]: [description] - """ - self.cur_time = cur_time # None for latest date - if not delay_prepare: - self.update_online_pred() - self.prepare_signals() - tasks = self.prepare_tasks(*args, **kwargs) - self.prepare_new_models(tasks, *args, **kwargs) - - return self.reset_online_tag() - - -class OnlineManagerR(OnlineManager): - """ - The implementation of OnlineManager based on (R)ecorder. - - """ - - def __init__(self, experiment_name: str, trainer: Trainer = None, need_log=True): - """ - init OnlineManagerR. - - Args: - experiment_name (str): the experiment name. - trainer (Trainer, optional): a instance of Trainer. Defaults to None. - need_log (bool, optional): print log or not. Defaults to True. - """ - if trainer is None: - trainer = TrainerR(experiment_name) - super().__init__(trainer=trainer, need_log=need_log) - self.exp_name = experiment_name - self.signal_rec = None - - def set_online_tag(self, tag, recorder: Union[Recorder, List]): - """ - Set `tag` to the model to sign whether online. - - Args: - tag (str): the tags in `ONLINE_TAG`, `NEXT_ONLINE_TAG`, `OFFLINE_TAG` - recorder (Union[Recorder, List]) - """ - if isinstance(recorder, Recorder): - recorder = [recorder] - for rec in recorder: - rec.set_tags(**{self.ONLINE_KEY: tag}) - if self.need_log: - self.logger.info(f"Set {len(recorder)} models to '{tag}'.") - - def get_online_tag(self, recorder: Recorder): - """ - Given a model and return its online tag. - - Args: - recorder (Recorder): a instance of recorder - - Returns: - str: the tag - """ - tags = recorder.list_tags() - return tags.get(OnlineManager.ONLINE_KEY, OnlineManager.OFFLINE_TAG) - - def reset_online_tag(self, recorder: Union[Recorder, List] = None): - """offline all models and set the recorders to 'online'. If no parameter and no 'next online' model, then do nothing. - - Args: - recorders (Union[Recorder, List], optional): - the recorders you want to reset to 'online'. If don't give, set 'next online' model to 'online' model. If there isn't any 'next online' model, then maintain existing 'online' model. - - Returns: - list: new online recorder. [] if there is no update. - """ - if recorder is None: - recorder = list( - list_recorders( - self.exp_name, lambda rec: self.get_online_tag(rec) == OnlineManager.NEXT_ONLINE_TAG - ).values() - ) - if isinstance(recorder, Recorder): - recorder = [recorder] - if len(recorder) == 0: - if self.need_log: - self.logger.info("No 'next online' model, just use current 'online' models.") - return [] - recs = list_recorders(self.exp_name) - self.set_online_tag(OnlineManager.OFFLINE_TAG, list(recs.values())) - self.set_online_tag(OnlineManager.ONLINE_TAG, recorder) - return recorder - - def get_signals(self): - """ - get signals from the recorder(named self.exp_name) of the experiment(named self.SIGNAL_EXP) - - Returns: - signals - """ - if self.signal_rec is None: - with R.start(experiment_name=self.SIGNAL_EXP, recorder_name=self.exp_name, resume=True): - self.signal_rec = R.get_recorder() - signals = None - try: - signals = self.signal_rec.load_object("signals") - except OSError: - self.logger.warn("Can not find `signals`, have you called `prepare_signals` before?") - return signals - - def online_models(self): - """ - Return online models. - - Returns: - list: the list of online models - """ - return list( - list_recorders(self.exp_name, lambda rec: self.get_online_tag(rec) == OnlineManager.ONLINE_TAG).values() - ) - - def update_online_pred(self): - """ - Update all online model predictions to the latest day in Calendar - """ - online_models = self.online_models() - for rec in online_models: - PredUpdater(rec, to_date=self.cur_time, need_log=self.need_log).update() - - if self.need_log: - self.logger.info(f"Finished updating {len(online_models)} online model predictions of {self.exp_name}.") - - def prepare_signals(self, over_write=False): - """ - Average the predictions of online models and offer a trading signals every routine. - The signals will be saved to `signal` file of a recorder named self.exp_name of a experiment using the name of `SIGNAL_EXP` - Even if the latest signal already exists, the latest calculation result will be overwritten. - NOTE: Given a prediction of a certain time, all signals before this time will be prepared well. - Args: - over_write (bool, optional): If True, the new signals will overwrite the file. If False, the new signals will append to the end of signals. Defaults to False. - """ - if self.signal_rec is None: - with R.start(experiment_name=self.SIGNAL_EXP, recorder_name=self.exp_name, resume=True): - self.signal_rec = R.get_recorder() - - pred = [] - try: - old_signals = self.signal_rec.load_object("signals") - except OSError: - old_signals = None - - for rec in self.online_models(): - pred.append(rec.load_object("pred.pkl")) +from qlib.workflow.task.collect import HyperCollector - signals = pd.concat(pred, axis=1).mean(axis=1).to_frame("score") - signals = signals.sort_index() - if old_signals is not None and not over_write: - old_max = old_signals.index.get_level_values("datetime").max() - new_signals = signals.loc[old_max:] - signals = pd.concat([old_signals, new_signals], axis=0) - else: - new_signals = signals - if self.need_log: - self.logger.info(f"Finished preparing new {len(new_signals)} signals to {self.SIGNAL_EXP}/{self.exp_name}.") - self.signal_rec.save_objects(**{"signals": signals}) - - -class RollingOnlineManager(OnlineManagerR): - """An implementation of OnlineManager based on Rolling.""" +class OnlineManager(Serializable): def __init__( self, - experiment_name: str, - rolling_gen: RollingGen, - trainer: Trainer = None, + strategy: Union[OnlineStrategy, List[OnlineStrategy]], + begin_time: Union[str, pd.Timestamp] = None, + freq="day", need_log=True, ): """ - init RollingOnlineManager. + Init OnlineManager. Args: - experiment_name (str): the experiment name. - rolling_gen (RollingGen): an instance of RollingGen - trainer (Trainer, optional): an instance of Trainer. Defaults to None. - collector (Collector, optional): an instance of Collector. Defaults to None. + strategy (Union[OnlineStrategy, List[OnlineStrategy]]): an instance of OnlineStrategy or a list of OnlineStrategy + begin_time (Union[str,pd.Timestamp], optional): the OnlineManager will begin at this time. Defaults to None. + freq (str, optional): data frequency. Defaults to "day". need_log (bool, optional): print log or not. Defaults to True. """ - if trainer is None: - trainer = TrainerR(experiment_name) - super().__init__(experiment_name=experiment_name, trainer=trainer, need_log=need_log) - self.ta = TimeAdjuster() - self.rg = rolling_gen - self.logger = get_module_logger(self.__class__.__name__) - - def get_collector(self, rec_key_func=None, rec_filter_func=None): - """ - Get the instance of collector to collect results. The returned collector must can distinguish results in different models. - Assumption: the models can be distinguished based on model name and rolling test segments. - If you do not want this assumption, please implement your own method or use another rec_key_func. - - Args: - rec_key_func (Callable): a function to get the key of a recorder. If None, use recorder id. - rec_filter_func (Callable, optional): filter the recorder by return True or False. Defaults to None. - """ - - def rec_key(recorder): - task_config = recorder.load_object("task") - model_key = task_config["model"]["class"] - rolling_key = task_config["dataset"]["kwargs"]["segments"]["test"] - return model_key, rolling_key - - if rec_key_func is None: - rec_key_func = rec_key - - return RecorderCollector(experiment=self.exp_name, rec_key_func=rec_key_func, rec_filter_func=rec_filter_func) - - def collect_artifact(self, rec_key_func=None, rec_filter_func=None): - """ - collecting artifact based on the collector and RollingGroup. - - Args: - rec_key_func (Callable): a function to get the key of a recorder. If None, use recorder id. - rec_filter_func (Callable, optional): filter the recorder by return True or False. Defaults to None. - - Returns: - dict: the artifact dict after rolling ensemble - """ - artifact = ens_workflow( - self.get_collector(rec_key_func=rec_key_func, rec_filter_func=rec_filter_func), RollingGroup() - ) - return artifact - - def first_train(self, task_configs: list): - """ - Use rolling_gen to generate different tasks based on task_configs and trained them. - - Args: - task_configs (list or dict): a list of task configs or a task config - - Returns: - Collector: a instance of a Collector. - """ - tasks = task_generator( - tasks=task_configs, - generators=self.rg, # generate different date segment - ) - self.prepare_new_models(tasks, tag=self.ONLINE_TAG) - return self.get_collector() - - def prepare_tasks(self): - """ - Prepare new tasks based on new date. - - Returns: - list: a list of new tasks. - """ - latest_records, max_test = self.list_latest_recorders( - lambda rec: self.get_online_tag(rec) == OnlineManager.ONLINE_TAG - ) - if max_test is None: - self.logger.warn(f"No latest online recorders, no new tasks.") - return [] - calendar_latest = D.calendar(end_time=self.cur_time)[-1] if self.cur_time is None else self.cur_time - if self.need_log: - self.logger.info( - f"The interval between current time {calendar_latest} and last rolling test begin time {max_test[0]} is {self.ta.cal_interval(calendar_latest, max_test[0])}, the rolling step is {self.rg.step}" - ) - if self.ta.cal_interval(calendar_latest, max_test[0]) >= self.rg.step: - old_tasks = [] - tasks_tmp = [] - for rid, rec in latest_records.items(): - task = rec.load_object("task") - old_tasks.append(deepcopy(task)) - test_begin = task["dataset"]["kwargs"]["segments"]["test"][0] - # modify the test segment to generate new tasks - task["dataset"]["kwargs"]["segments"]["test"] = (test_begin, calendar_latest) - tasks_tmp.append(task) - new_tasks_tmp = task_generator(tasks_tmp, self.rg) - new_tasks = [task for task in new_tasks_tmp if task not in old_tasks] - return new_tasks - return [] - - def list_latest_recorders(self, rec_filter_func=None): - """find latest recorders based on test segments. - - Args: - rec_filter_func (Callable, optional): recorder filter. Defaults to None. - - Returns: - dict, tuple: the latest recorders and the latest date of them - """ - recs_flt = list_recorders(self.exp_name, rec_filter_func) - if len(recs_flt) == 0: - return recs_flt, None - max_test = max(rec.load_object("task")["dataset"]["kwargs"]["segments"]["test"] for rec in recs_flt.values()) - latest_rec = {} - for rid, rec in recs_flt.items(): - if rec.load_object("task")["dataset"]["kwargs"]["segments"]["test"] == max_test: - latest_rec[rid] = rec - return latest_rec, max_test - - -class OnlineM(Serializable): - def __init__( - self, strategy: Union[OnlineStrategy, List[OnlineStrategy]], begin_time=None, freq="day", need_log=True - ): self.logger = get_module_logger(self.__class__.__name__) self.need_log = need_log if not isinstance(strategy, list): @@ -491,38 +44,37 @@ def __init__( self.freq = freq if begin_time is None: begin_time = D.calendar(freq=self.freq).max() - self.cur_time = pd.Timestamp(begin_time) + self.begin_time = pd.Timestamp(begin_time) + self.cur_time = self.begin_time self.history = {} def first_train(self): """ - Train a series of models firstly and set some of them into online models. + Run every strategy first_train method and record the online history """ for strategy in self.strategy: self.logger.info(f"Strategy `{strategy.name_id}` begins first training...") online_models = strategy.first_train() self.history.setdefault(strategy.name_id, {})[self.cur_time] = online_models - def routine(self, cur_time=None, task_kwargs={}, model_kwargs={}): + def routine(self, cur_time: Union[str, pd.Timestamp] = None, task_kwargs: dict = {}, model_kwargs: dict = {}): """ + Run typical update process for every strategy and record the online history. + The typical update process after a routine, such as day by day or month by month. update online prediction -> prepare signals -> prepare tasks -> prepare new models -> reset online models - NOTE: Assumption: if using simulator (delay_prepare is True), the prediction will be prepared well after every training, so there is no need to update predictions. - Args: - cur_time ([type], optional): [description]. Defaults to None. - delay_prepare (bool, optional): [description]. Defaults to False. - *args, **kwargs: will be passed to `prepare_tasks` and `prepare_new_models`. It can be some hyper parameter or training config. - - Returns: - [type]: [description] + cur_time (Union[str,pd.Timestamp], optional): run routine method in this time. Defaults to None. + task_kwargs (dict): the params for `prepare_tasks` + model_kwargs (dict): the params for `prepare_online_models` """ if cur_time is None: cur_time = D.calendar(freq=self.freq).max() self.cur_time = pd.Timestamp(cur_time) # None for latest date for strategy in self.strategy: - self.logger.info(f"Strategy `{strategy.name_id}` begins routine...") + if self.need_log: + self.logger.info(f"Strategy `{strategy.name_id}` begins routine...") if not strategy.trainer.is_delay(): strategy.prepare_signals() tasks = strategy.prepare_tasks(self.cur_time, **task_kwargs) @@ -530,13 +82,28 @@ def routine(self, cur_time=None, task_kwargs={}, model_kwargs={}): if len(online_models) > 0: self.history.setdefault(strategy.name_id, {})[self.cur_time] = online_models - def get_collector(self): + def get_collector(self) -> HyperCollector: + """ + Get the instance of HyperCollector to collect results from every strategy. + + Returns: + HyperCollector: the collector can collect other collectors. + """ collector_dict = {} for strategy in self.strategy: collector_dict[strategy.name_id] = strategy.get_collector() return HyperCollector(collector_dict) - def get_online_history(self, strategy_name_id): + def get_online_history(self, strategy_name_id: str) -> list: + """ + Get the online history based on strategy_name_id. + + Args: + strategy_name_id (str): the name_id of strategy + + Returns: + dict: a list like [(time, [online_models])] + """ history_dict = self.history[strategy_name_id] history = [] for time in sorted(history_dict): @@ -547,22 +114,20 @@ def get_online_history(self, strategy_name_id): def delay_prepare(self, delay_kwargs={}): """ Prepare all models and signals if there are something waiting for prepare. - NOTE: Assumption: the predictions of online models are between `time_segment`, or this method will work in a wrong way. Args: - rec_dict (str): an online models dict likes {(begin_time, end_time):[online models]}. - *args, **kwargs: will be passed to end_train which means will be passed to customized train method. + delay_kwargs: the params for `delay_prepare` """ for strategy in self.strategy: strategy.delay_prepare(self.get_online_history(strategy.name_id), **delay_kwargs) - def simulate(self, end_time, frequency="day", task_kwargs={}, model_kwargs={}, delay_kwargs={}): + def simulate(self, end_time, frequency="day", task_kwargs={}, model_kwargs={}, delay_kwargs={}) -> HyperCollector: """ - Starting from start time, this method will simulate every routine in OnlineManager. + Starting from cur time, this method will simulate every routine in OnlineManager. NOTE: Considering the parallel training, the models and signals can be perpared after all routine simulating. Returns: - Collector: the OnlineManager's collector + HyperCollector: the OnlineManager's collector """ cal = D.calendar(start_time=self.cur_time, end_time=end_time, freq=frequency) self.first_train() @@ -572,3 +137,12 @@ def simulate(self, end_time, frequency="day", task_kwargs={}, model_kwargs={}, d self.delay_prepare(delay_kwargs=delay_kwargs) self.logger.info(f"Finished preparing signals") return self.get_collector() + + def reset(self): + """ + NOTE: This method will reset all strategy! Be careful to use it. + """ + self.cur_time = self.begin_time + self.history = {} + for strategy in self.strategy: + strategy.reset() diff --git a/qlib/workflow/online/simulator.py b/qlib/workflow/online/simulator.py deleted file mode 100644 index ddaf2471c1..0000000000 --- a/qlib/workflow/online/simulator.py +++ /dev/null @@ -1,77 +0,0 @@ -from qlib.data import D -from qlib import get_module_logger -from qlib.workflow.online.manager import OnlineM - - -class OnlineSimulator: - """ - To simulate online serving in the past, like a "online serving backtest". - """ - - def __init__( - self, - start_time, - end_time, - online_manager: OnlineManager, - frequency="day", - ): - """ - init OnlineSimulator. - - Args: - start_time (str or pd.Timestamp): the start time of simulating. - end_time (str or pd.Timestamp): the end time of simulating. If None, then end_time is latest. - onlinemanager (OnlineManager): the instance of OnlineManager - frequency (str, optional): the data frequency. Defaults to "day". - """ - self.logger = get_module_logger(self.__class__.__name__) - self.cal = D.calendar(start_time=start_time, end_time=end_time, freq=frequency) - self.start_time = self.cal[0] - self.end_time = self.cal[-1] - self.olm = online_manager - if len(self.cal) == 0: - self.logger.warn(f"There is no need to simulate bacause start_time is larger than end_time.") - - # def simulate(self, *args, **kwargs): - # """ - # Starting from start time, this method will simulate every routine in OnlineManager. - # NOTE: Considering the parallel training, the models and signals can be perpared after all routine simulating. - - # Returns: - # Collector: the OnlineManager's collector - # """ - # self.rec_dict = {} - # tmp_begin = self.start_time - # tmp_end = None - # self.olm.first_train() - # prev_recorders = self.olm.online_models() - # for cur_time in self.cal: - # self.logger.info(f"Simulating at {str(cur_time)}......") - # recorders = self.olm.routine(cur_time, True, *args, **kwargs) - # if len(recorders) == 0: - # tmp_end = cur_time - # else: - # self.rec_dict[(tmp_begin, tmp_end)] = prev_recorders - # tmp_begin = cur_time - # prev_recorders = recorders - # self.rec_dict[(tmp_begin, self.end_time)] = prev_recorders - # # finished perparing models (and pred) and signals - # self.olm.delay_prepare(self.rec_dict) - # self.logger.info(f"Finished preparing signals") - # return self.olm.get_collector() - - def simulate(self, task_kwargs={}, model_kwargs={}): - """ - Starting from start time, this method will simulate every routine in OnlineManager. - NOTE: Considering the parallel training, the models and signals can be perpared after all routine simulating. - - Returns: - Collector: the OnlineManager's collector - """ - self.olm.first_train() - for cur_time in self.cal: - self.logger.info(f"Simulating at {str(cur_time)}......") - self.olm.routine(cur_time, task_kwargs={}, model_kwargs={}) - self.olm.delay_prepare() - self.logger.info(f"Finished preparing signals") - return self.olm.get_collector() diff --git a/qlib/workflow/online/strategy.py b/qlib/workflow/online/strategy.py index 5e4dcc024b..3782ee6523 100644 --- a/qlib/workflow/online/strategy.py +++ b/qlib/workflow/online/strategy.py @@ -1,11 +1,14 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. + """ -This module is working with OnlineManager, responsing for a set of strategy about how the models are updated and signals are perpared. +OnlineStrategy is a set of strategy of online serving. +It is working with OnlineManager, responsing how the tasks are generated, the models are updated and signals are perpared. """ from copy import deepcopy -from typing import List, Union +from typing import List, Tuple, Union + import pandas as pd from qlib.data.data import D from qlib.log import get_module_logger @@ -13,7 +16,8 @@ from qlib.model.trainer import Trainer, TrainerR from qlib.workflow import R from qlib.workflow.online.utils import OnlineTool, OnlineToolR -from qlib.workflow.task.collect import HyperCollector, RecorderCollector +from qlib.workflow.recorder import Recorder +from qlib.workflow.task.collect import Collector, HyperCollector, RecorderCollector from qlib.workflow.task.gen import RollingGen, task_generator from qlib.workflow.task.utils import TimeAdjuster, list_recorders @@ -21,7 +25,7 @@ class OnlineStrategy: def __init__(self, name_id: str, trainer: Trainer = None, need_log=True): """ - init OnlineManager. + Init OnlineStrategy. Args: name_id (str): a unique name or id @@ -33,12 +37,15 @@ def __init__(self, name_id: str, trainer: Trainer = None, need_log=True): self.logger = get_module_logger(self.__class__.__name__) self.need_log = need_log self.tool = OnlineTool() - self.history = {} - def prepare_signals(self, delay=False): + def prepare_signals(self, delay: bool = False): """ After perparing the data of last routine (a box in box-plot) which means the end of the routine, we can prepare trading signals for next routine. - Must use `pass` even though there is nothing to do. + + NOTE: Given a set prediction, all signals before these prediction end time will be prepared well. + Args: + delay: bool + If this method was called by `delay_prepare` """ raise NotImplementedError(f"Please implement the `prepare_signals` method.") @@ -46,6 +53,8 @@ def prepare_tasks(self, *args, **kwargs): """ After the end of a routine, check whether we need to prepare and train some new tasks. return the new tasks waiting for training. + + You can find last online models by OnlineTool.online_models. """ raise NotImplementedError(f"Please implement the `prepare_tasks` method.") @@ -53,6 +62,8 @@ def prepare_online_models(self, tasks, check_func=None, **kwargs): """ Use trainer to train a list of tasks and set the trained model to `online`. + NOTE: This method will first offline all models and online the online models prepared by this method. So you can find last online models by OnlineTool.online_models if you still need them. + Args: tasks (list): a list of tasks. tag (str): @@ -78,33 +89,43 @@ def prepare_online_models(self, tasks, check_func=None, **kwargs): def first_train(self): """ - Train a series of models firstly and set some of them into online models. + Train a series of models firstly and set some of them as online models. """ raise NotImplementedError(f"Please implement the `first_train` method.") - def get_collector(self): + def get_collector(self) -> Collector: """ - Return the collector. + Get the instance of collector to collect results of online serving. + + For example: + 1) collect predictions in Recorder + 2) collect signals in .txt file Returns: Collector """ raise NotImplementedError(f"Please implement the `get_collector` method.") - def delay_prepare(self, history, **kwargs): + def delay_prepare(self, history: list, **kwargs): """ Prepare all models and signals if there are something waiting for prepare. - NOTE: Assumption: the predictions of online models are between `time_segment`, or this method will work in a wrong way. + NOTE: Assumption: the predictions of online models need less than next begin_time, or this method will work in a wrong way. Args: - rec_dict (str): an online models dict likes {(begin_time, end_time):[online models]}. - *args, **kwargs: will be passed to end_train which means will be passed to customized train method. + history (list): an online models list likes [begin_time:[online models]]. + **kwargs: will be passed to end_train which means will be passed to customized train method. """ - for time_begin, recs_list in history: + for begin_time, recs_list in history: self.trainer.end_train(recs_list, **kwargs) self.tool.reset_online_tag(recs_list) self.prepare_signals(delay=True) + def reset(self): + """ + Delete all things and set them to default status. This method is convenient to explore the strategy for online simulation. + """ + pass + class RollingAverageStrategy(OnlineStrategy): @@ -122,7 +143,7 @@ def __init__( signal_exp_name="OnlineManagerSignals", ): """ - init OnlineManagerR. + Init RollingAverageStrategy. Assumption: the str of name_id, the experiment name and the trainer's experiment name are same one. @@ -139,11 +160,11 @@ def __init__( if not isinstance(task_template, list): task_template = [task_template] self.task_template = task_template - self.signal_rec = None self.signal_exp_name = signal_exp_name - self.ta = TimeAdjuster() self.rg = rolling_gen self.tool = OnlineToolR(self.exp_name) + self.ta = TimeAdjuster() + self.signal_rec = None # the recorder to record signals def get_collector(self, rec_key_func=None, rec_filter_func=None): """ @@ -180,12 +201,12 @@ def rec_key(recorder): ) return HyperCollector({"artifacts": artifacts_collector, "signals": signals_collector}) - def first_train(self): + def first_train(self) -> List[Recorder]: """ Use rolling_gen to generate different tasks based on task_template and trained them. Returns: - Collector: a instance of a Collector. + List[Recorder]: a list of Recorder. """ tasks = task_generator( tasks=self.task_template, @@ -193,12 +214,14 @@ def first_train(self): ) return self.prepare_online_models(tasks) - def prepare_tasks(self, cur_time): + def prepare_tasks(self, cur_time) -> List[dict]: """ Prepare new tasks based on cur_time (None for latest). + You can find last online models by OnlineToolR.online_models. + Returns: - list: a list of new tasks. + List[dict]: a list of new tasks. """ latest_records, max_test = self._list_latest(self.tool.online_models()) if max_test is None: @@ -224,7 +247,7 @@ def prepare_tasks(self, cur_time): return new_tasks return [] - def prepare_signals(self, delay=False, over_write=False): + def prepare_signals(self, delay=False, over_write=False) -> pd.DataFrame: """ Average the predictions of online models and offer a trading signals every routine. The signals will be saved to `signal` file of a recorder named self.exp_name of a experiment using the name of `SIGNAL_EXP` @@ -233,7 +256,7 @@ def prepare_signals(self, delay=False, over_write=False): Args: over_write (bool, optional): If True, the new signals will overwrite the file. If False, the new signals will append to the end of signals. Defaults to False. Returns: - object: the signals. + pd.DataFrame: the signals. """ if not delay: self.tool.update_online_pred() @@ -250,7 +273,7 @@ def prepare_signals(self, delay=False, over_write=False): for rec in self.tool.online_models(): pred.append(rec.load_object("pred.pkl")) - signals = pd.concat(pred, axis=1).mean(axis=1).to_frame("score") + signals: pd.DataFrame = pd.concat(pred, axis=1).mean(axis=1).to_frame("score") signals = signals.sort_index() if old_signals is not None and not over_write: old_max = old_signals.index.get_level_values("datetime").max() @@ -275,14 +298,19 @@ def prepare_signals(self, delay=False, over_write=False): # if self.signal_rec is None: # with R.start(experiment_name=self.signal_exp_name, recorder_name=self.exp_name, resume=True): # self.signal_rec = R.get_recorder() - # signals = None - # try: - # signals = self.signal_rec.load_object("signals") - # except OSError: - # self.logger.warn("Can not find `signals`, have you called `prepare_signals` before?") + # signals = self.signal_rec.load_object("signals") # return signals - def _list_latest(self, rec_list): + def _list_latest(self, rec_list: List[Recorder]): + """ + List latest recorder form rec_list + + Args: + rec_list (List[Recorder]): a list of Recorder + + Returns: + List[Recorder], pd.Timestamp: the latest recorders and its test end time + """ if len(rec_list) == 0: return rec_list, None max_test = max(rec.load_object("task")["dataset"]["kwargs"]["segments"]["test"] for rec in rec_list) @@ -291,3 +319,16 @@ def _list_latest(self, rec_list): if rec.load_object("task")["dataset"]["kwargs"]["segments"]["test"] == max_test: latest_rec.append(rec) return latest_rec, max_test + + def reset(self): + """ + NOTE: This method will delete all recorder in Experiment and reset the Trainer! + """ + self.trainer.reset() + # delete models + exp = R.get_exp(experiment_name=self.exp_name) + for rid in exp.list_recorders(): + exp.delete_recorder(rid) + # delete signals + for rid in list_recorders(self.signal_exp_name, lambda x: True if x.info["name"] == self.exp_name else False): + exp.delete_recorder(rid) diff --git a/qlib/workflow/online/update.py b/qlib/workflow/online/update.py index 5b58360d83..69ad553245 100644 --- a/qlib/workflow/online/update.py +++ b/qlib/workflow/online/update.py @@ -1,18 +1,20 @@ -from typing import Union, List -from qlib.data.dataset import DatasetH -from qlib.workflow import R -from qlib.data import D +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +""" +Update is a module to update artifacts such as predictions, when the stock data updating. +""" + +from abc import ABCMeta, abstractmethod + import pandas as pd from qlib import get_module_logger -from qlib.workflow import R -from qlib.model import Model -from qlib.model.trainer import task_train -from qlib.workflow.recorder import Recorder -from qlib.workflow.task.utils import list_recorders -from qlib.data.dataset.handler import DataHandlerLP +from qlib.data import D from qlib.data.dataset import DatasetH -from abc import ABCMeta, abstractmethod +from qlib.data.dataset.handler import DataHandlerLP +from qlib.model import Model from qlib.utils import get_date_by_shift +from qlib.workflow.recorder import Recorder class RMDLoader: @@ -25,19 +27,22 @@ def __init__(self, rec: Recorder): def get_dataset(self, start_time, end_time, segments=None) -> DatasetH: """ - load, config and setup dataset. - - This dataset is for inference - - Parameters - ---------- - start_time : - the start_time of underlying data - end_time : - the end_time of underlying data - segments : dict - the segments config for dataset - Due to the time series dataset (TSDatasetH), the test segments maybe different from start_time and end_time + Load, config and setup dataset. + + This dataset is for inference. + + Args: + start_time : + the start_time of underlying data + end_time : + the end_time of underlying data + segments : dict + the segments config for dataset + Due to the time series dataset (TSDatasetH), the test segments maybe different from start_time and end_time + + Returns: + DatasetH: the instance of DatasetH + """ if segments is None: segments = {"test": (start_time, end_time)} @@ -52,7 +57,7 @@ def get_model(self) -> Model: class RecordUpdater(metaclass=ABCMeta): """ - Updata a specific recorders + Update a specific recorders """ def __init__(self, record: Recorder, need_log=True, *args, **kwargs): @@ -75,16 +80,17 @@ class PredUpdater(RecordUpdater): def __init__(self, record: Recorder, to_date=None, hist_ref: int = 0, freq="day", need_log=True): """ - Parameters - ---------- - record : Recorder - to_date : - update to prediction to the `to_date` - hist_ref : int - Sometimes, the dataset will have historical depends. - Leave the problem to user to set the length of historical dependancy - NOTE: the start_time is not included in the hist_ref - # TODO: automate this step in the future. + Init PredUpdater. + + Args: + record : Recorder + to_date : + update to prediction to the `to_date` + hist_ref : int + Sometimes, the dataset will have historical depends. + Leave the problem to user to set the length of historical dependency + NOTE: the start_time is not included in the hist_ref + # TODO: automate this step in the future. """ super().__init__(record=record, need_log=need_log) @@ -101,9 +107,12 @@ def __init__(self, record: Recorder, to_date=None, hist_ref: int = 0, freq="day" def prepare_data(self) -> DatasetH: """ - # Load dataset + Load dataset Seperating this function will make it easier to reuse the dataset + + Returns: + DatasetH: the instance of DatasetH """ start_time_buffer = get_date_by_shift(self.last_end, -self.hist_ref + 1, clip_shift=False, freq=self.freq) start_time = get_date_by_shift(self.last_end, 1, freq=self.freq) @@ -113,9 +122,12 @@ def prepare_data(self) -> DatasetH: def update(self, dataset: DatasetH = None): """ - update the precition in a recorder + Update the precition in a recorder + + Args: + DatasetH: the instance of DatasetH. None for reprepare. """ - # FIXME: the problme below is not solved + # FIXME: the problem below is not solved # The model dumped on GPU instances can not be loaded on CPU instance. Follow exception will raised # RuntimeError: Attempting to deserialize object on a CUDA device but torch.cuda.is_available() is False. If you are running on a CPU-only machine, please use torch.load with map_location=torch.device('cpu') to map your storages to the CPU. # https://github.com/pytorch/pytorch/issues/16797 diff --git a/qlib/workflow/online/utils.py b/qlib/workflow/online/utils.py index 1cd89d6680..4d630a6656 100644 --- a/qlib/workflow/online/utils.py +++ b/qlib/workflow/online/utils.py @@ -1,7 +1,14 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + """ -This module is like a online backend, deciding which models are `online` models and how can change them +OnlineTool is a module to set and unset a series of `online` models. +The `online` models are some decisive models in some time point, which can be changed with the change of time. +This allows us to use efficient submodels as the market style changing. """ + from typing import List, Union + from qlib.log import get_module_logger from qlib.workflow.online.update import PredUpdater from qlib.workflow.recorder import Recorder @@ -12,60 +19,66 @@ class OnlineTool: ONLINE_KEY = "online_status" # the online status key in recorder ONLINE_TAG = "online" # the 'online' model - # NOTE: The meaning of this tag is that we can not assume the training models can be trained before we need its predition. Whenever finished training, it can be guaranteed that there are some online models. - NEXT_ONLINE_TAG = "next_online" # the 'next online' model, which can be 'online' model when call reset_online_model OFFLINE_TAG = "offline" # the 'offline' model, not for online serving def __init__(self, need_log=True): """ - init OnlineTool. + Init OnlineTool. Args: need_log (bool, optional): print log or not. Defaults to True. """ self.logger = get_module_logger(self.__class__.__name__) self.need_log = need_log - self.cur_time = None - def set_online_tag(self, tag, recorder): + def set_online_tag(self, tag, recorder: Union[list, object]): """ Set `tag` to the model to sign whether online. Args: - tag (str): the tags in `ONLINE_TAG`, `NEXT_ONLINE_TAG`, `OFFLINE_TAG` + tag (str): the tags in `ONLINE_TAG`, `OFFLINE_TAG` + recorder (Union[list,object]): the model's recorder """ raise NotImplementedError(f"Please implement the `set_online_tag` method.") - def get_online_tag(self): + def get_online_tag(self, recorder: object) -> str: """ - Given a model and return its online tag. + Given a model recorder and return its online tag. + + Args: + recorder (Object): the model's recorder + + Returns: + str: the online tag """ raise NotImplementedError(f"Please implement the `get_online_tag` method.") - def reset_online_tag(self, recorders=None): - """offline all models and set the recorders to 'online'. If no parameter and no 'next online' model, then do nothing. + def reset_online_tag(self, recorder: Union[list, object]): + """ + Offline all models and set the recorders to 'online'. Args: - recorders (List, optional): - the recorders you want to reset to 'online'. If don't give, set 'next online' model to 'online' model. If there isn't any 'next online' model, then maintain existing 'online' model. + recorder (Union[list,object]): + the recorder you want to reset to 'online'. - Returns: - list: new online recorder. [] if there is no update. """ raise NotImplementedError(f"Please implement the `reset_online_tag` method.") - def online_models(self): + def online_models(self) -> list: """ - Return `online` models. + Get current `online` models + + Returns: + list: a list of `online` models. """ raise NotImplementedError(f"Please implement the `online_models` method.") def update_online_pred(self, to_date=None): """ - Update the predictions of online models to a date. + Update the predictions of `online` models to a date. Args: - to_date (pd.Timestamp): the pred before this date will be updated. None for latest. + to_date (pd.Timestamp): the pred before this date will be updated. None for update to latest. """ raise NotImplementedError(f"Please implement the `update_online_pred` method.") @@ -74,12 +87,11 @@ def update_online_pred(self, to_date=None): class OnlineToolR(OnlineTool): """ The implementation of OnlineTool based on (R)ecorder. - """ def __init__(self, experiment_name: str, need_log=True): """ - init OnlineToolR. + Init OnlineToolR. Args: experiment_name (str): the experiment name. @@ -90,11 +102,11 @@ def __init__(self, experiment_name: str, need_log=True): def set_online_tag(self, tag, recorder: Union[Recorder, List]): """ - Set `tag` to the model to sign whether online. + Set `tag` to the model's recorder to sign whether online. Args: tag (str): the tags in `ONLINE_TAG`, `NEXT_ONLINE_TAG`, `OFFLINE_TAG` - recorder (Union[Recorder, List]) + recorder (Union[Recorder, List]): a list of Recorder or an instance of Recorder """ if isinstance(recorder, Recorder): recorder = [recorder] @@ -103,50 +115,40 @@ def set_online_tag(self, tag, recorder: Union[Recorder, List]): if self.need_log: self.logger.info(f"Set {len(recorder)} models to '{tag}'.") - def get_online_tag(self, recorder: Recorder): + def get_online_tag(self, recorder: Recorder) -> str: """ - Given a model and return its online tag. + Given a model recorder and return its online tag. Args: - recorder (Recorder): a instance of recorder + recorder (Recorder): an instance of recorder Returns: - str: the tag + str: the online tag """ tags = recorder.list_tags() return tags.get(self.ONLINE_KEY, self.OFFLINE_TAG) - def reset_online_tag(self, recorder: Union[Recorder, List] = None): - """offline all models and set the recorders to 'online'. If no parameter and no 'next online' model, then do nothing. + def reset_online_tag(self, recorder: Union[Recorder, List]): + """ + Offline all models and set the recorders to 'online'. Args: - recorders (Union[Recorder, List], optional): - the recorders you want to reset to 'online'. If don't give, set 'next online' model to 'online' model. If there isn't any 'next online' model, then maintain existing 'online' model. + recorder (Union[Recorder, List]): + the recorder you want to reset to 'online'. - Returns: - list: new online recorder. [] if there is no update. """ - if recorder is None: - recorder = list( - list_recorders(self.exp_name, lambda rec: self.get_online_tag(rec) == self.NEXT_ONLINE_TAG).values() - ) if isinstance(recorder, Recorder): recorder = [recorder] - if len(recorder) == 0: - if self.need_log: - self.logger.info("No 'next online' model, just use current 'online' models.") - return [] recs = list_recorders(self.exp_name) self.set_online_tag(self.OFFLINE_TAG, list(recs.values())) self.set_online_tag(self.ONLINE_TAG, recorder) - return recorder - def online_models(self): + def online_models(self) -> list: """ - Return online models. + Get current `online` models Returns: - list: the list of online models + list: a list of `online` models. """ return list(list_recorders(self.exp_name, lambda rec: self.get_online_tag(rec) == self.ONLINE_TAG).values()) @@ -155,7 +157,7 @@ def update_online_pred(self, to_date=None): Update the predictions of online models to a date. Args: - to_date (pd.Timestamp): the pred before this date will be updated. None for latest in Calendar. + to_date (pd.Timestamp): the pred before this date will be updated. None for update to latest time in Calendar. """ online_models = self.online_models() for rec in online_models: diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index eb0a20029c..d74d081846 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -1,9 +1,11 @@ -from abc import abstractmethod -from typing import Callable, Union -from qlib import init +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +""" +Collector can collect object from everywhere and process them such as merging, grouping, averaging and so on. +""" + from qlib.workflow import R -from qlib.workflow.task.utils import list_recorders -from qlib.utils.serial import Serializable import dill as pickle @@ -19,7 +21,7 @@ def __init__(self, process_list=[]): process_list = [process_list] self.process_list = process_list - def collect(self): + def collect(self) -> dict: """Collect the results and return a dict like {key: things} Returns: @@ -36,7 +38,7 @@ def collect(self): raise NotImplementedError(f"Please implement the `collect` method.") @staticmethod - def process_collect(collected_dict, process_list=[], *args, **kwargs): + def process_collect(collected_dict, process_list=[], *args, **kwargs) -> dict: """do a series of processing to the dict returned by collect and return a dict like {key: things} For example: you can group and ensemble. @@ -61,7 +63,7 @@ def process_collect(collected_dict, process_list=[], *args, **kwargs): result[artifact] = value return result - def __call__(self, *args, **kwargs): + def __call__(self, *args, **kwargs) -> dict: """ do the workflow including collect and process_collect @@ -124,7 +126,7 @@ def __init__(self, collector_dict, process_list=[]): super().__init__(process_list=process_list) self.collector_dict = collector_dict - def collect(self): + def collect(self) -> dict: collect_dict = {} for key, collector in self.collector_dict.items(): collect_dict[key] = collector() @@ -153,10 +155,10 @@ def __init__( artifacts_path (dict, optional): The artifacts name and its path in Recorder. Defaults to {"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"}. artifacts_key (str or List, optional): the artifacts key you want to get. If None, get all artifacts. """ + super().__init__(process_list=process_list) if isinstance(experiment, str): experiment = R.get_exp(experiment_name=experiment) self.experiment = experiment - self.process_list = process_list self.artifacts_path = artifacts_path if rec_key_func is None: rec_key_func = lambda rec: rec.info["id"] @@ -166,7 +168,7 @@ def __init__( self.artifacts_key = artifacts_key self._rec_filter_func = rec_filter_func - def collect(self, artifacts_key=None, rec_filter_func=None): + def collect(self, artifacts_key=None, rec_filter_func=None) -> dict: """Collect different artifacts based on recorder after filtering. Args: @@ -203,5 +205,11 @@ def collect(self, artifacts_key=None, rec_filter_func=None): return collect_dict - def get_exp_name(self): + def get_exp_name(self) -> str: + """ + Get experiment name + + Returns: + str: experiment name + """ return self.experiment.name diff --git a/qlib/workflow/task/gen.py b/qlib/workflow/task/gen.py index 158bc99168..c4c6bab7fb 100644 --- a/qlib/workflow/task/gen.py +++ b/qlib/workflow/task/gen.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """ -this is a task generator +Task generator can generate many tasks based on TaskGen and some task templates. """ import abc import copy @@ -113,7 +113,7 @@ def __init__(self, step: int = 40, rtype: str = ROLL_EX, modify_end_time=True): self.test_key = "test" self.train_key = "train" - def generate(self, task: dict): + def generate(self, task: dict) -> typing.List[dict]: """ Converting the task into a rolling task. @@ -158,6 +158,10 @@ def generate(self, task: dict): }, ] } + + Returns + ---------- + typing.List[dict]: a list of tasks """ res = [] @@ -196,16 +200,18 @@ def generate(self, task: dict): # update segments of this task t["dataset"]["kwargs"]["segments"] = copy.deepcopy(segments) - # if end_time < the end of test_segments, then change end_time to allow load more data - if ( - self.modify_end_time - and self.ta.cal_interval( + + try: + interval = self.ta.cal_interval( t["dataset"]["kwargs"]["handler"]["kwargs"]["end_time"], t["dataset"]["kwargs"]["segments"][self.test_key][1], ) - < 0 - ): - t["dataset"]["kwargs"]["handler"]["kwargs"]["end_time"] = copy.deepcopy(segments[self.test_key][1]) + # if end_time < the end of test_segments, then change end_time to allow load more data + if self.modify_end_time and interval < 0: + t["dataset"]["kwargs"]["handler"]["kwargs"]["end_time"] = copy.deepcopy(segments[self.test_key][1]) + except KeyError: + # Maybe the user dataset has no handler or end_time + pass prev_seg = segments res.append(t) return res diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 9d50d85638..3c3144fe89 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -1,31 +1,39 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. + """ -A task consists of 3 parts +TaskManager can fetch unused tasks automatically and manager the lifecycle of a set of tasks with error handling. +These features can run tasks concurrently and ensure every task will be used only once. +Task Manager will store all tasks in `MongoDB `_. +Users **MUST** finished the configuration of `MongoDB `_ when using this module. + +A task in TaskManager consists of 3 parts - tasks description: the desc will define the task - tasks status: the status of the task - tasks result information : A user can get the task with the task description and task result. - """ -from bson.binary import Binary +import concurrent import pickle -from pymongo.errors import InvalidDocument -from bson.objectid import ObjectId -from contextlib import contextmanager -import qlib -from tqdm.cli import tqdm import time -import concurrent +from contextlib import contextmanager +from typing import Callable, List + +import fire import pymongo -from qlib.config import C +from bson.binary import Binary +from bson.objectid import ObjectId +from pymongo.errors import InvalidDocument +from qlib import auto_init, get_module_logger +from tqdm.cli import tqdm + from .utils import get_mongodb -from qlib import get_module_logger, auto_init -import fire class TaskManager: - """TaskManager - here is what will a task looks like when it created by TaskManager + """ + TaskManager + + Here is what will a task looks like when it created by TaskManager .. code-block:: python @@ -42,6 +50,12 @@ class TaskManager: .. note:: Assumption: the data in MongoDB was encoded and the data out of MongoDB was decoded + + Here are four status which are: + STATUS_WAITING: waiting for train + STATUS_RUNNING: training + STATUS_PART_DONE: finished some step and waiting for next step. + STATUS_DONE: all work done """ STATUS_WAITING = "waiting" @@ -53,7 +67,7 @@ class TaskManager: def __init__(self, task_pool: str = None): """ - init Task Manager, remember to make the statement of MongoDB url and database name firstly. + Init Task Manager, remember to make the statement of MongoDB url and database name firstly. Parameters ---------- @@ -65,7 +79,7 @@ def __init__(self, task_pool: str = None): self.task_pool = getattr(self.mdb, task_pool) self.logger = get_module_logger(self.__class__.__name__) - def list(self): + def list(self) -> list: """ list the all collection(task_pool) of the db @@ -92,7 +106,9 @@ def _dict_to_str(self, flt): return {k: str(v) for k, v in flt.items()} def replace_task(self, task, new_task): - # assume that the data out of interface was decoded and the data in interface was encoded + """ + Use a new task to replace a old one + """ new_task = self._encode_task(new_task) query = {"_id": ObjectId(task["_id"])} try: @@ -121,7 +137,7 @@ def insert_task_def(self, task_def): Returns ------- - + pymongo.results.InsertOneResult """ task = self._encode_task( { @@ -133,9 +149,9 @@ def insert_task_def(self, task_def): insert_result = self.insert_task(task) return insert_result - def create_task(self, task_def_l, dry_run=False, print_nt=False): + def create_task(self, task_def_l, dry_run=False, print_nt=False) -> List[str]: """ - if the tasks in task_def_l is new, then insert new tasks into the task_pool + If the tasks in task_def_l is new, then insert new tasks into the task_pool Parameters ---------- @@ -145,6 +161,7 @@ def create_task(self, task_def_l, dry_run=False, print_nt=False): if insert those new tasks to task pool print_nt: bool if print new task + Returns ------- list @@ -165,7 +182,7 @@ def create_task(self, task_def_l, dry_run=False, print_nt=False): print(t) if dry_run: - return + return [] _id_list = [] for t in new_tasks: @@ -174,7 +191,17 @@ def create_task(self, task_def_l, dry_run=False, print_nt=False): return _id_list - def fetch_task(self, query={}, status=STATUS_WAITING): + def fetch_task(self, query={}, status=STATUS_WAITING) -> dict: + """ + Use query to fetch tasks + + Args: + query (dict, optional): query dict. Defaults to {}. + status (str, optional): [description]. Defaults to STATUS_WAITING. + + Returns: + dict: a task(document in collection) after decoding + """ query = query.copy() if "_id" in query: query["_id"] = ObjectId(query["_id"]) @@ -191,7 +218,7 @@ def fetch_task(self, query={}, status=STATUS_WAITING): @contextmanager def safe_fetch_task(self, query={}, status=STATUS_WAITING): """ - fetch task from task_pool using query with contextmanager + Fetch task from task_pool using query with contextmanager Parameters ---------- @@ -200,7 +227,7 @@ def safe_fetch_task(self, query={}, status=STATUS_WAITING): Returns ------- - + dict: a task(document in collection) after decoding """ task = self.fetch_task(query=query, status=status) try: @@ -231,7 +258,7 @@ def query(self, query={}, decode=True): Returns ------- - + dict: a task(document in collection) after decoding """ query = query.copy() if "_id" in query: @@ -240,16 +267,40 @@ def query(self, query={}, decode=True): yield self._decode_task(t) def re_query(self, _id): + """ + Use _id to query task. + + Args: + _id (str): _id of a document + + Returns: + dict: a task(document in collection) after decoding + """ t = self.task_pool.find_one({"_id": ObjectId(_id)}) return self._decode_task(t) - def commit_task_res(self, task, res, status=None): + def commit_task_res(self, task, res, status=STATUS_DONE): + """ + Commit the result to task['res']. + + Args: + task ([type]): [description] + res (object): the result you want to save + status (str, optional): STATUS_WAITING, STATUS_RUNNING, STATUS_DONE, STATUS_PART_DONE. Defaults to STATUS_DONE. + """ # A workaround to use the class attribute. if status is None: status = TaskManager.STATUS_DONE self.task_pool.update_one({"_id": task["_id"]}, {"$set": {"status": status, "res": Binary(pickle.dumps(res))}}) - def return_task(self, task, status=None): + def return_task(self, task, status=STATUS_WAITING): + """ + Return a task to status. Alway using in error handling. + + Args: + task ([type]): [description] + status (str, optional): STATUS_WAITING, STATUS_RUNNING, STATUS_DONE, STATUS_PART_DONE. Defaults to STATUS_WAITING. + """ if status is None: status = TaskManager.STATUS_WAITING update_dict = {"$set": {"status": status}} @@ -257,7 +308,7 @@ def return_task(self, task, status=None): def remove(self, query={}): """ - remove the task using query + Remove the task using query Parameters ---------- @@ -295,7 +346,7 @@ def reset_status(self, query, status): def prioritize(self, task, priority: int): """ - set priority for task + Set priority for task Parameters ---------- @@ -331,29 +382,37 @@ def __str__(self): def run_task( - task_func, - task_pool, - force_release=False, - before_status=TaskManager.STATUS_WAITING, - after_status=TaskManager.STATUS_DONE, - *args, + task_func: Callable, + task_pool: str, + force_release: bool = False, + before_status: str = TaskManager.STATUS_WAITING, + after_status: str = TaskManager.STATUS_DONE, **kwargs, ): """ While task pool is not empty (has WAITING tasks), use task_func to fetch and run tasks in task_pool + After running this method, here are 4 situations (before_status -> after_status): + STATUS_WAITING -> STATUS_DONE: use task["def"] as `task_func` param + STATUS_WAITING -> STATUS_PART_DONE: use task["def"] as `task_func` param + STATUS_PART_DONE -> STATUS_PART_DONE: use task["res"] as `task_func` param + STATUS_PART_DONE -> STATUS_DONE: use task["res"] as `task_func` param + Parameters ---------- - task_func : def (task_def, *args, **kwargs) -> - the function to run the task + task_func : Callable + def (task_def, **kwargs) -> + the function to run the task task_pool : str the name of the task pool (Collection in MongoDB) - force_release : + force_release : bool will the program force to release the resource - args : - args - kwargs : - kwargs + before_status : str: + the tasks in before_status will be fetched and trained. Can be STATUS_WAITING, STATUS_PART_DONE. + after_status : str: + the tasks after trained will become after_status. Can be STATUS_WAITING, STATUS_PART_DONE. + kwargs + the params for `task_func` """ tm = TaskManager(task_pool) @@ -364,19 +423,19 @@ def run_task( if task is None: break get_module_logger("run_task").info(task["def"]) - # when fetching `WAITING` task, use task_def to train + # when fetching `WAITING` task, use task["def"] to train if before_status == TaskManager.STATUS_WAITING: param = task["def"] - # when fetching `PART_DONE` task, use task_res to train for the result has been saved + # when fetching `PART_DONE` task, use task["res"] to train because the middle result has been saved to task["res"] elif before_status == TaskManager.STATUS_PART_DONE: param = task["res"] else: raise ValueError("The fetched task must be `STATUS_WAITING` or `STATUS_PART_DONE`!") if force_release: with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: - res = executor.submit(task_func, param, *args, **kwargs).result() + res = executor.submit(task_func, param, **kwargs).result() else: - res = task_func(param, *args, **kwargs) + res = task_func(param, **kwargs) tm.commit_task_res(task, res, status=after_status) ever_run = True diff --git a/qlib/workflow/task/utils.py b/qlib/workflow/task/utils.py index ce8e0dfa35..ed5e1a2359 100644 --- a/qlib/workflow/task/utils.py +++ b/qlib/workflow/task/utils.py @@ -1,5 +1,10 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. + +""" +Some tools for task management. +""" + import bisect import pandas as pd from qlib.data import D @@ -7,13 +12,14 @@ from qlib.config import C from qlib.log import get_module_logger from pymongo import MongoClient +from pymongo.database import Database from typing import Union -def get_mongodb(): - """ +def get_mongodb() -> Database: - get database in MongoDB, which means you need to declare the address and the name of database. + """ + Get database in MongoDB, which means you need to declare the address and the name of database. for example: Using qlib.init(): @@ -31,6 +37,8 @@ def get_mongodb(): "task_db_name" : "rolling_db" } + Returns: + Database: the Database instance """ try: cfg = C["mongo"] @@ -43,7 +51,8 @@ def get_mongodb(): def list_recorders(experiment, rec_filter_func=None): - """list all recorders which can pass the filter in a experiment. + """ + List all recorders which can pass the filter in a experiment. Args: experiment (str or Experiment): the name of a Experiment or a instance @@ -65,7 +74,7 @@ def list_recorders(experiment, rec_filter_func=None): class TimeAdjuster: """ - find appropriate date and adjust date. + Find appropriate date and adjust date. """ def __init__(self, future=True, end_time=None): @@ -88,15 +97,15 @@ def get(self, idx: int): return None return self.cals[idx] - def max(self): + def max(self) -> pd.Timestamp: """ Return the max calendar datetime """ return max(self.cals) - def align_idx(self, time_point, tp_type="start"): + def align_idx(self, time_point, tp_type="start") -> int: """ - align the index of time_point in the calendar + Align the index of time_point in the calendar Parameters ---------- @@ -116,9 +125,9 @@ def align_idx(self, time_point, tp_type="start"): raise NotImplementedError(f"This type of input is not supported") return idx - def cal_interval(self, time_point_A, time_point_B): + def cal_interval(self, time_point_A, time_point_B) -> int: """ - calculate the trading day interval + Calculate the trading day interval (time_point_A - time_point_B) Args: time_point_A : time_point_A @@ -129,20 +138,22 @@ def cal_interval(self, time_point_A, time_point_B): """ return self.align_idx(time_point_A) - self.align_idx(time_point_B) - def align_time(self, time_point, tp_type="start"): + def align_time(self, time_point, tp_type="start") -> pd.Timestamp: """ Align time_point to trade date of calendar - Parameters - ---------- - time_point - Time point - tp_type : str - time point type (`"start"`, `"end"`) + Args: + time_point + Time point + tp_type : str + time point type (`"start"`, `"end"`) + + Returns: + pd.Timestamp """ return self.cals[self.align_idx(time_point, tp_type=tp_type)] - def align_seg(self, segment: Union[dict, tuple]): + def align_seg(self, segment: Union[dict, tuple]) -> Union[dict, tuple]: """ align the given date to trade date @@ -162,7 +173,7 @@ def align_seg(self, segment: Union[dict, tuple]): Returns ------- - the start and end trade date (pd.Timestamp) between the given start and end date. + Union[dict, tuple]: the start and end trade date (pd.Timestamp) between the given start and end date. """ if isinstance(segment, dict): return {k: self.align_seg(seg) for k, seg in segment.items()} @@ -171,7 +182,7 @@ def align_seg(self, segment: Union[dict, tuple]): else: raise NotImplementedError(f"This type of input is not supported") - def truncate(self, segment: tuple, test_start, days: int): + def truncate(self, segment: tuple, test_start, days: int) -> tuple: """ truncate the segment based on the test_start date @@ -183,6 +194,10 @@ def truncate(self, segment: tuple, test_start, days: int): days : int The trading days to be truncated the data in this segment may need 'days' data + + Returns + --------- + tuple: new segment """ test_idx = self.align_idx(test_start) if isinstance(segment, tuple): @@ -198,7 +213,7 @@ def truncate(self, segment: tuple, test_start, days: int): SHIFT_SD = "sliding" SHIFT_EX = "expanding" - def shift(self, seg: tuple, step: int, rtype=SHIFT_SD): + def shift(self, seg: tuple, step: int, rtype=SHIFT_SD) -> tuple: """ shift the datatime of segment @@ -211,6 +226,10 @@ def shift(self, seg: tuple, step: int, rtype=SHIFT_SD): rtype : str rolling type ("sliding" or "expanding") + Returns + -------- + tuple: new segment + Raises ------ KeyError: From 84c56f13bd47ee45ae50ec74c5a154295cf55a43 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Thu, 6 May 2021 04:18:55 +0000 Subject: [PATCH 51/61] docs and bug fixed --- docs/advanced/task_management.rst | 48 +++++---- docs/component/online.rst | 41 ++++++++ docs/index.rst | 1 + docs/reference/api.rst | 55 +++++++++-- .../online_srv/online_management_simulate.py | 2 + .../online_srv/rolling_online_management.py | 4 +- qlib/data/dataset/__init__.py | 55 +++-------- qlib/data/dataset/handler.py | 8 +- qlib/model/ens/ensemble.py | 46 +++++++++ qlib/model/ens/group.py | 7 ++ qlib/model/trainer.py | 16 ++- qlib/workflow/online/manager.py | 51 +++++++--- qlib/workflow/online/strategy.py | 97 ++++++++++--------- qlib/workflow/online/update.py | 10 +- qlib/workflow/online/utils.py | 3 + qlib/workflow/task/collect.py | 5 +- qlib/workflow/task/manage.py | 10 +- 17 files changed, 313 insertions(+), 146 deletions(-) create mode 100644 docs/component/online.rst diff --git a/docs/advanced/task_management.rst b/docs/advanced/task_management.rst index 230a4e9d1a..a68c126276 100644 --- a/docs/advanced/task_management.rst +++ b/docs/advanced/task_management.rst @@ -1,4 +1,4 @@ -.. _task_managment: +.. _task_management: ================================= Task Management @@ -10,15 +10,17 @@ Introduction ============= The `Workflow <../component/introduction.html>`_ part introduces how to run research workflow in a loosely-coupled way. But it can only execute one ``task`` when you use ``qrun``. -To automatically generate and execute different tasks, ``Task Management`` provides a whole process including `Task Generating`_, `Task Storing`_, `Task Running`_ and `Task Collecting`_. +To automatically generate and execute different tasks, ``Task Management`` provides a whole process including `Task Generating`_, `Task Storing`_, `Task Training`_ and `Task Collecting`_. With this module, users can run their ``task`` automatically at different periods, in different losses, or even by different models. -An example of the entire process is shown `here `_. +This whole process can be used in `Online Serving <../component/online.html>`_. + +An example of the entire process is shown `here `_. Task Generating =============== A ``task`` consists of `Model`, `Dataset`, `Record` or anything added by users. -The specific task template(/definition/config) can be viewed in +The specific task template can be viewed in `Task Section <../component/workflow.html#task-section>`_. Even though the task template is fixed, users can customize their ``TaskGen`` to generate different ``task`` by task template. @@ -27,15 +29,16 @@ Here is the base class of ``TaskGen``: .. autoclass:: qlib.workflow.task.gen.TaskGen :members: -``Qlib`` provider a class `RollingGen `_ to generate a list of ``task`` of the dataset in different date segments. -This class allows users to verify the effect of data from different periods on the model in one experiment. +``Qlib`` provides a class `RollingGen `_ to generate a list of ``task`` of the dataset in different date segments. +This class allows users to verify the effect of data from different periods on the model in one experiment. More information in `here <../reference/api.html#TaskGen>`_. Task Storing =============== To achieve higher efficiency and the possibility of cluster operation, ``Task Manager`` will store all tasks in `MongoDB `_. +``TaskManager`` can fetch undone tasks automatically and manage the lifecycle of a set of tasks with error handling. Users **MUST** finished the configuration of `MongoDB `_ when using this module. -Users need to provide the URL and database name of ``task`` storing like this. +Users need to provide the MongoDB URL and database name for using ``TaskManager`` in `initialization <../start/initialization.html#Parameters>`_ or make statement like this. .. code-block:: python @@ -45,13 +48,12 @@ Users need to provide the URL and database name of ``task`` storing like this. "task_db_name" : "rolling_db" # database name } -The CRUD methods of ``task`` can be found in TaskManager. -More methods can be seen in the `Github `_. - .. autoclass:: qlib.workflow.task.manage.TaskManager :members: -Task Running +More information of ``Task Manager`` can be found in `here <../reference/api.html#TaskManager>`_. + +Task Training =============== After generating and storing those ``task``, it's time to run the ``task`` which are in the *WAITING* status. ``Qlib`` provides a method called ``run_task`` to run those ``task`` in task pool, however, users can also customize how tasks are executed. @@ -60,14 +62,24 @@ It will run the whole workflow defined by ``task``, which includes *Model*, *Dat .. autofunction:: qlib.workflow.task.manage.run_task +Meanwhile, ``Qlib`` provides a module called ``Trainer``. +``Trainer`` will train a list of tasks and return a list of model recorder. +``Qlib`` offer two kind of Trainer, TrainerR is the simplest way and TrainerRM is based on TaskManager to help manager tasks lifecycle automatically. +If you do not want to use ``Task Manager`` to manage tasks, then use TrainerR to train a list of tasks generated by ``TaskGen`` is enough. +More information is in `here <../reference/api.html#Trainer>`_. + Task Collecting =============== -To see the results of ``task`` after running or to update something, ``Qlib`` provides a ``TaskCollector`` to collect the tasks by filter condition (optional). -Here are some methods in this class. +To collect the results of ``task`` after training, ``Qlib`` provides `Collector <../reference/api.html#Collector>`_, `Group <../reference/api.html#Group>`_ and `Ensemble <../reference/api.html#Ensemble>`_ to collect the results in a readable, expandable and loosely-coupled way. -.. autoclass:: qlib.workflow.task.collect.TaskCollector - :members: +`Collector <../reference/api.html#Collector>`_ can collect object from everywhere and process them such as merging, grouping, averaging and so on. It has 2 step action including ``collect`` (collect anything in a dict) and ``process_collect`` (process collected dict). + +`Group <../reference/api.html#Group>`_ also has 2 steps including ``group`` (can group a set of object based on `group_func` and change them to a dict) and ``reduce`` (can make a dict become an ensemble based on some rule). +For example: {(A,B,C1): object, (A,B,C2): object} ---``group``---> {(A,B): {C1: object, C2: object}} ---``reduce``---> {(A,B): object} + +`Ensemble <../reference/api.html#Ensemble>`_ can merge the objects in an ensemble. +For example: {C1: object, C2: object} ---``Ensemble``---> object + +So the hierarchy is ``Collector``'s second step correspond to ``Group``. And ``Group``'s second step correspond to ``Ensemble``. -``Qlib`` provides a concrete `example `_, including a whole process of `Task Generating`_ (using `RollingGen `_), `Task Storing`_, `Task Running`_ and `Task Collecting`_. -Besides, the `example `_ uses a ``ModelUpdater`` inherited from ``TaskCollector``, which can update the inferences and retrain the model if it is out of date. -Actually, the model updating can be viewed as a subset of ``Online Serving``. \ No newline at end of file +For more information, please see `Collector <../reference/api.html#Collector>`_, `Group <../reference/api.html#Group>`_ and `Ensemble <../reference/api.html#Ensemble>`_, or the `example `_ \ No newline at end of file diff --git a/docs/component/online.rst b/docs/component/online.rst new file mode 100644 index 0000000000..e251731533 --- /dev/null +++ b/docs/component/online.rst @@ -0,0 +1,41 @@ +.. _online: + +================================= +Online Serving +================================= +.. currentmodule:: qlib + + +Introduction +============= +In addition to backtesting, one way to test a model is effective is to make predictions in real market conditions or even do real trading based on those predictions. +``Online Serving`` is a set of module for online models using latest data, +which including `Online Manager <#Online Manager>`_, `Online Strategy <#Online Strategy>`_, `Online Tool <#Online Tool>`_, `Updater <#Updater>`_. + +`Here `_ are several examples for reference, which demonstrate different features of ``Online Serving``. +If you have many models or `task` need to be managed, please consider `Task Management <../advanced/task_management.html>`_. +The `examples `_ maybe based on `Task Management <../advanced/task_management.html>`_ such as ``TrainerRM`` or ``Collector``. + +Online Manager +============= + +.. automodule:: qlib.workflow.online.manager + :members: + +Online Strategy +============= + +.. automodule:: qlib.workflow.online.strategy + :members: + +Online Tool +============= + +.. automodule:: qlib.workflow.online.utils + :members: + +Updater +============= + +.. automodule:: qlib.workflow.online.update + :members: \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 274dc8045e..803aa97d2d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -42,6 +42,7 @@ Document Structure Intraday Trading: Model&Strategy Testing Qlib Recorder: Experiment Management Analysis: Evaluation & Results Analysis + Online Serving: Online Management & Strategy & Tool .. toctree:: :maxdepth: 3 diff --git a/docs/reference/api.rst b/docs/reference/api.rst index 691dff7036..edba6228a0 100644 --- a/docs/reference/api.rst +++ b/docs/reference/api.rst @@ -154,36 +154,71 @@ Record Template .. automodule:: qlib.workflow.record_temp :members: - Task Management ==================== -RollingGen +TaskGen -------------------- -.. autoclass:: qlib.workflow.task.gen.RollingGen +.. automodule:: qlib.workflow.task.gen :members: TaskManager -------------------- -.. autoclass:: qlib.workflow.task.manage.TaskManager +.. automodule:: qlib.workflow.task.manage + :members: + +Trainer +-------------------- +.. automodule:: qlib.model.trainer :members: -TaskCollector +Collector -------------------- -.. autoclass:: qlib.workflow.task.collect.TaskCollector +.. automodule:: qlib.workflow.task.collect :members: -ModelUpdater +Group -------------------- -.. autoclass:: qlib.workflow.task.update.ModelUpdater +.. automodule:: qlib.model.ens.group :members: -TimeAdjuster +Ensemble -------------------- -.. autoclass:: qlib.workflow.task.utils.TimeAdjuster +.. automodule:: qlib.model.ens.ensemble :members: +Utils +-------------------- +.. automodule:: qlib.workflow.task.utils + :members: + + +Online Serving +==================== + + +Online Manager +-------------------- +.. automodule:: qlib.workflow.online.manager + :members: + +Online Strategy +-------------------- +.. automodule:: qlib.workflow.online.strategy + :members: + +Online Tool +-------------------- +.. automodule:: qlib.workflow.online.utils + :members: + +RecordUpdater +-------------------- +.. automodule:: qlib.workflow.online.update + :members: + + Utils ==================== diff --git a/examples/online_srv/online_management_simulate.py b/examples/online_srv/online_management_simulate.py index 16e985ccd3..7be46d999b 100644 --- a/examples/online_srv/online_management_simulate.py +++ b/examples/online_srv/online_management_simulate.py @@ -131,6 +131,8 @@ def main(self): self.rolling_online_manager.simulate(end_time=self.end_time) print("========== collect results ==========") print(self.rolling_online_manager.get_collector()()) + print("========== signals ==========") + print(self.rolling_online_manager.get_signals()) print("========== online history ==========") print(self.rolling_online_manager.get_online_history(self.exp_name)) diff --git a/examples/online_srv/rolling_online_management.py b/examples/online_srv/rolling_online_management.py index 950c9684d4..25b6fc4da3 100644 --- a/examples/online_srv/rolling_online_management.py +++ b/examples/online_srv/rolling_online_management.py @@ -86,7 +86,7 @@ def __init__( task_url="mongodb://10.0.0.4:27017/", task_db_name="rolling_db", rolling_step=550, - tasks=[task_xgboost_config, task_lgb_config], + tasks=[task_xgboost_config], # , task_lgb_config], ): mongo_conf = { "task_url": task_url, # your MongoDB url @@ -148,6 +148,8 @@ def routine(self): self.rolling_online_manager.routine() print("========== collect results ==========") print(self.collector()) + print("========== signals ==========") + print(self.rolling_online_manager.get_signals()) def main(self): self.first_run() diff --git a/qlib/data/dataset/__init__.py b/qlib/data/dataset/__init__.py index 4457dda5fa..4ae73c670e 100644 --- a/qlib/data/dataset/__init__.py +++ b/qlib/data/dataset/__init__.py @@ -27,7 +27,7 @@ def __init__(self, **kwargs): - setup data - The data related attributes' names should start with '_' so that it will not be saved on disk when serializing. - The data could specify the info to caculate the essential data for preparation + The data could specify the info to calculate the essential data for preparation """ self.setup_data(**kwargs) super().__init__() @@ -92,7 +92,7 @@ def __init__(self, handler: Union[Dict, DataHandler], segments: Dict[Text, Tuple handler : Union[dict, DataHandler] handler could be: - - insntance of `DataHandler` + - instance of `DataHandler` - config of `DataHandler`. Please refer to `DataHandler` @@ -114,7 +114,6 @@ def __init__(self, handler: Union[Dict, DataHandler], segments: Dict[Text, Tuple """ self.handler: DataHandler = init_instance_by_config(handler, accept_types=DataHandler) self.segments = segments.copy() - self.fetch_kwargs = {} super().__init__(**kwargs) def config(self, handler_kwargs: dict = None, **kwargs): @@ -124,7 +123,7 @@ def config(self, handler_kwargs: dict = None, **kwargs): Parameters ---------- handler_kwargs : dict - Config of DataHanlder, which could include the following arguments: + Config of DataHandler, which could include the following arguments: - arguments of DataHandler.conf_data, such as 'instruments', 'start_time' and 'end_time'. @@ -148,11 +147,11 @@ def setup_data(self, handler_kwargs: dict = None, **kwargs): Parameters ---------- handler_kwargs : dict - init arguments of DataHanlder, which could include the following arguments: + init arguments of DataHandler, which could include the following arguments: - init_type : Init Type of Handler - - enable_cache : wheter to enable cache + - enable_cache : whether to enable cache """ super().setup_data(**kwargs) @@ -172,7 +171,7 @@ def _prepare_seg(self, slc: slice, **kwargs): ---------- slc : slice """ - return self.handler.fetch(slc, **kwargs, **self.fetch_kwargs) + return self.handler.fetch(slc, **kwargs) def prepare( self, @@ -232,7 +231,7 @@ class TSDataSampler: (T)ime-(S)eries DataSampler This is the result of TSDatasetH - It works like `torch.data.utils.Dataset`, it provides a very convient interface for constructing time-series + It works like `torch.data.utils.Dataset`, it provides a very convenient interface for constructing time-series dataset based on tabular data. If user have further requirements for processing data, user could process them based on `TSDataSampler` or create @@ -289,29 +288,12 @@ def __init__(self, data: pd.DataFrame, start, end, step_len: int, fillna_type: s # the data type will be changed # The index of usable data is between start_idx and end_idx + self.start_idx, self.end_idx = self.data.index.slice_locs(start=pd.Timestamp(start), end=pd.Timestamp(end)) self.idx_df, self.idx_map = self.build_index(self.data) - self.data_index = deepcopy(self.data.index) - - if flt_data is not None: - self.flt_data = np.array(flt_data).reshape(-1) - self.idx_map = self.flt_idx_map(self.flt_data, self.idx_map) - self.data_index = self.data_index[np.where(self.flt_data == True)[0]] - - self.start_idx, self.end_idx = self.data_index.slice_locs(start=pd.Timestamp(start), end=pd.Timestamp(end)) self.idx_arr = np.array(self.idx_df.values, dtype=np.float64) # for better performance - + self.data_idx = deepcopy(self.data.index) del self.data # save memory - @staticmethod - def flt_idx_map(flt_data, idx_map): - idx = 0 - new_idx_map = {} - for i, exist in enumerate(flt_data): - if exist: - new_idx_map[idx] = idx_map[i] - idx += 1 - return new_idx_map - def get_index(self): """ Get the pandas index of the data, it will be useful in following scenarios @@ -461,7 +443,7 @@ class TSDatasetH(DatasetH): (T)ime-(S)eries Dataset (H)andler - Covnert the tabular data to Time-Series data + Convert the tabular data to Time-Series data Requirements analysis @@ -505,19 +487,8 @@ def _prepare_seg(self, slc: slice, **kwargs) -> TSDataSampler: """ split the _prepare_raw_seg is to leave a hook for data preprocessing before creating processing data """ - dtype = kwargs.pop("dtype") + dtype = kwargs.pop("dtype", None) start, end = slc.start, slc.stop - flt_col = kwargs.pop("flt_col", None) - # TSDatasetH will retrieve more data for complete - data = self._prepare_raw_seg(slc, **kwargs) - - flt_kwargs = deepcopy(kwargs) - if flt_col is not None: - flt_kwargs["col_set"] = flt_col - flt_data = self._prepare_raw_seg(slc, **flt_kwargs) - assert len(flt_data.columns) == 1 - else: - flt_data = None - - tsds = TSDataSampler(data=data, start=start, end=end, step_len=self.step_len, dtype=dtype, flt_data=flt_data) + data = self._prepare_raw_seg(slc=slc, **kwargs) + tsds = TSDataSampler(data=data, start=start, end=end, step_len=self.step_len, dtype=dtype) return tsds diff --git a/qlib/data/dataset/handler.py b/qlib/data/dataset/handler.py index f1fa39c3bd..63b49d78b5 100644 --- a/qlib/data/dataset/handler.py +++ b/qlib/data/dataset/handler.py @@ -36,7 +36,7 @@ class DataHandler(Serializable): The data handler try to maintain a handler with 2 level. `datetime` & `instruments`. - Any order of the index level can be suported (The order will be implied in the data). + Any order of the index level can be supported (The order will be implied in the data). The order <`datetime`, `instruments`> will be used when the dataframe index name is missed. Example of the data: @@ -77,7 +77,7 @@ def __init__( data_loader : Tuple[dict, str, DataLoader] data loader to load the data. init_data : - intialize the original data in the constructor. + initialize the original data in the constructor. fetch_orig : bool Return the original data instead of copy if possible. """ @@ -128,7 +128,7 @@ def config(self, **kwargs): def setup_data(self, enable_cache: bool = False): """ - Set Up the data in case of running intialization for multiple time + Set Up the data in case of running initialization for multiple time It is responsible for maintaining following variable 1) self._data @@ -453,7 +453,7 @@ def config(self, processor_kwargs: dict = None, **kwargs): def setup_data(self, init_type: str = IT_FIT_SEQ, **kwargs): """ - Set up the data in case of running intialization for multiple time + Set up the data in case of running initialization for multiple time Parameters ---------- diff --git a/qlib/model/ens/ensemble.py b/qlib/model/ens/ensemble.py index 7ccf98ab2e..1fb14a37b4 100644 --- a/qlib/model/ens/ensemble.py +++ b/qlib/model/ens/ensemble.py @@ -5,6 +5,7 @@ Ensemble can merge the objects in an Ensemble. For example, if there are many submodels predictions, we may need to merge them in an ensemble predictions. """ +from typing import Union import pandas as pd @@ -24,6 +25,30 @@ def __call__(self, ensemble_dict: dict, *args, **kwargs): raise NotImplementedError(f"Please implement the `__call__` method.") +class SingleKeyEnsemble(Ensemble): + + """ + Extract the object if there is only one key and value in dict. Make result more readable. + {Only key: Only value} -> Only value + If there are more than 1 key or less than 1 key, then do nothing. + Even you can run this recursively to make dict more readable. + NOTE: Default run recursively. + """ + + def __call__(self, ensemble_dict: Union[dict, object], recursion: bool = True) -> object: + if not isinstance(ensemble_dict, dict): + return ensemble_dict + if recursion: + tmp_dict = {} + for k, v in ensemble_dict.items(): + tmp_dict[k] = self(v, recursion) + ensemble_dict = tmp_dict + keys = list(ensemble_dict.keys()) + if len(keys) == 1: + ensemble_dict = ensemble_dict[keys[0]] + return ensemble_dict + + class RollingEnsemble(Ensemble): """Merge the rolling objects in an Ensemble""" @@ -47,3 +72,24 @@ def __call__(self, ensemble_dict: dict) -> pd.DataFrame: artifact = artifact[~artifact.index.duplicated(keep="last")] artifact = artifact.sort_index() return artifact + + +class AverageEnsemble(Ensemble): + def __call__(self, ensemble_dict: dict): + """ + Average a dict of same shape dataframe like `prediction` or `IC` into an ensemble. + + NOTE: The values of dict must be pd.DataFrame, and have the index "datetime" + + Args: + ensemble_dict (dict): a dict like {"A": pd.DataFrame, "B": pd.DataFrame}. + The key of the dict will be ignored. + + Returns: + pd.DataFrame: the complete result of averaging. + """ + values = list(ensemble_dict.values()) + results = pd.concat(values, axis=1) + results = results.mean(axis=1).to_frame("score") + results = results.sort_index() + return results diff --git a/qlib/model/ens/group.py b/qlib/model/ens/group.py index d53a55f4c8..d8f174105b 100644 --- a/qlib/model/ens/group.py +++ b/qlib/model/ens/group.py @@ -3,6 +3,13 @@ """ Group can group a set of object based on `group_func` and change them to a dict. +After group, we provide a method to reduce them. + +For example: + +group: {(A,B,C1): object, (A,B,C2): object} -> {(A,B): {C1: object, C2: object}} +reduce: {(A,B): {C1: object, C2: object}} -> {(A,B): object} + """ from qlib.model.ens.ensemble import Ensemble, RollingEnsemble diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index a0d252ab44..7680674a6d 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -3,12 +3,12 @@ """ The Trainer will train a list of tasks and return a list of model recorder. -There are two steps in each Trainer including `train`(make model recorder) and `end_train`(modify model recorder). +There are two steps in each Trainer including ``train``(make model recorder) and ``end_train``(modify model recorder). -This is concept called "DelayTrainer", which can be used in online simulating to parallel training. -In "DelayTrainer", the first step is only to save some necessary info to model recorder, and the second step which will be finished in the end can do some concurrent and time-consuming operations such as model fitting. +This is concept called ``DelayTrainer``, which can be used in online simulating to parallel training. +In ``DelayTrainer``, the first step is only to save some necessary info to model recorder, and the second step which will be finished in the end can do some concurrent and time-consuming operations such as model fitting. -`Qlib` offer two kind of Trainer, TrainerR is simplest and TrainerRM is based on TaskManager to help manager tasks lifecycle automatically. +``Qlib`` offer two kind of Trainer, ``TrainerR`` is the simplest way and ``TrainerRM`` is based on TaskManager to help manager tasks lifecycle automatically. """ import socket @@ -36,9 +36,6 @@ def begin_task_train(task_config: dict, experiment_name: str, recorder_name: str Returns: Recorder: the model recorder """ - # FIXME: recorder_id - if recorder_name is None: - recorder_name = str(time.time()) with R.start(experiment_name=experiment_name, recorder_name=recorder_name): R.log_params(**flatten_dict(task_config)) R.save_objects(**{"task": task_config}) # keep the original format and datatype @@ -58,7 +55,7 @@ def end_task_train(rec: Recorder, experiment_name: str) -> Recorder: Returns: Recorder: the model recorder """ - with R.start(experiment_name=experiment_name, recorder_name=rec.info["name"], resume=True): + with R.start(experiment_name=experiment_name, recorder_id=rec.info["id"], resume=True): task_config = R.load_object("task") # model & dataset initiation model: Model = init_instance_by_config(task_config["model"]) @@ -314,7 +311,8 @@ def end_train(self, recs: list, **kwargs) -> list: def reset(self): """ - NOTE: this method will delete all task in this task_pool! + .. note:: + this method will delete all task in this task_pool! """ tm = TaskManager(task_pool=self.task_pool) tm.remove() diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index 4e92900960..6c62fbce97 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -2,11 +2,14 @@ # Licensed under the MIT License. """ -OnlineManager can manage a set of OnlineStrategy and run them dynamically. +OnlineManager can manage a set of `Online Strategy <#Online Strategy>`_ and run them dynamically. With the change of time, the decisive models will be also changed. In this module, we call those contributing models as `online` models. In every routine(such as everyday or every minutes), the `online` models maybe changed and the prediction of them need to be updated. So this module provide a series methods to control this process. + +This module also provide a method to simulate `Online Strategy <#Online Strategy>`_ in the history. +Which means you can verify your strategy or find a better one. """ from typing import Dict, List, Union @@ -14,12 +17,18 @@ import pandas as pd from qlib import get_module_logger from qlib.data.data import D +from qlib.model.ens.ensemble import AverageEnsemble, SingleKeyEnsemble from qlib.utils.serial import Serializable from qlib.workflow.online.strategy import OnlineStrategy from qlib.workflow.task.collect import HyperCollector class OnlineManager(Serializable): + """ + OnlineManager can manage online models with `Online Strategy <#Online Strategy>`_. + It also provide a history recording which models are onlined at what time. + """ + def __init__( self, strategy: Union[OnlineStrategy, List[OnlineStrategy]], @@ -29,10 +38,11 @@ def __init__( ): """ Init OnlineManager. + One OnlineManager must have at least one OnlineStrategy. Args: strategy (Union[OnlineStrategy, List[OnlineStrategy]]): an instance of OnlineStrategy or a list of OnlineStrategy - begin_time (Union[str,pd.Timestamp], optional): the OnlineManager will begin at this time. Defaults to None. + begin_time (Union[str,pd.Timestamp], optional): the OnlineManager will begin at this time. Defaults to None for using latest date. freq (str, optional): data frequency. Defaults to "day". need_log (bool, optional): print log or not. Defaults to True. """ @@ -50,7 +60,7 @@ def __init__( def first_train(self): """ - Run every strategy first_train method and record the online history + Run every strategy first_train method and record the online history. """ for strategy in self.strategy: self.logger.info(f"Strategy `{strategy.name_id}` begins first training...") @@ -62,7 +72,7 @@ def routine(self, cur_time: Union[str, pd.Timestamp] = None, task_kwargs: dict = Run typical update process for every strategy and record the online history. The typical update process after a routine, such as day by day or month by month. - update online prediction -> prepare signals -> prepare tasks -> prepare new models -> reset online models + The process is: Prepare signals -> Prepare tasks -> Prepare online models. Args: cur_time (Union[str,pd.Timestamp], optional): run routine method in this time. Defaults to None. @@ -84,15 +94,15 @@ def routine(self, cur_time: Union[str, pd.Timestamp] = None, task_kwargs: dict = def get_collector(self) -> HyperCollector: """ - Get the instance of HyperCollector to collect results from every strategy. + Get the instance of `Collector <../advanced/task_management.html#Task Collecting>`_ to collect results from every strategy. Returns: - HyperCollector: the collector can collect other collectors. + HyperCollector: the collector to collect other collectors (using SingleKeyEnsemble() to make results more readable). """ collector_dict = {} for strategy in self.strategy: collector_dict[strategy.name_id] = strategy.get_collector() - return HyperCollector(collector_dict) + return HyperCollector(collector_dict, process_list=SingleKeyEnsemble()) def get_online_history(self, strategy_name_id: str) -> list: """ @@ -102,7 +112,7 @@ def get_online_history(self, strategy_name_id: str) -> list: strategy_name_id (str): the name_id of strategy Returns: - dict: a list like [(time, [online_models])] + list: a list like [(begin_time, [online_models])] """ history_dict = self.history[strategy_name_id] history = [] @@ -121,10 +131,27 @@ def delay_prepare(self, delay_kwargs={}): for strategy in self.strategy: strategy.delay_prepare(self.get_online_history(strategy.name_id), **delay_kwargs) + def get_signals(self) -> pd.DataFrame: + """ + Average all strategy signals as the online signals. + + Assumption: the signals from every strategy is pd.DataFrame. Override this function to change. + + Returns: + pd.DataFrame: signals + """ + signals_dict = {} + for strategy in self.strategy: + signals_dict[strategy.name_id] = strategy.get_signals() + return AverageEnsemble()(signals_dict) + def simulate(self, end_time, frequency="day", task_kwargs={}, model_kwargs={}, delay_kwargs={}) -> HyperCollector: """ - Starting from cur time, this method will simulate every routine in OnlineManager. - NOTE: Considering the parallel training, the models and signals can be perpared after all routine simulating. + Starting from current time, this method will simulate every routine in OnlineManager until end time. + + Considering the parallel training, the models and signals can be perpared after all routine simulating. + + The delay training way can be ``DelayTrainer`` and the delay preparing signals way can be ``delay_prepare``. Returns: HyperCollector: the OnlineManager's collector @@ -140,7 +167,9 @@ def simulate(self, end_time, frequency="day", task_kwargs={}, model_kwargs={}, d def reset(self): """ - NOTE: This method will reset all strategy! Be careful to use it. + This method will reset all strategy! + + **Be careful to use it.** """ self.cur_time = self.begin_time self.history = {} diff --git a/qlib/workflow/online/strategy.py b/qlib/workflow/online/strategy.py index 3782ee6523..0cae11b7fb 100644 --- a/qlib/workflow/online/strategy.py +++ b/qlib/workflow/online/strategy.py @@ -2,8 +2,7 @@ # Licensed under the MIT License. """ -OnlineStrategy is a set of strategy of online serving. -It is working with OnlineManager, responsing how the tasks are generated, the models are updated and signals are perpared. +OnlineStrategy is a set of strategy for online serving. """ from copy import deepcopy @@ -12,6 +11,7 @@ import pandas as pd from qlib.data.data import D from qlib.log import get_module_logger +from qlib.model.ens.ensemble import AverageEnsemble, SingleKeyEnsemble from qlib.model.ens.group import RollingGroup from qlib.model.trainer import Trainer, TrainerR from qlib.workflow import R @@ -23,9 +23,14 @@ class OnlineStrategy: + """ + OnlineStrategy is working with `Online Manager <#Online Manager>`_, responsing how the tasks are generated, the models are updated and signals are perpared. + """ + def __init__(self, name_id: str, trainer: Trainer = None, need_log=True): """ Init OnlineStrategy. + This module **MUST** use `Trainer <../reference/api.html#Trainer>`_ to finishing model training. Args: name_id (str): a unique name or id @@ -43,6 +48,7 @@ def prepare_signals(self, delay: bool = False): After perparing the data of last routine (a box in box-plot) which means the end of the routine, we can prepare trading signals for next routine. NOTE: Given a set prediction, all signals before these prediction end time will be prepared well. + Args: delay: bool If this method was called by `delay_prepare` @@ -52,7 +58,7 @@ def prepare_signals(self, delay: bool = False): def prepare_tasks(self, *args, **kwargs): """ After the end of a routine, check whether we need to prepare and train some new tasks. - return the new tasks waiting for training. + Return the new tasks waiting for training. You can find last online models by OnlineTool.online_models. """ @@ -66,10 +72,6 @@ def prepare_online_models(self, tasks, check_func=None, **kwargs): Args: tasks (list): a list of tasks. - tag (str): - `ONLINE_TAG` for first train or additional train - `NEXT_ONLINE_TAG` for reset online model when calling `reset_online_tag` - `OFFLINE_TAG` for train but offline those models check_func: the method to judge if a model can be online. The parameter is the model record and return True for online. None for online every models. @@ -95,7 +97,8 @@ def first_train(self): def get_collector(self) -> Collector: """ - Get the instance of collector to collect results of online serving. + Get the instance of `Collector <../advanced/task_management.html#Task Collecting>`_ to collect results of online serving. + For example: 1) collect predictions in Recorder @@ -109,7 +112,8 @@ def get_collector(self) -> Collector: def delay_prepare(self, history: list, **kwargs): """ Prepare all models and signals if there are something waiting for prepare. - NOTE: Assumption: the predictions of online models need less than next begin_time, or this method will work in a wrong way. + + Assumption: the predictions of online models need less than next begin_time, or this method will work in a wrong way. Args: history (list): an online models list likes [begin_time:[online models]]. @@ -120,6 +124,12 @@ def delay_prepare(self, history: list, **kwargs): self.tool.reset_online_tag(recs_list) self.prepare_signals(delay=True) + def get_signals(self): + """ + Get prepared signals. + """ + raise NotImplementedError(f"Please implement the `get_signals` method.") + def reset(self): """ Delete all things and set them to default status. This method is convenient to explore the strategy for online simulation. @@ -164,17 +174,20 @@ def __init__( self.rg = rolling_gen self.tool = OnlineToolR(self.exp_name) self.ta = TimeAdjuster() - self.signal_rec = None # the recorder to record signals + with R.start(experiment_name=self.signal_exp_name, recorder_name=self.exp_name, resume=True): + self.signal_rec = R.get_recorder() # the recorder to record signals + self.signal_rec.save_objects(**{"signals": None}) - def get_collector(self, rec_key_func=None, rec_filter_func=None): + def get_collector(self, process_list=[RollingGroup()], rec_key_func=None, rec_filter_func=None, artifacts_key=None): """ - Get the instance of collector to collect results. The returned collector must can distinguish results in different models. + Get the instance of `Collector <../advanced/task_management.html#Task Collecting>`_ to collect results. The returned collector must can distinguish results in different models. Assumption: the models can be distinguished based on model name and rolling test segments. If you do not want this assumption, please implement your own method or use another rec_key_func. Args: rec_key_func (Callable): a function to get the key of a recorder. If None, use recorder id. rec_filter_func (Callable, optional): filter the recorder by return True or False. Defaults to None. + artifacts_key (List[str], optional): the artifacts key you want to get. If None, get all artifacts. """ def rec_key(recorder): @@ -188,18 +201,13 @@ def rec_key(recorder): artifacts_collector = RecorderCollector( experiment=self.exp_name, - process_list=RollingGroup(), + process_list=process_list, rec_key_func=rec_key_func, rec_filter_func=rec_filter_func, + artifacts_key=artifacts_key, ) - signals_collector = RecorderCollector( - experiment=self.signal_exp_name, - rec_key_func=lambda rec: rec.info["name"], - rec_filter_func=lambda rec: rec.info["name"] == self.exp_name, - artifacts_path={"signals": "signals"}, - ) - return HyperCollector({"artifacts": artifacts_collector, "signals": signals_collector}) + return artifacts_collector def first_train(self) -> List[Recorder]: """ @@ -252,7 +260,11 @@ def prepare_signals(self, delay=False, over_write=False) -> pd.DataFrame: Average the predictions of online models and offer a trading signals every routine. The signals will be saved to `signal` file of a recorder named self.exp_name of a experiment using the name of `SIGNAL_EXP` Even if the latest signal already exists, the latest calculation result will be overwritten. - NOTE: Given a prediction of a certain time, all signals before this time will be prepared well. + + .. note:: + + Given a prediction of a certain time, all signals before this time will be prepared well. + Args: over_write (bool, optional): If True, the new signals will overwrite the file. If False, the new signals will append to the end of signals. Defaults to False. Returns: @@ -260,21 +272,17 @@ def prepare_signals(self, delay=False, over_write=False) -> pd.DataFrame: """ if not delay: self.tool.update_online_pred() - if self.signal_rec is None: - with R.start(experiment_name=self.signal_exp_name, recorder_name=self.exp_name, resume=True): - self.signal_rec = R.get_recorder() - pred = [] - try: - old_signals = self.signal_rec.load_object("signals") - except OSError: - old_signals = None - - for rec in self.tool.online_models(): - pred.append(rec.load_object("pred.pkl")) + # Get a collector to average online models predictions + online_collector = self.get_collector( + process_list=[AverageEnsemble()], + rec_filter_func=lambda x: True if self.tool.get_online_tag(x) == self.tool.ONLINE_TAG else False, + artifacts_key="pred", + ) + online_results = online_collector() + signals = online_results["pred"] - signals: pd.DataFrame = pd.concat(pred, axis=1).mean(axis=1).to_frame("score") - signals = signals.sort_index() + old_signals = self.get_signals() if old_signals is not None and not over_write: old_max = old_signals.index.get_level_values("datetime").max() new_signals = signals.loc[old_max:] @@ -288,18 +296,15 @@ def prepare_signals(self, delay=False, over_write=False) -> pd.DataFrame: self.signal_rec.save_objects(**{"signals": signals}) return signals - # def get_signals(self): - # """ - # get signals from the recorder(named self.exp_name) of the experiment(named self.SIGNAL_EXP) - - # Returns: - # signals - # """ - # if self.signal_rec is None: - # with R.start(experiment_name=self.signal_exp_name, recorder_name=self.exp_name, resume=True): - # self.signal_rec = R.get_recorder() - # signals = self.signal_rec.load_object("signals") - # return signals + def get_signals(self) -> object: + """ + Get signals from the recorder(named self.exp_name) of the experiment(named self.SIGNAL_EXP) + + Returns: + object: signals + """ + signals = self.signal_rec.load_object("signals") + return signals def _list_latest(self, rec_list: List[Recorder]): """ diff --git a/qlib/workflow/online/update.py b/qlib/workflow/online/update.py index 69ad553245..ab910ba8dc 100644 --- a/qlib/workflow/online/update.py +++ b/qlib/workflow/online/update.py @@ -2,7 +2,7 @@ # Licensed under the MIT License. """ -Update is a module to update artifacts such as predictions, when the stock data updating. +Updater is a module to update artifacts such as predictions, when the stock data is updating. """ from abc import ABCMeta, abstractmethod @@ -89,9 +89,13 @@ def __init__(self, record: Recorder, to_date=None, hist_ref: int = 0, freq="day" hist_ref : int Sometimes, the dataset will have historical depends. Leave the problem to user to set the length of historical dependency - NOTE: the start_time is not included in the hist_ref - # TODO: automate this step in the future. + + .. note:: + + the start_time is not included in the hist_ref + """ + # TODO: automate this hist_ref in the future. super().__init__(record=record, need_log=need_log) self.to_date = to_date diff --git a/qlib/workflow/online/utils.py b/qlib/workflow/online/utils.py index 4d630a6656..296ca3ea6c 100644 --- a/qlib/workflow/online/utils.py +++ b/qlib/workflow/online/utils.py @@ -16,6 +16,9 @@ class OnlineTool: + """ + OnlineTool. + """ ONLINE_KEY = "online_status" # the online status key in recorder ONLINE_TAG = "online" # the 'online' model diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index d74d081846..28320e2ceb 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -5,6 +5,7 @@ Collector can collect object from everywhere and process them such as merging, grouping, averaging and so on. """ +from qlib.model.ens.ensemble import SingleKeyEnsemble from qlib.workflow import R import dill as pickle @@ -81,7 +82,7 @@ def save(self, filepath): filepath (str): the path of file Returns: - bool: if successed + bool: if succeeded """ try: with open(filepath, "wb") as f: @@ -122,6 +123,8 @@ def __init__(self, collector_dict, process_list=[]): Args: collector_dict (dict): the dict like {collector_key, Collector} process_list (list or Callable): the list of processors or the instance of processor to process dict. + NOTE: process_list = [SingleKeyEnsemble()] can ignore key and use value directly if there is only one {k,v} in a dict. + This can make result more readable. If you want to maintain as it should be, just give a empty process list. """ super().__init__(process_list=process_list) self.collector_dict = collector_dict diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 3c3144fe89..c71be7d39b 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -52,9 +52,13 @@ class TaskManager: Assumption: the data in MongoDB was encoded and the data out of MongoDB was decoded Here are four status which are: + STATUS_WAITING: waiting for train + STATUS_RUNNING: training - STATUS_PART_DONE: finished some step and waiting for next step. + + STATUS_PART_DONE: finished some step and waiting for next step + STATUS_DONE: all work done """ @@ -393,9 +397,13 @@ def run_task( While task pool is not empty (has WAITING tasks), use task_func to fetch and run tasks in task_pool After running this method, here are 4 situations (before_status -> after_status): + STATUS_WAITING -> STATUS_DONE: use task["def"] as `task_func` param + STATUS_WAITING -> STATUS_PART_DONE: use task["def"] as `task_func` param + STATUS_PART_DONE -> STATUS_PART_DONE: use task["res"] as `task_func` param + STATUS_PART_DONE -> STATUS_DONE: use task["res"] as `task_func` param Parameters From 846c64f6c6e695a7fd13e805af57c68a5de887c9 Mon Sep 17 00:00:00 2001 From: blin <981921742@qq.com> Date: Thu, 6 May 2021 12:00:41 +0000 Subject: [PATCH 52/61] fix param --- qlib/data/dataset/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qlib/data/dataset/__init__.py b/qlib/data/dataset/__init__.py index 0bdb5018b4..8bcd6419a8 100644 --- a/qlib/data/dataset/__init__.py +++ b/qlib/data/dataset/__init__.py @@ -244,7 +244,7 @@ class TSDataSampler: """ - def __init__(self, data: pd.DataFrame, start, end, step_len: int, fillna_type: str = "none", dtype=None): + def __init__(self, data: pd.DataFrame, start, end, step_len: int, fillna_type: str = "none", dtype=None, flt_data=None): """ Build a dataset which looks like torch.data.utils.Dataset. @@ -317,7 +317,7 @@ def get_index(self): Get the pandas index of the data, it will be useful in following scenarios - Special sampler will be used (e.g. user want to sample day by day) """ - return self.data_idx[self.start_idx : self.end_idx] + return self.data_index[self.start_idx : self.end_idx] def config(self, **kwargs): # Config the attributes From 9dfd001f6fafbe077dcdc30feacd0dbeb7bf31e6 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Fri, 7 May 2021 09:59:15 +0000 Subject: [PATCH 53/61] online serving v10 --- docs/advanced/task_management.rst | 1 + .../online_srv/online_management_simulate.py | 21 +- .../online_srv/rolling_online_management.py | 13 +- qlib/model/ens/ensemble.py | 12 +- qlib/model/ens/group.py | 20 +- qlib/model/trainer.py | 135 ++++++++---- qlib/utils/__init__.py | 2 +- qlib/utils/serial.py | 36 +++- qlib/workflow/online/manager.py | 201 +++++++++++++----- qlib/workflow/online/strategy.py | 172 +++------------ qlib/workflow/online/utils.py | 4 +- qlib/workflow/task/collect.py | 81 +++---- qlib/workflow/task/gen.py | 53 +++-- qlib/workflow/task/manage.py | 8 +- 14 files changed, 420 insertions(+), 339 deletions(-) diff --git a/docs/advanced/task_management.rst b/docs/advanced/task_management.rst index a68c126276..d600494556 100644 --- a/docs/advanced/task_management.rst +++ b/docs/advanced/task_management.rst @@ -55,6 +55,7 @@ More information of ``Task Manager`` can be found in `here <../reference/api.htm Task Training =============== +#FIXME: Trainer After generating and storing those ``task``, it's time to run the ``task`` which are in the *WAITING* status. ``Qlib`` provides a method called ``run_task`` to run those ``task`` in task pool, however, users can also customize how tasks are executed. An easy way to get the ``task_func`` is using ``qlib.model.trainer.task_train`` directly. diff --git a/examples/online_srv/online_management_simulate.py b/examples/online_srv/online_management_simulate.py index 7be46d999b..5583ee1604 100644 --- a/examples/online_srv/online_management_simulate.py +++ b/examples/online_srv/online_management_simulate.py @@ -8,6 +8,7 @@ import fire import qlib from qlib.model.trainer import DelayTrainerRM +from qlib.workflow import R from qlib.workflow.online.manager import OnlineManager from qlib.workflow.online.strategy import RollingAverageStrategy from qlib.workflow.task.gen import RollingGen @@ -110,23 +111,29 @@ def __init__( } qlib.init(provider_uri=provider_uri, region=region, mongo=mongo_conf) self.rolling_gen = RollingGen( - step=rolling_step, rtype=RollingGen.ROLL_SD, modify_end_time=False - ) # The rolling tasks generator, modify_end_time is false because we just need simulate to 2018-10-31. + step=rolling_step, rtype=RollingGen.ROLL_SD, ds_extra_mod_func=None + ) # The rolling tasks generator, ds_extra_mod_func is None because we just need simulate to 2018-10-31 and needn't change handler end time. self.trainer = DelayTrainerRM(self.exp_name, self.task_pool) self.task_manager = TaskManager(self.task_pool) # A good way to manage all your tasks self.rolling_online_manager = OnlineManager( - RollingAverageStrategy( - exp_name, task_template=tasks, rolling_gen=self.rolling_gen, trainer=self.trainer, need_log=False - ), + RollingAverageStrategy(exp_name, task_template=tasks, rolling_gen=self.rolling_gen, need_log=False), + trainer=self.trainer, begin_time=self.start_time, need_log=False, ) self.tasks = tasks + # Reset all things to the first status, be careful to save important data + def reset(self): + TaskManager(self.task_pool).remove() + exp = R.get_exp(experiment_name=self.exp_name) + for rid in exp.list_recorders(): + exp.delete_recorder(rid) + # Run this to run all workflow automatically def main(self): print("========== reset ==========") - self.rolling_online_manager.reset() + self.reset() print("========== simulate ==========") self.rolling_online_manager.simulate(end_time=self.end_time) print("========== collect results ==========") @@ -134,7 +141,7 @@ def main(self): print("========== signals ==========") print(self.rolling_online_manager.get_signals()) print("========== online history ==========") - print(self.rolling_online_manager.get_online_history(self.exp_name)) + print(self.rolling_online_manager.history) if __name__ == "__main__": diff --git a/examples/online_srv/rolling_online_management.py b/examples/online_srv/rolling_online_management.py index 25b6fc4da3..ebf1ab59a7 100644 --- a/examples/online_srv/rolling_online_management.py +++ b/examples/online_srv/rolling_online_management.py @@ -18,8 +18,6 @@ from qlib.workflow.task.gen import RollingGen from qlib.workflow.task.manage import TaskManager from qlib.workflow.online.manager import OnlineManager -from qlib.workflow.task.utils import list_recorders -from qlib.model.trainer import TrainerRM data_handler_config = { "start_time": "2013-01-01", @@ -86,7 +84,7 @@ def __init__( task_url="mongodb://10.0.0.4:27017/", task_db_name="rolling_db", rolling_step=550, - tasks=[task_xgboost_config], # , task_lgb_config], + tasks=[task_xgboost_config, task_lgb_config], ): mongo_conf = { "task_url": task_url, # your MongoDB url @@ -103,7 +101,6 @@ def __init__( name_id, task, RollingGen(step=rolling_step, rtype=RollingGen.ROLL_SD), - TrainerRM(experiment_name=name_id, task_pool=name_id), ) ) @@ -116,9 +113,8 @@ def __init__( # Reset all things to the first status, be careful to save important data def reset(self): - print("========== reset ==========") for task in self.tasks: - name_id = task["model"]["class"] + "_" + str(self.rolling_step) + name_id = task["model"]["class"] TaskManager(name_id).remove() exp = R.get_exp(experiment_name=name_id) for rid in exp.list_recorders(): @@ -127,12 +123,9 @@ def reset(self): if os.path.exists(self._ROLLING_MANAGER_PATH): os.remove(self._ROLLING_MANAGER_PATH) - for rid in list_recorders("OnlineManagerSignals", lambda x: True if x.info["name"] == name_id else False): - exp.delete_recorder(rid) - def first_run(self): print("========== reset ==========") - self.rolling_online_manager.reset() + self.reset() print("========== first_run ==========") self.rolling_online_manager.first_train() print("========== dump ==========") diff --git a/qlib/model/ens/ensemble.py b/qlib/model/ens/ensemble.py index 1fb14a37b4..a7b837ea50 100644 --- a/qlib/model/ens/ensemble.py +++ b/qlib/model/ens/ensemble.py @@ -7,6 +7,7 @@ from typing import Union import pandas as pd +from qlib.utils import flatten_dict class Ensemble: @@ -77,19 +78,22 @@ def __call__(self, ensemble_dict: dict) -> pd.DataFrame: class AverageEnsemble(Ensemble): def __call__(self, ensemble_dict: dict): """ - Average a dict of same shape dataframe like `prediction` or `IC` into an ensemble. + Average and standardize a dict of same shape dataframe like `prediction` or `IC` into an ensemble. - NOTE: The values of dict must be pd.DataFrame, and have the index "datetime" + NOTE: The values of dict must be pd.DataFrame, and have the index "datetime". If it is a nested dict, then flat it. Args: ensemble_dict (dict): a dict like {"A": pd.DataFrame, "B": pd.DataFrame}. The key of the dict will be ignored. Returns: - pd.DataFrame: the complete result of averaging. + pd.DataFrame: the complete result of averaging and standardizing. """ + # need to flatten the nested dict + ensemble_dict = flatten_dict(ensemble_dict) values = list(ensemble_dict.values()) results = pd.concat(values, axis=1) - results = results.mean(axis=1).to_frame("score") + results = results.groupby("datetime").apply(lambda df: (df - df.mean()) / df.std()) + results = results.mean(axis=1) results = results.sort_index() return results diff --git a/qlib/model/ens/group.py b/qlib/model/ens/group.py index d8f174105b..a00a8ea0e0 100644 --- a/qlib/model/ens/group.py +++ b/qlib/model/ens/group.py @@ -36,20 +36,36 @@ def __init__(self, group_func=None, ens: Ensemble = None): self._ens_func = ens def group(self, *args, **kwargs) -> dict: - # TODO: such design is weird when `_group_func` is the only configurable part in the class + """ + Group a set of object and change them to a dict. + + For example: {(A,B,C1): object, (A,B,C2): object} -> {(A,B): {C1: object, C2: object}} + + Returns: + dict: grouped dict + """ if isinstance(getattr(self, "_group_func", None), Callable): return self._group_func(*args, **kwargs) else: raise NotImplementedError(f"Please specify valid `group_func`.") def reduce(self, *args, **kwargs) -> dict: + """ + Reduce grouped dict in some way. + + For example: {(A,B): {C1: object, C2: object}} -> {(A,B): object} + + Returns: + dict: reduced dict + """ if isinstance(getattr(self, "_ens_func", None), Callable): return self._ens_func(*args, **kwargs) else: raise NotImplementedError(f"Please specify valid `_ens_func`.") def __call__(self, ungrouped_dict: dict, n_jobs=1, verbose=0, *args, **kwargs) -> dict: - """Group the ungrouped_dict into different groups. + """ + Group the ungrouped_dict into different groups. Args: ungrouped_dict (dict): the ungrouped dict waiting for grouping like {name: things} diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index 7680674a6d..68b78d9df2 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -12,7 +12,6 @@ """ import socket -import time from typing import Callable, List from qlib.data.dataset import Dataset @@ -145,12 +144,6 @@ def is_delay(self) -> bool: """ return self.delay - def reset(self): - """ - Reset the Trainer status. - """ - pass - class TrainerR(Trainer): """ @@ -160,42 +153,52 @@ class TrainerR(Trainer): Assumption: models were defined by `task` and the results will saved to `Recorder` """ - def __init__(self, experiment_name: str, train_func: Callable = task_train): + # Those tag will help you distinguish whether the Recorder has finished traning + STATUS_KEY = "train_status" + STATUS_BEGIN = "begin_task_train" + STATUS_END = "end_task_train" + + def __init__(self, experiment_name: str = None, train_func: Callable = task_train): """ Init TrainerR. Args: - experiment_name (str): the name of experiment. + experiment_name (str, optional): the default name of experiment. train_func (Callable, optional): default training method. Defaults to `task_train`. """ super().__init__() self.experiment_name = experiment_name self.train_func = train_func - def train(self, tasks: list, train_func: Callable = None, **kwargs) -> List[Recorder]: + def train(self, tasks: list, train_func: Callable = None, experiment_name: str = None, **kwargs) -> List[Recorder]: """ Given a list of `task`s and return a list of trained Recorder. The order can be guaranteed. Args: tasks (list): a list of definition based on `task` dict train_func (Callable): the train method which need at least `task`s and `experiment_name`. None for default training method. + experiment_name (str): the experiment name, None for use default name. kwargs: the params for train_func. Returns: list: a list of Recorders """ + if len(tasks) == 0: + return [] if train_func is None: train_func = self.train_func + if experiment_name is None: + experiment_name = self.experiment_name recs = [] for task in tasks: - rec = train_func(task, self.experiment_name, **kwargs) - rec.set_tags(**{"train_status": "begin_task_train"}) + rec = train_func(task, experiment_name, **kwargs) + rec.set_tags(**{self.STATUS_KEY: self.STATUS_BEGIN}) recs.append(rec) return recs - def end_train(self, recs: list, **kwargs) -> list: + def end_train(self, recs: list, **kwargs) -> List[Recorder]: for rec in recs: - rec.set_tags(**{"train_status": "end_task_train"}) + rec.set_tags(**{self.STATUS_KEY: self.STATUS_END}) return recs @@ -204,12 +207,12 @@ class DelayTrainerR(TrainerR): A delayed implementation based on TrainerR, which means `train` method may only do some preparation and `end_train` method can do the real model fitting. """ - def __init__(self, experiment_name, train_func=begin_task_train, end_train_func=end_task_train): + def __init__(self, experiment_name: str = None, train_func=begin_task_train, end_train_func=end_task_train): """ Init TrainerRM. Args: - experiment_name (str): the name of experiment. + experiment_name (str): the default name of experiment. train_func (Callable, optional): default train method. Defaults to `begin_task_train`. end_train_func (Callable, optional): default end_train method. Defaults to `end_task_train`. """ @@ -217,7 +220,7 @@ def __init__(self, experiment_name, train_func=begin_task_train, end_train_func= self.end_train_func = end_train_func self.delay = True - def end_train(self, recs, end_train_func=None, **kwargs) -> List[Recorder]: + def end_train(self, recs, end_train_func=None, experiment_name: str = None, **kwargs) -> List[Recorder]: """ Given a list of Recorder and return a list of trained Recorder. This class will finish real data loading and model fitting. @@ -225,6 +228,7 @@ def end_train(self, recs, end_train_func=None, **kwargs) -> List[Recorder]: Args: recs (list): a list of Recorder, the tasks have been saved to them end_train_func (Callable, optional): the end_train method which need at least `recorder`s and `experiment_name`. Defaults to None for using self.end_train_func. + experiment_name (str): the experiment name, None for use default name. kwargs: the params for end_train_func. Returns: @@ -232,9 +236,13 @@ def end_train(self, recs, end_train_func=None, **kwargs) -> List[Recorder]: """ if end_train_func is None: end_train_func = self.end_train_func + if experiment_name is None: + experiment_name = self.experiment_name for rec in recs: - end_train_func(rec, **kwargs) - rec.set_tags(**{"train_status": "end_task_train"}) + if rec.list_tags()[self.STATUS_KEY] == self.STATUS_END: + continue + end_train_func(rec, experiment_name, **kwargs) + rec.set_tags(**{self.STATUS_KEY: self.STATUS_END}) return recs @@ -246,13 +254,18 @@ class TrainerRM(Trainer): Assumption: `task` will be saved to TaskManager and `task` will be fetched and trained from TaskManager """ - def __init__(self, experiment_name: str, task_pool: str, train_func=task_train): + # Those tag will help you distinguish whether the Recorder has finished traning + STATUS_KEY = "train_status" + STATUS_BEGIN = "begin_task_train" + STATUS_END = "end_task_train" + + def __init__(self, experiment_name: str = None, task_pool: str = None, train_func=task_train): """ Init TrainerR. Args: - experiment_name (str): the name of experiment. - task_pool (str): task pool name in TaskManager. + experiment_name (str): the default name of experiment. + task_pool (str): task pool name in TaskManager. None for use same name as experiment_name. train_func (Callable, optional): default training method. Defaults to `task_train`. """ super().__init__() @@ -264,6 +277,7 @@ def train( self, tasks: list, train_func: Callable = None, + experiment_name: str = None, before_status: str = TaskManager.STATUS_WAITING, after_status: str = TaskManager.STATUS_DONE, **kwargs, @@ -277,6 +291,7 @@ def train( Args: tasks (list): a list of definition based on `task` dict train_func (Callable): the train method which need at least `task`s and `experiment_name`. None for default training method. + experiment_name (str): the experiment name, None for use default name. before_status (str): the tasks in before_status will be fetched and trained. Can be STATUS_WAITING, STATUS_PART_DONE. after_status (str): the tasks after trained will become after_status. Can be STATUS_WAITING, STATUS_PART_DONE. kwargs: the params for train_func. @@ -284,14 +299,21 @@ def train( Returns: list: a list of Recorders """ + if len(tasks) == 0: + return [] if train_func is None: train_func = self.train_func - tm = TaskManager(task_pool=self.task_pool) + if experiment_name is None: + experiment_name = self.experiment_name + task_pool = self.task_pool + if task_pool is None: + task_pool = experiment_name + tm = TaskManager(task_pool=task_pool) _id_list = tm.create_task(tasks) # all tasks will be saved to MongoDB run_task( train_func, - self.task_pool, - experiment_name=self.experiment_name, + task_pool, + experiment_name=experiment_name, before_status=before_status, after_status=after_status, **kwargs, @@ -300,23 +322,15 @@ def train( recs = [] for _id in _id_list: rec = tm.re_query(_id)["res"] - rec.set_tags(**{"train_status": "begin_task_train"}) + rec.set_tags(**{self.STATUS_KEY: self.STATUS_BEGIN}) recs.append(rec) return recs def end_train(self, recs: list, **kwargs) -> list: for rec in recs: - rec.set_tags(**{"train_status": "end_task_train"}) + rec.set_tags(**{self.STATUS_KEY: self.STATUS_END}) return recs - def reset(self): - """ - .. note:: - this method will delete all task in this task_pool! - """ - tm = TaskManager(task_pool=self.task_pool) - tm.remove() - class DelayTrainerRM(TrainerRM): """ @@ -324,30 +338,57 @@ class DelayTrainerRM(TrainerRM): """ - def __init__(self, experiment_name, task_pool: str, train_func=begin_task_train, end_train_func=end_task_train): + def __init__( + self, + experiment_name: str = None, + task_pool: str = None, + train_func=begin_task_train, + end_train_func=end_task_train, + ): + """ + Init DelayTrainerRM. + + Args: + experiment_name (str): the default name of experiment. + task_pool (str): task pool name in TaskManager. None for use same name as experiment_name. + train_func (Callable, optional): default train method. Defaults to `begin_task_train`. + end_train_func (Callable, optional): default end_train method. Defaults to `end_task_train`. + """ super().__init__(experiment_name, task_pool, train_func) self.end_train_func = end_train_func self.delay = True - def train(self, tasks: list, train_func=None, **kwargs): + def train(self, tasks: list, train_func=None, experiment_name: str = None, **kwargs): """ Same as `train` of TrainerRM, after_status will be STATUS_PART_DONE. Args: tasks (list): a list of definition based on `task` dict train_func (Callable): the train method which need at least `task`s and `experiment_name`. Defaults to None for using self.train_func. + experiment_name (str): the experiment name, None for use default name. Returns: list: a list of Recorders """ - return super().train(tasks, train_func=train_func, after_status=TaskManager.STATUS_PART_DONE, **kwargs) + if len(tasks) == 0: + return [] + return super().train( + tasks, + train_func=train_func, + experiment_name=experiment_name, + after_status=TaskManager.STATUS_PART_DONE, + **kwargs, + ) - def end_train(self, recs, end_train_func=None, **kwargs): + def end_train(self, recs, end_train_func=None, experiment_name: str = None, **kwargs): """ Given a list of Recorder and return a list of trained Recorder. This class will finish real data loading and model fitting. + NOTE: This method will train all STATUS_PART_DONE tasks in task pool, not only the ``recs``. + Args: recs (list): a list of Recorder, the tasks have been saved to them. end_train_func (Callable, optional): the end_train method which need at least `recorder`s and `experiment_name`. Defaults to None for using self.end_train_func. + experiment_name (str): the experiment name, None for use default name. kwargs: the params for end_train_func. Returns: @@ -356,13 +397,23 @@ def end_train(self, recs, end_train_func=None, **kwargs): if end_train_func is None: end_train_func = self.end_train_func + if experiment_name is None: + experiment_name = self.experiment_name + task_pool = self.task_pool + if task_pool is None: + task_pool = experiment_name + tasks = [] + for rec in recs: + tasks.append(rec.load_object("task")) + run_task( end_train_func, - self.task_pool, - experiment_name=self.experiment_name, + task_pool, + tasks=tasks, + experiment_name=experiment_name, before_status=TaskManager.STATUS_PART_DONE, **kwargs, ) for rec in recs: - rec.set_tags(**{"train_status": "end_task_train"}) + rec.set_tags(**{self.STATUS_KEY: self.STATUS_END}) return recs diff --git a/qlib/utils/__init__.py b/qlib/utils/__init__.py index 3ebc6fc1ca..8583e946f2 100644 --- a/qlib/utils/__init__.py +++ b/qlib/utils/__init__.py @@ -732,7 +732,7 @@ def flatten_dict(d, parent_key="", sep="."): """ items = [] for k, v in d.items(): - new_key = parent_key + sep + k if parent_key else k + new_key = parent_key + sep + str(k) if parent_key else k if isinstance(v, collections.abc.MutableMapping): items.extend(flatten_dict(v, new_key, sep=sep).items()) else: diff --git a/qlib/utils/serial.py b/qlib/utils/serial.py index 52d326c2ab..9c5fc9ac27 100644 --- a/qlib/utils/serial.py +++ b/qlib/utils/serial.py @@ -3,6 +3,7 @@ from pathlib import Path import pickle +import dill from typing import Union @@ -14,6 +15,8 @@ class Serializable: - For examples, a learnable Datahandler just wants to save the parameters without data when dumping to disk """ + pickle_backend = "pickle" # another optional value is "dill" which can pickle more things of python. + def __init__(self): self._dump_all = False self._exclude = [] @@ -74,4 +77,35 @@ def config(self, dump_all: bool = None, exclude: list = None, recursive=False): def to_pickle(self, path: Union[Path, str], dump_all: bool = None, exclude: list = None): self.config(dump_all=dump_all, exclude=exclude) with Path(path).open("wb") as f: - pickle.dump(self, f) + if self.pickle_backend == "pickle": + pickle.dump(self, f) + elif self.pickle_backend == "dill": + dill.dump(self, f) + else: + raise ValueError("Unknown pickle backend, please use 'pickle' or 'dill'.") + + @classmethod + def load(cls, filepath): + """ + load the collector from a file + + Args: + filepath (str): the path of file + + Raises: + TypeError: the pickled file must be `Collector` + + Returns: + Collector: the instance of Collector + """ + with open(filepath, "rb") as f: + if cls.pickle_backend == "pickle": + object = pickle.load(f) + elif cls.pickle_backend == "dill": + object = dill.load(f) + else: + raise ValueError("Unknown pickle backend, please use 'pickle' or 'dill'.") + if isinstance(object, cls): + return object + else: + raise TypeError(f"The instance of {type(object)} is not a valid `{type(cls)}`!") diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index 6c62fbce97..a282865e65 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -12,15 +12,17 @@ Which means you can verify your strategy or find a better one. """ -from typing import Dict, List, Union +from typing import Callable, Dict, List, Union import pandas as pd from qlib import get_module_logger from qlib.data.data import D -from qlib.model.ens.ensemble import AverageEnsemble, SingleKeyEnsemble +from qlib.model.ens.ensemble import AverageEnsemble +from qlib.model.trainer import DelayTrainerR, Trainer +from qlib.utils import flatten_dict from qlib.utils.serial import Serializable from qlib.workflow.online.strategy import OnlineStrategy -from qlib.workflow.task.collect import HyperCollector +from qlib.workflow.task.collect import MergeCollector class OnlineManager(Serializable): @@ -32,6 +34,7 @@ class OnlineManager(Serializable): def __init__( self, strategy: Union[OnlineStrategy, List[OnlineStrategy]], + trainer: Trainer = None, begin_time: Union[str, pd.Timestamp] = None, freq="day", need_log=True, @@ -43,6 +46,7 @@ def __init__( Args: strategy (Union[OnlineStrategy, List[OnlineStrategy]]): an instance of OnlineStrategy or a list of OnlineStrategy begin_time (Union[str,pd.Timestamp], optional): the OnlineManager will begin at this time. Defaults to None for using latest date. + trainer (Trainer): the trainer to train task. None for using DelayTrainerR. freq (str, optional): data frequency. Defaults to "day". need_log (bool, optional): print log or not. Defaults to True. """ @@ -56,96 +60,166 @@ def __init__( begin_time = D.calendar(freq=self.freq).max() self.begin_time = pd.Timestamp(begin_time) self.cur_time = self.begin_time - self.history = {} + # The history of online models, which is a dict like {begin_time, {strategy, [online_models]}} + # begin_time means when online_models are onlined + self.history = {} + if trainer is None: + trainer = DelayTrainerR() + self.trainer = trainer + self.signals = None - def first_train(self): + def first_train(self, strategies:List[OnlineStrategy]=None, model_kwargs: dict = {}): """ - Run every strategy first_train method and record the online history. + Get tasks from every strategy's first_tasks method and train them. + If using DelayTrainer, it can finish training all together after every strategy's first_tasks. + + Args: + strategies (List[OnlineStrategy]): the strategies list (need this param when adding strategies). None for use default strategies. + model_kwargs (dict): the params for `prepare_online_models` """ - for strategy in self.strategy: + models_list = [] + if strategies is None: + strategies = self.strategy + for strategy in strategies: self.logger.info(f"Strategy `{strategy.name_id}` begins first training...") - online_models = strategy.first_train() - self.history.setdefault(strategy.name_id, {})[self.cur_time] = online_models + tasks = strategy.first_tasks() + models = self.trainer.train(tasks, experiment_name=strategy.name_id) + models_list.append(models) - def routine(self, cur_time: Union[str, pd.Timestamp] = None, task_kwargs: dict = {}, model_kwargs: dict = {}): + for strategy, models in zip(strategies, models_list): + self.prepare_online_models(strategy, models, model_kwargs=model_kwargs) + + def routine( + self, + cur_time: Union[str, pd.Timestamp] = None, + delay: bool = False, + task_kwargs: dict = {}, + model_kwargs: dict = {}, + signal_kwargs: dict = {}, + ): """ Run typical update process for every strategy and record the online history. The typical update process after a routine, such as day by day or month by month. The process is: Prepare signals -> Prepare tasks -> Prepare online models. + If using DelayTrainer, it can finish training all together after every strategy's prepare_tasks. + Args: cur_time (Union[str,pd.Timestamp], optional): run routine method in this time. Defaults to None. + delay (bool): if delay prepare signals and models task_kwargs (dict): the params for `prepare_tasks` model_kwargs (dict): the params for `prepare_online_models` + signal_kwargs (dict): the params for `prepare_signals` """ if cur_time is None: cur_time = D.calendar(freq=self.freq).max() self.cur_time = pd.Timestamp(cur_time) # None for latest date + models_list = [] for strategy in self.strategy: + if not delay: + strategy.tool.update_online_pred() if self.need_log: self.logger.info(f"Strategy `{strategy.name_id}` begins routine...") - if not strategy.trainer.is_delay(): - strategy.prepare_signals() + tasks = strategy.prepare_tasks(self.cur_time, **task_kwargs) - online_models = strategy.prepare_online_models(tasks, **model_kwargs) - if len(online_models) > 0: - self.history.setdefault(strategy.name_id, {})[self.cur_time] = online_models + models = self.trainer.train(tasks) + models_list.append(models) + + if not delay: + self.prepare_signals(**signal_kwargs) + + for strategy, models in zip(self.strategy, models_list): + self.prepare_online_models(strategy, models, delay=delay, model_kwargs=model_kwargs) - def get_collector(self) -> HyperCollector: + def prepare_online_models( + self, strategy: OnlineStrategy, models: list, delay: bool = False, model_kwargs: dict = {} + ): + """ + Prepare online model for strategy, including end_train, reset_online_tag and add history. + + Args: + strategy (OnlineStrategy): the instance of strategy. + models (list): a list of models. + delay (bool, optional): if delay prepare models. Defaults to False. + model_kwargs (dict, optional): the params for `prepare_online_models`. + """ + if not delay: + models = self.trainer.end_train(models, experiment_name=strategy.name_id) + online_models = strategy.prepare_online_models(models, **model_kwargs) + else: + # just set every models as online models temporarily before ``prepare_online_models`` + online_models = models + if len(online_models) > 0: + strategy.tool.reset_online_tag(online_models) + self.history.setdefault(self.cur_time, {})[strategy] = online_models + + def get_collector(self) -> MergeCollector: """ Get the instance of `Collector <../advanced/task_management.html#Task Collecting>`_ to collect results from every strategy. Returns: - HyperCollector: the collector to collect other collectors (using SingleKeyEnsemble() to make results more readable). + MergeCollector: the collector to merge other collectors. """ collector_dict = {} for strategy in self.strategy: collector_dict[strategy.name_id] = strategy.get_collector() - return HyperCollector(collector_dict, process_list=SingleKeyEnsemble()) + return MergeCollector(collector_dict, process_list=[]) - def get_online_history(self, strategy_name_id: str) -> list: + def add_strategy(self, strategy: Union[OnlineStrategy, List[OnlineStrategy]]): """ - Get the online history based on strategy_name_id. + Add some new strategies to online manager. Args: - strategy_name_id (str): the name_id of strategy - - Returns: - list: a list like [(begin_time, [online_models])] + strategy (Union[OnlineStrategy, List[OnlineStrategy]]): a list of OnlineStrategy """ - history_dict = self.history[strategy_name_id] - history = [] - for time in sorted(history_dict): - models = history_dict[time] - history.append((time, models)) - return history + if not isinstance(strategy, list): + strategy = [strategy] + self.first_train(strategy) + self.strategy.extend(strategy) - def delay_prepare(self, delay_kwargs={}): + def prepare_signals(self, prepare_func: Callable = AverageEnsemble(), over_write=False): """ - Prepare all models and signals if there are something waiting for prepare. + After perparing the data of last routine (a box in box-plot) which means the end of the routine, we can prepare trading signals for next routine. + + NOTE: Given a set prediction, all signals before these prediction end time will be prepared well. + + Even if the latest signal already exists, the latest calculation result will be overwritten. + + .. note:: + + Given a prediction of a certain time, all signals before this time will be prepared well. Args: - delay_kwargs: the params for `delay_prepare` - """ - for strategy in self.strategy: - strategy.delay_prepare(self.get_online_history(strategy.name_id), **delay_kwargs) + prepare_func (Callable, optional): Get signals from a dict after collecting. Defaults to AverageEnsemble(), the results after mergecollector must be {xxx:pred}. + over_write (bool, optional): If True, the new signals will overwrite. If False, the new signals will append to the end of signals. Defaults to False. - def get_signals(self) -> pd.DataFrame: + Returns: + pd.DataFrame: the signals. """ - Average all strategy signals as the online signals. + signals = prepare_func(self.get_collector()()) + old_signals = self.signals + if old_signals is not None and not over_write: + old_max = old_signals.index.get_level_values("datetime").max() + new_signals = signals.loc[old_max:] + signals = pd.concat([old_signals, new_signals], axis=0) + else: + new_signals = signals + if self.need_log: + self.logger.info(f"Finished preparing new {len(new_signals)} signals.") + self.signals = signals + return new_signals - Assumption: the signals from every strategy is pd.DataFrame. Override this function to change. + def get_signals(self) -> pd.Series: + """ + Get prepared online signals. Returns: - pd.DataFrame: signals + pd.Series: signals """ - signals_dict = {} - for strategy in self.strategy: - signals_dict[strategy.name_id] = strategy.get_signals() - return AverageEnsemble()(signals_dict) + return self.signals - def simulate(self, end_time, frequency="day", task_kwargs={}, model_kwargs={}, delay_kwargs={}) -> HyperCollector: + def simulate(self, end_time, frequency="day", task_kwargs={}, model_kwargs={}, signal_kwargs={}): """ Starting from current time, this method will simulate every routine in OnlineManager until end time. @@ -153,6 +227,13 @@ def simulate(self, end_time, frequency="day", task_kwargs={}, model_kwargs={}, d The delay training way can be ``DelayTrainer`` and the delay preparing signals way can be ``delay_prepare``. + Args: + end_time: the time the simulation will end + frequency: the calendar frequency + task_kwargs (dict): the params for `prepare_tasks` + model_kwargs (dict): the params for `prepare_online_models` + signal_kwargs (dict): the params for `prepare_signals` + Returns: HyperCollector: the OnlineManager's collector """ @@ -160,18 +241,30 @@ def simulate(self, end_time, frequency="day", task_kwargs={}, model_kwargs={}, d self.first_train() for cur_time in cal: self.logger.info(f"Simulating at {str(cur_time)}......") - self.routine(cur_time, task_kwargs=task_kwargs, model_kwargs=model_kwargs) - self.delay_prepare(delay_kwargs=delay_kwargs) + self.routine( + cur_time, + delay=self.trainer.is_delay(), + task_kwargs=task_kwargs, + model_kwargs=model_kwargs, + signal_kwargs=signal_kwargs, + ) + # delay prepare the models and signals + if self.trainer.is_delay(): + self.delay_prepare(model_kwargs=model_kwargs, signal_kwargs=signal_kwargs) self.logger.info(f"Finished preparing signals") return self.get_collector() - def reset(self): + def delay_prepare(self, model_kwargs={}, signal_kwargs={}): """ - This method will reset all strategy! + Prepare all models and signals if there are something waiting for prepare. - **Be careful to use it.** + Args: + model_kwargs: the params for `prepare_online_models` + signal_kwargs: the params for `prepare_signals` """ - self.cur_time = self.begin_time - self.history = {} - for strategy in self.strategy: - strategy.reset() + for cur_time, strategy_models in self.history.items(): + self.cur_time = cur_time + for strategy, models in strategy_models.items(): + self.prepare_online_models(strategy, models, delay=False, model_kwargs=model_kwargs) + # NOTE: Assumption: the predictions of online models need less than next cur_time, or this method will work in a wrong way. + self.prepare_signals(**signal_kwargs) diff --git a/qlib/workflow/online/strategy.py b/qlib/workflow/online/strategy.py index 0cae11b7fb..1184553bd0 100644 --- a/qlib/workflow/online/strategy.py +++ b/qlib/workflow/online/strategy.py @@ -7,19 +7,14 @@ from copy import deepcopy from typing import List, Tuple, Union - -import pandas as pd from qlib.data.data import D from qlib.log import get_module_logger -from qlib.model.ens.ensemble import AverageEnsemble, SingleKeyEnsemble from qlib.model.ens.group import RollingGroup -from qlib.model.trainer import Trainer, TrainerR -from qlib.workflow import R from qlib.workflow.online.utils import OnlineTool, OnlineToolR from qlib.workflow.recorder import Recorder -from qlib.workflow.task.collect import Collector, HyperCollector, RecorderCollector +from qlib.workflow.task.collect import Collector, RecorderCollector from qlib.workflow.task.gen import RollingGen, task_generator -from qlib.workflow.task.utils import TimeAdjuster, list_recorders +from qlib.workflow.task.utils import TimeAdjuster class OnlineStrategy: @@ -27,7 +22,7 @@ class OnlineStrategy: OnlineStrategy is working with `Online Manager <#Online Manager>`_, responsing how the tasks are generated, the models are updated and signals are perpared. """ - def __init__(self, name_id: str, trainer: Trainer = None, need_log=True): + def __init__(self, name_id: str, need_log=True): """ Init OnlineStrategy. This module **MUST** use `Trainer <../reference/api.html#Trainer>`_ to finishing model training. @@ -38,34 +33,22 @@ def __init__(self, name_id: str, trainer: Trainer = None, need_log=True): need_log (bool, optional): print log or not. Defaults to True. """ self.name_id = name_id - self.trainer = trainer self.logger = get_module_logger(self.__class__.__name__) self.need_log = need_log - self.tool = OnlineTool() - - def prepare_signals(self, delay: bool = False): - """ - After perparing the data of last routine (a box in box-plot) which means the end of the routine, we can prepare trading signals for next routine. + self.tool = OnlineTool(need_log) - NOTE: Given a set prediction, all signals before these prediction end time will be prepared well. - - Args: - delay: bool - If this method was called by `delay_prepare` + def prepare_tasks(self, cur_time, **kwargs) -> List[dict]: """ - raise NotImplementedError(f"Please implement the `prepare_signals` method.") - - def prepare_tasks(self, *args, **kwargs): - """ - After the end of a routine, check whether we need to prepare and train some new tasks. + After the end of a routine, check whether we need to prepare and train some new tasks based on cur_time (None for latest).. Return the new tasks waiting for training. You can find last online models by OnlineTool.online_models. """ raise NotImplementedError(f"Please implement the `prepare_tasks` method.") - def prepare_online_models(self, tasks, check_func=None, **kwargs): + def prepare_online_models(self, models, cur_time=None, check_func=None, **kwargs): """ + A typically implementation, but maybe you will need old models by online_tool. Use trainer to train a list of tasks and set the trained model to `online`. NOTE: This method will first offline all models and online the online models prepared by this method. So you can find last online models by OnlineTool.online_models if you still need them. @@ -78,64 +61,34 @@ def prepare_online_models(self, tasks, check_func=None, **kwargs): **kwargs: will be passed to end_train which means will be passed to customized train method. """ - if check_func is None: - check_func = lambda x: True - online_models = [] - if len(tasks) > 0: - new_models = self.trainer.train(tasks, **kwargs) - for model in new_models: - if check_func(model): + if check_func is not None: + online_models = [] + for model in models: + if check_func(model, cur_time): online_models.append(model) - self.tool.reset_online_tag(online_models) - return online_models + models = online_models + self.tool.reset_online_tag(models) + return models - def first_train(self): + def first_tasks(self) -> List[dict]: """ - Train a series of models firstly and set some of them as online models. + Generate a series of tasks firstly and return them. """ - raise NotImplementedError(f"Please implement the `first_train` method.") + raise NotImplementedError(f"Please implement the `first_tasks` method.") def get_collector(self) -> Collector: """ - Get the instance of `Collector <../advanced/task_management.html#Task Collecting>`_ to collect results of online serving. - + Get the instance of `Collector <../advanced/task_management.html#Task Collecting>`_ to collect different results of this strategy. For example: 1) collect predictions in Recorder - 2) collect signals in .txt file + 2) collect signals in a txt file Returns: Collector """ raise NotImplementedError(f"Please implement the `get_collector` method.") - def delay_prepare(self, history: list, **kwargs): - """ - Prepare all models and signals if there are something waiting for prepare. - - Assumption: the predictions of online models need less than next begin_time, or this method will work in a wrong way. - - Args: - history (list): an online models list likes [begin_time:[online models]]. - **kwargs: will be passed to end_train which means will be passed to customized train method. - """ - for begin_time, recs_list in history: - self.trainer.end_train(recs_list, **kwargs) - self.tool.reset_online_tag(recs_list) - self.prepare_signals(delay=True) - - def get_signals(self): - """ - Get prepared signals. - """ - raise NotImplementedError(f"Please implement the `get_signals` method.") - - def reset(self): - """ - Delete all things and set them to default status. This method is convenient to explore the strategy for online simulation. - """ - pass - class RollingAverageStrategy(OnlineStrategy): @@ -148,9 +101,7 @@ def __init__( name_id: str, task_template: Union[dict, List[dict]], rolling_gen: RollingGen, - trainer: Trainer = None, need_log=True, - signal_exp_name="OnlineManagerSignals", ): """ Init RollingAverageStrategy. @@ -161,22 +112,16 @@ def __init__( name_id (str): a unique name or id. Will be also the name of Experiment. task_template (Union[dict,List[dict]]): a list of task_template or a single template, which will be used to generate many tasks using rolling_gen. rolling_gen (RollingGen): an instance of RollingGen - trainer (Trainer, optional): a instance of Trainer. Defaults to None. need_log (bool, optional): print log or not. Defaults to True. - signal_exp_path (str): a specific experiment to save signals of different experiment. """ - super().__init__(name_id=name_id, trainer=trainer, need_log=need_log) + super().__init__(name_id=name_id, need_log=need_log) self.exp_name = self.name_id if not isinstance(task_template, list): task_template = [task_template] self.task_template = task_template - self.signal_exp_name = signal_exp_name self.rg = rolling_gen - self.tool = OnlineToolR(self.exp_name) + self.tool = OnlineToolR(self.exp_name, need_log) self.ta = TimeAdjuster() - with R.start(experiment_name=self.signal_exp_name, recorder_name=self.exp_name, resume=True): - self.signal_rec = R.get_recorder() # the recorder to record signals - self.signal_rec.save_objects(**{"signals": None}) def get_collector(self, process_list=[RollingGroup()], rec_key_func=None, rec_filter_func=None, artifacts_key=None): """ @@ -209,18 +154,17 @@ def rec_key(recorder): return artifacts_collector - def first_train(self) -> List[Recorder]: + def first_tasks(self) -> List[dict]: """ - Use rolling_gen to generate different tasks based on task_template and trained them. + Use rolling_gen to generate different tasks based on task_template. Returns: - List[Recorder]: a list of Recorder. + List[dict]: a list of tasks """ - tasks = task_generator( + return task_generator( tasks=self.task_template, generators=self.rg, # generate different date segment ) - return self.prepare_online_models(tasks) def prepare_tasks(self, cur_time) -> List[dict]: """ @@ -255,57 +199,6 @@ def prepare_tasks(self, cur_time) -> List[dict]: return new_tasks return [] - def prepare_signals(self, delay=False, over_write=False) -> pd.DataFrame: - """ - Average the predictions of online models and offer a trading signals every routine. - The signals will be saved to `signal` file of a recorder named self.exp_name of a experiment using the name of `SIGNAL_EXP` - Even if the latest signal already exists, the latest calculation result will be overwritten. - - .. note:: - - Given a prediction of a certain time, all signals before this time will be prepared well. - - Args: - over_write (bool, optional): If True, the new signals will overwrite the file. If False, the new signals will append to the end of signals. Defaults to False. - Returns: - pd.DataFrame: the signals. - """ - if not delay: - self.tool.update_online_pred() - - # Get a collector to average online models predictions - online_collector = self.get_collector( - process_list=[AverageEnsemble()], - rec_filter_func=lambda x: True if self.tool.get_online_tag(x) == self.tool.ONLINE_TAG else False, - artifacts_key="pred", - ) - online_results = online_collector() - signals = online_results["pred"] - - old_signals = self.get_signals() - if old_signals is not None and not over_write: - old_max = old_signals.index.get_level_values("datetime").max() - new_signals = signals.loc[old_max:] - signals = pd.concat([old_signals, new_signals], axis=0) - else: - new_signals = signals - if self.need_log: - self.logger.info( - f"Finished preparing new {len(new_signals)} signals to {self.signal_exp_name}/{self.exp_name}." - ) - self.signal_rec.save_objects(**{"signals": signals}) - return signals - - def get_signals(self) -> object: - """ - Get signals from the recorder(named self.exp_name) of the experiment(named self.SIGNAL_EXP) - - Returns: - object: signals - """ - signals = self.signal_rec.load_object("signals") - return signals - def _list_latest(self, rec_list: List[Recorder]): """ List latest recorder form rec_list @@ -324,16 +217,3 @@ def _list_latest(self, rec_list: List[Recorder]): if rec.load_object("task")["dataset"]["kwargs"]["segments"]["test"] == max_test: latest_rec.append(rec) return latest_rec, max_test - - def reset(self): - """ - NOTE: This method will delete all recorder in Experiment and reset the Trainer! - """ - self.trainer.reset() - # delete models - exp = R.get_exp(experiment_name=self.exp_name) - for rid in exp.list_recorders(): - exp.delete_recorder(rid) - # delete signals - for rid in list_recorders(self.signal_exp_name, lambda x: True if x.info["name"] == self.exp_name else False): - exp.delete_recorder(rid) diff --git a/qlib/workflow/online/utils.py b/qlib/workflow/online/utils.py index 296ca3ea6c..c79a5dc00f 100644 --- a/qlib/workflow/online/utils.py +++ b/qlib/workflow/online/utils.py @@ -17,7 +17,7 @@ class OnlineTool: """ - OnlineTool. + OnlineTool will manage `online` models in an experiment which includes the models recorder. """ ONLINE_KEY = "online_status" # the online status key in recorder @@ -92,7 +92,7 @@ class OnlineToolR(OnlineTool): The implementation of OnlineTool based on (R)ecorder. """ - def __init__(self, experiment_name: str, need_log=True): + def __init__(self, experiment_name:str, need_log=True): """ Init OnlineToolR. diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index 28320e2ceb..b40ee01646 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -5,14 +5,16 @@ Collector can collect object from everywhere and process them such as merging, grouping, averaging and so on. """ -from qlib.model.ens.ensemble import SingleKeyEnsemble +from typing import Callable, Dict, List +from qlib.utils.serial import Serializable from qlib.workflow import R -import dill as pickle -class Collector: +class Collector(Serializable): """The collector to collect different results""" + pickle_backend = "dill" # use dill to dump user method + def __init__(self, process_list=[]): """ Args: @@ -74,65 +76,42 @@ def __call__(self, *args, **kwargs) -> dict: collected = self.collect() return self.process_collect(collected, self.process_list, *args, **kwargs) - def save(self, filepath): - """ - save the collector into a file - - Args: - filepath (str): the path of file - - Returns: - bool: if succeeded - """ - try: - with open(filepath, "wb") as f: - pickle.dump(self, f) - except Exception: - return False - return True - - @staticmethod - def load(filepath): - """ - load the collector from a file - Args: - filepath (str): the path of file +class MergeCollector(Collector): + """ + A collector to collect the results of other Collectors - Raises: - TypeError: the pickled file must be `Collector` + For example: - Returns: - Collector: the instance of Collector - """ - with open(filepath, "rb") as f: - collector = pickle.load(f) - if isinstance(collector, Collector): - return collector - else: - raise TypeError(f"The instance of {type(collector)} is not a valid `Collector`!") + We have 2 collector, which named A and B. + A can collect {"prediction": pd.Series} and B can collect {"IC": {"Xgboost": pd.Series, "LSTM": pd.Series}}. + Then after this class's collect, we can collect {"A_prediction": pd.Series, "B_IC": {"Xgboost": pd.Series, "LSTM": pd.Series}} + ...... -class HyperCollector(Collector): - """ - A collector to collect the results of other Collectors """ - def __init__(self, collector_dict, process_list=[]): + def __init__(self, collector_dict: Dict[str, Collector], process_list: List[Callable] = []): """ Args: - collector_dict (dict): the dict like {collector_key, Collector} - process_list (list or Callable): the list of processors or the instance of processor to process dict. - NOTE: process_list = [SingleKeyEnsemble()] can ignore key and use value directly if there is only one {k,v} in a dict. - This can make result more readable. If you want to maintain as it should be, just give a empty process list. + collector_dict (Dict[str,Collector]): the dict like {collector_key, Collector} + process_list (List[Callable]): the list of processors or the instance of processor to process dict. """ super().__init__(process_list=process_list) self.collector_dict = collector_dict def collect(self) -> dict: + """ + Collect all result of collector_dict and change the outermost key to "``collector_key``_``key``" (like merge them, but rename every key) + + Returns: + dict: the dict after collecting. + """ collect_dict = {} - for key, collector in self.collector_dict.items(): - collect_dict[key] = collector() + for collector_key, collector in self.collector_dict.items(): + tmp_dict = collector() + for key, value in tmp_dict.items(): + collect_dict[collector_key + "_" + str(key)] = value return collect_dict @@ -145,7 +124,7 @@ def __init__( process_list=[], rec_key_func=None, rec_filter_func=None, - artifacts_path={"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"}, + artifacts_path={"pred": "pred.pkl"}, artifacts_key=None, ): """init RecorderCollector @@ -203,7 +182,11 @@ def collect(self, artifacts_key=None, rec_filter_func=None) -> dict: if self.ART_KEY_RAW == key: artifact = rec else: - artifact = rec.load_object(self.artifacts_path[key]) + # only collect existing artifact + try: + artifact = rec.load_object(self.artifacts_path[key]) + except Exception: + continue collect_dict.setdefault(key, {})[rec_key] = artifact return collect_dict diff --git a/qlib/workflow/task/gen.py b/qlib/workflow/task/gen.py index c4c6bab7fb..7e08c76f48 100644 --- a/qlib/workflow/task/gen.py +++ b/qlib/workflow/task/gen.py @@ -5,7 +5,7 @@ """ import abc import copy -import typing +from typing import List, Union, Callable from .utils import TimeAdjuster @@ -64,7 +64,7 @@ class TaskGen(metaclass=abc.ABCMeta): """ @abc.abstractmethod - def generate(self, task: dict) -> typing.List[dict]: + def generate(self, task: dict) -> List[dict]: """ generate different tasks based on a task template @@ -87,11 +87,34 @@ def __call__(self, *args, **kwargs): return self.generate(*args, **kwargs) +def handler_mod(task: dict, rg): + """ + Help to modify the handler end time when using RollingGen + + Args: + task (dict): a task template + rg (RollingGen): an instance of RollingGen + """ + try: + interval = rg.ta.cal_interval( + task["dataset"]["kwargs"]["handler"]["kwargs"]["end_time"], + task["dataset"]["kwargs"]["segments"][rg.test_key][1], + ) + # if end_time < the end of test_segments, then change end_time to allow load more data + if interval < 0: + task["dataset"]["kwargs"]["handler"]["kwargs"]["end_time"] = copy.deepcopy( + task["dataset"]["kwargs"]["segments"][rg.test_key][1] + ) + except KeyError: + # Maybe dataset do not have handler, then do nothing. + pass + + class RollingGen(TaskGen): ROLL_EX = TimeAdjuster.SHIFT_EX # fixed start date, expanding end date ROLL_SD = TimeAdjuster.SHIFT_SD # fixed segments size, slide it from start date - def __init__(self, step: int = 40, rtype: str = ROLL_EX, modify_end_time=True): + def __init__(self, step: int = 40, rtype: str = ROLL_EX, ds_extra_mod_func: Union[None, Callable] = handler_mod): """ Generate tasks for rolling @@ -101,19 +124,19 @@ def __init__(self, step: int = 40, rtype: str = ROLL_EX, modify_end_time=True): step to rolling rtype : str rolling type (expanding, sliding) - modify_end_time: bool - Whether the data set configuration needs to be modified when the required scope exceeds the original data set scope + ds_extra_mod_func: Callable + A method like: handler_mod(task: dict, rg: RollingGen) + Do some extra action after generating a task. For example, use ``handler_mod`` to modify the end time of handler of dataset. """ self.step = step self.rtype = rtype - self.modify_end_time = modify_end_time - # TODO: Ask pengrong to update future date in dataset + self.ds_extra_mod_func = ds_extra_mod_func self.ta = TimeAdjuster(future=True) self.test_key = "test" self.train_key = "train" - def generate(self, task: dict) -> typing.List[dict]: + def generate(self, task: dict) -> List[dict]: """ Converting the task into a rolling task. @@ -200,18 +223,8 @@ def generate(self, task: dict) -> typing.List[dict]: # update segments of this task t["dataset"]["kwargs"]["segments"] = copy.deepcopy(segments) - - try: - interval = self.ta.cal_interval( - t["dataset"]["kwargs"]["handler"]["kwargs"]["end_time"], - t["dataset"]["kwargs"]["segments"][self.test_key][1], - ) - # if end_time < the end of test_segments, then change end_time to allow load more data - if self.modify_end_time and interval < 0: - t["dataset"]["kwargs"]["handler"]["kwargs"]["end_time"] = copy.deepcopy(segments[self.test_key][1]) - except KeyError: - # Maybe the user dataset has no handler or end_time - pass prev_seg = segments + if self.ds_extra_mod_func is not None: + self.ds_extra_mod_func(t, self) res.append(t) return res diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index c71be7d39b..025dfa85c2 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -388,6 +388,7 @@ def __str__(self): def run_task( task_func: Callable, task_pool: str, + tasks: List[dict] = None, force_release: bool = False, before_status: str = TaskManager.STATUS_WAITING, after_status: str = TaskManager.STATUS_DONE, @@ -413,6 +414,8 @@ def (task_def, **kwargs) -> the function to run the task task_pool : str the name of the task pool (Collection in MongoDB) + tasks: List[dict] + will only train these tasks config, None for train all tasks. force_release : bool will the program force to release the resource before_status : str: @@ -425,9 +428,12 @@ def (task_def, **kwargs) -> tm = TaskManager(task_pool) ever_run = False + query = {} + if tasks is not None: + query = {"filter": {"$in": tasks}} while True: - with tm.safe_fetch_task(status=before_status) as task: + with tm.safe_fetch_task(status=before_status, query=query) as task: if task is None: break get_module_logger("run_task").info(task["def"]) From bec65ddf94a014349a16931833ac85c7f78ebc5d Mon Sep 17 00:00:00 2001 From: binlins <981921742@qq.com> Date: Fri, 7 May 2021 11:47:47 +0000 Subject: [PATCH 54/61] add document and reindex --- qlib/data/dataset/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/qlib/data/dataset/__init__.py b/qlib/data/dataset/__init__.py index 8bcd6419a8..a8b10a258d 100644 --- a/qlib/data/dataset/__init__.py +++ b/qlib/data/dataset/__init__.py @@ -200,6 +200,12 @@ def prepare( The data to fetch: DK_* Default is DK_I, which indicate fetching data for **inference**. + kwargs : + The parameters that kwargs may contain: + flt_col : str + It only exists in TSDatasetH, can be used to add a column of data(True or False) to filter data. + This parameter is only supported when it is an instance of TSDatasetH. + Returns ------- Union[List[pd.DataFrame], pd.DataFrame]: @@ -293,7 +299,7 @@ def __init__(self, data: pd.DataFrame, start, end, step_len: int, fillna_type: s self.data_index = deepcopy(self.data.index) if flt_data is not None: - self.flt_data = np.array(flt_data).reshape(-1) + self.flt_data = np.array(flt_data.reindex(self.data_index)).reshape(-1) self.idx_map = self.flt_idx_map(self.flt_data, self.idx_map) self.data_index = self.data_index[np.where(self.flt_data == True)[0]] From 08edb92461779e7b2ba666b1bd2c14e27969f848 Mon Sep 17 00:00:00 2001 From: binlins <981921742@qq.com> Date: Fri, 7 May 2021 12:56:58 +0000 Subject: [PATCH 55/61] add flt_data doc --- qlib/data/dataset/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qlib/data/dataset/__init__.py b/qlib/data/dataset/__init__.py index a8b10a258d..2173d87ae4 100644 --- a/qlib/data/dataset/__init__.py +++ b/qlib/data/dataset/__init__.py @@ -272,6 +272,11 @@ def __init__(self, data: pd.DataFrame, start, end, step_len: int, fillna_type: s ffill with previous sample ffill+bfill: ffill with previous samples first and fill with later samples second + flt_data : pd.Series + a column of data(True or False) to filter data. + None: + kepp all data + """ self.start = start self.end = end From 370b6aad74a5f36984e5311b925eda218d7b8490 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Sun, 9 May 2021 11:58:06 +0000 Subject: [PATCH 56/61] logger & doc --- .../online_srv/online_management_simulate.py | 4 +- qlib/data/dataset/__init__.py | 12 ++--- qlib/model/trainer.py | 17 ++++--- qlib/workflow/online/manager.py | 44 +++++++++++-------- qlib/workflow/online/strategy.py | 28 +++++------- qlib/workflow/online/update.py | 13 +++--- qlib/workflow/online/utils.py | 19 +++----- 7 files changed, 69 insertions(+), 68 deletions(-) diff --git a/examples/online_srv/online_management_simulate.py b/examples/online_srv/online_management_simulate.py index 5583ee1604..3a87e01c49 100644 --- a/examples/online_srv/online_management_simulate.py +++ b/examples/online_srv/online_management_simulate.py @@ -114,12 +114,10 @@ def __init__( step=rolling_step, rtype=RollingGen.ROLL_SD, ds_extra_mod_func=None ) # The rolling tasks generator, ds_extra_mod_func is None because we just need simulate to 2018-10-31 and needn't change handler end time. self.trainer = DelayTrainerRM(self.exp_name, self.task_pool) - self.task_manager = TaskManager(self.task_pool) # A good way to manage all your tasks self.rolling_online_manager = OnlineManager( - RollingAverageStrategy(exp_name, task_template=tasks, rolling_gen=self.rolling_gen, need_log=False), + RollingAverageStrategy(exp_name, task_template=tasks, rolling_gen=self.rolling_gen), trainer=self.trainer, begin_time=self.start_time, - need_log=False, ) self.tasks = tasks diff --git a/qlib/data/dataset/__init__.py b/qlib/data/dataset/__init__.py index 75abc0cbbe..021311d479 100644 --- a/qlib/data/dataset/__init__.py +++ b/qlib/data/dataset/__init__.py @@ -200,7 +200,7 @@ def prepare( The data to fetch: DK_* Default is DK_I, which indicate fetching data for **inference**. - kwargs : + kwargs : The parameters that kwargs may contain: flt_col : str It only exists in TSDatasetH, can be used to add a column of data(True or False) to filter data. @@ -250,7 +250,9 @@ class TSDataSampler: """ - def __init__(self, data: pd.DataFrame, start, end, step_len: int, fillna_type: str = "none", dtype=None, flt_data=None): + def __init__( + self, data: pd.DataFrame, start, end, step_len: int, fillna_type: str = "none", dtype=None, flt_data=None + ): """ Build a dataset which looks like torch.data.utils.Dataset. @@ -518,17 +520,17 @@ def _prepare_seg(self, slc: slice, **kwargs) -> TSDataSampler: """ dtype = kwargs.pop("dtype", None) start, end = slc.start, slc.stop - flt_col = kwargs.pop('flt_col', None) + flt_col = kwargs.pop("flt_col", None) # TSDatasetH will retrieve more data for complete data = self._prepare_raw_seg(slc, **kwargs) flt_kwargs = deepcopy(kwargs) if flt_col is not None: - flt_kwargs['col_set'] = flt_col + flt_kwargs["col_set"] = flt_col flt_data = self._prepare_raw_seg(slc, **flt_kwargs) assert len(flt_data.columns) == 1 else: flt_data = None tsds = TSDataSampler(data=data, start=start, end=end, step_len=self.step_len, dtype=dtype, flt_data=flt_data) - return tsds \ No newline at end of file + return tsds diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index 68b78d9df2..0b64d3b308 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -5,7 +5,7 @@ The Trainer will train a list of tasks and return a list of model recorder. There are two steps in each Trainer including ``train``(make model recorder) and ``end_train``(modify model recorder). -This is concept called ``DelayTrainer``, which can be used in online simulating to parallel training. +This is concept called ``DelayTrainer``, which can be used in online simulating for parallel training. In ``DelayTrainer``, the first step is only to save some necessary info to model recorder, and the second step which will be finished in the end can do some concurrent and time-consuming operations such as model fitting. ``Qlib`` offer two kind of Trainer, ``TrainerR`` is the simplest way and ``TrainerRM`` is based on TaskManager to help manager tasks lifecycle automatically. @@ -103,7 +103,8 @@ def task_train(task_config: dict, experiment_name: str) -> Recorder: class Trainer: """ - The trainer which can train a list of model + The trainer can train a list of model. + There are Trainer and DelayTrainer, which can be distinguished by when it will finish real training. """ def __init__(self): @@ -113,6 +114,9 @@ def train(self, tasks: list, *args, **kwargs) -> list: """ Given a list of model definition, begin a training and return the models. + For Trainer, it finish real training in this method. + For DelayTrainer, it only do some preparation in this method. + Args: tasks: a list of tasks @@ -126,6 +130,9 @@ def end_train(self, models: list, *args, **kwargs) -> list: Given a list of models, finished something in the end of training if you need. The models maybe Recorder, txt file, database and so on. + For Trainer, it do some finishing touches in this method. + For DelayTrainer, it finish real training in this method. + Args: models: a list of models @@ -326,7 +333,7 @@ def train( recs.append(rec) return recs - def end_train(self, recs: list, **kwargs) -> list: + def end_train(self, recs: list, **kwargs) -> List[Recorder]: for rec in recs: rec.set_tags(**{self.STATUS_KEY: self.STATUS_END}) return recs @@ -358,7 +365,7 @@ def __init__( self.end_train_func = end_train_func self.delay = True - def train(self, tasks: list, train_func=None, experiment_name: str = None, **kwargs): + def train(self, tasks: list, train_func=None, experiment_name: str = None, **kwargs) -> List[Recorder]: """ Same as `train` of TrainerRM, after_status will be STATUS_PART_DONE. Args: @@ -378,7 +385,7 @@ def train(self, tasks: list, train_func=None, experiment_name: str = None, **kwa **kwargs, ) - def end_train(self, recs, end_train_func=None, experiment_name: str = None, **kwargs): + def end_train(self, recs, end_train_func=None, experiment_name: str = None, **kwargs) -> List[Recorder]: """ Given a list of Recorder and return a list of trained Recorder. This class will finish real data loading and model fitting. diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index a282865e65..e41c3f20a8 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -12,11 +12,13 @@ Which means you can verify your strategy or find a better one. """ +import logging from typing import Callable, Dict, List, Union import pandas as pd from qlib import get_module_logger from qlib.data.data import D +from qlib.log import set_global_logger_level from qlib.model.ens.ensemble import AverageEnsemble from qlib.model.trainer import DelayTrainerR, Trainer from qlib.utils import flatten_dict @@ -37,7 +39,6 @@ def __init__( trainer: Trainer = None, begin_time: Union[str, pd.Timestamp] = None, freq="day", - need_log=True, ): """ Init OnlineManager. @@ -48,10 +49,8 @@ def __init__( begin_time (Union[str,pd.Timestamp], optional): the OnlineManager will begin at this time. Defaults to None for using latest date. trainer (Trainer): the trainer to train task. None for using DelayTrainerR. freq (str, optional): data frequency. Defaults to "day". - need_log (bool, optional): print log or not. Defaults to True. """ self.logger = get_module_logger(self.__class__.__name__) - self.need_log = need_log if not isinstance(strategy, list): strategy = [strategy] self.strategy = strategy @@ -60,19 +59,18 @@ def __init__( begin_time = D.calendar(freq=self.freq).max() self.begin_time = pd.Timestamp(begin_time) self.cur_time = self.begin_time - # The history of online models, which is a dict like {begin_time, {strategy, [online_models]}} - # begin_time means when online_models are onlined - self.history = {} + # OnlineManager will recorder the history of online models, which is a dict like {begin_time, {strategy, [online_models]}}. begin_time means when online_models are onlined. + self.history = {} if trainer is None: trainer = DelayTrainerR() self.trainer = trainer self.signals = None - def first_train(self, strategies:List[OnlineStrategy]=None, model_kwargs: dict = {}): + def first_train(self, strategies: List[OnlineStrategy] = None, model_kwargs: dict = {}): """ Get tasks from every strategy's first_tasks method and train them. If using DelayTrainer, it can finish training all together after every strategy's first_tasks. - + Args: strategies (List[OnlineStrategy]): the strategies list (need this param when adding strategies). None for use default strategies. model_kwargs (dict): the params for `prepare_online_models` @@ -119,8 +117,7 @@ def routine( for strategy in self.strategy: if not delay: strategy.tool.update_online_pred() - if self.need_log: - self.logger.info(f"Strategy `{strategy.name_id}` begins routine...") + self.logger.info(f"Strategy `{strategy.name_id}` begins routine...") tasks = strategy.prepare_tasks(self.cur_time, **task_kwargs) models = self.trainer.train(tasks) @@ -144,19 +141,20 @@ def prepare_online_models( delay (bool, optional): if delay prepare models. Defaults to False. model_kwargs (dict, optional): the params for `prepare_online_models`. """ + if not models: + return if not delay: models = self.trainer.end_train(models, experiment_name=strategy.name_id) online_models = strategy.prepare_online_models(models, **model_kwargs) else: # just set every models as online models temporarily before ``prepare_online_models`` online_models = models - if len(online_models) > 0: - strategy.tool.reset_online_tag(online_models) - self.history.setdefault(self.cur_time, {})[strategy] = online_models + self.history.setdefault(self.cur_time, {})[strategy] = online_models def get_collector(self) -> MergeCollector: """ Get the instance of `Collector <../advanced/task_management.html#Task Collecting>`_ to collect results from every strategy. + This collector can be a basis as the signals preparation. Returns: MergeCollector: the collector to merge other collectors. @@ -205,20 +203,23 @@ def prepare_signals(self, prepare_func: Callable = AverageEnsemble(), over_write signals = pd.concat([old_signals, new_signals], axis=0) else: new_signals = signals - if self.need_log: - self.logger.info(f"Finished preparing new {len(new_signals)} signals.") + self.logger.info(f"Finished preparing new {len(new_signals)} signals.") self.signals = signals return new_signals - def get_signals(self) -> pd.Series: + def get_signals(self) -> Union[pd.Series, pd.DataFrame]: """ Get prepared online signals. Returns: - pd.Series: signals + Union[pd.Series, pd.DataFrame]: pd.Series for only one signals every datetime. + pd.DataFrame for multiple signals, for example, buy and sell operation use different trading signal. """ return self.signals + SIM_LOG_LEVEL = logging.INFO + 1 + SIM_LOG_NAME = "SIMULATE_INFO" + def simulate(self, end_time, frequency="day", task_kwargs={}, model_kwargs={}, signal_kwargs={}): """ Starting from current time, this method will simulate every routine in OnlineManager until end time. @@ -239,8 +240,13 @@ def simulate(self, end_time, frequency="day", task_kwargs={}, model_kwargs={}, s """ cal = D.calendar(start_time=self.cur_time, end_time=end_time, freq=frequency) self.first_train() + + simulate_level = self.SIM_LOG_LEVEL + set_global_logger_level(simulate_level) + logging.addLevelName(simulate_level, self.SIM_LOG_NAME) + for cur_time in cal: - self.logger.info(f"Simulating at {str(cur_time)}......") + self.logger.log(level=simulate_level, msg=f"Simulating at {str(cur_time)}......") self.routine( cur_time, delay=self.trainer.is_delay(), @@ -251,6 +257,8 @@ def simulate(self, end_time, frequency="day", task_kwargs={}, model_kwargs={}, s # delay prepare the models and signals if self.trainer.is_delay(): self.delay_prepare(model_kwargs=model_kwargs, signal_kwargs=signal_kwargs) + + set_global_logger_level(logging.INFO) self.logger.info(f"Finished preparing signals") return self.get_collector() diff --git a/qlib/workflow/online/strategy.py b/qlib/workflow/online/strategy.py index 1184553bd0..9f657427d8 100644 --- a/qlib/workflow/online/strategy.py +++ b/qlib/workflow/online/strategy.py @@ -22,7 +22,7 @@ class OnlineStrategy: OnlineStrategy is working with `Online Manager <#Online Manager>`_, responsing how the tasks are generated, the models are updated and signals are perpared. """ - def __init__(self, name_id: str, need_log=True): + def __init__(self, name_id: str): """ Init OnlineStrategy. This module **MUST** use `Trainer <../reference/api.html#Trainer>`_ to finishing model training. @@ -30,12 +30,10 @@ def __init__(self, name_id: str, need_log=True): Args: name_id (str): a unique name or id trainer (Trainer, optional): a instance of Trainer. Defaults to None. - need_log (bool, optional): print log or not. Defaults to True. """ self.name_id = name_id self.logger = get_module_logger(self.__class__.__name__) - self.need_log = need_log - self.tool = OnlineTool(need_log) + self.tool = OnlineTool() def prepare_tasks(self, cur_time, **kwargs) -> List[dict]: """ @@ -46,20 +44,21 @@ def prepare_tasks(self, cur_time, **kwargs) -> List[dict]: """ raise NotImplementedError(f"Please implement the `prepare_tasks` method.") - def prepare_online_models(self, models, cur_time=None, check_func=None, **kwargs): + def prepare_online_models(self, models, cur_time=None, check_func=None) -> List[object]: """ A typically implementation, but maybe you will need old models by online_tool. - Use trainer to train a list of tasks and set the trained model to `online`. + Select some models as the online models from the trained models. - NOTE: This method will first offline all models and online the online models prepared by this method. So you can find last online models by OnlineTool.online_models if you still need them. + NOTE: This method offline all models and online the online models prepared by this method (if have). So you can find last online models by OnlineTool.online_models if you still need them. Args: tasks (list): a list of tasks. check_func: the method to judge if a model can be online. The parameter is the model record and return True for online. None for online every models. - **kwargs: will be passed to end_train which means will be passed to customized train method. + Returns: + List[object]: a list of selected models. """ if check_func is not None: online_models = [] @@ -101,7 +100,6 @@ def __init__( name_id: str, task_template: Union[dict, List[dict]], rolling_gen: RollingGen, - need_log=True, ): """ Init RollingAverageStrategy. @@ -112,15 +110,14 @@ def __init__( name_id (str): a unique name or id. Will be also the name of Experiment. task_template (Union[dict,List[dict]]): a list of task_template or a single template, which will be used to generate many tasks using rolling_gen. rolling_gen (RollingGen): an instance of RollingGen - need_log (bool, optional): print log or not. Defaults to True. """ - super().__init__(name_id=name_id, need_log=need_log) + super().__init__(name_id=name_id) self.exp_name = self.name_id if not isinstance(task_template, list): task_template = [task_template] self.task_template = task_template self.rg = rolling_gen - self.tool = OnlineToolR(self.exp_name, need_log) + self.tool = OnlineToolR(self.exp_name) self.ta = TimeAdjuster() def get_collector(self, process_list=[RollingGroup()], rec_key_func=None, rec_filter_func=None, artifacts_key=None): @@ -180,10 +177,9 @@ def prepare_tasks(self, cur_time) -> List[dict]: self.logger.warn(f"No latest online recorders, no new tasks.") return [] calendar_latest = D.calendar(end_time=cur_time)[-1] if cur_time is None else cur_time - if self.need_log: - self.logger.info( - f"The interval between current time {calendar_latest} and last rolling test begin time {max_test[0]} is {self.ta.cal_interval(calendar_latest, max_test[0])}, the rolling step is {self.rg.step}" - ) + self.logger.info( + f"The interval between current time {calendar_latest} and last rolling test begin time {max_test[0]} is {self.ta.cal_interval(calendar_latest, max_test[0])}, the rolling step is {self.rg.step}" + ) if self.ta.cal_interval(calendar_latest, max_test[0]) >= self.rg.step: old_tasks = [] tasks_tmp = [] diff --git a/qlib/workflow/online/update.py b/qlib/workflow/online/update.py index ab910ba8dc..a69e1005fd 100644 --- a/qlib/workflow/online/update.py +++ b/qlib/workflow/online/update.py @@ -60,10 +60,9 @@ class RecordUpdater(metaclass=ABCMeta): Update a specific recorders """ - def __init__(self, record: Recorder, need_log=True, *args, **kwargs): + def __init__(self, record: Recorder, *args, **kwargs): self.record = record self.logger = get_module_logger(self.__class__.__name__) - self.need_log = need_log @abstractmethod def update(self, *args, **kwargs): @@ -78,7 +77,7 @@ class PredUpdater(RecordUpdater): Update the prediction in the Recorder """ - def __init__(self, record: Recorder, to_date=None, hist_ref: int = 0, freq="day", need_log=True): + def __init__(self, record: Recorder, to_date=None, hist_ref: int = 0, freq="day"): """ Init PredUpdater. @@ -96,7 +95,7 @@ def __init__(self, record: Recorder, to_date=None, hist_ref: int = 0, freq="day" """ # TODO: automate this hist_ref in the future. - super().__init__(record=record, need_log=need_log) + super().__init__(record=record) self.to_date = to_date self.hist_ref = hist_ref @@ -138,8 +137,7 @@ def update(self, dataset: DatasetH = None): start_time = get_date_by_shift(self.last_end, 1, freq=self.freq) if start_time >= self.to_date: - if self.need_log: - self.logger.info(f"The prediction in {self.record.info['id']} are latest. No need to update.") + self.logger.info(f"The prediction in {self.record.info['id']} are latest. No need to update.") return # load dataset @@ -157,5 +155,4 @@ def update(self, dataset: DatasetH = None): self.record.save_objects(**{"pred.pkl": cb_pred}) - if self.need_log: - self.logger.info(f"Finish updating new {new_pred.shape[0]} predictions in {self.record.info['id']}.") + self.logger.info(f"Finish updating new {new_pred.shape[0]} predictions in {self.record.info['id']}.") diff --git a/qlib/workflow/online/utils.py b/qlib/workflow/online/utils.py index c79a5dc00f..c3af9d1cab 100644 --- a/qlib/workflow/online/utils.py +++ b/qlib/workflow/online/utils.py @@ -24,15 +24,11 @@ class OnlineTool: ONLINE_TAG = "online" # the 'online' model OFFLINE_TAG = "offline" # the 'offline' model, not for online serving - def __init__(self, need_log=True): + def __init__(self): """ Init OnlineTool. - - Args: - need_log (bool, optional): print log or not. Defaults to True. """ self.logger = get_module_logger(self.__class__.__name__) - self.need_log = need_log def set_online_tag(self, tag, recorder: Union[list, object]): """ @@ -92,15 +88,14 @@ class OnlineToolR(OnlineTool): The implementation of OnlineTool based on (R)ecorder. """ - def __init__(self, experiment_name:str, need_log=True): + def __init__(self, experiment_name: str): """ Init OnlineToolR. Args: experiment_name (str): the experiment name. - need_log (bool, optional): print log or not. Defaults to True. """ - super().__init__(need_log=need_log) + super().__init__() self.exp_name = experiment_name def set_online_tag(self, tag, recorder: Union[Recorder, List]): @@ -115,8 +110,7 @@ def set_online_tag(self, tag, recorder: Union[Recorder, List]): recorder = [recorder] for rec in recorder: rec.set_tags(**{self.ONLINE_KEY: tag}) - if self.need_log: - self.logger.info(f"Set {len(recorder)} models to '{tag}'.") + self.logger.info(f"Set {len(recorder)} models to '{tag}'.") def get_online_tag(self, recorder: Recorder) -> str: """ @@ -164,7 +158,6 @@ def update_online_pred(self, to_date=None): """ online_models = self.online_models() for rec in online_models: - PredUpdater(rec, to_date=to_date, need_log=self.need_log).update() + PredUpdater(rec, to_date=to_date).update() - if self.need_log: - self.logger.info(f"Finished updating {len(online_models)} online model predictions of {self.exp_name}.") + self.logger.info(f"Finished updating {len(online_models)} online model predictions of {self.exp_name}.") From d71a666904edf8ad97a5e68eb6ba2af9e99063ff Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Thu, 13 May 2021 09:43:42 +0000 Subject: [PATCH 57/61] Online serving V11 --- .../online_srv/online_management_simulate.py | 6 +-- .../online_srv/rolling_online_management.py | 4 +- qlib/data/dataset/__init__.py | 5 ++- qlib/model/ens/ensemble.py | 4 +- qlib/model/trainer.py | 2 +- qlib/utils/__init__.py | 30 ++++++++----- qlib/utils/serial.py | 33 ++++++++------ qlib/workflow/online/manager.py | 45 +++++++++---------- qlib/workflow/online/strategy.py | 33 ++++++-------- qlib/workflow/online/update.py | 4 +- qlib/workflow/online/utils.py | 6 ++- qlib/workflow/task/collect.py | 32 ++++++++----- qlib/workflow/task/manage.py | 28 ++++++------ 13 files changed, 131 insertions(+), 101 deletions(-) diff --git a/examples/online_srv/online_management_simulate.py b/examples/online_srv/online_management_simulate.py index 3a87e01c49..48433c6d5c 100644 --- a/examples/online_srv/online_management_simulate.py +++ b/examples/online_srv/online_management_simulate.py @@ -7,10 +7,10 @@ import fire import qlib -from qlib.model.trainer import DelayTrainerRM +from qlib.model.trainer import DelayTrainerR, DelayTrainerRM, TrainerR, TrainerRM from qlib.workflow import R from qlib.workflow.online.manager import OnlineManager -from qlib.workflow.online.strategy import RollingAverageStrategy +from qlib.workflow.online.strategy import RollingStrategy from qlib.workflow.task.gen import RollingGen from qlib.workflow.task.manage import TaskManager @@ -115,7 +115,7 @@ def __init__( ) # The rolling tasks generator, ds_extra_mod_func is None because we just need simulate to 2018-10-31 and needn't change handler end time. self.trainer = DelayTrainerRM(self.exp_name, self.task_pool) self.rolling_online_manager = OnlineManager( - RollingAverageStrategy(exp_name, task_template=tasks, rolling_gen=self.rolling_gen), + RollingStrategy(exp_name, task_template=tasks, rolling_gen=self.rolling_gen), trainer=self.trainer, begin_time=self.start_time, ) diff --git a/examples/online_srv/rolling_online_management.py b/examples/online_srv/rolling_online_management.py index ebf1ab59a7..e15daeb298 100644 --- a/examples/online_srv/rolling_online_management.py +++ b/examples/online_srv/rolling_online_management.py @@ -14,7 +14,7 @@ import fire import qlib from qlib.workflow import R -from qlib.workflow.online.strategy import RollingAverageStrategy +from qlib.workflow.online.strategy import RollingStrategy from qlib.workflow.task.gen import RollingGen from qlib.workflow.task.manage import TaskManager from qlib.workflow.online.manager import OnlineManager @@ -97,7 +97,7 @@ def __init__( for task in tasks: name_id = task["model"]["class"] # NOTE: Assumption: The model class can specify only one strategy strategy.append( - RollingAverageStrategy( + RollingStrategy( name_id, task, RollingGen(step=rolling_step, rtype=RollingGen.ROLL_SD), diff --git a/qlib/data/dataset/__init__.py b/qlib/data/dataset/__init__.py index 021311d479..206561aed9 100644 --- a/qlib/data/dataset/__init__.py +++ b/qlib/data/dataset/__init__.py @@ -172,7 +172,10 @@ def _prepare_seg(self, slc: slice, **kwargs): ---------- slc : slice """ - return self.handler.fetch(slc, **kwargs, **self.fetch_kwargs) + if hasattr(self, "fetch_kwargs"): + return self.handler.fetch(slc, **kwargs, **self.fetch_kwargs) + else: + return self.handler.fetch(slc, **kwargs) def prepare( self, diff --git a/qlib/model/ens/ensemble.py b/qlib/model/ens/ensemble.py index a7b837ea50..6040517e25 100644 --- a/qlib/model/ens/ensemble.py +++ b/qlib/model/ens/ensemble.py @@ -7,7 +7,7 @@ from typing import Union import pandas as pd -from qlib.utils import flatten_dict +from qlib.utils import FLATTEN_TUPLE, flatten_dict class Ensemble: @@ -90,7 +90,7 @@ def __call__(self, ensemble_dict: dict): pd.DataFrame: the complete result of averaging and standardizing. """ # need to flatten the nested dict - ensemble_dict = flatten_dict(ensemble_dict) + ensemble_dict = flatten_dict(ensemble_dict, sep=FLATTEN_TUPLE) values = list(ensemble_dict.values()) results = pd.concat(values, axis=1) results = results.groupby("datetime").apply(lambda df: (df - df.mean()) / df.std()) diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index 0b64d3b308..f261a4b4e0 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -416,7 +416,7 @@ def end_train(self, recs, end_train_func=None, experiment_name: str = None, **kw run_task( end_train_func, task_pool, - tasks=tasks, + query={"filter": {"$in": tasks}}, # only train these tasks experiment_name=experiment_name, before_status=TaskManager.STATUS_PART_DONE, **kwargs, diff --git a/qlib/utils/__init__.py b/qlib/utils/__init__.py index 8583e946f2..2ff6877373 100644 --- a/qlib/utils/__init__.py +++ b/qlib/utils/__init__.py @@ -716,23 +716,33 @@ def lazy_sort_index(df: pd.DataFrame, axis=0) -> pd.DataFrame: return df.sort_index(axis=axis) +FLATTEN_TUPLE = "_FLATTEN_TUPLE" + + def flatten_dict(d, parent_key="", sep="."): - """flatten_dict. + """ + Flatten a nested dict. + >>> flatten_dict({'a': 1, 'c': {'a': 2, 'b': {'x': 5, 'y' : 10}}, 'd': [1, 2, 3]}) >>> {'a': 1, 'c.a': 2, 'c.b.x': 5, 'd': [1, 2, 3], 'c.b.y': 10} - Parameters - ---------- - d : - d - parent_key : - parent_key - sep : - sep + >>> flatten_dict({'a': 1, 'c': {'a': 2, 'b': {'x': 5, 'y' : 10}}, 'd': [1, 2, 3]}, sep=FLATTEN_TUPLE) + >>> {'a': 1, ('c','a'): 2, ('c','b','x'): 5, 'd': [1, 2, 3], ('c','b','y'): 10} + + Args: + d (dict): the dict waiting for flatting + parent_key (str, optional): the parent key, will be a prefix in new key. Defaults to "". + sep (str, optional): the separator for string connecting. FLATTEN_TUPLE for tuple connecting. + + Returns: + dict: flatten dict """ items = [] for k, v in d.items(): - new_key = parent_key + sep + str(k) if parent_key else k + if sep == FLATTEN_TUPLE: + new_key = (parent_key, k) if parent_key else k + else: + new_key = parent_key + sep + k if parent_key else k if isinstance(v, collections.abc.MutableMapping): items.extend(flatten_dict(v, new_key, sep=sep).items()) else: diff --git a/qlib/utils/serial.py b/qlib/utils/serial.py index 9c5fc9ac27..352949198a 100644 --- a/qlib/utils/serial.py +++ b/qlib/utils/serial.py @@ -16,9 +16,10 @@ class Serializable: """ pickle_backend = "pickle" # another optional value is "dill" which can pickle more things of python. + default_dump_all = False # if dump all things def __init__(self): - self._dump_all = False + self._dump_all = self.default_dump_all self._exclude = [] def __getstate__(self) -> dict: @@ -77,12 +78,7 @@ def config(self, dump_all: bool = None, exclude: list = None, recursive=False): def to_pickle(self, path: Union[Path, str], dump_all: bool = None, exclude: list = None): self.config(dump_all=dump_all, exclude=exclude) with Path(path).open("wb") as f: - if self.pickle_backend == "pickle": - pickle.dump(self, f) - elif self.pickle_backend == "dill": - dill.dump(self, f) - else: - raise ValueError("Unknown pickle backend, please use 'pickle' or 'dill'.") + self.get_backend().dump(self, f) @classmethod def load(cls, filepath): @@ -99,13 +95,24 @@ def load(cls, filepath): Collector: the instance of Collector """ with open(filepath, "rb") as f: - if cls.pickle_backend == "pickle": - object = pickle.load(f) - elif cls.pickle_backend == "dill": - object = dill.load(f) - else: - raise ValueError("Unknown pickle backend, please use 'pickle' or 'dill'.") + object = cls.get_backend().load(f) if isinstance(object, cls): return object else: raise TypeError(f"The instance of {type(object)} is not a valid `{type(cls)}`!") + + @classmethod + def get_backend(cls): + """ + Return the backend of a Serializable class. The value will be "pickle" or "dill". + + Returns: + str: The value of "pickle" or "dill" + """ + if cls.pickle_backend == "pickle": + return pickle + elif cls.pickle_backend == "dill": + return dill + else: + raise ValueError("Unknown pickle backend, please use 'pickle' or 'dill'.") + diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index e41c3f20a8..63169b58d5 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -35,7 +35,7 @@ class OnlineManager(Serializable): def __init__( self, - strategy: Union[OnlineStrategy, List[OnlineStrategy]], + strategies: Union[OnlineStrategy, List[OnlineStrategy]], trainer: Trainer = None, begin_time: Union[str, pd.Timestamp] = None, freq="day", @@ -45,15 +45,15 @@ def __init__( One OnlineManager must have at least one OnlineStrategy. Args: - strategy (Union[OnlineStrategy, List[OnlineStrategy]]): an instance of OnlineStrategy or a list of OnlineStrategy + strategies (Union[OnlineStrategy, List[OnlineStrategy]]): an instance of OnlineStrategy or a list of OnlineStrategy begin_time (Union[str,pd.Timestamp], optional): the OnlineManager will begin at this time. Defaults to None for using latest date. trainer (Trainer): the trainer to train task. None for using DelayTrainerR. freq (str, optional): data frequency. Defaults to "day". """ self.logger = get_module_logger(self.__class__.__name__) - if not isinstance(strategy, list): - strategy = [strategy] - self.strategy = strategy + if not isinstance(strategies, list): + strategies = [strategies] + self.strategies = strategies self.freq = freq if begin_time is None: begin_time = D.calendar(freq=self.freq).max() @@ -77,7 +77,7 @@ def first_train(self, strategies: List[OnlineStrategy] = None, model_kwargs: dic """ models_list = [] if strategies is None: - strategies = self.strategy + strategies = self.strategies for strategy in strategies: self.logger.info(f"Strategy `{strategy.name_id}` begins first training...") tasks = strategy.first_tasks() @@ -114,21 +114,22 @@ def routine( cur_time = D.calendar(freq=self.freq).max() self.cur_time = pd.Timestamp(cur_time) # None for latest date models_list = [] - for strategy in self.strategy: + for strategy in self.strategies: + self.logger.info(f"Strategy `{strategy.name_id}` begins routine...") if not delay: strategy.tool.update_online_pred() - self.logger.info(f"Strategy `{strategy.name_id}` begins routine...") tasks = strategy.prepare_tasks(self.cur_time, **task_kwargs) models = self.trainer.train(tasks) + self.logger.info(f"Finished training {len(models)} models.") models_list.append(models) + for strategy, models in zip(self.strategies, models_list): + self.prepare_online_models(strategy, models, delay=delay, model_kwargs=model_kwargs) + if not delay: self.prepare_signals(**signal_kwargs) - for strategy, models in zip(self.strategy, models_list): - self.prepare_online_models(strategy, models, delay=delay, model_kwargs=model_kwargs) - def prepare_online_models( self, strategy: OnlineStrategy, models: list, delay: bool = False, model_kwargs: dict = {} ): @@ -141,14 +142,9 @@ def prepare_online_models( delay (bool, optional): if delay prepare models. Defaults to False. model_kwargs (dict, optional): the params for `prepare_online_models`. """ - if not models: - return if not delay: models = self.trainer.end_train(models, experiment_name=strategy.name_id) - online_models = strategy.prepare_online_models(models, **model_kwargs) - else: - # just set every models as online models temporarily before ``prepare_online_models`` - online_models = models + online_models = strategy.prepare_online_models(models, **model_kwargs) self.history.setdefault(self.cur_time, {})[strategy] = online_models def get_collector(self) -> MergeCollector: @@ -160,21 +156,21 @@ def get_collector(self) -> MergeCollector: MergeCollector: the collector to merge other collectors. """ collector_dict = {} - for strategy in self.strategy: + for strategy in self.strategies: collector_dict[strategy.name_id] = strategy.get_collector() return MergeCollector(collector_dict, process_list=[]) - def add_strategy(self, strategy: Union[OnlineStrategy, List[OnlineStrategy]]): + def add_strategy(self, strategies: Union[OnlineStrategy, List[OnlineStrategy]]): """ Add some new strategies to online manager. Args: strategy (Union[OnlineStrategy, List[OnlineStrategy]]): a list of OnlineStrategy """ - if not isinstance(strategy, list): - strategy = [strategy] - self.first_train(strategy) - self.strategy.extend(strategy) + if not isinstance(strategies, list): + strategies = [strategies] + self.first_train(strategies) + self.strategies.extend(strategies) def prepare_signals(self, prepare_func: Callable = AverageEnsemble(), over_write=False): """ @@ -258,7 +254,8 @@ def simulate(self, end_time, frequency="day", task_kwargs={}, model_kwargs={}, s if self.trainer.is_delay(): self.delay_prepare(model_kwargs=model_kwargs, signal_kwargs=signal_kwargs) - set_global_logger_level(logging.INFO) + # FIXME: get logging level firstly and restore it here + set_global_logger_level(logging.DEBUG) self.logger.info(f"Finished preparing signals") return self.get_collector() diff --git a/qlib/workflow/online/strategy.py b/qlib/workflow/online/strategy.py index 9f657427d8..04c854f797 100644 --- a/qlib/workflow/online/strategy.py +++ b/qlib/workflow/online/strategy.py @@ -44,28 +44,23 @@ def prepare_tasks(self, cur_time, **kwargs) -> List[dict]: """ raise NotImplementedError(f"Please implement the `prepare_tasks` method.") - def prepare_online_models(self, models, cur_time=None, check_func=None) -> List[object]: + def prepare_online_models(self, models, cur_time=None) -> List[object]: """ - A typically implementation, but maybe you will need old models by online_tool. - Select some models as the online models from the trained models. + Select some models from trained models and set them to online models. + This is a typically implementation to online all trained models, you can override it to implement complex method. + You can find last online models by OnlineTool.online_models if you still need them. - NOTE: This method offline all models and online the online models prepared by this method (if have). So you can find last online models by OnlineTool.online_models if you still need them. + NOTE: Reset all online models to trained model. If there is no trained models, then do nothing. Args: - tasks (list): a list of tasks. - check_func: the method to judge if a model can be online. - The parameter is the model record and return True for online. - None for online every models. + models (list): a list of models. + cur_time (pd.Dataframe): current time from OnlineManger. None for latest. Returns: - List[object]: a list of selected models. - """ - if check_func is not None: - online_models = [] - for model in models: - if check_func(model, cur_time): - online_models.append(model) - models = online_models + List[object]: a list of online models. + """ + if not models: + return self.tool.online_models() self.tool.reset_online_tag(models) return models @@ -89,10 +84,10 @@ def get_collector(self) -> Collector: raise NotImplementedError(f"Please implement the `get_collector` method.") -class RollingAverageStrategy(OnlineStrategy): +class RollingStrategy(OnlineStrategy): """ - This example strategy always use latest rolling model as online model and prepare trading signals using the average prediction of online models + This example strategy always use latest rolling model as online model. """ def __init__( @@ -102,7 +97,7 @@ def __init__( rolling_gen: RollingGen, ): """ - Init RollingAverageStrategy. + Init RollingStrategy. Assumption: the str of name_id, the experiment name and the trainer's experiment name are same one. diff --git a/qlib/workflow/online/update.py b/qlib/workflow/online/update.py index a69e1005fd..9cb294169f 100644 --- a/qlib/workflow/online/update.py +++ b/qlib/workflow/online/update.py @@ -137,7 +137,9 @@ def update(self, dataset: DatasetH = None): start_time = get_date_by_shift(self.last_end, 1, freq=self.freq) if start_time >= self.to_date: - self.logger.info(f"The prediction in {self.record.info['id']} are latest. No need to update.") + self.logger.info( + f"The prediction in {self.record.info['id']} are latest ({start_time}). No need to update to {self.to_date}." + ) return # load dataset diff --git a/qlib/workflow/online/utils.py b/qlib/workflow/online/utils.py index c3af9d1cab..3c2774cec9 100644 --- a/qlib/workflow/online/utils.py +++ b/qlib/workflow/online/utils.py @@ -158,6 +158,10 @@ def update_online_pred(self, to_date=None): """ online_models = self.online_models() for rec in online_models: - PredUpdater(rec, to_date=to_date).update() + hist_ref = 0 + task = rec.load_object("task") + if task["dataset"]["class"] == "TSDatasetH": + hist_ref = task["dataset"]["kwargs"]["step_len"] + PredUpdater(rec, to_date=to_date, hist_ref=hist_ref).update() self.logger.info(f"Finished updating {len(online_models)} online model predictions of {self.exp_name}.") diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index b40ee01646..1080d07f45 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -91,18 +91,21 @@ class MergeCollector(Collector): """ - def __init__(self, collector_dict: Dict[str, Collector], process_list: List[Callable] = []): + def __init__(self, collector_dict: Dict[str, Collector], process_list: List[Callable] = [], merge_func=None): """ Args: collector_dict (Dict[str,Collector]): the dict like {collector_key, Collector} process_list (List[Callable]): the list of processors or the instance of processor to process dict. + merge_func (Callable): a method to generate outermost key. The given params are ``collector_key`` from collector_dict and ``key`` from every collector after collecting. + None for use tuple to connect them, such as "ABC"+("a","b") -> ("ABC", ("a","b")). """ super().__init__(process_list=process_list) self.collector_dict = collector_dict + self.merge_func = merge_func def collect(self) -> dict: """ - Collect all result of collector_dict and change the outermost key to "``collector_key``_``key``" (like merge them, but rename every key) + Collect all result of collector_dict and change the outermost key to a recombination key. Returns: dict: the dict after collecting. @@ -111,7 +114,10 @@ def collect(self) -> dict: for collector_key, collector in self.collector_dict.items(): tmp_dict = collector() for key, value in tmp_dict.items(): - collect_dict[collector_key + "_" + str(key)] = value + if self.merge_func is not None: + collect_dict[self.merge_func(collector_key, key)] = value + else: + collect_dict[(collector_key, key)] = value return collect_dict @@ -146,16 +152,18 @@ def __init__( rec_key_func = lambda rec: rec.info["id"] if artifacts_key is None: artifacts_key = list(self.artifacts_path.keys()) - self._rec_key_func = rec_key_func + self.rec_key_func = rec_key_func self.artifacts_key = artifacts_key - self._rec_filter_func = rec_filter_func + self.rec_filter_func = rec_filter_func - def collect(self, artifacts_key=None, rec_filter_func=None) -> dict: + def collect(self, artifacts_key=None, rec_filter_func=None, only_exist=True) -> dict: """Collect different artifacts based on recorder after filtering. Args: artifacts_key (str or List, optional): the artifacts key you want to get. If None, use default. rec_filter_func (Callable, optional): filter the recorder by return True or False. If None, use default. + only_exist (bool, optional): if only collect the artifacts when a recorder really have. + If True, the recorder with exception when loading will not be collected. But if False, it will raise the exception. Returns: dict: the dict after collected like {artifact: {rec_key: object}} @@ -163,7 +171,7 @@ def collect(self, artifacts_key=None, rec_filter_func=None) -> dict: if artifacts_key is None: artifacts_key = self.artifacts_key if rec_filter_func is None: - rec_filter_func = self._rec_filter_func + rec_filter_func = self.rec_filter_func if isinstance(artifacts_key, str): artifacts_key = [artifacts_key] @@ -177,16 +185,18 @@ def collect(self, artifacts_key=None, rec_filter_func=None) -> dict: recs_flt[rid] = rec for _, rec in recs_flt.items(): - rec_key = self._rec_key_func(rec) + rec_key = self.rec_key_func(rec) for key in artifacts_key: if self.ART_KEY_RAW == key: artifact = rec else: - # only collect existing artifact try: artifact = rec.load_object(self.artifacts_path[key]) - except Exception: - continue + except Exception as e: + if only_exist: + # only collect existing artifact + continue + raise e collect_dict.setdefault(key, {})[rec_key] = artifact return collect_dict diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 025dfa85c2..40f8682959 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -155,7 +155,8 @@ def insert_task_def(self, task_def): def create_task(self, task_def_l, dry_run=False, print_nt=False) -> List[str]: """ - If the tasks in task_def_l is new, then insert new tasks into the task_pool + If the tasks in task_def_l is new, then insert new tasks into the task_pool, and record inserted_id. + If a task is not new, then query its _id. Parameters ---------- @@ -169,9 +170,10 @@ def create_task(self, task_def_l, dry_run=False, print_nt=False) -> List[str]: Returns ------- list - a list of the _id of new tasks + a list of the _id of task_def_l """ new_tasks = [] + _id_list = [] for t in task_def_l: try: r = self.task_pool.find_one({"filter": t}) @@ -179,6 +181,14 @@ def create_task(self, task_def_l, dry_run=False, print_nt=False) -> List[str]: r = self.task_pool.find_one({"filter": self._dict_to_str(t)}) if r is None: new_tasks.append(t) + if not dry_run: + insert_result = self.insert_task_def(t) + _id_list.append(insert_result.inserted_id) + else: + _id_list.append(None) + else: + _id_list.append(self._decode_task(r)["_id"]) + self.logger.info(f"Total Tasks: {len(task_def_l)}, New Tasks: {len(new_tasks)}") if print_nt: # print new task @@ -188,11 +198,6 @@ def create_task(self, task_def_l, dry_run=False, print_nt=False) -> List[str]: if dry_run: return [] - _id_list = [] - for t in new_tasks: - insert_result = self.insert_task_def(t) - _id_list.append(insert_result.inserted_id) - return _id_list def fetch_task(self, query={}, status=STATUS_WAITING) -> dict: @@ -388,7 +393,7 @@ def __str__(self): def run_task( task_func: Callable, task_pool: str, - tasks: List[dict] = None, + query: dict = {}, force_release: bool = False, before_status: str = TaskManager.STATUS_WAITING, after_status: str = TaskManager.STATUS_DONE, @@ -414,8 +419,8 @@ def (task_def, **kwargs) -> the function to run the task task_pool : str the name of the task pool (Collection in MongoDB) - tasks: List[dict] - will only train these tasks config, None for train all tasks. + query: dict + will use this dict to query task_pool when fetching task force_release : bool will the program force to release the resource before_status : str: @@ -428,9 +433,6 @@ def (task_def, **kwargs) -> tm = TaskManager(task_pool) ever_run = False - query = {} - if tasks is not None: - query = {"filter": {"$in": tasks}} while True: with tm.safe_fetch_task(status=before_status, query=query) as task: From ebd01e0de5dee65e2bed9e11a9f0f22711a269df Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Fri, 14 May 2021 06:44:16 +0000 Subject: [PATCH 58/61] Online Serving V11 --- docs/advanced/serial.rst | 3 + docs/advanced/task_management.rst | 27 +++-- docs/component/online.rst | 6 +- docs/reference/api.rst | 5 +- .../model_rolling/task_manager_rolling.py | 4 +- .../online_srv/online_management_simulate.py | 8 +- .../online_srv/rolling_online_management.py | 4 +- examples/online_srv/update_online_pred.py | 6 +- qlib/model/ens/ensemble.py | 55 +++++---- qlib/model/ens/group.py | 21 ++-- qlib/model/trainer.py | 80 ++++++++----- qlib/utils/__init__.py | 2 +- qlib/utils/serial.py | 19 +++- qlib/workflow/online/manager.py | 106 +++++++++--------- qlib/workflow/online/strategy.py | 45 ++++---- qlib/workflow/online/update.py | 8 +- qlib/workflow/online/utils.py | 15 +-- qlib/workflow/task/collect.py | 42 ++++--- qlib/workflow/task/gen.py | 23 ++-- qlib/workflow/task/manage.py | 52 +++++++-- qlib/workflow/task/utils.py | 25 +++-- 21 files changed, 326 insertions(+), 230 deletions(-) diff --git a/docs/advanced/serial.rst b/docs/advanced/serial.rst index 8c0f837467..e0840069bf 100644 --- a/docs/advanced/serial.rst +++ b/docs/advanced/serial.rst @@ -14,6 +14,9 @@ Serializable Class ``Qlib`` provides a base class ``qlib.utils.serial.Serializable``, whose state can be dumped into or loaded from disk in `pickle` format. When users dump the state of a ``Serializable`` instance, the attributes of the instance whose name **does not** start with `_` will be saved on the disk. +However, users can use ``config`` method or override ``default_dump_all`` attribute to prevent this feature. + +Users can also override ``pickle_backend`` attribute to choose a pickle backend. The supported value is "pickle" (default and common) and "dill" (dump more things such as function, more information in `here `_). Example ========================== diff --git a/docs/advanced/task_management.rst b/docs/advanced/task_management.rst index d600494556..56a3137f9f 100644 --- a/docs/advanced/task_management.rst +++ b/docs/advanced/task_management.rst @@ -19,7 +19,7 @@ An example of the entire process is shown `here `_. Even though the task template is fixed, users can customize their ``TaskGen`` to generate different ``task`` by task template. @@ -30,15 +30,15 @@ Here is the base class of ``TaskGen``: :members: ``Qlib`` provides a class `RollingGen `_ to generate a list of ``task`` of the dataset in different date segments. -This class allows users to verify the effect of data from different periods on the model in one experiment. More information in `here <../reference/api.html#TaskGen>`_. +This class allows users to verify the effect of data from different periods on the model in one experiment. More information is `here <../reference/api.html#TaskGen>`_. Task Storing =============== To achieve higher efficiency and the possibility of cluster operation, ``Task Manager`` will store all tasks in `MongoDB `_. ``TaskManager`` can fetch undone tasks automatically and manage the lifecycle of a set of tasks with error handling. -Users **MUST** finished the configuration of `MongoDB `_ when using this module. +Users **MUST** finish the configuration of `MongoDB `_ when using this module. -Users need to provide the MongoDB URL and database name for using ``TaskManager`` in `initialization <../start/initialization.html#Parameters>`_ or make statement like this. +Users need to provide the MongoDB URL and database name for using ``TaskManager`` in `initialization <../start/initialization.html#Parameters>`_ or make a statement like this. .. code-block:: python @@ -55,8 +55,7 @@ More information of ``Task Manager`` can be found in `here <../reference/api.htm Task Training =============== -#FIXME: Trainer -After generating and storing those ``task``, it's time to run the ``task`` which are in the *WAITING* status. +After generating and storing those ``task``, it's time to run the ``task`` which is in the *WAITING* status. ``Qlib`` provides a method called ``run_task`` to run those ``task`` in task pool, however, users can also customize how tasks are executed. An easy way to get the ``task_func`` is using ``qlib.model.trainer.task_train`` directly. It will run the whole workflow defined by ``task``, which includes *Model*, *Dataset*, *Record*. @@ -64,16 +63,20 @@ It will run the whole workflow defined by ``task``, which includes *Model*, *Dat .. autofunction:: qlib.workflow.task.manage.run_task Meanwhile, ``Qlib`` provides a module called ``Trainer``. -``Trainer`` will train a list of tasks and return a list of model recorder. -``Qlib`` offer two kind of Trainer, TrainerR is the simplest way and TrainerRM is based on TaskManager to help manager tasks lifecycle automatically. + +.. autoclass:: qlib.model.trainer.Trainer + :members: + +``Trainer`` will train a list of tasks and return a list of model recorders. +``Qlib`` offer two kinds of Trainer, TrainerR is the simplest way and TrainerRM is based on TaskManager to help manager tasks lifecycle automatically. If you do not want to use ``Task Manager`` to manage tasks, then use TrainerR to train a list of tasks generated by ``TaskGen`` is enough. -More information is in `here <../reference/api.html#Trainer>`_. +`Here <../reference/api.html#Trainer>`_ are the details about different ``Trainer``. Task Collecting =============== To collect the results of ``task`` after training, ``Qlib`` provides `Collector <../reference/api.html#Collector>`_, `Group <../reference/api.html#Group>`_ and `Ensemble <../reference/api.html#Ensemble>`_ to collect the results in a readable, expandable and loosely-coupled way. -`Collector <../reference/api.html#Collector>`_ can collect object from everywhere and process them such as merging, grouping, averaging and so on. It has 2 step action including ``collect`` (collect anything in a dict) and ``process_collect`` (process collected dict). +`Collector <../reference/api.html#Collector>`_ can collect objects from everywhere and process them such as merging, grouping, averaging and so on. It has 2 step action including ``collect`` (collect anything in a dict) and ``process_collect`` (process collected dict). `Group <../reference/api.html#Group>`_ also has 2 steps including ``group`` (can group a set of object based on `group_func` and change them to a dict) and ``reduce`` (can make a dict become an ensemble based on some rule). For example: {(A,B,C1): object, (A,B,C2): object} ---``group``---> {(A,B): {C1: object, C2: object}} ---``reduce``---> {(A,B): object} @@ -81,6 +84,6 @@ For example: {(A,B,C1): object, (A,B,C2): object} ---``group``---> {(A,B): {C1: `Ensemble <../reference/api.html#Ensemble>`_ can merge the objects in an ensemble. For example: {C1: object, C2: object} ---``Ensemble``---> object -So the hierarchy is ``Collector``'s second step correspond to ``Group``. And ``Group``'s second step correspond to ``Ensemble``. +So the hierarchy is ``Collector``'s second step corresponds to ``Group``. And ``Group``'s second step correspond to ``Ensemble``. -For more information, please see `Collector <../reference/api.html#Collector>`_, `Group <../reference/api.html#Group>`_ and `Ensemble <../reference/api.html#Ensemble>`_, or the `example `_ \ No newline at end of file +For more information, please see `Collector <../reference/api.html#Collector>`_, `Group <../reference/api.html#Group>`_ and `Ensemble <../reference/api.html#Ensemble>`_, or the `example `_. \ No newline at end of file diff --git a/docs/component/online.rst b/docs/component/online.rst index e251731533..66331901f7 100644 --- a/docs/component/online.rst +++ b/docs/component/online.rst @@ -9,12 +9,12 @@ Online Serving Introduction ============= In addition to backtesting, one way to test a model is effective is to make predictions in real market conditions or even do real trading based on those predictions. -``Online Serving`` is a set of module for online models using latest data, +``Online Serving`` is a set of modules for online models using the latest data, which including `Online Manager <#Online Manager>`_, `Online Strategy <#Online Strategy>`_, `Online Tool <#Online Tool>`_, `Updater <#Updater>`_. `Here `_ are several examples for reference, which demonstrate different features of ``Online Serving``. -If you have many models or `task` need to be managed, please consider `Task Management <../advanced/task_management.html>`_. -The `examples `_ maybe based on `Task Management <../advanced/task_management.html>`_ such as ``TrainerRM`` or ``Collector``. +If you have many models or `task` needs to be managed, please consider `Task Management <../advanced/task_management.html>`_. +The `examples `_ are based on some components in `Task Management <../advanced/task_management.html>`_ such as ``TrainerRM`` or ``Collector``. Online Manager ============= diff --git a/docs/reference/api.rst b/docs/reference/api.rst index edba6228a0..57f61f18b1 100644 --- a/docs/reference/api.rst +++ b/docs/reference/api.rst @@ -226,4 +226,7 @@ Serializable -------------------- .. automodule:: qlib.utils.serial.Serializable - :members: \ No newline at end of file + :members: + + + \ No newline at end of file diff --git a/examples/model_rolling/task_manager_rolling.py b/examples/model_rolling/task_manager_rolling.py index 1753198858..4f3ac04b15 100644 --- a/examples/model_rolling/task_manager_rolling.py +++ b/examples/model_rolling/task_manager_rolling.py @@ -2,8 +2,8 @@ # Licensed under the MIT License. """ -This example shows how a TrainerRM work based on TaskManager with rolling tasks. -After training, how to collect the rolling results will be showed in task_collecting. +This example shows how a TrainerRM works based on TaskManager with rolling tasks. +After training, how to collect the rolling results will be shown in task_collecting. """ from pprint import pprint diff --git a/examples/online_srv/online_management_simulate.py b/examples/online_srv/online_management_simulate.py index 48433c6d5c..c09b10aa77 100644 --- a/examples/online_srv/online_management_simulate.py +++ b/examples/online_srv/online_management_simulate.py @@ -2,7 +2,7 @@ # Licensed under the MIT License. """ -This examples is about how can simulate the OnlineManager based on rolling tasks. +This example is about how can simulate the OnlineManager based on rolling tasks. """ import fire @@ -112,8 +112,8 @@ def __init__( qlib.init(provider_uri=provider_uri, region=region, mongo=mongo_conf) self.rolling_gen = RollingGen( step=rolling_step, rtype=RollingGen.ROLL_SD, ds_extra_mod_func=None - ) # The rolling tasks generator, ds_extra_mod_func is None because we just need simulate to 2018-10-31 and needn't change handler end time. - self.trainer = DelayTrainerRM(self.exp_name, self.task_pool) + ) # The rolling tasks generator, ds_extra_mod_func is None because we just need to simulate to 2018-10-31 and needn't change the handler end time. + self.trainer = DelayTrainerRM(self.exp_name, self.task_pool) # Also can be TrainerR, TrainerRM, DelayTrainerR self.rolling_online_manager = OnlineManager( RollingStrategy(exp_name, task_template=tasks, rolling_gen=self.rolling_gen), trainer=self.trainer, @@ -138,8 +138,6 @@ def main(self): print(self.rolling_online_manager.get_collector()()) print("========== signals ==========") print(self.rolling_online_manager.get_signals()) - print("========== online history ==========") - print(self.rolling_online_manager.history) if __name__ == "__main__": diff --git a/examples/online_srv/rolling_online_management.py b/examples/online_srv/rolling_online_management.py index e15daeb298..e5c37dac63 100644 --- a/examples/online_srv/rolling_online_management.py +++ b/examples/online_srv/rolling_online_management.py @@ -2,7 +2,7 @@ # Licensed under the MIT License. """ -This example show how OnlineManager works with rolling tasks. +This example shows how OnlineManager works with rolling tasks. There are two parts including first train and routine. Firstly, the OnlineManager will finish the first training and set trained models to `online` models. Next, the OnlineManager will finish a routine process, including update online prediction -> prepare signals -> prepare tasks -> prepare new models -> reset online models @@ -154,7 +154,7 @@ def main(self): # python rolling_online_management.py first_run ####### to update the models and predictions after the trading time, use the command below - # python rolling_online_management.py after_day + # python rolling_online_management.py routine ####### to define your own parameters, use `--` # python rolling_online_management.py first_run --exp_name='your_exp_name' --rolling_step=40 diff --git a/examples/online_srv/update_online_pred.py b/examples/online_srv/update_online_pred.py index 6e2725c7a2..228bc0dacb 100644 --- a/examples/online_srv/update_online_pred.py +++ b/examples/online_srv/update_online_pred.py @@ -2,10 +2,10 @@ # Licensed under the MIT License. """ -This example show how OnlineTool works when we need update prediction. +This example shows how OnlineTool works when we need update prediction. There are two parts including first_train and update_online_pred. -Firstly, we will finish the training and set the trained model to `online` model. -Next, we will finish updating online prediction. +Firstly, we will finish the training and set the trained models to the `online` models. +Next, we will finish updating online predictions. """ import fire import qlib diff --git a/qlib/model/ens/ensemble.py b/qlib/model/ens/ensemble.py index 6040517e25..0f48ce728e 100644 --- a/qlib/model/ens/ensemble.py +++ b/qlib/model/ens/ensemble.py @@ -2,7 +2,7 @@ # Licensed under the MIT License. """ -Ensemble can merge the objects in an Ensemble. For example, if there are many submodels predictions, we may need to merge them in an ensemble predictions. +Ensemble module can merge the objects in an Ensemble. For example, if there are many submodels predictions, we may need to merge them into an ensemble prediction. """ from typing import Union @@ -11,29 +11,41 @@ class Ensemble: - """Merge the objects in an Ensemble.""" + """Merge the ensemble_dict into an ensemble object. - def __call__(self, ensemble_dict: dict, *args, **kwargs): - """Merge the ensemble_dict into an ensemble object. - For example: {Rollinga_b: object, Rollingb_c: object} -> object + For example: {Rollinga_b: object, Rollingb_c: object} -> object + When calling this class: + Args: - ensemble_dict (dict): the ensemble dict waiting for merging like {name: things} + ensemble_dict (dict): the ensemble dict like {name: things} waiting for merging Returns: object: the ensemble object - """ + """ + + def __call__(self, ensemble_dict: dict, *args, **kwargs): raise NotImplementedError(f"Please implement the `__call__` method.") class SingleKeyEnsemble(Ensemble): """ - Extract the object if there is only one key and value in dict. Make result more readable. + Extract the object if there is only one key and value in the dict. Make the result more readable. {Only key: Only value} -> Only value - If there are more than 1 key or less than 1 key, then do nothing. + + If there is more than 1 key or less than 1 key, then do nothing. Even you can run this recursively to make dict more readable. - NOTE: Default run recursively. + + NOTE: Default runs recursively. + + When calling this class: + + Args: + ensemble_dict (dict): the dict. The key of the dict will be ignored. + + Returns: + dict: the readable dict. """ def __call__(self, ensemble_dict: Union[dict, object], recursion: bool = True) -> object: @@ -52,12 +64,11 @@ def __call__(self, ensemble_dict: Union[dict, object], recursion: bool = True) - class RollingEnsemble(Ensemble): - """Merge the rolling objects in an Ensemble""" + """Merge a dict of rolling dataframe like `prediction` or `IC` into an ensemble. - def __call__(self, ensemble_dict: dict) -> pd.DataFrame: - """Merge a dict of rolling dataframe like `prediction` or `IC` into an ensemble. + NOTE: The values of dict must be pd.DataFrame, and have the index "datetime". - NOTE: The values of dict must be pd.DataFrame, and have the index "datetime" + When calling this class: Args: ensemble_dict (dict): a dict like {"A": pd.DataFrame, "B": pd.DataFrame}. @@ -65,7 +76,9 @@ def __call__(self, ensemble_dict: dict) -> pd.DataFrame: Returns: pd.DataFrame: the complete result of rolling. - """ + """ + + def __call__(self, ensemble_dict: dict) -> pd.DataFrame: artifact_list = list(ensemble_dict.values()) artifact_list.sort(key=lambda x: x.index.get_level_values("datetime").min()) artifact = pd.concat(artifact_list) @@ -76,11 +89,12 @@ def __call__(self, ensemble_dict: dict) -> pd.DataFrame: class AverageEnsemble(Ensemble): - def __call__(self, ensemble_dict: dict): - """ - Average and standardize a dict of same shape dataframe like `prediction` or `IC` into an ensemble. + """ + Average and standardize a dict of same shape dataframe like `prediction` or `IC` into an ensemble. + + NOTE: The values of dict must be pd.DataFrame, and have the index "datetime". If it is a nested dict, then flat it. - NOTE: The values of dict must be pd.DataFrame, and have the index "datetime". If it is a nested dict, then flat it. + When calling this class: Args: ensemble_dict (dict): a dict like {"A": pd.DataFrame, "B": pd.DataFrame}. @@ -88,7 +102,8 @@ def __call__(self, ensemble_dict: dict): Returns: pd.DataFrame: the complete result of averaging and standardizing. - """ + """ + def __call__(self, ensemble_dict: dict) -> pd.DataFrame: # need to flatten the nested dict ensemble_dict = flatten_dict(ensemble_dict, sep=FLATTEN_TUPLE) values = list(ensemble_dict.values()) diff --git a/qlib/model/ens/group.py b/qlib/model/ens/group.py index a00a8ea0e0..93903f4334 100644 --- a/qlib/model/ens/group.py +++ b/qlib/model/ens/group.py @@ -2,7 +2,7 @@ # Licensed under the MIT License. """ -Group can group a set of object based on `group_func` and change them to a dict. +Group can group a set of objects based on `group_func` and change them to a dict. After group, we provide a method to reduce them. For example: @@ -21,10 +21,11 @@ class Group: """Group the objects based on dict""" def __init__(self, group_func=None, ens: Ensemble = None): - """init Group. + """ + Init Group. Args: - group_func (Callable, optional): Given a dict and return the group key and one of group elements. + group_func (Callable, optional): Given a dict and return the group key and one of the group elements. For example: {(A,B,C1): object, (A,B,C2): object} -> {(A,B): {C1: object, C2: object}} @@ -37,7 +38,7 @@ def __init__(self, group_func=None, ens: Ensemble = None): def group(self, *args, **kwargs) -> dict: """ - Group a set of object and change them to a dict. + Group a set of objects and change them to a dict. For example: {(A,B,C1): object, (A,B,C2): object} -> {(A,B): {C1: object, C2: object}} @@ -51,7 +52,7 @@ def group(self, *args, **kwargs) -> dict: def reduce(self, *args, **kwargs) -> dict: """ - Reduce grouped dict in some way. + Reduce grouped dict. For example: {(A,B): {C1: object, C2: object}} -> {(A,B): object} @@ -63,7 +64,7 @@ def reduce(self, *args, **kwargs) -> dict: else: raise NotImplementedError(f"Please specify valid `_ens_func`.") - def __call__(self, ungrouped_dict: dict, n_jobs=1, verbose=0, *args, **kwargs) -> dict: + def __call__(self, ungrouped_dict: dict, n_jobs:int=1, verbose:int=0, *args, **kwargs) -> dict: """ Group the ungrouped_dict into different groups. @@ -72,10 +73,12 @@ def __call__(self, ungrouped_dict: dict, n_jobs=1, verbose=0, *args, **kwargs) - Returns: dict: grouped_dict like {G1: object, G2: object} + n_jobs: how many progress you need. + verbose: the print mode for Parallel. """ # NOTE: The multiprocessing will raise error if you use `Serializable` - # Because the `Serializable` will affect the behaviours of pickle + # Because the `Serializable` will affect the behaviors of pickle grouped_dict = self.group(ungrouped_dict, *args, **kwargs) key_l = [] @@ -87,12 +90,12 @@ def __call__(self, ungrouped_dict: dict, n_jobs=1, verbose=0, *args, **kwargs) - class RollingGroup(Group): - """group the rolling dict""" + """Group the rolling dict""" def group(self, rolling_dict: dict) -> dict: """Given an rolling dict likes {(A,B,R): things}, return the grouped dict likes {(A,B): {R:things}} - NOTE: There is a assumption which is the rolling key is at the end of key tuple, because the rolling results always need to be ensemble firstly. + NOTE: There is an assumption which is the rolling key is at the end of the key tuple, because the rolling results always need to be ensemble firstly. Args: rolling_dict (dict): an rolling dict. If the key is not a tuple, then do nothing. diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index f261a4b4e0..0c9c9e2c2a 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -2,13 +2,13 @@ # Licensed under the MIT License. """ -The Trainer will train a list of tasks and return a list of model recorder. +The Trainer will train a list of tasks and return a list of model recorders. There are two steps in each Trainer including ``train``(make model recorder) and ``end_train``(modify model recorder). -This is concept called ``DelayTrainer``, which can be used in online simulating for parallel training. -In ``DelayTrainer``, the first step is only to save some necessary info to model recorder, and the second step which will be finished in the end can do some concurrent and time-consuming operations such as model fitting. +This is a concept called ``DelayTrainer``, which can be used in online simulating for parallel training. +In ``DelayTrainer``, the first step is only to save some necessary info to model recorders, and the second step which will be finished in the end can do some concurrent and time-consuming operations such as model fitting. -``Qlib`` offer two kind of Trainer, ``TrainerR`` is the simplest way and ``TrainerRM`` is based on TaskManager to help manager tasks lifecycle automatically. +``Qlib`` offer two kinds of Trainer, ``TrainerR`` is the simplest way and ``TrainerRM`` is based on TaskManager to help manager tasks lifecycle automatically. """ import socket @@ -25,7 +25,7 @@ def begin_task_train(task_config: dict, experiment_name: str, recorder_name: str = None) -> Recorder: """ - Begin a task training to start a recorder and save the task config. + Begin task training to start a recorder and save the task config. Args: task_config (dict): the config of a task @@ -94,7 +94,7 @@ def task_train(task_config: dict, experiment_name: str) -> Recorder: Returns ---------- - Recorder : The instance of the recorder + Recorder: The instance of the recorder """ recorder = begin_task_train(task_config, experiment_name) recorder = end_task_train(recorder, experiment_name) @@ -103,7 +103,7 @@ def task_train(task_config: dict, experiment_name: str) -> Recorder: class Trainer: """ - The trainer can train a list of model. + The trainer can train a list of models. There are Trainer and DelayTrainer, which can be distinguished by when it will finish real training. """ @@ -112,10 +112,10 @@ def __init__(self): def train(self, tasks: list, *args, **kwargs) -> list: """ - Given a list of model definition, begin a training and return the models. + Given a list of task definitions, begin training, and return the models. - For Trainer, it finish real training in this method. - For DelayTrainer, it only do some preparation in this method. + For Trainer, it finishes real training in this method. + For DelayTrainer, it only does some preparation in this method. Args: tasks: a list of tasks @@ -127,11 +127,11 @@ def train(self, tasks: list, *args, **kwargs) -> list: def end_train(self, models: list, *args, **kwargs) -> list: """ - Given a list of models, finished something in the end of training if you need. - The models maybe Recorder, txt file, database and so on. + Given a list of models, finished something at the end of training if you need. + The models may be Recorder, txt file, database, and so on. - For Trainer, it do some finishing touches in this method. - For DelayTrainer, it finish real training in this method. + For Trainer, it does some finishing touches in this method. + For DelayTrainer, it finishes real training in this method. Args: models: a list of models @@ -155,9 +155,9 @@ def is_delay(self) -> bool: class TrainerR(Trainer): """ Trainer based on (R)ecorder. - It will train a list of tasks and return a list of model recorder in a linear way. + It will train a list of tasks and return a list of model recorders in a linear way. - Assumption: models were defined by `task` and the results will saved to `Recorder` + Assumption: models were defined by `task` and the results will be saved to `Recorder`. """ # Those tag will help you distinguish whether the Recorder has finished traning @@ -182,13 +182,13 @@ def train(self, tasks: list, train_func: Callable = None, experiment_name: str = Given a list of `task`s and return a list of trained Recorder. The order can be guaranteed. Args: - tasks (list): a list of definition based on `task` dict - train_func (Callable): the train method which need at least `task`s and `experiment_name`. None for default training method. + tasks (list): a list of definitions based on `task` dict + train_func (Callable): the training method which needs at least `tasks` and `experiment_name`. None for the default training method. experiment_name (str): the experiment name, None for use default name. kwargs: the params for train_func. Returns: - list: a list of Recorders + List[Recorder]: a list of Recorders """ if len(tasks) == 0: return [] @@ -204,6 +204,15 @@ def train(self, tasks: list, train_func: Callable = None, experiment_name: str = return recs def end_train(self, recs: list, **kwargs) -> List[Recorder]: + """ + Set STATUS_END tag to the recorders. + + Args: + recs (list): a list of trained recorders. + + Returns: + List[Recorder]: the same list as the param. + """ for rec in recs: rec.set_tags(**{self.STATUS_KEY: self.STATUS_END}) return recs @@ -231,15 +240,15 @@ def end_train(self, recs, end_train_func=None, experiment_name: str = None, **kw """ Given a list of Recorder and return a list of trained Recorder. This class will finish real data loading and model fitting. - + Args: recs (list): a list of Recorder, the tasks have been saved to them - end_train_func (Callable, optional): the end_train method which need at least `recorder`s and `experiment_name`. Defaults to None for using self.end_train_func. + end_train_func (Callable, optional): the end_train method which needs at least `recorder`s and `experiment_name`. Defaults to None for using self.end_train_func. experiment_name (str): the experiment name, None for use default name. kwargs: the params for end_train_func. - + Returns: - list: a list of Recorders + List[Recorder]: a list of Recorders """ if end_train_func is None: end_train_func = self.end_train_func @@ -256,7 +265,7 @@ def end_train(self, recs, end_train_func=None, experiment_name: str = None, **kw class TrainerRM(Trainer): """ Trainer based on (R)ecorder and Task(M)anager. - It can train a list of tasks and return a list of model recorder in a multiprocessing way. + It can train a list of tasks and return a list of model recorders in a multiprocessing way. Assumption: `task` will be saved to TaskManager and `task` will be fetched and trained from TaskManager """ @@ -296,15 +305,15 @@ def train( Users can customize their train_func to realize multiple processes or even multiple machines. Args: - tasks (list): a list of definition based on `task` dict - train_func (Callable): the train method which need at least `task`s and `experiment_name`. None for default training method. + tasks (list): a list of definitions based on `task` dict + train_func (Callable): the training method which needs at least `task`s and `experiment_name`. None for the default training method. experiment_name (str): the experiment name, None for use default name. before_status (str): the tasks in before_status will be fetched and trained. Can be STATUS_WAITING, STATUS_PART_DONE. after_status (str): the tasks after trained will become after_status. Can be STATUS_WAITING, STATUS_PART_DONE. kwargs: the params for train_func. Returns: - list: a list of Recorders + List[Recorder]: a list of Recorders """ if len(tasks) == 0: return [] @@ -334,6 +343,15 @@ def train( return recs def end_train(self, recs: list, **kwargs) -> List[Recorder]: + """ + Set STATUS_END tag to the recorders. + + Args: + recs (list): a list of trained recorders. + + Returns: + List[Recorder]: the same list as the param. + """ for rec in recs: rec.set_tags(**{self.STATUS_KEY: self.STATUS_END}) return recs @@ -368,12 +386,14 @@ def __init__( def train(self, tasks: list, train_func=None, experiment_name: str = None, **kwargs) -> List[Recorder]: """ Same as `train` of TrainerRM, after_status will be STATUS_PART_DONE. + Args: tasks (list): a list of definition based on `task` dict train_func (Callable): the train method which need at least `task`s and `experiment_name`. Defaults to None for using self.train_func. experiment_name (str): the experiment name, None for use default name. + Returns: - list: a list of Recorders + List[Recorder]: a list of Recorders """ if len(tasks) == 0: return [] @@ -390,7 +410,7 @@ def end_train(self, recs, end_train_func=None, experiment_name: str = None, **kw Given a list of Recorder and return a list of trained Recorder. This class will finish real data loading and model fitting. - NOTE: This method will train all STATUS_PART_DONE tasks in task pool, not only the ``recs``. + NOTE: This method will train all STATUS_PART_DONE tasks in the task pool, not only the ``recs``. Args: recs (list): a list of Recorder, the tasks have been saved to them. @@ -399,7 +419,7 @@ def end_train(self, recs, end_train_func=None, experiment_name: str = None, **kw kwargs: the params for end_train_func. Returns: - list: a list of Recorders + List[Recorder]: a list of Recorders """ if end_train_func is None: diff --git a/qlib/utils/__init__.py b/qlib/utils/__init__.py index 2ff6877373..77857182d9 100644 --- a/qlib/utils/__init__.py +++ b/qlib/utils/__init__.py @@ -719,7 +719,7 @@ def lazy_sort_index(df: pd.DataFrame, axis=0) -> pd.DataFrame: FLATTEN_TUPLE = "_FLATTEN_TUPLE" -def flatten_dict(d, parent_key="", sep="."): +def flatten_dict(d, parent_key="", sep=".") -> dict: """ Flatten a nested dict. diff --git a/qlib/utils/serial.py b/qlib/utils/serial.py index 352949198a..c7c51bac2c 100644 --- a/qlib/utils/serial.py +++ b/qlib/utils/serial.py @@ -3,10 +3,12 @@ from pathlib import Path import pickle +import typing import dill from typing import Union + class Serializable: """ Serializable will change the behaviors of pickle. @@ -16,7 +18,7 @@ class Serializable: """ pickle_backend = "pickle" # another optional value is "dill" which can pickle more things of python. - default_dump_all = False # if dump all things + default_dump_all = False # if dump all things def __init__(self): self._dump_all = self.default_dump_all @@ -76,6 +78,14 @@ def config(self, dump_all: bool = None, exclude: list = None, recursive=False): del self.__dict__[self.FLAG_KEY] def to_pickle(self, path: Union[Path, str], dump_all: bool = None, exclude: list = None): + """ + Dump self to a pickle file. + + Args: + path (Union[Path, str]): the path to dump + dump_all (bool, optional): if need to dump all things. Defaults to None. + exclude (list, optional): will exclude the attributes in this list when dumping. Defaults to None. + """ self.config(dump_all=dump_all, exclude=exclude) with Path(path).open("wb") as f: self.get_backend().dump(self, f) @@ -83,7 +93,7 @@ def to_pickle(self, path: Union[Path, str], dump_all: bool = None, exclude: list @classmethod def load(cls, filepath): """ - load the collector from a file + Load the collector from a filepath. Args: filepath (str): the path of file @@ -104,10 +114,10 @@ def load(cls, filepath): @classmethod def get_backend(cls): """ - Return the backend of a Serializable class. The value will be "pickle" or "dill". + Return the real backend of a Serializable class. The pickle_backend value can be "pickle" or "dill". Returns: - str: The value of "pickle" or "dill" + module: pickle or dill module based on pickle_backend """ if cls.pickle_backend == "pickle": return pickle @@ -115,4 +125,3 @@ def get_backend(cls): return dill else: raise ValueError("Unknown pickle backend, please use 'pickle' or 'dill'.") - diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index 63169b58d5..7d1c723f3e 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -4,12 +4,20 @@ """ OnlineManager can manage a set of `Online Strategy <#Online Strategy>`_ and run them dynamically. -With the change of time, the decisive models will be also changed. In this module, we call those contributing models as `online` models. -In every routine(such as everyday or every minutes), the `online` models maybe changed and the prediction of them need to be updated. -So this module provide a series methods to control this process. +With the change of time, the decisive models will be also changed. In this module, we call those contributing models `online` models. +In every routine(such as every day or every minute), the `online` models may be changed and the prediction of them needs to be updated. +So this module provides a series of methods to control this process. -This module also provide a method to simulate `Online Strategy <#Online Strategy>`_ in the history. +This module also provides a method to simulate `Online Strategy <#Online Strategy>`_ in history. Which means you can verify your strategy or find a better one. + +There are total 3 situations for using the different trainer: + +1: Online: Only use Trainer. + +2: Simulate with temporal dependence: Only use Trainer. + +3: Simulate without temporal dependence: Use Trainer or DelayTrainer. """ import logging @@ -20,7 +28,7 @@ from qlib.data.data import D from qlib.log import set_global_logger_level from qlib.model.ens.ensemble import AverageEnsemble -from qlib.model.trainer import DelayTrainerR, Trainer +from qlib.model.trainer import DelayTrainerR, Trainer, TrainerR from qlib.utils import flatten_dict from qlib.utils.serial import Serializable from qlib.workflow.online.strategy import OnlineStrategy @@ -30,9 +38,12 @@ class OnlineManager(Serializable): """ OnlineManager can manage online models with `Online Strategy <#Online Strategy>`_. - It also provide a history recording which models are onlined at what time. + It also provides a history recording of which models are online at what time. """ + STATUS_SIMULATING = "simulating" # when calling `simulate` + STATUS_NORMAL = "normal" # the normal status + def __init__( self, strategies: Union[OnlineStrategy, List[OnlineStrategy]], @@ -46,8 +57,8 @@ def __init__( Args: strategies (Union[OnlineStrategy, List[OnlineStrategy]]): an instance of OnlineStrategy or a list of OnlineStrategy - begin_time (Union[str,pd.Timestamp], optional): the OnlineManager will begin at this time. Defaults to None for using latest date. - trainer (Trainer): the trainer to train task. None for using DelayTrainerR. + begin_time (Union[str,pd.Timestamp], optional): the OnlineManager will begin at this time. Defaults to None for using the latest date. + trainer (Trainer): the trainer to train task. None for using TrainerR. freq (str, optional): data frequency. Defaults to "day". """ self.logger = get_module_logger(self.__class__.__name__) @@ -59,12 +70,13 @@ def __init__( begin_time = D.calendar(freq=self.freq).max() self.begin_time = pd.Timestamp(begin_time) self.cur_time = self.begin_time - # OnlineManager will recorder the history of online models, which is a dict like {begin_time, {strategy, [online_models]}}. begin_time means when online_models are onlined. + # OnlineManager will recorder the history of online models, which is a dict like {pd.Timestamp, {strategy, [online_models]}}. self.history = {} if trainer is None: - trainer = DelayTrainerR() + trainer = TrainerR() self.trainer = trainer self.signals = None + self.status = self.STATUS_NORMAL def first_train(self, strategies: List[OnlineStrategy] = None, model_kwargs: dict = {}): """ @@ -75,37 +87,36 @@ def first_train(self, strategies: List[OnlineStrategy] = None, model_kwargs: dic strategies (List[OnlineStrategy]): the strategies list (need this param when adding strategies). None for use default strategies. model_kwargs (dict): the params for `prepare_online_models` """ - models_list = [] if strategies is None: strategies = self.strategies for strategy in strategies: + self.logger.info(f"Strategy `{strategy.name_id}` begins first training...") tasks = strategy.first_tasks() models = self.trainer.train(tasks, experiment_name=strategy.name_id) - models_list.append(models) + models = self.trainer.end_train(models, experiment_name=strategy.name_id) + self.logger.info(f"Finished training {len(models)} models.") - for strategy, models in zip(strategies, models_list): - self.prepare_online_models(strategy, models, model_kwargs=model_kwargs) + online_models = strategy.prepare_online_models(models, **model_kwargs) + self.history.setdefault(self.cur_time, {})[strategy] = online_models def routine( self, cur_time: Union[str, pd.Timestamp] = None, - delay: bool = False, task_kwargs: dict = {}, model_kwargs: dict = {}, signal_kwargs: dict = {}, ): """ - Run typical update process for every strategy and record the online history. + Typical update process for every strategy and record the online history. The typical update process after a routine, such as day by day or month by month. - The process is: Prepare signals -> Prepare tasks -> Prepare online models. + The process is: Update predictions -> Prepare tasks -> Prepare online models -> Prepare signals. If using DelayTrainer, it can finish training all together after every strategy's prepare_tasks. Args: cur_time (Union[str,pd.Timestamp], optional): run routine method in this time. Defaults to None. - delay (bool): if delay prepare signals and models task_kwargs (dict): the params for `prepare_tasks` model_kwargs (dict): the params for `prepare_online_models` signal_kwargs (dict): the params for `prepare_signals` @@ -113,40 +124,23 @@ def routine( if cur_time is None: cur_time = D.calendar(freq=self.freq).max() self.cur_time = pd.Timestamp(cur_time) # None for latest date - models_list = [] + for strategy in self.strategies: self.logger.info(f"Strategy `{strategy.name_id}` begins routine...") - if not delay: + if self.status == self.STATUS_NORMAL: strategy.tool.update_online_pred() tasks = strategy.prepare_tasks(self.cur_time, **task_kwargs) models = self.trainer.train(tasks) + if self.status == self.STATUS_NORMAL or not self.trainer.is_delay(): + models = self.trainer.end_train(models, experiment_name=strategy.name_id) self.logger.info(f"Finished training {len(models)} models.") - models_list.append(models) + online_models = strategy.prepare_online_models(models, **model_kwargs) + self.history.setdefault(self.cur_time, {})[strategy] = online_models - for strategy, models in zip(self.strategies, models_list): - self.prepare_online_models(strategy, models, delay=delay, model_kwargs=model_kwargs) - - if not delay: + if not self.trainer.is_delay(): self.prepare_signals(**signal_kwargs) - def prepare_online_models( - self, strategy: OnlineStrategy, models: list, delay: bool = False, model_kwargs: dict = {} - ): - """ - Prepare online model for strategy, including end_train, reset_online_tag and add history. - - Args: - strategy (OnlineStrategy): the instance of strategy. - models (list): a list of models. - delay (bool, optional): if delay prepare models. Defaults to False. - model_kwargs (dict, optional): the params for `prepare_online_models`. - """ - if not delay: - models = self.trainer.end_train(models, experiment_name=strategy.name_id) - online_models = strategy.prepare_online_models(models, **model_kwargs) - self.history.setdefault(self.cur_time, {})[strategy] = online_models - def get_collector(self) -> MergeCollector: """ Get the instance of `Collector <../advanced/task_management.html#Task Collecting>`_ to collect results from every strategy. @@ -162,7 +156,7 @@ def get_collector(self) -> MergeCollector: def add_strategy(self, strategies: Union[OnlineStrategy, List[OnlineStrategy]]): """ - Add some new strategies to online manager. + Add some new strategies to OnlineManager. Args: strategy (Union[OnlineStrategy, List[OnlineStrategy]]): a list of OnlineStrategy @@ -174,9 +168,9 @@ def add_strategy(self, strategies: Union[OnlineStrategy, List[OnlineStrategy]]): def prepare_signals(self, prepare_func: Callable = AverageEnsemble(), over_write=False): """ - After perparing the data of last routine (a box in box-plot) which means the end of the routine, we can prepare trading signals for next routine. + After preparing the data of the last routine (a box in box-plot) which means the end of the routine, we can prepare trading signals for the next routine. - NOTE: Given a set prediction, all signals before these prediction end time will be prepared well. + NOTE: Given a set prediction, all signals before these prediction end times will be prepared well. Even if the latest signal already exists, the latest calculation result will be overwritten. @@ -185,7 +179,7 @@ def prepare_signals(self, prepare_func: Callable = AverageEnsemble(), over_write Given a prediction of a certain time, all signals before this time will be prepared well. Args: - prepare_func (Callable, optional): Get signals from a dict after collecting. Defaults to AverageEnsemble(), the results after mergecollector must be {xxx:pred}. + prepare_func (Callable, optional): Get signals from a dict after collecting. Defaults to AverageEnsemble(), the results collected by MergeCollector must be {xxx:pred}. over_write (bool, optional): If True, the new signals will overwrite. If False, the new signals will append to the end of signals. Defaults to False. Returns: @@ -209,18 +203,18 @@ def get_signals(self) -> Union[pd.Series, pd.DataFrame]: Returns: Union[pd.Series, pd.DataFrame]: pd.Series for only one signals every datetime. - pd.DataFrame for multiple signals, for example, buy and sell operation use different trading signal. + pd.DataFrame for multiple signals, for example, buy and sell operations use different trading signals. """ return self.signals SIM_LOG_LEVEL = logging.INFO + 1 SIM_LOG_NAME = "SIMULATE_INFO" - def simulate(self, end_time, frequency="day", task_kwargs={}, model_kwargs={}, signal_kwargs={}): + def simulate(self, end_time, frequency="day", task_kwargs={}, model_kwargs={}, signal_kwargs={}) -> Union[pd.Series, pd.DataFrame]: """ - Starting from current time, this method will simulate every routine in OnlineManager until end time. + Starting from the current time, this method will simulate every routine in OnlineManager until the end time. - Considering the parallel training, the models and signals can be perpared after all routine simulating. + Considering the parallel training, the models and signals can be prepared after all routine simulating. The delay training way can be ``DelayTrainer`` and the delay preparing signals way can be ``delay_prepare``. @@ -232,8 +226,10 @@ def simulate(self, end_time, frequency="day", task_kwargs={}, model_kwargs={}, s signal_kwargs (dict): the params for `prepare_signals` Returns: - HyperCollector: the OnlineManager's collector + Union[pd.Series, pd.DataFrame]: pd.Series for only one signals every datetime. + pd.DataFrame for multiple signals, for example, buy and sell operations use different trading signals. """ + self.status = self.STATUS_SIMULATING cal = D.calendar(start_time=self.cur_time, end_time=end_time, freq=frequency) self.first_train() @@ -245,7 +241,6 @@ def simulate(self, end_time, frequency="day", task_kwargs={}, model_kwargs={}, s self.logger.log(level=simulate_level, msg=f"Simulating at {str(cur_time)}......") self.routine( cur_time, - delay=self.trainer.is_delay(), task_kwargs=task_kwargs, model_kwargs=model_kwargs, signal_kwargs=signal_kwargs, @@ -257,11 +252,12 @@ def simulate(self, end_time, frequency="day", task_kwargs={}, model_kwargs={}, s # FIXME: get logging level firstly and restore it here set_global_logger_level(logging.DEBUG) self.logger.info(f"Finished preparing signals") - return self.get_collector() + self.status = self.STATUS_NORMAL + return self.get_signals() def delay_prepare(self, model_kwargs={}, signal_kwargs={}): """ - Prepare all models and signals if there are something waiting for prepare. + Prepare all models and signals if something is waiting for preparation. Args: model_kwargs: the params for `prepare_online_models` @@ -270,6 +266,6 @@ def delay_prepare(self, model_kwargs={}, signal_kwargs={}): for cur_time, strategy_models in self.history.items(): self.cur_time = cur_time for strategy, models in strategy_models.items(): - self.prepare_online_models(strategy, models, delay=False, model_kwargs=model_kwargs) + models = self.trainer.end_train(models, experiment_name=strategy.name_id) # NOTE: Assumption: the predictions of online models need less than next cur_time, or this method will work in a wrong way. self.prepare_signals(**signal_kwargs) diff --git a/qlib/workflow/online/strategy.py b/qlib/workflow/online/strategy.py index 04c854f797..491b191dd6 100644 --- a/qlib/workflow/online/strategy.py +++ b/qlib/workflow/online/strategy.py @@ -2,7 +2,7 @@ # Licensed under the MIT License. """ -OnlineStrategy is a set of strategy for online serving. +OnlineStrategy module is an element of online serving. """ from copy import deepcopy @@ -19,7 +19,7 @@ class OnlineStrategy: """ - OnlineStrategy is working with `Online Manager <#Online Manager>`_, responsing how the tasks are generated, the models are updated and signals are perpared. + OnlineStrategy is working with `Online Manager <#Online Manager>`_, responding to how the tasks are generated, the models are updated and signals are prepared. """ def __init__(self, name_id: str): @@ -28,7 +28,7 @@ def __init__(self, name_id: str): This module **MUST** use `Trainer <../reference/api.html#Trainer>`_ to finishing model training. Args: - name_id (str): a unique name or id + name_id (str): a unique name or id. trainer (Trainer, optional): a instance of Trainer. Defaults to None. """ self.name_id = name_id @@ -40,29 +40,29 @@ def prepare_tasks(self, cur_time, **kwargs) -> List[dict]: After the end of a routine, check whether we need to prepare and train some new tasks based on cur_time (None for latest).. Return the new tasks waiting for training. - You can find last online models by OnlineTool.online_models. + You can find the last online models by OnlineTool.online_models. """ raise NotImplementedError(f"Please implement the `prepare_tasks` method.") - def prepare_online_models(self, models, cur_time=None) -> List[object]: + def prepare_online_models(self, trained_models, cur_time=None) -> List[object]: """ Select some models from trained models and set them to online models. - This is a typically implementation to online all trained models, you can override it to implement complex method. - You can find last online models by OnlineTool.online_models if you still need them. + This is a typical implementation to online all trained models, you can override it to implement the complex method. + You can find the last online models by OnlineTool.online_models if you still need them. - NOTE: Reset all online models to trained model. If there is no trained models, then do nothing. + NOTE: Reset all online models to trained models. If there are no trained models, then do nothing. Args: models (list): a list of models. - cur_time (pd.Dataframe): current time from OnlineManger. None for latest. + cur_time (pd.Dataframe): current time from OnlineManger. None for the latest. Returns: List[object]: a list of online models. """ - if not models: + if not trained_models: return self.tool.online_models() - self.tool.reset_online_tag(models) - return models + self.tool.reset_online_tag(trained_models) + return trained_models def first_tasks(self) -> List[dict]: """ @@ -87,7 +87,7 @@ def get_collector(self) -> Collector: class RollingStrategy(OnlineStrategy): """ - This example strategy always use latest rolling model as online model. + This example strategy always uses the latest rolling model sas online models. """ def __init__( @@ -99,11 +99,11 @@ def __init__( """ Init RollingStrategy. - Assumption: the str of name_id, the experiment name and the trainer's experiment name are same one. + Assumption: the str of name_id, the experiment name, and the trainer's experiment name are the same. Args: - name_id (str): a unique name or id. Will be also the name of Experiment. - task_template (Union[dict,List[dict]]): a list of task_template or a single template, which will be used to generate many tasks using rolling_gen. + name_id (str): a unique name or id. Will be also the name of the Experiment. + task_template (Union[dict, List[dict]]): a list of task_template or a single template, which will be used to generate many tasks using rolling_gen. rolling_gen (RollingGen): an instance of RollingGen """ super().__init__(name_id=name_id) @@ -117,9 +117,10 @@ def __init__( def get_collector(self, process_list=[RollingGroup()], rec_key_func=None, rec_filter_func=None, artifacts_key=None): """ - Get the instance of `Collector <../advanced/task_management.html#Task Collecting>`_ to collect results. The returned collector must can distinguish results in different models. - Assumption: the models can be distinguished based on model name and rolling test segments. - If you do not want this assumption, please implement your own method or use another rec_key_func. + Get the instance of `Collector <../advanced/task_management.html#Task Collecting>`_ to collect results. The returned collector must distinguish results in different models. + + Assumption: the models can be distinguished based on the model name and rolling test segments. + If you do not want this assumption, please implement your method or use another rec_key_func. Args: rec_key_func (Callable): a function to get the key of a recorder. If None, use recorder id. @@ -160,9 +161,9 @@ def first_tasks(self) -> List[dict]: def prepare_tasks(self, cur_time) -> List[dict]: """ - Prepare new tasks based on cur_time (None for latest). + Prepare new tasks based on cur_time (None for the latest). - You can find last online models by OnlineToolR.online_models. + You can find the last online models by OnlineToolR.online_models. Returns: List[dict]: a list of new tasks. @@ -198,7 +199,7 @@ def _list_latest(self, rec_list: List[Recorder]): rec_list (List[Recorder]): a list of Recorder Returns: - List[Recorder], pd.Timestamp: the latest recorders and its test end time + List[Recorder], pd.Timestamp: the latest recorders and their test end time """ if len(rec_list) == 0: return rec_list, None diff --git a/qlib/workflow/online/update.py b/qlib/workflow/online/update.py index 9cb294169f..561f7e18ae 100644 --- a/qlib/workflow/online/update.py +++ b/qlib/workflow/online/update.py @@ -2,7 +2,7 @@ # Licensed under the MIT License. """ -Updater is a module to update artifacts such as predictions, when the stock data is updating. +Updater is a module to update artifacts such as predictions when the stock data is updating. """ from abc import ABCMeta, abstractmethod @@ -87,7 +87,7 @@ def __init__(self, record: Recorder, to_date=None, hist_ref: int = 0, freq="day" update to prediction to the `to_date` hist_ref : int Sometimes, the dataset will have historical depends. - Leave the problem to user to set the length of historical dependency + Leave the problem to users to set the length of historical dependency .. note:: @@ -112,7 +112,7 @@ def prepare_data(self) -> DatasetH: """ Load dataset - Seperating this function will make it easier to reuse the dataset + Separating this function will make it easier to reuse the dataset Returns: DatasetH: the instance of DatasetH @@ -125,7 +125,7 @@ def prepare_data(self) -> DatasetH: def update(self, dataset: DatasetH = None): """ - Update the precition in a recorder + Update the prediction in a recorder. Args: DatasetH: the instance of DatasetH. None for reprepare. diff --git a/qlib/workflow/online/utils.py b/qlib/workflow/online/utils.py index 3c2774cec9..f3ef13aa93 100644 --- a/qlib/workflow/online/utils.py +++ b/qlib/workflow/online/utils.py @@ -3,8 +3,8 @@ """ OnlineTool is a module to set and unset a series of `online` models. -The `online` models are some decisive models in some time point, which can be changed with the change of time. -This allows us to use efficient submodels as the market style changing. +The `online` models are some decisive models in some time points, which can be changed with the change of time. +This allows us to use efficient submodels as the market-style changing. """ from typing import List, Union @@ -17,7 +17,7 @@ class OnlineTool: """ - OnlineTool will manage `online` models in an experiment which includes the models recorder. + OnlineTool will manage `online` models in an experiment that includes the model recorders. """ ONLINE_KEY = "online_status" # the online status key in recorder @@ -74,10 +74,10 @@ def online_models(self) -> list: def update_online_pred(self, to_date=None): """ - Update the predictions of `online` models to a date. + Update the predictions of `online` models to to_date. Args: - to_date (pd.Timestamp): the pred before this date will be updated. None for update to latest. + to_date (pd.Timestamp): the pred before this date will be updated. None for updating to the latest. """ raise NotImplementedError(f"Please implement the `update_online_pred` method.") @@ -151,15 +151,16 @@ def online_models(self) -> list: def update_online_pred(self, to_date=None): """ - Update the predictions of online models to a date. + Update the predictions of online models to to_date. Args: - to_date (pd.Timestamp): the pred before this date will be updated. None for update to latest time in Calendar. + to_date (pd.Timestamp): the pred before this date will be updated. None for updating to latest time in Calendar. """ online_models = self.online_models() for rec in online_models: hist_ref = 0 task = rec.load_object("task") + # Special treatment of historical dependencies if task["dataset"]["class"] == "TSDatasetH": hist_ref = task["dataset"]["kwargs"]["step_len"] PredUpdater(rec, to_date=to_date, hist_ref=hist_ref).update() diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index 1080d07f45..3a8bd1f2c2 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -2,7 +2,7 @@ # Licensed under the MIT License. """ -Collector can collect object from everywhere and process them such as merging, grouping, averaging and so on. +Collector module can collect objects from everywhere and process them such as merging, grouping, averaging and so on. """ from typing import Callable, Dict, List @@ -17,15 +17,18 @@ class Collector(Serializable): def __init__(self, process_list=[]): """ + Init Collector. + Args: - process_list (list, optional): process_list (list or Callable): the list of processors or the instance of processor to process dict. + process_list (list or Callable): the list of processors or the instance of a processor to process dict. """ if not isinstance(process_list, list): process_list = [process_list] self.process_list = process_list def collect(self) -> dict: - """Collect the results and return a dict like {key: things} + """ + Collect the results and return a dict like {key: things} Returns: dict: the dict after collecting. @@ -42,13 +45,14 @@ def collect(self) -> dict: @staticmethod def process_collect(collected_dict, process_list=[], *args, **kwargs) -> dict: - """do a series of processing to the dict returned by collect and return a dict like {key: things} - For example: you can group and ensemble. + """ + Do a series of processing to the dict returned by collect and return a dict like {key: things} + For example, you can group and ensemble. Args: collected_dict (dict): the dict return by `collect` - process_list (list or Callable): the list of processors or the instance of processor to process dict. - The processor order is same as the list order. + process_list (list or Callable): the list of processors or the instance of a processor to process dict. + The processor order is the same as the list order. For example: [Group1(..., Ensemble1()), Group2(..., Ensemble2())] Returns: @@ -68,7 +72,7 @@ def process_collect(collected_dict, process_list=[], *args, **kwargs) -> dict: def __call__(self, *args, **kwargs) -> dict: """ - do the workflow including collect and process_collect + Do the workflow including ``collect`` and ``process_collect`` Returns: dict: the dict after collecting and processing. @@ -93,11 +97,13 @@ class MergeCollector(Collector): def __init__(self, collector_dict: Dict[str, Collector], process_list: List[Callable] = [], merge_func=None): """ + Init MergeCollector. + Args: collector_dict (Dict[str,Collector]): the dict like {collector_key, Collector} process_list (List[Callable]): the list of processors or the instance of processor to process dict. merge_func (Callable): a method to generate outermost key. The given params are ``collector_key`` from collector_dict and ``key`` from every collector after collecting. - None for use tuple to connect them, such as "ABC"+("a","b") -> ("ABC", ("a","b")). + None for using tuple to connect them, such as "ABC"+("a","b") -> ("ABC", ("a","b")). """ super().__init__(process_list=process_list) self.collector_dict = collector_dict @@ -105,7 +111,7 @@ def __init__(self, collector_dict: Dict[str, Collector], process_list: List[Call def collect(self) -> dict: """ - Collect all result of collector_dict and change the outermost key to a recombination key. + Collect all results of collector_dict and change the outermost key to a recombination key. Returns: dict: the dict after collecting. @@ -133,11 +139,12 @@ def __init__( artifacts_path={"pred": "pred.pkl"}, artifacts_key=None, ): - """init RecorderCollector + """ + Init RecorderCollector. Args: - experiment (Experiment or str): an instance of a Experiment or the name of a Experiment - process_list (list or Callable): the list of processors or the instance of processor to process dict. + experiment (Experiment or str): an instance of an Experiment or the name of an Experiment + process_list (list or Callable): the list of processors or the instance of a processor to process dict. rec_key_func (Callable): a function to get the key of a recorder. If None, use recorder id. rec_filter_func (Callable, optional): filter the recorder by return True or False. Defaults to None. artifacts_path (dict, optional): The artifacts name and its path in Recorder. Defaults to {"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"}. @@ -157,12 +164,13 @@ def __init__( self.rec_filter_func = rec_filter_func def collect(self, artifacts_key=None, rec_filter_func=None, only_exist=True) -> dict: - """Collect different artifacts based on recorder after filtering. + """ + Collect different artifacts based on recorder after filtering. Args: - artifacts_key (str or List, optional): the artifacts key you want to get. If None, use default. - rec_filter_func (Callable, optional): filter the recorder by return True or False. If None, use default. - only_exist (bool, optional): if only collect the artifacts when a recorder really have. + artifacts_key (str or List, optional): the artifacts key you want to get. If None, use the default. + rec_filter_func (Callable, optional): filter the recorder by return True or False. If None, use the default. + only_exist (bool, optional): if only collect the artifacts when a recorder really has. If True, the recorder with exception when loading will not be collected. But if False, it will raise the exception. Returns: diff --git a/qlib/workflow/task/gen.py b/qlib/workflow/task/gen.py index 7e08c76f48..cdebf50494 100644 --- a/qlib/workflow/task/gen.py +++ b/qlib/workflow/task/gen.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. """ -Task generator can generate many tasks based on TaskGen and some task templates. +TaskGenerator module can generate many tasks based on TaskGen and some task templates. """ import abc import copy @@ -10,7 +10,8 @@ def task_generator(tasks, generators) -> list: - """Use a list of TaskGen and a list of task templates to generate different tasks. + """ + Use a list of TaskGen and a list of task templates to generate different tasks. For examples: @@ -47,7 +48,7 @@ def task_generator(tasks, generators) -> list: class TaskGen(metaclass=abc.ABCMeta): """ - the base class for generate different tasks + The base class for generating different tasks Example 1: @@ -66,7 +67,7 @@ class TaskGen(metaclass=abc.ABCMeta): @abc.abstractmethod def generate(self, task: dict) -> List[dict]: """ - generate different tasks based on a task template + Generate different tasks based on a task template Parameters ---------- @@ -87,7 +88,7 @@ def __call__(self, *args, **kwargs): return self.generate(*args, **kwargs) -def handler_mod(task: dict, rg): +def handler_mod(task: dict, rolling_gen): """ Help to modify the handler end time when using RollingGen @@ -96,14 +97,14 @@ def handler_mod(task: dict, rg): rg (RollingGen): an instance of RollingGen """ try: - interval = rg.ta.cal_interval( + interval = rolling_gen.ta.cal_interval( task["dataset"]["kwargs"]["handler"]["kwargs"]["end_time"], - task["dataset"]["kwargs"]["segments"][rg.test_key][1], + task["dataset"]["kwargs"]["segments"][rolling_gen.test_key][1], ) # if end_time < the end of test_segments, then change end_time to allow load more data if interval < 0: task["dataset"]["kwargs"]["handler"]["kwargs"]["end_time"] = copy.deepcopy( - task["dataset"]["kwargs"]["segments"][rg.test_key][1] + task["dataset"]["kwargs"]["segments"][rolling_gen.test_key][1] ) except KeyError: # Maybe dataset do not have handler, then do nothing. @@ -126,7 +127,7 @@ def __init__(self, step: int = 40, rtype: str = ROLL_EX, ds_extra_mod_func: Unio rolling type (expanding, sliding) ds_extra_mod_func: Callable A method like: handler_mod(task: dict, rg: RollingGen) - Do some extra action after generating a task. For example, use ``handler_mod`` to modify the end time of handler of dataset. + Do some extra action after generating a task. For example, use ``handler_mod`` to modify the end time of the handler of a dataset. """ self.step = step self.rtype = rtype @@ -142,7 +143,7 @@ def generate(self, task: dict) -> List[dict]: Parameters ---------- - task : dict + task: dict A dict describing a task. For example. .. code-block:: python @@ -184,7 +185,7 @@ def generate(self, task: dict) -> List[dict]: Returns ---------- - typing.List[dict]: a list of tasks + List[dict]: a list of tasks """ res = [] diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 40f8682959..658eec4d6e 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -2,7 +2,7 @@ # Licensed under the MIT License. """ -TaskManager can fetch unused tasks automatically and manager the lifecycle of a set of tasks with error handling. +TaskManager can fetch unused tasks automatically and manage the lifecycle of a set of tasks with error handling. These features can run tasks concurrently and ensure every task will be used only once. Task Manager will store all tasks in `MongoDB `_. Users **MUST** finished the configuration of `MongoDB `_ when using this module. @@ -10,7 +10,7 @@ A task in TaskManager consists of 3 parts - tasks description: the desc will define the task - tasks status: the status of the task -- tasks result information : A user can get the task with the task description and task result. +- tasks result: A user can get the task with the task description and task result. """ import concurrent import pickle @@ -44,7 +44,7 @@ class TaskManager: 'res': pickle serialized task result, } - The tasks manager assume that you will only update the tasks you fetched. + The tasks manager assumes that you will only update the tasks you fetched. The mongo fetch one and update will make it date updating secure. .. note:: @@ -53,7 +53,7 @@ class TaskManager: Here are four status which are: - STATUS_WAITING: waiting for train + STATUS_WAITING: waiting for training STATUS_RUNNING: training @@ -85,7 +85,7 @@ def __init__(self, task_pool: str = None): def list(self) -> list: """ - list the all collection(task_pool) of the db + List the all collection(task_pool) of the db Returns: list @@ -112,6 +112,10 @@ def _dict_to_str(self, flt): def replace_task(self, task, new_task): """ Use a new task to replace a old one + + Args: + task: old task + new_task: new task """ new_task = self._encode_task(new_task) query = {"_id": ObjectId(task["_id"])} @@ -122,7 +126,15 @@ def replace_task(self, task, new_task): self.task_pool.replace_one(query, new_task) def insert_task(self, task): + """ + Insert a task. + + Args: + task: the task waiting for insert + Returns: + pymongo.results.InsertOneResult + """ try: insert_result = self.task_pool.insert_one(task) except InvalidDocument: @@ -132,7 +144,7 @@ def insert_task(self, task): def insert_task_def(self, task_def): """ - insert a task to task_pool + Insert a task to task_pool Parameters ---------- @@ -155,8 +167,8 @@ def insert_task_def(self, task_def): def create_task(self, task_def_l, dry_run=False, print_nt=False) -> List[str]: """ - If the tasks in task_def_l is new, then insert new tasks into the task_pool, and record inserted_id. - If a task is not new, then query its _id. + If the tasks in task_def_l are new, then insert new tasks into the task_pool, and record inserted_id. + If a task is not new, then just query its _id. Parameters ---------- @@ -169,7 +181,7 @@ def create_task(self, task_def_l, dry_run=False, print_nt=False) -> List[str]: Returns ------- - list + List[str] a list of the _id of task_def_l """ new_tasks = [] @@ -202,7 +214,7 @@ def create_task(self, task_def_l, dry_run=False, print_nt=False) -> List[str]: def fetch_task(self, query={}, status=STATUS_WAITING) -> dict: """ - Use query to fetch tasks + Use query to fetch tasks. Args: query (dict, optional): query dict. Defaults to {}. @@ -257,6 +269,7 @@ def task_fetcher_iter(self, query={}): def query(self, query={}, decode=True): """ + Query task in collection. This function may raise exception `pymongo.errors.CursorNotFound: cursor id not found` if it takes too long to iterate the generator Parameters @@ -330,7 +343,16 @@ def remove(self, query={}): query["_id"] = ObjectId(query["_id"]) self.task_pool.delete_many(query) - def task_stat(self, query={}): + def task_stat(self, query={}) -> dict: + """ + Count the tasks in every status. + + Args: + query (dict, optional): the query dict. Defaults to {}. + + Returns: + dict + """ query = query.copy() if "_id" in query: query["_id"] = ObjectId(query["_id"]) @@ -341,6 +363,12 @@ def task_stat(self, query={}): return status_stat def reset_waiting(self, query={}): + """ + Reset all running task into waiting status. Can be used when some running task exit unexpected. + + Args: + query (dict, optional): the query dict. Defaults to {}. + """ query = query.copy() # default query if "status" not in query: @@ -400,7 +428,7 @@ def run_task( **kwargs, ): """ - While task pool is not empty (has WAITING tasks), use task_func to fetch and run tasks in task_pool + While the task pool is not empty (has WAITING tasks), use task_func to fetch and run tasks in task_pool After running this method, here are 4 situations (before_status -> after_status): diff --git a/qlib/workflow/task/utils.py b/qlib/workflow/task/utils.py index ed5e1a2359..89059e9f88 100644 --- a/qlib/workflow/task/utils.py +++ b/qlib/workflow/task/utils.py @@ -19,8 +19,9 @@ def get_mongodb() -> Database: """ - Get database in MongoDB, which means you need to declare the address and the name of database. - for example: + Get database in MongoDB, which means you need to declare the address and the name of a database at first. + + For example: Using qlib.init(): @@ -52,10 +53,10 @@ def get_mongodb() -> Database: def list_recorders(experiment, rec_filter_func=None): """ - List all recorders which can pass the filter in a experiment. + List all recorders which can pass the filter in an experiment. Args: - experiment (str or Experiment): the name of a Experiment or a instance + experiment (str or Experiment): the name of an Experiment or an instance rec_filter_func (Callable, optional): return True to retain the given recorder. Defaults to None. Returns: @@ -82,11 +83,17 @@ def __init__(self, future=True, end_time=None): self.cals = D.calendar(future=future, end_time=end_time) def set_end_time(self, end_time=None): + """ + Set end time. None for use calendar's end time. + + Args: + end_time + """ self.cals = D.calendar(future=self._future, end_time=end_time) def get(self, idx: int): """ - Get datetime by index + Get datetime by index. Parameters ---------- @@ -105,7 +112,7 @@ def max(self) -> pd.Timestamp: def align_idx(self, time_point, tp_type="start") -> int: """ - Align the index of time_point in the calendar + Align the index of time_point in the calendar. Parameters ---------- @@ -155,7 +162,7 @@ def align_time(self, time_point, tp_type="start") -> pd.Timestamp: def align_seg(self, segment: Union[dict, tuple]) -> Union[dict, tuple]: """ - align the given date to trade date + Align the given date to the trade date for example: @@ -184,7 +191,7 @@ def align_seg(self, segment: Union[dict, tuple]) -> Union[dict, tuple]: def truncate(self, segment: tuple, test_start, days: int) -> tuple: """ - truncate the segment based on the test_start date + Truncate the segment based on the test_start date Parameters ---------- @@ -215,7 +222,7 @@ def truncate(self, segment: tuple, test_start, days: int) -> tuple: def shift(self, seg: tuple, step: int, rtype=SHIFT_SD) -> tuple: """ - shift the datatime of segment + Shift the datatime of segment Parameters ---------- From aef3f186c16ea3ea514710b915d6cd11cc9991f9 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Fri, 14 May 2021 06:58:02 +0000 Subject: [PATCH 59/61] format code --- examples/online_srv/online_management_simulate.py | 2 +- qlib/model/ens/ensemble.py | 5 +++-- qlib/model/ens/group.py | 2 +- qlib/model/trainer.py | 4 ++-- qlib/utils/serial.py | 1 - qlib/workflow/online/manager.py | 8 +++++--- qlib/workflow/online/strategy.py | 2 +- qlib/workflow/task/collect.py | 2 +- qlib/workflow/task/utils.py | 2 +- 9 files changed, 15 insertions(+), 13 deletions(-) diff --git a/examples/online_srv/online_management_simulate.py b/examples/online_srv/online_management_simulate.py index c09b10aa77..4bb5022ee0 100644 --- a/examples/online_srv/online_management_simulate.py +++ b/examples/online_srv/online_management_simulate.py @@ -113,7 +113,7 @@ def __init__( self.rolling_gen = RollingGen( step=rolling_step, rtype=RollingGen.ROLL_SD, ds_extra_mod_func=None ) # The rolling tasks generator, ds_extra_mod_func is None because we just need to simulate to 2018-10-31 and needn't change the handler end time. - self.trainer = DelayTrainerRM(self.exp_name, self.task_pool) # Also can be TrainerR, TrainerRM, DelayTrainerR + self.trainer = DelayTrainerRM(self.exp_name, self.task_pool) # Also can be TrainerR, TrainerRM, DelayTrainerR self.rolling_online_manager = OnlineManager( RollingStrategy(exp_name, task_template=tasks, rolling_gen=self.rolling_gen), trainer=self.trainer, diff --git a/qlib/model/ens/ensemble.py b/qlib/model/ens/ensemble.py index 0f48ce728e..4fa6a5ec63 100644 --- a/qlib/model/ens/ensemble.py +++ b/qlib/model/ens/ensemble.py @@ -16,9 +16,9 @@ class Ensemble: For example: {Rollinga_b: object, Rollingb_c: object} -> object When calling this class: - + Args: - ensemble_dict (dict): the ensemble dict like {name: things} waiting for merging + ensemble_dict (dict): the ensemble dict like {name: things} waiting for merging Returns: object: the ensemble object @@ -103,6 +103,7 @@ class AverageEnsemble(Ensemble): Returns: pd.DataFrame: the complete result of averaging and standardizing. """ + def __call__(self, ensemble_dict: dict) -> pd.DataFrame: # need to flatten the nested dict ensemble_dict = flatten_dict(ensemble_dict, sep=FLATTEN_TUPLE) diff --git a/qlib/model/ens/group.py b/qlib/model/ens/group.py index 93903f4334..7f45b06a5c 100644 --- a/qlib/model/ens/group.py +++ b/qlib/model/ens/group.py @@ -64,7 +64,7 @@ def reduce(self, *args, **kwargs) -> dict: else: raise NotImplementedError(f"Please specify valid `_ens_func`.") - def __call__(self, ungrouped_dict: dict, n_jobs:int=1, verbose:int=0, *args, **kwargs) -> dict: + def __call__(self, ungrouped_dict: dict, n_jobs: int = 1, verbose: int = 0, *args, **kwargs) -> dict: """ Group the ungrouped_dict into different groups. diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index 0c9c9e2c2a..fd76e67284 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -240,13 +240,13 @@ def end_train(self, recs, end_train_func=None, experiment_name: str = None, **kw """ Given a list of Recorder and return a list of trained Recorder. This class will finish real data loading and model fitting. - + Args: recs (list): a list of Recorder, the tasks have been saved to them end_train_func (Callable, optional): the end_train method which needs at least `recorder`s and `experiment_name`. Defaults to None for using self.end_train_func. experiment_name (str): the experiment name, None for use default name. kwargs: the params for end_train_func. - + Returns: List[Recorder]: a list of Recorders """ diff --git a/qlib/utils/serial.py b/qlib/utils/serial.py index c7c51bac2c..263e632deb 100644 --- a/qlib/utils/serial.py +++ b/qlib/utils/serial.py @@ -8,7 +8,6 @@ from typing import Union - class Serializable: """ Serializable will change the behaviors of pickle. diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index 7d1c723f3e..f2a5765602 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -41,8 +41,8 @@ class OnlineManager(Serializable): It also provides a history recording of which models are online at what time. """ - STATUS_SIMULATING = "simulating" # when calling `simulate` - STATUS_NORMAL = "normal" # the normal status + STATUS_SIMULATING = "simulating" # when calling `simulate` + STATUS_NORMAL = "normal" # the normal status def __init__( self, @@ -210,7 +210,9 @@ def get_signals(self) -> Union[pd.Series, pd.DataFrame]: SIM_LOG_LEVEL = logging.INFO + 1 SIM_LOG_NAME = "SIMULATE_INFO" - def simulate(self, end_time, frequency="day", task_kwargs={}, model_kwargs={}, signal_kwargs={}) -> Union[pd.Series, pd.DataFrame]: + def simulate( + self, end_time, frequency="day", task_kwargs={}, model_kwargs={}, signal_kwargs={} + ) -> Union[pd.Series, pd.DataFrame]: """ Starting from the current time, this method will simulate every routine in OnlineManager until the end time. diff --git a/qlib/workflow/online/strategy.py b/qlib/workflow/online/strategy.py index 491b191dd6..a54eb32bfe 100644 --- a/qlib/workflow/online/strategy.py +++ b/qlib/workflow/online/strategy.py @@ -118,7 +118,7 @@ def __init__( def get_collector(self, process_list=[RollingGroup()], rec_key_func=None, rec_filter_func=None, artifacts_key=None): """ Get the instance of `Collector <../advanced/task_management.html#Task Collecting>`_ to collect results. The returned collector must distinguish results in different models. - + Assumption: the models can be distinguished based on the model name and rolling test segments. If you do not want this assumption, please implement your method or use another rec_key_func. diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index 3a8bd1f2c2..9410c2b9c2 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -98,7 +98,7 @@ class MergeCollector(Collector): def __init__(self, collector_dict: Dict[str, Collector], process_list: List[Callable] = [], merge_func=None): """ Init MergeCollector. - + Args: collector_dict (Dict[str,Collector]): the dict like {collector_key, Collector} process_list (List[Callable]): the list of processors or the instance of processor to process dict. diff --git a/qlib/workflow/task/utils.py b/qlib/workflow/task/utils.py index 89059e9f88..174b4b9bfc 100644 --- a/qlib/workflow/task/utils.py +++ b/qlib/workflow/task/utils.py @@ -20,7 +20,7 @@ def get_mongodb() -> Database: """ Get database in MongoDB, which means you need to declare the address and the name of a database at first. - + For example: Using qlib.init(): From a986379debea6793d1dd6a0a242340bcbc17fcb9 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Fri, 14 May 2021 11:31:50 +0000 Subject: [PATCH 60/61] bug fixed --- .../online_srv/rolling_online_management.py | 58 +++++++++++++------ qlib/workflow/online/manager.py | 25 ++++++-- qlib/workflow/recorder.py | 11 ++++ 3 files changed, 71 insertions(+), 23 deletions(-) diff --git a/examples/online_srv/rolling_online_management.py b/examples/online_srv/rolling_online_management.py index e5c37dac63..25b8b2a0c0 100644 --- a/examples/online_srv/rolling_online_management.py +++ b/examples/online_srv/rolling_online_management.py @@ -3,20 +3,19 @@ """ This example shows how OnlineManager works with rolling tasks. -There are two parts including first train and routine. +There are four parts including first train, routine 1, add strategy and routine 2. Firstly, the OnlineManager will finish the first training and set trained models to `online` models. -Next, the OnlineManager will finish a routine process, including update online prediction -> prepare signals -> prepare tasks -> prepare new models -> reset online models +Next, the OnlineManager will finish a routine process, including update online prediction -> prepare tasks -> prepare new models -> prepare signals +Then, we will add some new strategies to the OnlineManager. This will finish first training of new strategies. +Finally, the OnlineManager will finish second routine and update all strategies. """ import os -from pathlib import Path -import pickle import fire import qlib from qlib.workflow import R from qlib.workflow.online.strategy import RollingStrategy from qlib.workflow.task.gen import RollingGen -from qlib.workflow.task.manage import TaskManager from qlib.workflow.online.manager import OnlineManager data_handler_config = { @@ -84,7 +83,8 @@ def __init__( task_url="mongodb://10.0.0.4:27017/", task_db_name="rolling_db", rolling_step=550, - tasks=[task_xgboost_config, task_lgb_config], + tasks=[task_xgboost_config], + add_tasks=[task_lgb_config], ): mongo_conf = { "task_url": task_url, # your MongoDB url @@ -92,11 +92,12 @@ def __init__( } qlib.init(provider_uri=provider_uri, region=region, mongo=mongo_conf) self.tasks = tasks + self.add_tasks = add_tasks self.rolling_step = rolling_step - strategy = [] + strategies = [] for task in tasks: name_id = task["model"]["class"] # NOTE: Assumption: The model class can specify only one strategy - strategy.append( + strategies.append( RollingStrategy( name_id, task, @@ -104,8 +105,7 @@ def __init__( ) ) - self.rolling_online_manager = OnlineManager(strategy) - self.collector = self.rolling_online_manager.get_collector() + self.rolling_online_manager = OnlineManager(strategies) _ROLLING_MANAGER_PATH = ( ".RollingOnlineExample" # the OnlineManager will dump to this file, for it can be loaded when calling routine. @@ -113,40 +113,60 @@ def __init__( # Reset all things to the first status, be careful to save important data def reset(self): - for task in self.tasks: + for task in self.tasks + self.add_tasks: name_id = task["model"]["class"] - TaskManager(name_id).remove() exp = R.get_exp(experiment_name=name_id) for rid in exp.list_recorders(): exp.delete_recorder(rid) - if os.path.exists(self._ROLLING_MANAGER_PATH): - os.remove(self._ROLLING_MANAGER_PATH) + if os.path.exists(self._ROLLING_MANAGER_PATH): + os.remove(self._ROLLING_MANAGER_PATH) def first_run(self): print("========== reset ==========") self.reset() print("========== first_run ==========") self.rolling_online_manager.first_train() + print("========== collect results ==========") + print(self.rolling_online_manager.get_collector()()) print("========== dump ==========") self.rolling_online_manager.to_pickle(self._ROLLING_MANAGER_PATH) - print("========== collect results ==========") - print(self.collector()) def routine(self): print("========== load ==========") - with Path(self._ROLLING_MANAGER_PATH).open("rb") as f: - self.rolling_online_manager = pickle.load(f) + self.rolling_online_manager = OnlineManager.load(self._ROLLING_MANAGER_PATH) print("========== routine ==========") self.rolling_online_manager.routine() print("========== collect results ==========") - print(self.collector()) + print(self.rolling_online_manager.get_collector()()) print("========== signals ==========") print(self.rolling_online_manager.get_signals()) + print("========== dump ==========") + self.rolling_online_manager.to_pickle(self._ROLLING_MANAGER_PATH) + + def add_strategy(self): + print("========== load ==========") + self.rolling_online_manager = OnlineManager.load(self._ROLLING_MANAGER_PATH) + print("========== add strategy ==========") + strategies = [] + for task in self.add_tasks: + name_id = task["model"]["class"] # NOTE: Assumption: The model class can specify only one strategy + strategies.append( + RollingStrategy( + name_id, + task, + RollingGen(step=self.rolling_step, rtype=RollingGen.ROLL_SD), + ) + ) + self.rolling_online_manager.add_strategy(strategies=strategies) + print("========== dump ==========") + self.rolling_online_manager.to_pickle(self._ROLLING_MANAGER_PATH) def main(self): self.first_run() self.routine() + self.add_strategy() + self.routine() if __name__ == "__main__": diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index f2a5765602..6947d6678c 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -262,12 +262,29 @@ def delay_prepare(self, model_kwargs={}, signal_kwargs={}): Prepare all models and signals if something is waiting for preparation. Args: - model_kwargs: the params for `prepare_online_models` + model_kwargs: the params for `end_train` signal_kwargs: the params for `prepare_signals` """ + last_models = {} + signals_time = D.calendar()[0] + need_prepare = False for cur_time, strategy_models in self.history.items(): self.cur_time = cur_time + for strategy, models in strategy_models.items(): - models = self.trainer.end_train(models, experiment_name=strategy.name_id) - # NOTE: Assumption: the predictions of online models need less than next cur_time, or this method will work in a wrong way. - self.prepare_signals(**signal_kwargs) + # only new online models need to prepare + if last_models.setdefault(strategy, set()) != set(models): + models = self.trainer.end_train(models, experiment_name=strategy.name_id, **model_kwargs) + strategy.tool.reset_online_tag(models) + need_prepare = True + last_models[strategy] = set(models) + + if need_prepare: + # NOTE: Assumption: the predictions of online models need less than next cur_time, or this method will work in a wrong way. + self.prepare_signals(**signal_kwargs) + if signals_time > cur_time: + self.logger.warn( + f"The signals have already parpred to {signals_time} by last preparation, but current time is only {cur_time}. This may be because the online models predict more than they should, which can cause signals to be contaminated by the offline models." + ) + need_prepare = False + signals_time = self.signals.index.get_level_values("datetime").max() diff --git a/qlib/workflow/recorder.py b/qlib/workflow/recorder.py index b9b2fd1b36..0c9abf7318 100644 --- a/qlib/workflow/recorder.py +++ b/qlib/workflow/recorder.py @@ -39,6 +39,9 @@ def __repr__(self): def __str__(self): return str(self.info) + def __hash__(self) -> int: + return hash(self.info["id"]) + @property def info(self): output = dict() @@ -232,6 +235,14 @@ def __repr__(self): client=self.client, ) + def __hash__(self) -> int: + return hash(self.info["id"]) + + def __eq__(self, o: object) -> bool: + if isinstance(o, MLflowRecorder): + return self.info["id"] == o.info["id"] + return False + @property def uri(self): return self._uri From 8c3a08b18da81b49e56d9e158fbc5572c54a5b64 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Mon, 17 May 2021 07:27:55 +0000 Subject: [PATCH 61/61] Finally! --- docs/_static/img/online_serving.png | Bin 0 -> 450088 bytes docs/component/online.rst | 5 +++++ qlib/workflow/online/manager.py | 24 +++++++++++++++++++----- 3 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 docs/_static/img/online_serving.png diff --git a/docs/_static/img/online_serving.png b/docs/_static/img/online_serving.png new file mode 100644 index 0000000000000000000000000000000000000000..8647ebe7736cf41b1716cb2c0491d866844a211e GIT binary patch literal 450088 zcmeFa2T)a6*DZR`R$2w=Hj4sniwa0k1eA7==Q8?AN9EO)YipQG8i;_L!O_g|hM$`5%?f&Xt=els_oqhxRL72)8}dVl~YCll5}VTRs?>f7rh#me#~% zk7TimIirB#>&W=4O24jo{`(=S-`AXV+1Pr%cx38cWoQA-i=t=LnZ0AR#eus*OGmP` zhXPENjyKtPv-wDD!>GQ0DFp|FQ-A*}{1mof5zl{qsVw0_CimSdOk&yF?|wmkP-3N` z|M8bDY}>>Z|M*L}h+hnU{2k?iH-+}cUyAvx3Zwe*m(dFUGsuNC`Ja_sSP2;K|9@>V zCOw?`kB*)m=`>AFI~oc_`j7Z+QO}OiN$f=JkDVy^#0%ewcymO&B)9;F%p5pxLQU8$3$I1Wh0{6mo3fx|KN5% zn~L@5Okd{|hnV!c|HN^M9L@hk`~}mBR)~;@P++zD+qk;C+$%mlK0KE~`MAXN{iBl` z`Am<$JKe2rSGB{(;+-OmW__=COnf`*(vts6dzdmdTV3xx{BYBF)0hx{OruGAYs{iW zJc3kY`%?D)u!$%9`o|NzdUq_Hy+9RkMn!Sn3l^sN_Qq1tk^R|hZJwmC<-^U=iE5_FuF@O^%QhxKRVO#zKcqKUT8Rw7i z^~58BK3%jbPAvZ6{l5}StNcLeM}xnznpl{%yw83S?H}xZJ~F>Q{9_uX@g-pPk;wxZ zZZX9lG3>ngS<7Fhd3)&pTU%aM z=2=x$#V}jvwOzWvO?x<;VKy@(KqA7kGFH_f-*a0>Z*N1lotBo?cIg1`b4_m|B9A0J z3=nD@>$|i2w8b+j>igN*PSmnX`=zojJ7kL1et)N53|)HkUteF#Kh>RN*%g=C+ut_( zd40|uS?9@e!D(9?-WCh~y$u&mi;Sg}>J6AUv1qJ=KrQX{n^kQWmKgB?+u!XQ%7zQF%wlnOmXD z1T#uXO4v1@o;~(?4=*pTk(pUZZ$@83+el<&WJN^<&7>`5X#{s3MOys7Hpo)+{_*Ld z`aY+Wd{eNW0Bwg{k_k;XJoMauqH0g zN#t;0xT_FzKPhPsOiv>-Av;@ymX?;)CuQ1o7{=-D?tb3JX3So8VRD@zRt>(48f-pg zZ{OZJ`O-B)i7f)3Pfgt$$akh7({^YuS9EqawlKdeLQ+vt(d*z!e&+H>=|U5osSlga zzuhleI_ofGE(<fAXp+HQG@5~49!*1(ZxSS^>*Kumz zdF{5|pe9jIwsEh1(VfiZZCv?(EnCg4SKyu5t;bcEJ=GiN>gtM(8=jh$?McU)VHTVC zOy7StNyGx2IdeuW$H~#ycy?~7*1>}Z>FMc1tjyCiWsV*_3Ns5esI07vc>H)kt<5wp zDw|0|LnG91$3Z95FQ#7_5^Ouvs#g-kFYf6$TC;FT_++?-E``2F{t`@HV*s<^ng2|wovKTXSG-kAo=_PbYGH#LqQaTl3vHwdn5Xb35F z8cmvb=i&6E=JXK{#$YaoPb>I)o|G%-=!Ei`R5d%#wmDnmZ}V$z4K#1)o?WtHU9)P= zq_StTw7H#P!&Jjq>)%`5^P=VNSzq|ADJCy>dSb|Oq*Ar5Op=Ri?qrtpz~|Scla6pY zchTAL%&GdmlG{fa{db>!Bh=USwx-6v&#}**zb|tr5r$e^GWC&K^iHWm>zdi185)l*> z9D25xt~tFczL5M{X#C^lnTCuD1E1?`+k#6c-@iM`6kIMN$dTYQ61&G~dc2~l%53zh zMOx6#_@{q{PSv%AtR3r+m6Vj6?FgMUO7d_XaMLM@5}oNVa5O~@nCQPM8l+*Km0X%6 zBQ)?z^TMYqh{N4OB7KV3ktDPF^rBEvQPH`H(xHsTnfS)E(1}Sy1Qjg%o6Mdw=7l#l z?n$iHwjW9Jn0=^|BVuk_sKHlzZZI{h==FM?pz5q%D-c|Se-n$^GpQ|tjT17xm`m{` z5x4bRCB=IigVd7rTV8WHookpVv1x7gk}Y{o&D6~#F8AyAvBQO-<;I?pyfdlwzAx!S zg_vgAkM24B=0^C@MXrAn`#~xgN=eP)7-}spw5rxhRa5#?9LZ&m{b={}XlhO|YgCYD zYip}WX=itLQF66*;3o6T$Bk_!me_1R;mpO!SeNNRznmh3fHI_63BB5I3IhW}+UN%p%eLSz^H4Jg z9?9Gi7FStSZmSrcVA8$fly!Mzh?)@N3&Bo zbAleQNfptQtSsT2$;yQJu$bUH(`8w}8r7_vU{}d^*f2ykO|t|sZ?K7JZXfwzRlHWl z>#3bS626>T0YfE%Tu9%cgNRT3SDBnHbaqZxF1L<8GoAFE7U*&@qa!jf@zm?z%H{!l7cjm#Q45yk8<6v~PD*sFU4FyvwOiEz8czX`)c4b#|(+ zZ^+Kv+`KofEo2%7D1|ta)3({Z`!WASvsd~1_xr-sn6M!Z9g;c)#HHh|mohuAIK{4$ zErh+I8>~!l;+2KDx`f#Ok)EDD_>yUM7nWU{@xsT8XWUDL%8_yhy+Ws%8)rvThu)v| zptVauOqp)pA(~&EJ(^r+)96^bDbyBlAs^Nhw1V^LAF7(?UaK6{h0R#bj8wO|=}e!R z#TL`Bhzu3gU=?-R8fSYN6VG>)QH3f#frFXEr)>|#C5Lb$-N4U#N}pKwe85H4PQ2P7 zLQE&S^j3u!2LJ=|Q1xJW=4Vs;Yjf*@He4ZHQ^zSVzYO#TX@vv3i@a zjdx;{*woT2^pMtkCWhL^ri2tDrSh+ePJiltoXfa4z_O*lhs3DCsm3`GPYb>H*JE&(P~68!MLxkJV|~)LUuL zNS?{p!4^#qM~M~#FFvl%X>&24?;Z>MUQvD~R&QA$dmTPoHHQ{=L6XN$h9vB;Phqr8&MK&!-XMM~LyBnXbGg)y;fXYOm9n;k2! zt<9)$6c!dv8!XsqiGf)T9OkSdt9_N4!8IAmY6$F)skEb(`mtt zbn`tO;H@&a0Lj8sbDM1k^LGqkr-bQ5r~AWNJKEdxVAD~ENPBV_Y%~#psjV0vCI0|b zukS)udZ?Mky;J339)DYq7sB%HylH34-M|P5kg+Rj8P;b? z@YXHDLw8QQ=O13L<(==fW6Zv^{oTDGM8&;H2BjrPoYif@!NI{K-+TLI4@b!6%94np zY1a{%RDH4LBhDP8Q zgeWwaw>7ORJ|oe%@_|#S4&ognqS$7K{^v2VwJD;BxD6wM95Qcn-fjNwM=!Z;2b*b_ zM1)`j#9_^HW=0<#e|*ZVt=?g%m}G>cMyII}+pMUl9k6)S*)iw2)KW3|7^MPj`yL;R z&7kbQxRI%;)L`nd!ml~@%s^KSG7bTm(f8cx`@2kzc0ytX0&p6>$D{|asgY^(;;&`= zozeFa+d}O1p;WQ@71|EJJ8FsWLC`3x-_-1ihOHh#o^#_Kb9s{5gPxnuz50D|KvGtg zV^UVm_-n3t*aJ`810dIdj0zaeZBP>QBsF!+zKR%Jsqk2H`#q16{!cHddFUS9WCQQ* z{&PRqw^7a|xY$5+s%AcfkUZGX&_Lp6M@L6~l*mZT^i1QNV`o>F8{_Zq!QI3vS$!n< zk#8*DY&2*l;j^}UW~?{*FbuzDI%*TiRHI{SXP0 zT#g-*NwsDfw04ngAvqQ;1w|pkf&_?n+eu4H!!&%8N(6QuWpX?Z2#^r9my-GGT3Vni6iVq#+Q337Df%H9VTBBY35V|{>m$C5{- zB2&%6eYUbIMaDlGWF4Us9)9a2+IJCgtCS$8TJxN&Mi(h~7XiURcAYW&)BP{^STyGW ze2UCiO?2tZ1r&1S2vuuZO6Pp?Lk?<7EaWVFVpP7Aq1Dznq+C{1m~2ko?X85bg@lB# zt3P4!)3R1Wt`8mAboav(BSq|?_volK75)2@OE-Qm^_>uNhpQ2joM_#f<4;f;m=FP9 zB-gi`W|}(QFdVrz0<&!2eK8IkL@DCKhYy*EdNS|>PqU%o+1cX#>B(~sXd`Ne@AFt_YJGAsuqw8u*cG{&wnC~$(9KAWoF_H z4pr>Q$;mbLeOZ<-mhB+KjNr^E+ErH8*7`{DaO;@3xcctN0-v3r^g-a&=GJkMk=qS| ztyCIjdTpX&Vmj0^qGMwV5tnYm0XRt{N@6kXW7*fnfKP6J;-NP_r7hU-ny0}D#${jL};i; zDrnz;njl=-0zgE+z?+pAEoWx`3z36-Shsmr^(dolvrCgKcr*XNtJOm_rA~Za zJ7k@zga{?PdzCSO4z!yPU!uZ@+>;egv>WM!;=aYcC%M|Cm6B2kjqftuo9}AKO0j-9 z(=GgS(?gb!wweL@_mh|pV-_|DvDGJJ*u$~=^kqAbU5}bf8*M@tcfUfA@Cb66?BH*` zb9$2II&(f&V;+(qZkmN&^kX(MHZGtR>Ay%)jFr{F@)2WG)1n12}%;G$_&sXX(>EainiOt|Ih#*11cEYcjVW1$fn27!%>9A* z$U!<`o*lvG6ZUmh#6Q5Zsv^4zg-7Pyp9V}$_efo>=Uh;ROqoMI8 z4+8`fe*r7wBR4h-WVYSdu=|u-hKad3h!cWWRvPvjI4BgplOfYaC=*Da#|Wc*&OY@&dv|p zK5ze;7f>LsZyUiHHbaKc8LhWzj1w?__4n`mE#8WR8<6Dg*kD=6k=p#}DpLuzVJXkh zvK>LUFb?wpD+M+m;@+V2+dV`k4MWF>Pk|uNzomIbQDVC+G%tJqo54>!GB!3IEV0Sn zgD^xqYmZHD-SF^m^Jh1%lG7f-z8-)a{va5)vY9q*P)7<^#z2oFhE_Lg_1USCT|DD7 ztwAx+*%w6j7)$x5Q?jiSP}=DPeJ+b zeoz>mBq!@VzqBN6vNM)=PtRZwuoL3(w2ET`5st|S5?Xwf*nGSTTd=41V`F!6aBz?; zIXOe$%ZBzMLi0qnipb6*S*-LK0UHDle-TGJv-FNgQUOis!b~NQas>nh%j@bgr49D- z^6CM6YOH+~6B}FGI&`r|y#KpyhZ6?!5bN%_3$T$1kO=57meD8;^a6iLnHOOQA8?+| zb(V5;bbNPjQRx=3#cTAt-#?z{waE$e-(#>&;3$wDc6UC;#Hz#<)TLRbvK<(Dvp0Kp zREQn%%-#2g2x~_Cc)=&E4;QI{BrtW;k7&BGA``kG2)t{yn)yU6+6g}_@{$$tCsmWm z2g)Qp5?VX`iOx9>qF_oF4YXu?Wb{;Jd4Ir+8;rsNH*emo)$;{`PDx22d|EBlOpBl| zc8wHvqEwSS&90SxK0~O_y{f)Gc$=i}u)X7O`#b*TTL)9vMhB&k{A!%%X7*sGMHce_ zqp4*(*g`@~;s1>ooGw;c!^+&!(vsw;GXhNSLzZp|t4T3LqWR{fp#pNUZ1wi$z{Y8P zf=nIS1RL{-YZ5sWg4!``4)PbBF4iw5W`L3Fy&7zAI@B+#ThA8n^=~QkYeVR1{G5dp zT;gvK;tx1&pVtNvu%Y$W0^r${4R4Pp=}W`SdEUPZQH5-*s;Ww&R+?p-KJ-$x6jL>V zCy3~xmTX*U9s0?iq!g%e2WAJ=P>w<2JeSd)og_nCJP@}u=qQUYGBP6h)@EEv@Iabo z!>Qwj718n=fgAZkHGmp#Z#@d49#h%{5QW>I_dEE9H8S>k`ROePOsqSDpz-whfO?Yt&Fj~%4`ca_Q2VeO ztuf#|-!7*Xr;?L(bghf-s11b@&4o8dLV-FAA(_?2bpQT~;G6tN@v(1Ka3v-tdItt- z#PExRCKQFQzWQ=YY;vn$E-w1yT+@+y0P{&;qPq_tKBVUg-5pp=`5iFL$$4c8A$I7|ZV4h9dxFw$M4`0^@UHTe78zyJPQA$KV?b>XSUN_)-X z5Ly&1ED~pD`kV`UtW_^)U%2q}L#lZK&{hY`Ug^}SQ!%G|tTVc;?WJGE@$v;qKYsi; zBBZpWMCHhlBN5fSGzXU~S;B_bKxU<}@}WbAE2U(Z?!JNwX`V=fXZ%gV~Co?_}BBx7);ZSA^s#xs*6 zomFvxy-%A~@^5yh&-e26jaI?_G1-n&phYUibx$9oYl+LQ7>@b$#`n7BF_m14G!2X~yP&$A|JlCCJ=eg`k zW1mX(oVoTCxQ4qs=LjT)wh`1aRqeD=YvU>^lv?yP4Gqoz1X9djMJ2*HlwZTrZtt7Y{7_cHvW3O}WFPwrSQ(D~= zxk8$kZxM5ct8-?R4N61ehNquA;mI1Q(g5o4vV(JY#Y~#(aQALF84^PkC8yp&v6k=MnDx4Stj7p`@mjl}n`Rth({Lf!1 z@^c#-7gxd&%Zbb79TgvvO(Z$q!GY-aoei5s#UNTM!+QCO6^e3lx3Ky}`j!Khw?jg- z<4p2SJ!bjy&jTv4%8VXS1zk#4^L5LBv{;jB*h5Rll7`@Yf;PQ?Gzwy3Vppn*OG~RU z^YgRQ6RS$8mn@0F1}4__D3`1ZqvC#`fr9)$ujn(d8rY*nM*Xb2!j_8{FD|E{$!r&4 zOsI~{%*3U*KgkV zrn#54?Ht?gllb10@9@#1h5AVoaV|$ZSFKuAarYo~PeWG5t05hSfh#K+g`!t72q=vo z2|RdprP0)APlWc)-Me=i*QX~xRGBqitf~EAqK$g*)V~2)EKbmV^dZVMRB>uaZyM#< zbRlFyNQzpsYL$XRcXPfMz0ie6txM8BB6rLE0x z-}dd>kI;#XhdBXc`lMCmGC4m4&MWLVUv(@c%YLjA>{v-&Uf#b6TVibdV0OU6*;`<{ zkdP(}Fz&Xds82wEx_*arX^3zkM>7JXgtWBs$&)9gb7K{W8$n~15N?c+T+d7D>%5$6 z{tOm%&W>!VOoO(9dnvBakM2)D?zN z31(qs6}N$uc{p_4h%*92C*oZ-h_+xHI>pf z<%m`}JDF~nn@Rgr?ks8I;JNMT%9r1F1AQ?giB+pt-vf1u1_2G=w~VyzYly3>)67Cu zN5Hc623S*MRh-6@;P$iP*a4SDU0q#6AP0+mo0Ah~VR#ie&U33u`GaM)?bu<0(sP8b z`V-?y2}0AZ2M%tRx)TPGKmjpto4e@Qk1i0I!okU6^$LBt{iP}ka_q;jzp{8D+&@nXO?c)>Svs0CgpI@z}re<6G@4x*PCgM2BA|jHlt|qZ+?b>aKH221D z&aBr!4nuy4Mb$wDA&EP%n49LH>q*3lE)+}DADy_^;5_Guh4W39g&k*cg&|FP18Bgt z2c|(^FkChO*fW?CKJ4kl{%PQXPytdfGIzs;CR$>z6^?agsdZ*HK9;lnFX<*hv)u~M za!#w?k&0AsyY%3JOAOzrTMvWHMm(dv4c#g%~}!_v}fw zXp!L1d@6^(ZR9-MLz>8VRo1f4Gr+kcq{fbsB=BL%x?A|fJ;&<^(@QBN}F3iEa^ zaR!0&Z_(iJ@a@~T#(J#cXf~?Iw6zT0cP{+Z5b;Lzt3{*YqzvSth-8dWVueOPHB7)V z^?c(rKU$bvS04^aiGptBl80g^k@aZi@sNxT!uAz0* zeKV^KlSi&%C+|*iWWl?4@2+2)Hm!v8pbfWOyLN31j8DLMHq*MdE(W&P1uga95=Nuf zco9@qF*`FQ>0s)5vVFLt+^*H%;E7qiKs|a*B#s@sRd42pAc1#`pyDeH7|S-YJ5Xvn zv=DI*;nCx9noSh?MP9>6*{PjtpZy!{|J}anm&VBtbaI>s^uo=f*6vhc*|Fm&D0ao) zzi;KS>D8=1UoO3zj&96G>7(aCDq5Gq>(yPAnB9{L1E(;0pM7smJ#K%q$-Dwf%N|&) z;vmGsqXdvqLEZz&dCt%L$-DO7h89n%Vmt5`xJ{XC*?cRYdC*SJzHo z4KsNlD>upW#tH`xTvA7I*_i4kGa$JkI%CvPNA(%Y&&!kBzkk0^TlE!yT`L5T^tek$aWaE*Q$`9p_Z z9nLM*PXhYC2SrKo-Me=YC6Ild67v8aQDrp%(*4}j6w5Ofg&MVvAqKsimNsrweH(Jv zwfLOb$xvZgpS8elq&2W&2uK6*BEE_-nd6T?#IW3PqdH297cZ^^-cfKawy?19zI|K% z)CBC*maOyUl}95+$qqGJXY3446+&2|-=lxyi2G)>CX=gIu5^H4y+famN=NRBZj;qI14c}Cnz4iOa+Hmo zo&B5?9L)*E^anr+h`^$POo*&o4!Rmbdf^U~oZ}cqMeHq92cN*eoiBH>`Rw83lp!54 z6&W_>YV5ue#Xe5#TRIwZazdOYjkYl}dynrkz64rH0urv1XO^mJbo1}Oe}(kwi1g=+ z{35y8^7E@@hz8LD@?Eb8|Abzo0g#{^(06tpkLmke#Ua8Pj#Hx?=ZYYiJaL|LO4E#A zRT@~VpKRW!9V}=QU(qlzF)?^S$frb;(JTKx3Rp&vDaooGICwDDQ8++qJOAdD(!m1C zBuoL=#c6#neJm_qf9Rt}yXw&&MaZxKUs%3!CFeQNR)yoo@1Tm10E_oc56x|QM~ESb zuV8HtlvLbahXY=dJcb%Zw9;L1P(W;SjGwlhcpS>0Nz75{Qro=%DS3>_m}gLRe{g=8 zbXQdzuT`fKVLpIa%7`-(oBc3M=w1&S=32P8xfRA5opYeVaRQgB8kJV4I5Zq?jr2dae5$k zTZy!|xIBO^cEot-lad>*qp2Aba^dr`ZQHgPL6!TEa|O z@w7+kaobAjlMFmYdupT5ei;6W7lCFe&4w_Dt`ZO#DtqfQS}f8Eip-ML3mFu(wrtr# zFKqV&={g4WhGVD=ye$b9gh6Wc*15P)v`{Xh-*v*pENet1*{D3StWoyc)$jNC`0)fS zlZYk|Y)qL$XlWGLZg3$s8&gBWXwqu-0JT%)+5UK{#Y^+_leEsW1Gp2*!Ch2a>%zkR zus@Ch)o7@s$5-*Q<%*=}17P!}31t=J7_-1+|Tz0aRNUpF_*10zL0?#N_Ug5SQVGqowmL)X%6`Z!st5CY(djS<{thb(sBPr&M@q}!V3kU=gUJuc}B1)EN zq_Tt#Pa*fVG>x`@mPPUCX(t&)+)$v+uH*7er@9ud)h>=0mNwd*$ESQ*W zeTIz*$ee)VF47X&|Yk%sRS_IJr^jslF^5 z*1a*bexH*1D6Y_jKKgATbeIQ1tdn)zqLGWm`mt}Vz7cB096wv65xfKVpH=sGU|_&* zcA|~fVf?IGoWqf=3l_=S6V*3Rhbh5F)Y>)@fQXU^(F_Xm@@_E`(T~J!^o{OcRvhtX z)8c%sn<1C()ZTYF!t2+@Ek-3REJj6p);30ps(1-#L5l^esHhGolSP^)@*kuQh!{#| z&pvvnViMuTZ~Wm(kW7@~7nK7xB|;BU%p%kO5uwa z2R6EHfwqD$2xmkN@-J>xx96mJd8@WZ#mUKOAmQrp>1)MTOmaV+9dG%to95oQ9VqD07FYzBvpYwgp+}>q@vx-V<5#_LwKcAY>PF={vjj z1G;(P3iW2yB=!Lf#lnyikFz`FyV4(^k`8XF&DI`>9z7aff4NyQ{-mx5Amf|a`!dEQW<2N<(GrCA|xqfcn=Jbiyz_ke0 zKpkC89-=DHgjM3yBQ8_Zb(T8MCM~67d9c^k;B#B5Nl*PFaJ=JLmq3$>m<+XpJP5GzVN(z^c=m z1OmcqcE|+DA(Sx_dxw81!9^=^I|=6(u_rfRyaE=@e-i-;fv6ps3U&lrAW^p>-O7Xv zf7!BS&;X)uUOj#)J}782>0tekWuFQ;LO!U_IGK%)PX!t$5e=d0MU9VNz+OP_ljuvf z>Qb43zVh(O(S>0~vw}wfCzPReXbo1J-?Ehj_Dj8T^INbJf?UAeSp)<=G|FGJ^H_w+ z0?MsAh!ZBYDQAF}X5GKp#RxJ@0{#$ePwTn6K`w>}Ka9%Q5rl@K$1B#czeRY_X}trg zO@w3Pci~s=GZm^Lh$0pc9>k%Ot%*8-JQfIkqRHupKQSYVTVckxUVf5uaL7c*-$~ky z+=qax!Kb&aOWfeP#UuA#ceg3#Aps35ekGI0LrBQr|cKww}j0KmLv59t(AvK!OT6-`fF$jQm-Eob*3hexAb%CP5Clr);3}u@54|X8{*pduqc=$Y8^d&L1cj)NcK@y05 zQVm&Dw}nO3d}=#zY5(P=6TAQQP?vxzBopVO(Z54hjgkp2L;pzOWVhAAKYis(J3w{AnMz&=-iO#Bs!f6f;7HF5kjPdg|!He0VWv+A2<5=>>@Hj zhGAxDTRA+9$fZO&K%!!?I`eW@^LYWVeqsV7!+?n>kPqT0SvxYPs-dA_TIqgUh<4W2 zDcR$nS7c2O7D9JEJuoCspIXa-|DwuS#Yvc-|0`OWq1k)m%t#!P9Sh)0-Ao{_$(yeC z^NvKJ$X#l0Hzer@t2XOyaU0lybQK%dro;h_M*x{h@$>WB;T+ug;dXi9p`tzMfTz<8 ztD{p{zBI2@tU&un&NRblR;1g zvGH{q9TMD?73+=B)FptVaWg?jO*=uAs0ajagVCeF6IWm&u5jdttEZP&^2wJ=30kS{ za~czb0P4+e0NbD!IB%Fdx^R}Rf+M!_0U@OgnYICY%|jY&+ZZMsM9ws1s-fZU76~&t z=*)yJH0!>+LQo&@^@o>SB9SlA#&!?4gs>>S(N(@Bnbga$UnVdpgcc+bGD?KW_-Y;J zEQyvqdT~0+#L(ctBb~7cDj5!~i-6ro{zrNC9TqhT=bem&{l4DKWil3K5g=@zf)aaF zfB#wWGQign&}$uzp}U=5K{{^6vzf6y+AH)R&ZfA2eYqGRLJCG;&{i*l8>4RyS|dyrn#DKi_gq_NL? z@9jHxO6KS1gYZ_Ndj^8ekc8eZU+>F9gE^O@xiJn%qU*|f?YF?(=iA?I5k@Dk0OUWo zU?c(_iYlG7+8o~)!u$m~{mF9H7=*^X8}wfR%yJ(&%>-&AueylkbCh15nAUKE4a&Hl zR5FA#Urv?*2G6o>TZBr10cT!uaV7Mm2n5M?FyrJDoLQWIJZj$4J9fn5hT~SMHCIVo@7CB0qWO-N=;EB)aA@g z<&e?<`XS`i;_6fu;)&m%Ni`2aFToroaALI#`Vh#%Fq$CSFH$qUMeiZciDwjGlL9?L>X(<&mVrrFeSURKA_1jGXw}X{BFM5EQ9wnCeVrR~e%DAR z>UqfovVdT9pyh_kCMz>jh(YLroJ>?w-J$@@!UPUP7-?FUdYnoKU3Kd!NypE{PtF3Y5L0Q1a%k7PK3V_T|us#H9z`ELhOfS61zB_oL!T2exR$chqr6ZvdzY^UGZY#S;V~B;?Gd$7hYIBbtIru5EjAoE;j73bG#258wgP=gyte zi#OWfsuHgm10hUT zAx6N)&dsROfBzB_<~#N3S0da%8|AVLD%LNj-CGxiQca>l*I4_zgC0VIe?x0>lRP7# z;fz?{iMVy8s<@~~Nk&G-pGXPAP}~4G{P;^u`MN+*-vPZ$h1uYsw-ev(BGc&&}`z9O2{0%4mSTsMm?vv9gbTKqZrVIW+X*<`P-1(tiED} zZ$=&#lK%Pn&ht=5*tTzvJkS)#V-!yEoUzBddq-n|F!g*(HGgAZAOTa=u>K1w^5xej zyb#DD&4lbWq&dhP(pN@(??Zwm!rFW_O4G?HTR}nLJj`Wz!a@u&9A8RHe;-XR+Nq6B z8K~Dce~lohC{>~hdWL~6TJgXA10xl@)tZW*EH;QI)|etS#P&}@=`Fvtn5!?qCgdSLa&qL$^0zqEKc}TY~0%L}s7y%r-GRZ&| zA5*#DjvGozNnOcB^+gNz7pD||-Mtiy0Ht9`;ZRhNz=*vwWI*oh#(Lx4zkk1c{eb3) zWcA7f9Z`@UKYmWcWfW_@3R@mjd^1Go38}R{ zl%nMr4D)^x8t=kTU^>cn+Go$6y^^bG9EP~uo>FV(#f(y)Bb=4=l!TU3CC3^ky!d7b=D&?5MHj^F znB);iK5LnnbUJK7r1dRFM5xQVM%aLJNxjqkOM44}b3UWUd7kxVTsE3W7vLKI4&h_j ziKH5ALKslyTBbcT_YPd?jNUK5SHd;5bBJR>* zd-8MK7sv%-Ro!D8uSv&2PZME9fJFobV@Ia1QxKbhB%f^ACX~0|TV)gmCL5 zWxX+6EV4xY+m49&PieYBErkj_lb8yAn5H>6;owx0!;CaNiGVZBu)uCV?m)v!H5xHV zg$wOK=q89kZw;w1oyQUDDM3cL*#GV%=Rp+t*SWpsvd_@u)H5(BY#JEXoAZPA%q=TJei+GEbqd zLm&w)ZR^8hyXIdm0*MBtjKXM^CYThG-VSgxkhgXs9Uwt$%c$?|`&%sd?oS2`>nAzv z^{D}yHD53HOwSNeCqT$HQDkn~(l$$5Tp?7%akxTJw!?LkS-+(~Snq%LEKJ6~SRW!# zzA?v~b?eio`-pD?EhUG<+K=@nXD|5PGiOj-QfU;%sS^otaaIv095gfpVIcIKshnyS z6}qnjS@_~2p!fJV;jVWV(L)siMvX;)vbwd<4s$jJki86Djw7hs5~0lVKE#swatS5@ zBGTLTm&=^of9xzw26>(|RKcR_9>amqR_t|h14JyVTMN3{*%)!u#R&F=Y`%8Qnq!{v zPd_gwWdw-6&R1r$=%0H3WHm}&=jO(u0YCscihmPUm^4)twpa`-@Mnrx#0J;95K+n0 z?I38ld8DFks#Iu*RINH-Uo-!8LKlJR`{Ju1KucqTc3NseGayv+=7!w|NVKkCM2QvC zt_0OXy>}<%b=KXxfdRPpXey;*8if=&XV(w<=VZl%G%V^d>& zq>Y4G9Jc{IssQO)vu>Tr>!S;#E1C`mstCd>4&qnDa6H#++!%9h^^W8m0*0e4;Wx37EfRO+w4{))!lccM~Af+IuQpM{ADF{J(-U0igG}sC3!I$*=fHENhqJ)3iwVJSBpf7-!1d|6**r|#o=@qUc-5aEWgD z2N?>AAXU@6?6>*m2}CZT!(G*~gpoBk3`vu6JwG4xIvhZdlaRRXO0$u)`^7-ZB6?YM zmogF%dDuu6PGJyT5#TCqYOhI^6ajA_8Gm}^D=E?~fKX0xgse$CxdVDwS0Br3(z;8? zA<|40&abqL*y$y!TQ%L>+u7o$qk>X^Bs&jvzmCJz`GuVR6P;e9_LGeh4|XGTGXE~G z@e#1u83roaL+8&wA@w+Pq>%nlhyr@26(1qVve8&;!Bzspf$hFO+bVp zoTyTL)7L9<@-jIN-)08>uXLlk&qs3*Euf^|0S`=xD=Lx_SJ2hHPl|)+u`0&4yIx*c z(?ds(c7Yy@*(}6LT+k&Hjzjt@h%|<}ocm^e1q1srz-6NQqj!Z%Km<)tP#o`tkPSP#& znDY_m_IH}@D|MRnVxs2p?H=&FHn;6xuc3sAoZYvXp3n3SsYO7-L%B1$YAh3r$_uL{ zIIe1J;q4UqLNU9T0M`IN)_!2Fn&~b!1Cz)S)*fVQaVHjdBn`ZX232fOI zGG`fne`)|lrDmv|e7;mNb*54<+X;7|_n%8+P06yanXg3|fF>NCZ9hC|%W|F<3osk~idGPd` z;kXpEtTwH{XewkWPQSu=KzDXoQjIB~&aAt49P88+g1k${X0gv zP+)m<>&<|NwzwywEBf!hp~BzFSF&W1_&jg{Z6}c46Jr_JacUYqC5G`}oZjT_(4Kydavtg-Bq9K0k3HB+XkDdMiNZU{#uS zUr_v8igJ|0f;oP34T>BSTHp@gQARqX^-xb=U&0fWZyqEWB4(jdPA(Gg&qY#Q-Y9Kl z{q_P2-&eMc*g;C#A{`>6#^gNx7&}v>pCn-29Zlrt{*TY+ZD^Th;=;8yK2TY9GoM-F zA2hRYC47B0hmQ0SJhG^r2sG$Z*d`7T5B0bTj%ol4E<~u7WctF%=^5|Qx9ss`30(*> z-&}|?S6Eb3fzzd=hXBk~#&ZoLV>~%JI*CW_M8A4B+>t!Y#uKH;mW4z4PO_`sSXA|pGo`4z~Pkf2TQ@qt;?r+U2;Uo%r_Jmo zNDSRio+xvp#|($YNW-61Z(U8}0A>1@FUH8ZXFDqMHoyLG0rKNhW@H-xY-J)>WCLWO ztq*OhIQ|z2aeZnxgNx@^8@jlAc1T${a?ar=HbgPLj$#8k*`MHC0BHsnKFKrEuy;Rm;i;Pu`o? z$mh#VzlN>(>L14+)C(V+_y!OCbGeE&Xo$f=gSU0ySrhCcPIxwp5uPbP+L90lm{u=h zq5JBt8xB^Q+}%CXwtyu6xd^_IV}RsKumEXC1@M#f#D__X8hV$%hP1FhzdA?%hDz43 z+q&pVm9UF8=GU-;UG7$u0B(?1~ili$b;H)=rDty7`HdHzJKb zq&dr7IxTK+q~*r8ES=IlE>nrA zC`PiR9yk(jlO=gkEbJ96)tbv+*0*(oBKuE+PgxqkPvwUXldTC{>Fs}#E@C^>-?d+; zP$I8?{cKnVd)UdJ?yywura}WquB0&@PhZg~!|7?c&)ouxpb0rN)J;|G2g8p@Ll z-~BmL6FTEBGLbK9jM7&`{N)MV{{5_bct)mFb#|u38pM|_diHfOZcI#`d2swcvIAM# z*Eor!%Dn0}Gly2XQfj?*pjqa?k+k~WReN7%ddZ4UysGFkZ#9T7T=a>>>Sz8lzdb<> zoV1tdBYStCza0=b_cL(4u}ubBc;q#{Sx&1GkUeui!-@6H?XIju-nwu9KeSZ!)JF)#hL9GvPE9ZR;9z_fJkvDpcdrLx0uwa9S;qx)#%MCT#bs zr3;$McXxHE-Y20o|C{T3gO~8ETJf_KfhY67{AwmNg$Cb!e&GlGiVS3nuHqj}d#)tozwxPP*rEU(iihehqg&)=lEm*R= z=E<1uQ>>zq)$86hZ!340$96@PB|cw(!HF=KqanQpAkD+;idG z-+x`l^1oPM^0sG-UjH=BeYwQ(u%8E%FMnOJ=6_*q^>!_wU}~ZwN6j%8twFC5++#Q8|KbPjLc<|ANZDW)c@(+GZ7I_bz=m~-rMT&i#S zQp&UK7hQKY$L-Sy8T28a7du%bRH{ZfL6yJVu6Y@kU%9GQCmTioPPs4Jx~+aElWl18XSP+y2=+!L)8&a-)~Oq+j~{ z|6j(5MMWNqp1pW6%|cn@@x{t1C!WU$P3iN`j9@(|k1qQwH-8-Sxn|{^jt5mNIk$vf z$@&d-OTUsguJCNBzDV4~dd)Ly?lnG)HD*MGU`yV0J4Hv(kI@cObB zpFU+Br?71PXY^~Loii>;&K}|pnzxhvd<_wwvKm=QvRJN?@_F%~N<7IeF4CTkIpr|6 zdE-xp3=LG~@R&oU2$6ZF+Smw@A@h`ZCi9#kQ!-B>=^=CG%-^-1 zr`qrFdw<_|eE0GG@$S8)b+3D^YdEj-Ixp{r9KSI7=OV9O;r|HL=(a)&qcd8X=VQSb zo!*$0TEeH*E>!DW-&cHK{t|w$jl02$b-=Os;99gaoG88L@9Gy>fouNb7?a_TF&#R_ z+`o=-c;5PDukflO{0VA)BTQjBrh|MTIeXdn`e=g<9RL;L&lgL`?Q4^HxH%T$Y(T0`n(E@1Nid@=aq^PS_Yv`)q7t(et&8@Xh7 zq|q&X0`CDgHcaMNcm=VihGNO*C)Pd@KhHv}^tz}PS`>JTTioKrn4?Elk}-;>e^_1& zCNxB(?0Q(1IDCwK!$>>ltlqM5OB{ztx3@9IQ~$zYkG#0PeU8ZTW}E6t^2O1!4vys{ zKHaJIu>G;M>sP%rA1Ka!bqm)ht-ak&xtm3j){p;#B=oz!hmY?{@T@$aOqw6Yx=%Kd zr`NWn8e`@@FZXBOpWp9VO55w*9Y96YrO`SAz80aDKjBV-Id2v|N zQkii`%#<^p7Ms1u;x!}CI-gUJL5oTFGjYg2EWK9gnvaJ`a2s4!W#7>>a&6aQjuG@Z zFmDaOSWSu{yESR$L@3bZe*%leDl+jTgrwDc?hR)^0A1X!62>zL%V+t#fugW2<*<96 zqkwqZAD9;+2kk7?65N%q44QP18l4m^`uICWkv$?$%IKI{tJM~Jk?+Ed(7DpnfxmIz z!&-hXBzEEu7TPHcxmJ0v_D@WL2~JMy%c_d{B-jRuS!cy^+a7OKv@XvKbl4}4poJO# z^mOcwqz*R|Q)j=hhmPrXO6eOO&B&d%n z`d_t+KH4HRT*^i>8-tS~tVsXAS0oWqyP!lAvEyr4#$J?M9mIC=n+uKR0gT`HvTCh6 zZjiOroZnrx!DW9Cy;j@PLW;$R(jjL>#`pl1q3UY}$v);LL>Ol8JlC)m>fG@0RW0M^ zZXNemnXk>&xqczOl;sN3((v1us0zoaAdv!=7sj#pTqOSuPlU{xS^9i5KOf$2%2L3c z;@-Fj*NgW@j)TF3kod^p*u+^K%HHxs%dMHc+PH`-TpHv$jA4FpSMW=odX_}55bGD* zB3MCH{)evX{s%DJhe*d_7~)=waHg^z=sR0y zb^adYP;19Ymeo?A+_3`CC{wYmRE>0 z;wPI`dFfV({`Cv0`bIWfUgZE!LrT8DRW$QzpSt+~lDdh1ogaaMc=kd$4{dak8KW(j}f99G8 z3l(Rpws2>1Cz%3k1XVHmAg2?DV3TzXB6*80zO8i6vHgNYznYsjv(MuePfU2Ziearj zle#ACNAhwihxO(RyG^-ao<8jcTLkqh_zjg|IgarkjaoTu%G=VVT{5F(UM|^R3j#=rtXT=W8MD175Rv z>?Y-UjC~p4vYOih20!584{nN|>*VfG(dFuObDR-yu*(pCBINTUY{7WE{G46;jBUU$ zvNhHDlItoHI?m*5+O-;wGHr667^_0$qsG`Lw4DMU{6MZm^=r0KCZGPT_9fe$bSYtv z2k2TCv{34DS$>zD)kA?z>3jFH7>3(5%MI2Md4+8_p7VZZwo?i-4&XodiS(g#tKBf7 z3oCvOiz-PD{n-wL`$PNl{?7uw&x;XsX6-7E?=y?*lh^GP=hyEi?*i{yTcSF1MF~xq=LkgY?EBpWxy(YUhYlL z9%rSJk2m7)-2EqHw`2GNu>^~X<#(kkYU8p2RY`9?_~SPqtIE_rCQiCl+juo!AAD{^ z;LlHw-_jivImsUT`RVumX`od~pN?HtQP=enj`wBl+O)yHEH3ZkUmiH`$AxgT+pVG4 zXB^n7n`YHoiCe{L8F!|`_IZuV= z?zI5{sf~PclF6Nah*PF@XJlq(3N4kxUl_BnR<|zg5wxk?kYB!!};;Y(-$2zU2Ay-R24(lt$P9)t19GzhU&K37#RzvBMxlXmjjg#@8zhhvOocKCt_A8~peUtSY4%ofAR# z*(*G{%sB_Mz9!PMeaP34w@i^cc3_~M`wTI^PKHE-@!KF#!{GO9U7JUVvbaM17l|-d znh&>W$a8Cz8g96s>u!^c3OKV9m)Wst=JdK1eLHxV5C6QRaM)0WrT%VfdmDyma!~cV3?)IyACu8{yYpIgId8ZG?k~;1eNlP;BUpb5M-2J#G>@@>vow9LcJ3eAY zMg(Ill`w9-IXn6wlGo;zq>DYX2)`aaW~ds+v6_`-CYXcygh1N3{Dh~Hm5&zrefNkn zan`Notj-bzVl|YQy6MJgQWh15$dC05M7SebmeD^VhiQB)gMuu4{^@zrPbzr`T24>; zUhmv+!-c8w@$m2*1&;<(60vJr9j#W5>^6~xrTR>9ik!Un@r$_X&l#4dA7J#pt&x0j ztGFw;IZyjH_$yyJ?^0gy8Rii#$Ab{gP+wX0DUd3LqB)Pas}vXLOx!3_~)JK#z6`*mbLqZmw@IlcoG z<7O5Yxn|r8)MYCj7UNRz@!Dt$g^3C-szd!qdklCmW#Q3F$YM)AtY}r~tdir=4j;U} zsO(4YqHZi^b`8L=)DAV5n%5k|>^8YiU8Ls@M_|<5S-VYP72qsB0;z!P>f#7`P zbptZ{+~h;<2J?lTet^|4!=^?SB~xP(BuLdH#<#M*Eca>|^A?Z>ocSKt_VLBBcC>zs zay^*IHn(31CPN?Ld{EfML%k&n#OK4Q(VKoZz-5V@bx4296Ct-|`u$xHAvhhcXeU;Q zHK66Q@vGW$Gt!SO(j~@tQhi$P&oJMVvxa8tqR8tJQo@V)FhCvWpBEue)g}C*!z5v? zZ;51Oj$cu($Sf!CPVAhn?dHG zJP2iAC}~Z21Gu1{CZj=AD(V&mH9rk=l&XEo`B-lp7Q1w%Ip3Xyp>N}rQm2{)3M5tE zc5S>HC)Su$I>g01x|@G20EJ0=6*3g*W51ly&H&zVy~SnS)@Z=;#E3z3v3-JJJH9B9 z!c!@I^jiec9WUNXU8!$AEn1lT2EWy*`;?GqSa^Bc*Sl{k=u|jq(F>#XvZD2FK0@n7 z8RI9uZvYgVK^Jj?xp1tinb|D60CKM1R7o8a>lqe9({|Wf&Ba7AZ}z+KzXIpks#|_# zFY_3Y5!{w|96xx@GZ7xaS6fS(r?U#$QVnj9&rZzDV5}-0{!jOmaJOh@&9bx&X9MnO z@Bhm^c_FhKb6CfDngP0Js}knGL%~0!Xgol?tWQCkly_b#b{B^ z;N{HG??%m5n#RhVYGJ1xbvCt!P%aHEv&xAT?aRMKXd2rz>qs8oyek56P@k&Aa;jEZfz56luWoAJ_6Oiv9UP zy+?SlY6S>tg%5*Di;G#J+vS}D!9RkQ`rC;mI|r{D+`{93$8)GLLQf;nWwno zQ2@d2+aqyU)&j+*-|)S1 zJW8e~{Oii*PonTg{U!kau-;TydrCKxF4h(r1E}ff`k;Y=vC88zT|e}%h2Fd%FH{1; zLdCCMeFd&j%+jbOx14fvax%1j$H%#?YZRB3W>r>dMaoJ6r5CimQ^UD*bAj7;AKJBm z<5kpkKKSO>&8hf)RF(u1V0vw>F0l2GXu|O5I{zIA7$9Vd3Tuk%Sl+AxCy5jlr$GTH z2lY+{2cNhljN2#y4Y)TJU;(Am?vLENRJAe8W7B-gCi2U}fOWC6x|z~rqWQw2YpM+GE~#nvcc;PxmtKV`C`8ft)Ag7vl4a`bU5So@ zcnmxG3fn*{uFP+z-vt6%S?Wz46zP>J^18(kG!3Mf4|)JpxsT1patZGL1n>jhD4^nJG+<`Rq z1Se=(K+TOX&!~m_sq%tZR4SO%S0G`<@I0+Xu>g7w;nuV{PPvNZT|buyI&2y+v?2nu z+)seuGE1yHTSO};(pN_!0zH(;f#%Zt@nbg75lvBv$J8`p`eZ*GJ!fNG!0Q0P4ie{sO9g=TX95G+Ib-PFZ zhgOH1JWQlfLh!vaml^t-bqg9x+%Xt4d51{WsIeT!{lyCqWAk@zhkkCN3m2)9^?nl3 za{!YjuG>DWQZiTVN4K@~QmDJl`Xv2UHv17k9u`8!}~R+|*LwbBJ-m7%w6 z<)R=;%qwVT9q3{Zl{jx+)>Xicn0s}Lvt9M^| zaax9TOre=_;PQi9TtT_2TW5J$HPw_gQWnpjKqHdwLo?bINHJ%5p(9Z7T=0wa{&Um;_o3VOqZS{vo6T*Rv!9o~ z|2^e*{T$gI)eBQJ2`}(WXKBqE2hx>-o}$V?dhuMx%ek_QHeFL70|~iPsych>!oq_Q zn3u6tcSI~<5j2u#W};jL!QJYK0bFfh9l-(KvGFjUt>X^v3C6RFxZgrGVdl(iLZtGS z-bFUD*%$!zFl`ucxUX9+L%VhFMYdn4xb$&}7oR#}OTG&`H4tgv;)?IATgYq2=hP)T zQchWYfR;txmJHlg!mgi6`pE?tR3TtBzpGd%mhJf+3`spH3c+NhSs$SWmKyjgnDoV8 z)IPXJkO(S^(FHZZ_2eHf3^nJ^LSl@1dY?iKa5x-r2Ya8cqk90nhERIU})G(N`wxpbGbAmooGa%8-OvLs44#2E|BQroF=KK3yV z77g24QRR$4eW*w>ZFUicU|bRuRPWi9>js5LA6w`V!;5Tz^44t`M{qTh!VdYjjM&8f zqOf8Zblm5ami*zo{ZbZ#32>;6;?6(=%G>pfzf;~xg6!s&w~_s;joT>`?9@j*y8~L) zk|8KC6m)|yxI>;{JB|=)Y3>s>kC(JD+*oH8`|d&mv8W%IOG$Z!6eCVvXZ3YfWuu4ArFwUgQ+=R(eTNh?rz>E#}4S7OAVcQc9=f&um{I6 zu?xrMpLx>-d)rGl1i}Ag8We6eC`g~L)Txw=3qMyIcCO=Qb3@AB1x(u^h!GPZMl97Y zHsX&DFnzhA^Dl5TNYfM^MIFWc!O6kgVRK4v zkGRKD35rs!F_AL&UrvbQL)mbFo+QPQ)W?PRVo@qT8$PXkfl`DzJgD+xZxxweg0qcV ze%Q@tH#cw*Wae4oJL0Wf9Uawy5Kp}^0CJ949UZary8ve(8K)6wqo4Wu^%*pI+K)oR z^GqJFTGoIquO255-;q^Zj74LK(w`hjE1lnf0Zb4iME40zJ~!dZMC0z@6uZNW&`a|x zV8v0B{uYd~$l2T5quS;@gqh%B!0C4lTr|y=p`Hy@x{=azg2hf=@n#HJ3+^<@%c;kt_luN0sg4{cwpAQ4X1R7+8MV$@BvTn`VXv!6}9bx#SAq-J@=rA;M~$wt&<0|U9Pllq4@`v>~3hv5xMVm#`Y43mi*XqKR_3V zEj0BY)r5YCf%3q$Qh8q--*lKCN`V=<*Uq0m|2)pa(C|78YU*H*=>6^B0i*$n1VSH_ zjUTocslXz!@mDYx?|BU)yQGtxaBLrRGiO0Bv2MBcgV)+JGXA+$%Pj&P9(h0#jCGof zoY*T5dkIz-8im!6Oe;@hn|bdM&wh$!(UF)hhhfjH%2eDNeQ=ywcO%;akFnbCJjC%_ zgb*1+Y;>A*leLdovHZ4Im9|*bqDTv)y2uYTQCgzYHQ-c-r2di=jXc+>;gO-k@Cf%u3u{VDgcpu|= zt_&K6`*qJYSZNub&vVzE{(ARkDv!#P1oUZ1LxJWX#`q(2#Pom(hRG<;#prymufkbn zj+AYgT8XtaM`}V3Kfhs_0yp&mj{3cq{V>NCX_w7I>*#A)?$bRWP@I9L0U=>zh%^DO z0^}(*Fpv}p2ULJ2+gckhndQ_-xlI}(+g5K>FNO_^?$OM^a2eEmqLk6|_Y{m%?S~1b zV(5!Ni)q-{VuWcFI6~*G=`=f-{eig30G4=poR};J#`p{x)>0+|(_cX<6!L*?BvlEQ zMGcF^0tpYnDxlip4$^lspvQNc-G1h%E=(8Mf>E7`GMwetWy0Ab=M{+R9vew_>#!PM zdT|uvwW6AXHJMgD7BClm*~FNF-c@z{oeq&1XMWllsLwy}h`qWN!FF-=vGsH_<)pEG ztlz7BpUi%bdVKX!Az@F6h&oR(X*>8902wJJ9Sy|2x@Q&i3h+@7p%B5kc7w_N_{K3p zHA2a8`V>BI66wkQ(8VDgN4JGjenLfU4^R^AO{^(Vdj|>n{*I zu@Y3#(8vIVUR9430gq%D&3FrVcx7CTm&4@K)=-c7_Ikkn=Wvinf^Ozy)F*^w3BWoc zO6zHF;NXLb8sl!S?rsaF@;9sWE?<3LiBgY1RYT%}8wVq{lle zA`Sr0@=zUxv*Gb|06zadb%#~=ttL2v0G83Ugs$KxgalqCXFHoDt%E1Ruf%NiyTMQ3 zJxjYrxb;V>0u^n@)ICRDw~n4yc+1+VEk@Q2*^M+)1s-qlCZs3f(D; z3TabMO>vX{#d&Bte4WAtxga#=;g5ZI_$ex~oo_WDUkHsVw#Z4tAjHUTOQY)cy!`z3 zEd`yOw|Bvd&^n9U3pX+Ltt=IqiJ?(FGL@(pdj=qgyEym$~J-89XqEV;tpY^7mL1~q)~_53#1J^&Ul zvhjIam%SpY)%rR#c&MkpXDko?%~1_#x(y+NfchEA9s-V)!nS&j#Ei0H^R^#n-`JC7 z@X=MW{wf*CxHxjCZu-smGTTS1Fzp&=l=vqG9~#4gDRreqV8N$VR4&2efOXAyz~P~A zEc5_9YnlkYdm1&`KufKBGYlX@`9P@Y-nh@~anJ{cUjNvTV?GT?;Cjb>KYD7zxmtM! zG*B-hH5Tc#?g=_wP~=5ZIMU&Ay~goca+6#`EyHM_C{#9rqJ6=ZK{Ce2&280E0J%kR zvD{_ya!`52OU534^4$I+$u-oR5FjwhoYw7p+D435P8r5fpq}#O%^j8|H?eroba;t+ zAGV6&P{3zGDm{(6MaEI7P!}MQJQqwge;nhXC-1S3K)ANYg}h?t90sYv%Rdezc#~R| z;)~kj6>n2;wt_aU-@~9OYav=8>+3KxjEQ47_$Bz8W2B_=jtihkq5=bKh}NuxV=G{2F-Y%z ze!RQ_Nn9^6K}$1f*C6&H^KWvW-mkw&c9O!U+p1weZMU+^RG7 zKOt7te<4=)6m4My`@=&hl&>6uy2m$aAoWT3g3yp`895HN3C1plT#!6>0=84$F)%fi z5#(J=H=vOu=v?r-znego#g*vbqsJYfZHSspb9AbmUlR#|$}LP+Q{s^-9I4n!-vbo z=(WEJlttuE{34wiE*RR!LIt%%Fr8pvo4Hdro~(k} z0gQ-@|K=zs7w}$qHam%aDxjltie{hH70@_g&=|HZ{}@dVo&Qkvi2oJ?XG{}@8r>E> z_{cLVI@-UY!xgM;DqsuoujZz50BeJs79mb>O5ZHogNQDOq5VOhS6R9Ja!d_2p!<&4 zdB}h3%$It#6ZE<(4F#*h2i3>Af$u}0s6YV(%RYE6t#k+|8@6U99pVTp_KZBCYFTwEE4CqGtx>7On0EH2BLH9mR1m7fFRmo=9z}ea8Z!u8BglJ82 z4J;*ORgxq3YT!Tlz11h9TU=P^131eaLiLh_VdyOrU(kOku)4M;S2lMTlWO^yNWG5) zzjZ+435#F4Qr-&1EaRrb6tiyHz??uuA1@vfA6E)9iu8}^S9Qeb2RFEYcV}D+LI%8b zR%O`gaV{glMvwNroW`ViX_Cu%t`FM&8{Q`KqpOf9FYp~Za+q5HF;rMvs6tFr-x74XOs{i;&tp_)VJKM>wPuO*{)YFA$$ zn$Ut|N2M>EOgpo&y^BQMUB$*;RerG+0O(x0d z9py8EMN!>Kk-QWtD{E@+rVDEP9Kl8Y!p7lHsU|E*LwH25{Br87k@7CSHq$_(lM0~4 zQP{wErKF7vD3c_Rf2?Qp3V#sXpk)f@YWUL7_8USiX_^%3zPEm|Xb_GLPu;&P8btqQ zrh+h{u~7;RhsnnY5N+rMBpjt3Pb)pqi2}tY8}|Zf*03nsagxbYS25igV$5W*sbxZB z@F@&;aLc9m;D$SO{_it%0Itj>m`I6>P#F!*)6YSbQ3`xz^jCDZtIHG9Co|O~b=(xN zm|yS;r*1VU6kJYJZvu+IVTj}E9*l=uEURY2)dB{O{N)VWofmjh|1B&>DSus`iZlrL z8Y?dVc?R4Dy_sI-FVz3&$j4Ej_Sk3}8Kr>GF;nfBM@s>uwW2We7uqa8m9<{xEc`;fmvwZ>t ze32m#YSLPaCQ4SJ*7hEkqa@j_+Wg|XTd%h!)>!#LIS_?k0TO<*Z{V0U*H0?pe-Mn<=%+NV~e8K0AJ z=0O>V`0x{+r{Y!lAT+Cj#a~v_}BFavQxW3+Ed=3SbxMEH+5HUJ!hMVKj{|k9vIZcH4^A`x+9nl!&)491R?$Kas6vQl*IyIw z3Mm~iRp!M=B!W``3W%ETqbbJYQ9_cWdlEo0{Fj>vbyCMc7m7vq>j2)p95kQ#+dJrE zPE;6uDPmYE8p-4W>GD>}x#zWeFI}Y$Ra#$-e5a-=u)KCS>kD9 z`2_gCKZcq?$kd#|U=In#b9#xZ;^rgoRtizq2t)ZAJvZPYdeP*D2)ycnRNC`uS=A+g zE^-P6Zbz0cjRm<~OS*>`Z!NW7ufp%nSMIIonEw8pzGz418PNA!KWSk3#`N%?k2T!< z_RCqymK3Jid=12ximeUFTR$iyYV@;4^oy>WuPB5%XHF4A6rjh=>@#tE%=uUY($0sf zj&@Hu4Ch2!9J6{6vb2qD+>)c;!lacT;~mcD(kU<^4~E~V^*9|la-+c2K|12sTPuAQc(?M13S6$%%-re8r@@0CFnR#b!N(2Vc}0cS7> ztbhLg)WsDhp+iW=-GQnFo$;*4erk!# zf1UeF{ZEre2aPOQ%w(b#EV_(i+d?!t7LWJjNk{WOHoPY zsH>aS#v7dWMp~Jm18Wlf8aG69)cdLNA=&tH5A#RY z)R;JG;0hPWROgK#))A&rDj!F(NqtIdvQ1m*i3?rlhdLU9Th_Xy6{bAO)G%Gt#<(Gx zRnOr7Btp*-?LC&v%U_eNRsIH8_N&&+*dBq2j2*F0{J3WffJSdloLbGjYXsMP6A^`R zh-tE@3Yoj-!^tH_+jnRCv@G`fK(~kd_uodBW*aze74R4|lF+(>1c$3Dl%7$r0`{0# zSVv7Q0*zv`fS~}jK}QZA;Lxd3hpF<5UqwM9jzc~FlV_7o>4R7pSDkFmZ#IbfJW;&{ zQ39(snsC+`g@wzWK^p)%fWl;WtJCMDvcB9qS)rsI7A7C&C@Y9!q0_klNhKl9TgA9H%_^W1N~pjAA5dn=2vZjiv7&?%_ZfLC-Kv7M z8l7QeJ5Wpq$~DWM?zR#iz#K6?P3qLJg3A+=<;hk^+o*k*%(FCtr^c=VFnj4}EqlW_ zq$!7o!S>5<#;l#R!FalnG-T*CEG_@0#Zgpl7?DI40bsn*ff2Q)8QEh; zcrWFz95C_$kO-L8-mU$>E8(FcEj8lb>dOlad{FWuiokc+EQ6i*bYKGmTPFz9w2-Xr zj=RT|O^Lq}V7_L+{lT`qp5ESRm@GWFs82IJ&%lPHc+g$}%WtkAr1>Qp;(!#sj%e?| zT=qOb%OBM0Isjn=O`JbOCw|Hw0P26;yc}){8&eS5o+1_in!o z<#GtZkGNib-i+j2dr^VMJXZzsWDstGgW>aT-hJxXRj8ctc3e(ALg%PA;GSwa10SRJ z+HujBKz2a$&bCGt-P6!)2_zkA2gAL-X6f!`Ld9Kr6NhnB<7M!d)M(&+d2!E!8AH-U z7MJy56_I@9m*S?v<~(v%@su%_SqgJ?*0k-qR@41+WW%W@+r;RmaPzyaEe3nc+(LY1 zRonh2Eu`}V{&k#Q^wogAu~r6_&T=VV`Gq86S*e9OJ|eA-&<5Te^Czq8*7f!8dkNQP znwVMLBj)3tDG|=4&m;MV^+`rdu~2cSC|A+IvB5BUPEHyFIfFdZ!ql`A6v>;ao{TN_ z!`SHr5V&33Sc2#qn!)j!l<56k#cId2VFoZ@l#C-*4aa5{#JkT6eR7OVfg#{FauPgQ6&z^W5T@XLv(3>0tgGKW`i-3_8IZm z--TS?xyc~a6!AZ{VDfSXC=hn@U26xinHEx6c?YS>0QZtHa@g(y;U3LXJ=y7ah_n>GdE?+s~T$K zS#H7A(AQeL{U9CrthrP7MYZDIT}H@}TK4S2lnD_++e4{!$w9lO7AqBXW3oy$mcima z_-S}NgHf+V3+fISwLq~CnFOTglO3gdXH}eC4#+7hcfW7|Ngqfgg5Qg7Xk%3uJkA?1 zK1WsGJXDlGl08Uf_$w3$RBqhx(Q)528gX4|7X}GCJJ@o712#N6(9FO)?whzHu5`DA zLB#lsFb%AXK7razxoVKwMp`lmhy&D97KYMB!ZcZ~uYlSKekzOs zc2pt^j{v1odhQ~NVg5UAYrW9ZzQu)VAs+LEpc&#eepL6v4e_nps1Ow?YpildssU#` z{WqCBias{B0Zd1g{=EtDy=%y_A=`>{3jk^VfaJG8GFu};+u9dgRoo%5CODCHosyKt zwq1xY$xV=4;egJkwzk7&SsZ?9P$n~gTq%Y80=56?5QS!;;#sOXe2P( zASCNt={b|yLPhy*ImS0qd8|()thLc%8gQGKR^=S4R*jJHS8b%P@)R={Dc}O;M|YFm zUVF?%@vXW}rO@)pNl~t5KARF-Ae1$ZFK76&{{{T=zc9D1Ur{Lp5`)+*?s4P*|C(im z#^y&6OFjZ;<21^f2o{fKb`j#fkQK+vQ82Cz-gn_{L`s=9Cme8@L`9P}5s# zG6|$E4&}MZu^7`RNEOg51G|-W0m|LWFu=HCCNql*bazm zVNTO3Cu>PGY%pOalC3A-`q}TOPVe>mwBvYoXXd|$?evb7W>7)qQ13nQB~M7(cVWsK z@{VKXTVape=@lh$vMw!P?ua=pbFNxYGS|$WFq#nYX*X8h?kUCfXNtlg;vQ0rVtPwP z8f)KLnfKw*Gh!yjz10hv#e5vaDU=u&FU^i;r3=i*=G}}3w8bJm)V<#(PZyPbuuo2) z$osRaq%U1xd$1P=P^mhhv}Flm5Pe8xCxPGYEftHV&me%k;L7h-{gnB~I)<#W1!EO4@cQ>SVF9n2xTnGZr?*JzgYJdjz z+Q6CunfU2z^jnIzPF*-qH+Jzx5e)OJZ~Y6iZbOusX+7!1bA}>(&~d{?dcD>d;wFiL z*Bn#L3HSVUw?i8t%)=R*Ux0<-F4vAQDGAZY2|>M-EfV`%w&!A*xN)V54Rk1 z2qw{Yr-`kxAfg`_t@8^!not~~-yUePg_$g^iyJ}B7b&@PvXL%41isH2>VqgT(iWjd zP525ICotU9q!GH0GMKW?x&C9+*LEz!WB3l8gDOM($JEv{I?W z>8Y0KoT9eWiD78d%O?0~!RqCHx*Wf{)yKGd{zq$uk#7pqvMh3&u&>sj7w-Txie1Wozr^C_QKaY6+k(8Ktr5mQ_@benAkY5rI z2y#3iod~BWP<+pE_Y0GFCvbc+S(H5zY2rDk0%>R0cWtEsAOmFU*DQfc1hJOR?OJ*ewM=+I+JzUY zm{E>bva?%DuXySO>M=7Q_TPuJ-_1Zv{ruj_+V-yS=9lMoFko!yizS%!cs*8>pDS*7kVeGR@e=)5Dc?rTA2gQz77W`@QDrjDj|Bb-mj&YiPIi z%zx8*|EzXN5Hcj0``||56yqD3E>>ZSutGRDqF4{<1sC}zTJ8ku#b&gM-M6MM-_$!8 zUmh073OcHZ(Yu=Ypz`Jek0QXdW>h=87ZFzebBiT5fka$aNd!=m-s5ku`0t;dU~k_p z7;Az}L^B|wqM|~$5`sr{uJVGI6%xBd(!sU>p_sgn5rSzXL~ibIFz@=%rgpea@*+aw z8q8?W1ySCw3-~JWGyr%#8#NoEv9?{e;M#_FR70%+4<6L+&DLhZ9MSZt%ZkTAvk>QWSc|!h5%$~`0sKOj{cADB6(LW@(_j{Mi8~fi|e9BAaeh=qf2EVT?9LB zU&lZTl-_%>cHvO40bew>R0fkQgKy9Qc2PFgAwY5sZuSQXHSNUFQ3ujLR8>^Ig3dpu zWBt3abtK*xC_KY#F_3?StLA$W3SA&%yaR)zVVKr-sC*iXbMEcxLfP>&I{mM`j@rS* zLC`|fAmYlQLbDnG2uTE=rh+ED#F7i*1JHffXA0LjexrVi5U3xsEBQ z$_qmtQw(D>A)()0I}ODG7-OFT)$y^1J2Nx)VHCicXU8HwlI}Xh$qH~Gkq1ZDuty%6 zQj3O2AWAeiek5Qx4I`GiU_KZ$SLJAPGJ!>fvt|Ow0Ge?Kvac@<0)?cYzAdbW6k)2K zGhJ3oELjPiaWVQ19LcI}Cx&fZZ2q1)^2WrU*ZbH8@>P7}C_`mxSBiJXTHchf4Aoq{ zgH-<#~F9@tH|D%XlVkzYe2f~gb3Cn>F$_`Z1qmJDgi z`%A>daYoPzV{d&Vm9@t?H6&JnIbwzixcYM+^rr4(74{PdRN9Jz#C`N>>iuNWO(p7+ zkhENrYZ^00O*8YEo?Q_u(=By*+n;KHwY^9XtSBPUuKbg$6al+T#Z8z*O!wJw`&34k33Q zjFC2lfj-RS#4O~*8L6pKPzo|yu{+QJ*qJXv&u^ho%682R5l?NByP$K>x-w_c@6tH# zWLKSX*RwUNUUwl^rC$AJ0RwZF0}>|+o2zHa%yO5*9U|=0A^Yg<*o=vxz~X~a#t5pO zg|+>R@1sl{MSH0!_qDB`vt}Bj)>O|<|3&BPm2^UCn{A)IU1`V|dcuXBuYbM~LyZZ| zGS5`gir+~uycQcinE6CJcWc8{P2Ms9YCZ!j(AJlG_ZgX_* z3TThBo{T!%P|xCose3b(qw!L4c$0cyRpyurU1M8@C2yjSh$TcL~)pb#_2z@oa<|uJIfnCyFAJ;96QY5tP?scfc0)L z)w+SH`*wyio;5n0t;8XKXKB6x`m-!%#%zl&7yC4iuMYM6iQf{EpIb7Jy?tCM_1>3w zU6+b4bYo_ntpNsAN72tnJd~=0oV-yN*XE%l>6mN88Sk5UZ-q-n7plUq^co&6_n&zH z#0F}|Sv9zpVKty*U{E)mgH4}5SbngZy{Oza$hq_*AwFbHi#um&bikpDw`fMg!(#gA zfzy=r;k;k6Jc6Kv#L!ziA!g)VLC8pR3CczR5Cq;pw-BMddZ^=-aTdZHzbui8bVsU|k!cw%%~&GkW^Wx);Q$c@(w zN5gou-CMO;D{e2jnGrYyjzf{ws@I`)u%5%u!b9R;F@<2%fXIl%6l4+D2h04%slT&% z2-6tRfK$^O+EzJHP~-0vNv||kB7!G2g~O}Y%v?L#vYuU=`^Extei66=SGzmHy0w7J zz+`WdZ&r-2ky-tB7+A0@5T_T(yJEU#Q9S}L=c$kVO^d=je@}<=R_8iJi}ej7v{f(i zQ8L(!y2qYV{MbDzo6z_UNN35Ol&=2{X$(jrXc$k7beQHpKPr4cFX^GveToNSGV%&J zKwurn8CCNz`H7a=ZbpY|PPTn~c6j4*rlGjQ=pj$vMy*RX2w-U`0!!1rh9hb(|}&9o-a^sFR0Gx&`bl?>sLLVb|T9^kiMWFs^JG)kYo z2w6_Pd>B?OOX5AQV$yg7d)qGGh(BVXSMNWg4H5ttzvSs$pz2E--X7b@;90u!6JV5q zM1$4j>qXjG#j5{|HAwNX#=!l^c7{s60TgTK;Q4}}*I=f8&ZGMeRzqzoy?$vU@Aa^n z)0hNiPTl0I0Zc7cWgJDF>sn#|8@$;3L!+z3Unoj}A8F~4`672WJ*9D^y$UjU=D~gG zX?%-1__>~Z6?umgkXk_4n*||I{XP?<3MnGKUGr=1kucgB#2ijIjog}CD-?-^`@s&R z0-W~rh=X3{)f@qo9LwS0A>P!^W6D_xt-W>Eq^jBD2AH zp2Zbw6%~!it|be<1!r`%;PlCf$Z%RFLx4P|WIij}a%>H{Fg-^ebKGlDwn+zTBFEgf z_IFnS{#D5vw1l)CRTvlvvVwZaNLJXKrWrb7$k+bD%%A^gy$?O!-IGrF0j-tA^r%^hd#lxI zEblhFpvXy9g#Qzq3fITOi;fEJUpV$E?}Osv;UDme>e@~gv7pCY*1_5<5Q!K~{tLep zPG2Z@?Q12%+gQh<r{Q19XwozxP!e?j54=c1r?!NivF^!V>INQycpn|r@^0RA;RSJ=b^7tPe_h(m3UwVUDLtIo@2Fp{w#G(~q z&Z9~v9(4JWGH1S8H% z`TttbvQ2g&wsX+v`svi$67hupEzf5>3P?RX%v_9Ud1ClHdQlm@FVN;t{Qh* zR^tjGnT;3upT;xTDV__A=cR!}$Bm+vDQ|OBX$eKWsDkowdRbtJ=eqQoXj@X8p$1bY z?>+u3Z(MzoMuGY3j->4deHhgcYD1T}s9huFMXeAz4vhKa`BGL|3zV^mGmm%N5c;2y%Dnfxk9-mq zYJPaRpVRVc67^dXU)BXELl-sn15V;QsxeQI5&|Bi}E( zE27BOJ?GJW0)!^=9+f`0?OplLAxz8P-CD_t!$j6Z(HupaZJy2oV@Ui6FtM~aR!bZ^ zG~!`9w#*=itt&&9ucq-Ws%&-D1#X8@%@8X1HSZ1*(L_)HwRWILv-$lcbkg8CE%Y;J zPKZI(DhQFOE?(Lq;%ihVKj0Z+y_v?$H*pOgGbU{iF>LD)grlGiKv1fQiy$A{OhDwn z(=;_h=P6zkW+zl$I=0}Zol#FPU#KeGW2ER$VOS3QLtrrkGr&m}I7y zhF_JgUzLJi%mvACnQGb;QYYO0RR!kcVaD@0d}NpmjVd_0B0?~|GAk(S-o47?zJc2 zkt*vNfq0OoMH|MI-~tmJ6HV2RhEloI=ErA0xtNcSH8qo&qaLO^)N^Im%2o1E8r=*6 z1ikd_=G)9D`&8{wAl!2frVR9ru?h5`h{66Se>|t<{E6^#SI)$N(Mx~|p^g(p z5=)#p(ab1Y!|l6~0gI5Y_IIPu9iFG;KYfMbgMU|8?rKNevHR3ekRv1&Ql{)4Js(2j z1;PyCxS%J^T)%_*t^H-@AI!q!sT`g3PpdW;8fJ2)s+>!Aq{`b8e)MXbBQA|muG?fX zO>pz_b}sj4<)#u6y(>BCreh6hNpnpEoQuE}N(^_sptcy6KlVathjY0ef61L)8cS6% z5ySsNvv5D;dodU?ZCsoGmkb?~$kwTl8Pk0)WVhZD_i=^SHg0i#R?-2bU1_R?bEFt% zGRRR`hdG~#e9(_9UC&l3oTY-bo<1IYjWeR*j%g%~ z-vbrb#cq6IoedF$D0;)oWzGHdVd&hy0xb9_1)oCW9{;RC~k`~mIAAQ zXsZe8;7#EEQiRUy{FlWix)t_)nFPW9Wh>iuBoqFE=?pA(MZ?+&By`HZ(5=-h zA3`6Bn;#WG^Eb&6W6U()<`PQc|9Gb?q?MSu(K-(5h(7D0X(mmUc+xIJps$O>&!*+o z^F-g8Q&|irIEL(ZtNtng$+h8>MQ7YLwa^a2^B|tEtuhaN<92H>g+`w-ZGtA*k-IhOm3lj7a!Dl z=PfW3rN!x^rk@4P4D-S)eC2NGw=_0z?x1=yMJ8=4cKfpwrtyggH@3d72)C*NI!a6P zkhC)qf;w9watLlpYs(W{JAn4~qkc#IGp3gig3deq$CjYVh_dn3-ZKp^2b~qUpIiX* z%o`{fmiv{r7c6@b>|X&bTiDFlzxEFlc8Y%3zi0S6o_SwP8NdxAPJvfyPlgw6+ounM zJD59VE+jWPl+13kseQGeT7i{f7?13sni)m5h^lw;YaW`hB2C}iG?dxEbDHI6xTKc^ zg6lt4`_O5YJZ_r>TGB4aNjivUuYNMtat5S-3KSt9i0blu1OcBZYM{OROP5gPQpsO4 zZ_amS8&a`svS=|nOrT3-olCN66+(0gsbMwd-qt%RkDA|UoKQt!ZL;SIWi=OD=Pvsc z7Zl-;kkKVDe|gP3(E%3yb&c_C@wDjh!1EtDhwqO#8FHmmsB$m&u)l)3>RG$e{nvO% zg9wN@x;BmnnyU_vR!IJb^}DD2mqUIpPjtuI-l9!4{AJ66lKisj`$BMhYuS~#t6rF= z{(38>%K~&j>}h{`*naOyWFjZwYx}o>@E<-yucTKlw2{kTBY+5ai6=Y)hUX%q)hZez z6s^Qf5oMSV+$0#%J0u?V^kwd{jyse4%!sqMNPdZy4MCHT1@)cb+z&MhY1n!oSCHEj zMm>iI+)hJX^r{?FGhX5pk`w~gGI5q?qZ^Oncqh?NjBSS!EWSWI%RR|rBTlm6qyeNg zd>qAr@NlDw8ljl%`vd2KZ39jU8@EyUWvoqhgRy)yEgK$A<SvKZl)ODv!r}zf53i7ieKz{GPvNBcupg zJ1>LupidL#|1HIlktqHt#nT;By_cWC_9XcFFAA&pKcTx2L_+l1OoJCN>pTO)l|JkL zMZ%PSBO#GpE3C)i54)~{E-yF(G8oSnn*z@Mil4CIo!tZ(70TnUkjw}&Z4$S1RsW5I zx!V?l+a=$C-$4+dvAF-EFWElKE$J$^SlsgcKasFa@}E0O-~u_-VV}p_XQ0pOvrg6U zn}NdXKdicRj(wHy$|WfCC^T$)WecwO{HKa6Gl3g$^^jW1~v9GZm7NR%GieG!~TcB%e z$8G6UN144p7;%sNP1}4(_qFg| zD~sO~xEEv0MHwuAi*41B@Bb9ryxGhK<7lv{kU?MuZ^G|Ce^Ui~FzCElnqWT}L#=@$ z3YJ;H*MA{D0<`}C8-VBc`QKRkr^E*SqY=(@San}f3!*hBvRy|a|bN<>h!J7;GFJ*_Y;)oUYhQX_2POzF;uL5|(t&6X_S31r0 zo6{X8e$lt8R*Nh(I)(20Z5_o^-P8D8a?wm|9|-PKwPy93gMNI`;_UmY#an-UKLtsH zn;$RTeyp6x>ET@;T@_;c?OltGXI1Pw?H*R91jy6yTlLoq2Ru1~whCQ#V64TmytX|6 zdJ6Pvm+<2sFBM?8VWLsF1#=6;-;BTq+$0{~2+q@&9rKZ|B@0Wt+j~8Mut&7Vi6T<{T#o?)(9n%Zz2sQ`#pUCIWZp@ z^|(7hM?&yVPj8pyO&QOG3YSqhTrLPw3;MQG`W=naOnxm4bFx%lq*SbYQ2Wxxaio)+ z+;g0BsrgQTC)>9bR*DX31VrD&$N}mC1<>|jl~hcxQ0%sAMWLci(o1(o&rIBfDvQKj zu+u66mx`=ZcP1t|-<+5ajVAU8H0VVeE2?d~N_z+&7TY)P!gmu$9C$Xh@a7GQ-ic8K z{4jDB$b^jI27I8DwSsJvd!H9feRVofZ@$_v$13<4|0%fyE6s1zmC9|4R>1boc49)( zQiLTyi+(-!|jPZzGG*BB{|MaJE z$oiRc;J41qGdVRq|9b&4bY6Zn9P2QqqvKA=PEdlroy{hsTEr@-p7risUN(J_>@E~X zq)^A-r`7mq4c^5C+j^p|;;xDAO_G#ouPLCG10&yh2>*${aE$OuLq_OB^~&2p3Ii6% z;X{m9`s%Jw`p+58uU~!t>M)bbGg9%izY4lZznT4Q$$(EmbdWBQLhrLPm)@S% zq`~#CcAM0~M?VCpI7<~?g1NW-i*Z!Z45#zor@p%N?#ENM;?i6%Z$r-K3woCBkJnCr zeg1rDi-pycYioV6F5h5U5uRrsF6Pr-z=CtLIp=(`JFiJQ{Nl{zusj*lAt8QDu+684=O%-RBjW6VdQ z?wL|XN&_B#4$t;fgm2X-%Bt67ojN2Tv2gJ!53ozf@;XWaw%~sAq@i_L6_R|zw8C$; zOXUWON8z`3g6<6&ve*SZxif3ae^WFl$)9G+VWv~fLYwlaJU%=3Ts+HyQzln50Gzo& zUhY0gk^*LcJsa1rlEwFPYQhlX^x50y;bvhI|B9v~uJ@X?bk`$t6oVRp9;D0U9Vh3EMe=tSK6Q)IT(1ZoP zz{dRfXJbzFTRl2x-8k9!5Kk}cxFjoCh` zW!p|LXi`t{T_G1f+)8-IgzXDui;f2CN*Zl-raH3wxzg9b;AR%**Icj7N0fq<&%m!S5%%d;8!O`LtK- zpZc)orAW30Yqkb&x~f2e3f=QhQ+#$~b^fPMGfyJfc9>ef%L#gT+%BwQM#t!iRBG_q zPkJ~sy`j0B53t@lVT(%QShwrlXI=0dJUDHayBEFP^zhH4Y7_w0=A@yG#UqD4ZiRK{ z=LO?rSdu2*j%V6>o-?i6uoyei^hN9mt0HsS;8czwB=XM)de@W6Bx=vWe;GN?x}n(P zyWNYG+o@yX!)w>!61iI6{xRo9J(M59jr(sFTz3!N-SPcusC-#>bMD{#-RT6Li&`mk zSlI64u+SwUvE(hnpHKGRY@hn*52Vvm2C~RWJbv@8+&t>~!=z!$Jq(IdgQW(Q?CV&E z-AIecCkaFEJ)SM=flDqlik(U__L%|u`ArX43t;t*q3y-a``3oEPmo}BUAY?LjMlLt zpq&qX1U0&q3^ZE7%&!nmp(%o4%;{aTDdJEm1H z^V)R?$G_0w{(^XKzDA?4-#f7iL=3R;zn`>(yIr>W$t~TDJ)3QED>}ZpORlAAY&<=< zzomyI*a@3-fy5D#`%N;rN42XJqB{mH=LomkM-~@+8jp&e<^XB7;dbc%-@CFk=Xqc2 z3Vz*yPJ~G-g{w`Bk<0G zyqV#0J8CHUT*1hhS7}rYGmJ|9b6XeiCw&6hXAS+E_J;E|HI{pYI`4vm#8Y{-;HvtT z>jWLQ?Uah~`$hk3_x+DT9(2e9TK|J^gJsVgU_u5Uww?Pd)=qhVOc0s*S6+g!W%Et6 zn1xqdwvU{FBv5bZjQJ>D;`<2>&aR1uuhi6%ZzHe$VxjK)m7o3gCzDiHt*!MEy9P_# zc^IiG^`AFt){c3=+mi6%so$gi(`yA($)Jt#wNlX@z@NvzJMfb&nV$Tak_LB`&#~YR zmAyUKgFX=;r8y8VV@n_02B8Mbhq`TPxv0G#OS71I@_~~4K%2Bc&*yyS=~DA6kGmBk zEXkL3{Ok!sJC|pbyK+-^$|Ds%NmHs)$Ld2WU-mW0-lGrN`i@vLe9!u~r|VNHjUUB9 z>wDqFfJ~uz)|B@?BSu$;UPV~xyW13_XM9T)Lv!v#uJhpco!-_aIS{h2Q_(b#&y+SZ z^74s?5(c+fK0_PP0Qu8H@h{$F#7n7D|L{CTHf)fS0u&^#$Lee?Ln9fJhPPs4MM}m+ zd_!A~7t&7wO52x&mw)&N#$%YmuHEr>m9RAZ53cSI0qd!zFX^EJ3BBD$Gvk~&s^H#kVD11WGo3c5Gt3ZKl9lp@og03~9J=XWm{Wih*>d*oz(?Br zZsADyyMv*!sNRu}@Wu4_jV>R0@KMO!?i}AvS*$^Uj{kL_yT=~Ifh;M*I5#ZdlGH5~ zHPUGEn2XkZEG&uHn>#JPzkaH(L$oe{G9wY;xYQ0vvW|LdA zqVu{6#qwSrD@v3t=xvnMA}5#K_P{Cod7vRu==nB^j!6)#3wPqrSrWdjfl6Nu9HHY+ z$6jIKd-!S2)lEq?uvR+Z-O8tav~Uo*Y2SCCGM!DeG65mawyNBD;5hU>hagppPxyq&z!TZB6$jH5SOFbq^t;Ign-n#v{zfx&@ zI8Q=1pZ@YRhJU3RDIEL17LLDb4zfXiKRf2cl-lk)CH-P6RdVdqD5fyiDTLUa6{alV zWzKCi=JIRs9X;-y&c2My2o;EB-%Vdy7nde;%uiCtfX=TA!7w@J`j>a@(2P8-O3pu$ zC)^7yr^|Fg>&6!)cYLPikWO;~VKy*cRIFtul(#lPQO~_BneDl@;^zIxqF=AZo9qC% z`}yXV=lYLuQ7JkWeIiila_Wh8i@((Z>0i`-A)NlAM)>a4p=*hQeVhegPcCz^gJgYX@A{-&~Y~s`j|@`mG+ZK7oG%dOg6fN5fZK; z31iL1L$yec7oTKsRNU|SKjHt#=E0i@z6u(|x{>d!W}#Fh;nt>7%xsJ>v0%N3;Y%Ch zcl?dHF_ICH*wrRDnovQHhjA3O4$XDH!>sIuG+<5(k`W~rwa`yI%)q62~E?&tQJaO??3kR>JzZSLgH^o_M7S_#>`yKFO$<9H9YgC~*Zv+quhfjh?aY+z)0_)>U8UV`bpL=3;y=ZtjhK z;Y+Q{%J+G9r|Pyb|1opv>7_vaOmyaTtkM22r#^Y5+)x&~9NF&}R|5{qPCGX}4L{7- zHSELry$L@n>vrB7X zu_u=D-SvK2D0Is9ZNKpuxvExv`)rILyXy;s(;=DW71B=jddC7(8@|0jMTgT%5#|C) zQ9oBSWQ(&xjz->`D;04WK!tMu^Yg$Dts`YMc%aq6ohr-Mwirk7Fip&)pZrl+D_Q(J zz$@nug`gL&QIX!V`q5CwMyW`k82%d7sq2?fBBr2pEx^{r&dDtg^i>qyL(GCoYMrwv=B}hZ=8aI(^l|Pzkd6FXYrc>3PVQE&KKlMGmG$$Qe zj1m^6Mn+;tdR+Ng>a{QCv$uQFbh)^42Wc}4R=8}M$x>2oRQg^v3@tNmsIxtcUs&cA ztQOqsg*Z$<&>MbUKxm{DHT0F(w~?1F$%xW_|d&|*GNusjN#WqF+`jp;DKinh7lxlTAnJ)FSe#IQ!NiFy<4Dt7C~cL)6_?A{kILP`jiof{54Nj1xO zSu(hg?mh1be$NL`f(Dj}U9vDK6)Ph2cTeL!BqNHj)o;8;#2tvwiO9tC>(+ob!IH`r zPimlSay+1r{JV9;QMdK~Yx6*i7NgM?GLah;Q)Pn~PArzYW`rn3KJK~=gS&1hg|dQq zDC$;{3`AWzH}Vyb2`265->Yu2zBMAvKHzftrOcAWhvh30lTe2=5sNSW9b@4c-k%K$ z`)&TMB%Va>mK$6B1F1Gm9aZXR9v|^IBHEC+Z_AN%xw86jEsvQE%X-yO^z@-b!LYYy z#~_!6;B}k{57F#Fs@cG`Rk-u0Xw(7?PlsMYkJmU)73F#cN7#B@V%b8~=*}z5)Qu6rZn2{oQkpbH90JB?KPKXoC*>gw zxSUWWO&9LlRO}aMhG!sp7O_iFYhCpf7pQgIpJ^CETuH1+q+t=`&!^&yvFcbK$21aLVk-O_%i^ZTJ3}CK z=UkllIlfRXvn1J~HfeHbyP>$46%unPk7MHks_K&uq%<+nUmrz;1bK{}d6JUKoBy&V zA07SlGGP(5rrX_)HdB`|=ryf`yeYqN9$k+5leWzxUt+C<;Y}mbM2(5O;;;DM84)gg zMOO3ZXQjEfPT5I^9Qnai6V?=jBysEp`FHHK_)cSZ zsH#tWevHhTe#^;|4*q_H{ELsfeee7Q3qBn)9p+J$ip;BMtQ}|&H^h-2x3u~p8qEXM z1^vLYW;s;2t;&j!ys6iyrjfJ*ap38#M6ttB-fTFNMzw-j#8)wHsqc&?sl#PvP;|2n zI@;5z!aKoJbpC!~Qsp*cq+74M#WQUBE$>?ko_!sPcu!ihX)h|KmWM|hZD=rz&`>YKe@ z$V@oZZN|W}d%juaTB3LLWVkZM;)zC^V40R3`AL7a;%vxMMJe&_9g^wjzrm>K4r5QM z{_}wI_I%`Nn8g#^FY+hE9Juwa>(~W2F+pxP;!ufrj~T4_$a?BJZg|sCKt=yec$&bo zuQ3W=1T#Rg1b-@F>-s1jVv-X#C;%0vQv78_p`xdOW0y}Yt$H%Nzqm&u5UKO*W#Oik7j>BY-K4mDL@x1wXhn=$`InzH1ooS-Dw%YmEA=(rBhb>tZfI<2ss4$JWzg!uu0L zH$D9`v~IIEo|_P?(QUse9}s$2T3nE&v8iy?{GK)7{S9_JIsYCkanuH>$X|ZBTebYl zh;(q86eAs<2Se%HU!!tjVAb^o`@K{&>0Qj%k1b7dW@)xzygE}n@oo|;>b)1aML)w* z&^Kr_JU7Msv4hpq`H+c?Cfo1w8FnUrR9^K)CJ#ORFBzp*T_GwGPMgtaqyk^qy7DGt z_ngvk9?%90J~ldy7~Jn%4j%(fBnaps)0U5VdUKR2P31|SBPR5i zOFTLieX?Ujx|T80%h!nAVO!g7U9+;9WSo6OUXFcGz8K-)PW3$()o=f+Q|fc8uS?J! zE5(eVPLgyFAK~OQ!f+-UUBpOD#QgrZRM+nf&#vt1J!?tNw5bpuh{O{ZkBQchFvgI* zzvN=1W9WynVwpz~`Cw;ueMo3{%v>q{CI;E5Gr#WhUctm(%YayDS ziXIsc)Z0c6Q6o0699A5^=&cM0ACG+2h`lnseNz6u|B>;JECKBNaGb!v5tPAd_OxgG zzN?fiGcKr?NCm}ZV>#a|sKHsquR?X=P?1r1*V#vV%i%Mi8BH3G6sh@(K5fU9FNh!g z-E{vK-mIYBnX7}bGu;hz(kKzBu4NL5L@F4r;)pNCi(?oHD>0P_lp*ffambC%MhbjF zGyJa?;DZnkVTl_VYf4mc1baw{?*Xr6)$HiBDO~3T8})HdCwuYdG1zCqpX7t45jB_| z^{_T(DvRxrF0YM|?YF&CYLV13E1z=FyKh>KX`Q;i@Xs*Q z+B%i?tu>)}pE1y#l*g$yWu>C0gOhXV#m>w8K+DhD=px5IRnU$S`BdS+Q6hCiu3A(K2c+~;ReS_^qit|#W6>H zV8z+T@SCPYA{k!4YHmb2^%vk`(A>rD)b=2K_X~-(FB%tL_JP&|!Y3Gpre=1g=yX|Pq?^TqEeXtceXfCvW4OL9WRKI{-`I zwicNiE7rP6yB1-$QFgaaT^aJUpyL1q{hOJ7PW2KRcH z(%p4+9)Ges6(x3Gai-P37z}7@v|kbTOvT4x%L-52bsu+YSm1YEWjUu5+IC}F-0i*g z(kIWp@M(V0)?1Gk%wcm(D$|u`gYiLeQ@>~Uw%3sviVu*9O6gdKxkup0CM_*5`Ku$2 zttlp~6ASsSkTHqe#6baWb?d5G$DMfw+Zi{wP=HcPKZ}(Y!ULccxN+SNcGxEdria1| zcqTO{Mmra_k9tBY@Dk_t#GJg6mQu$$QILHZ@uEWdWE8&h+8@Ns80UwD?zKS%A4X-t zC2ZhkY{Um1VKr3J`TyLhP>TKs|H(AYv!3O1-|s99ao=A-j#_D=-l2Am<17U`)%34yS8`jRi!&Rx^MMR@E@@Dz;o*t3%*uDtwMG*u; zCLkXQJdEeE==&O9uW{`!AJu5qd67_S`=W?k`Em^#FdzCXzA_1iEFy8`#-V6HWwrDF z6c{=GQ(&;a$?hF>y%eq=TK9*fc~ni~yt*(v4Bf#so2n%ZYcFw0Ih=gC+eEDt9o188 zgy74&JW8RU+2|x#L0osY$j~E2|CEag;i?d_wS95{5!F~;n`yFX>_QmAz_8}Q;?ML! zdrra%sa+C@%0;+->$C7kC0H@G^%iPkyWod9;WlfNAiqeeKn(r^dkG#+l8#nnrk?7= zT)wL&_L-Qu_q!3a$H_(?3^d;nW&4LBXD)8~e6`H-oLk!2{jl;`rviScpp-$R@&ou> zR>697tT?Vh&X$((A>E%UeWqxixBaDw})YvaZ;tN-X|q7*oX^ zAV`_skTW0T$=Tq;X>Nar?j$p5sRu3dNk6XHq+7i%cD+9FH}}K|^WK3GK1fZaM_zpy zIzWD;$#z5*HG;}2dg1SO1Yi{qX9;3y%kT(K4;NQRmNJV$2Cu)r>8K9e ztec^l_B(v!P9x;iIYraxA!4pb?k55NsdKyI%He!jQZZ}FrB;aTLOXEp;B0CswsC=) za-<jv z(a^SjAyhzCErlYE>V{|)N$nRrKs~4G-f)1L^n4ogJRf7&8h_d#R{?o1BOB2k9JOIQ z(w+daK}q9yLwRxhMt+Ge{XFjh7KXOx?=$wK0IXNPJrBE2g=G55rIWc);#bE&X?aG` z>uvkx7frB``RiB*R#Z4&|ILHh9p?ro5LOnoYN=n2=mr+H;3=jI+Sn>&67#re>o&bc z1eJHC`)`T!Wfq~!<%LvEUJ5!kg)zVFt=o~9>~`n{9h~XecBmDB^x|k>55(xpZDXf? zZ(;M3EC29?-2%pLlgUpUeB3!lo@o?cTU*TuqBkb7Xv%9OfHmhvzoql9+leb>$*qz)G zb+W@MGeH`MHV___)wK{}i_XM9CtvrkRx|&GWc7gkmuJXy-ZN+?hJ7eUJ6 zvZLGi6jG94HBqcZ9mfXUF6oz^b{9+HtO>6D2OE1Y$_7G&yjjx9-WZbI57m^_q8Lt4 zhH{(h{uf!IUMB~usjl3)hFX!y?{_nkqpuILAT{tqLqhjE=bnxNP)_~Z5!nop05U9r z=BcokZnM$=O;yHh+Jn%L(1YEpM57f=i@@2w={Ax|9NsDBZH=GhXN%^4J=+N2j6Ou3 z7-Q0ICKT-O-vW;F2hTe@)nHpeMou;-23;^>=Or3#t~8eAe=0F?dm~?`M;%upXm-Oi z@RL44-WVoBipBT`aH7tV5!Jghn#tYWrQMzzb(IiVl_yOtb(lIjUO(5m~WbnTEF|F`)}`dWCPDyC7LTd zc{+R+P_o*5+O?G0s#&5LF@Bhrj-*Dg6a9=|ZO6$oj5E{X&tT0EN)P8-)C(3yQM$5q z--_zCmH;RA-N5I=h1>+sZIW2X`)x>W+^cPo@~CYrM&G+L#T&>iY+GXNc(ohX(6hCo zwtAhl!f4G7E*u_F2jrs~V^ZYWG~}jMcY;NOlY)^BF$Y~%D0J!-12*Mg`(UdlC5E|@ z{dnGaMF-?4E}1fx$l(V2sOkg1`-Zqjk?Y(F#ssQd3&rePBDmdi3LBOy!q3s+ZAz)c zdB0I$n9bjprt%>(9M*pG>FQK2`m6rWVF zL;)lZdnbTt3A+j_E)ZIc~bkvsj{Nnax5RZTio;78E_j~*tq&@UT zTYsO_J~!5~iSh}@Axn{$1Q@dh<5A)y8+*WlMD zYL}g0^&XBVb+VaJ?aa#gPfHXyNpW~J+2@{oS~Xo+ZwCn^@c0nShm822-H|YTA6jSE zy+mD&Fj_ul@HavGr)qMIoy+x4+%V`r6BoW8sAd5KD=eGe-7S*;;@k3_#J^AN?acXF zjl51_?X)~icC)jMaFk_Extgb#M|Ard(6S z5~78A?gTY*xXq&;dU7jHsi=v{qN4F!4J!QS3_3@to@%2zbuG6Al~Ke2Jmn_hIlkj+(Wa7~$z5;vrvA8rQR z;xG;C+iC<8tWphD^LsOs(D;9wu{iliOjh!WRM2-6iy(kyaJ8+9>*AK;0OBn6Dm;J} zvvU=8hHo<`_yZc+b``(nuDUi20!719rIg&2jYQm6TGzQu3q>n zo8#iDIB85Ay73bYF;742} z%Gz2GE@|2nT2@AUU;pKWp)y`UUcerS`dnsDH{5%sN6}RFvSYFmB`$}yHiaf-+l47Q z*g_j*4Bf*qXe#fKD_fe;&5~;E{Phn#g)RDV;bM)6F6I5pT^;FEz{R%<_gp?IQsX`3Zn;+E7RSP9tG0*GjV_2cs5o;wpjSQR|#5vNT+c54Bk z^yVt5?(pE=4>?ACYX3+Q>_u*!!^Jd(i=n3uAZU^aQPnFwx@%a#9L{KsHn_bM<@Xlg zgRQBFCUNhuZkvrmrMQWzBh(U*#&a!#Z#VA`8ClmzkQaP6dR@Kg7Y%CADw}UP20mNQ zxxF_89OUD**Pzxhk+~O16*A}dKW-DhSxHecTu%>ZyT3caPZZ;}IrdCTXBMDbq|lQJ zDH(gpP&G$U4>iSi3wUf?giK^pE)1uWd7qi`pCeGI(+oQTucG!-1IisNI|NI4m(M)Z z14Ikw&YM0SP$aaTUioOAb@Q>OHCASE3dx6t6E6k*ZrAcpPrIgwT?0%phZ7;Swh&wK z(sO)D808m&|3AD$_fdh9@~Ji^wy5JuMJ)%4`obly=|l~Pjb)2ph|*gZXxP=Cq!?W+ z`^U*j!UZB{QkPCS*s)3;bmiS=U|f3otP60H$!EF`&*^EIVd)20@Kg{cAw>`EB!qoU zyr5_)oZwm6OEN$_yE60=qNWt~qc+%^#)+DMRy@<|Uw4O$M%G*mwu*>5X}t*_uC!$=z8JOh;4qz9CT>?*z)p%-r;Z?=IVmE-s=R zmYpw`grmO3F06DV>!BQOpk(J2D$ebnfG3*%x+h4O^alCna*hhSsm`2W(smwMf{8n* ztfIVk>F%22occMI1)m-hsChg$K6GF74z{+fRdrhR_SyH4NWH-0qoT}uWLQ?~E?Smm zLfiKa_b1h_+pGr~q{QsQMos8)w_~`bzb;9`#9nz3E#54vu8|~P#ZZm%A<%sob%5<@ zfRvs(ly_}pqAa_6EY=_Ol=~db&Xp=FzZNrb)NAWx?;B+6qBy8%Zzr%2bgekR&h7w( z!-HXp`UQJRekbApuakh{;xX4PVfZ%FZ-xsSdY(`S7GS`sL<}^zQh&fG2*xPr(a}~P zJ78Ddj^rbAQ{~F0;bgH38Dj|IKKKPB!1dm7=TrVvf8e_H>Du2Fr|Nk zEKE>5uy-WI3?7RHU>Q@#ojRm8ofm$^se$yTC;`sAFp4!G4>x5LBRVCb4qj+G`z{;) zWc(9tiL75p{8zMv95cjqD+S9e@I_b>$8?uA3KUR}7}wH5U&H?pBQ&u9(6nsPYS~}Z z%ZnMx`ECC>`jfZ{qCk(wO}2coDPg1q;!>hUY|7rt!Jt;YAW#H3Ze7e!SAAJAf2e{J$Zad5 zTm>SJn-edry4hTb?8ue*h||K)X;?O{jUjP147Q$=eH{g$AOXPsE$WTT)_h0pe|{|L zMB6&LY12hVD~-Juo7hz!IK&OY#L-khKnSQt6a$cp$1IPo8cSidmv_t70|jJyUi(|@ z3985bi2l<&9Ge|PtqO*uVcj0#iYo!x{zxWF+}T!6+twT+n%4}n1EqrHaD193)hNb8 za{)aL*?x#Xn|<4xzi{%MB84f8#TilZ_*c&A_Q%jTos>u3grBM)!SC$0Xidg2JKKzF z#T?jI=DSNhJT%?0Xmmm7ENS`0c7Q0B9)ToijW9vAq&N@B~`&?f$6z1A;%I6s1~O?iSaq%Ey1W z2z%qY*MH;=0{{Wg;VgzXAt07q*!llb2ekw&&ww>?s^cL9wTC9V1?Pq$)!fF^z$bi` zG(jTO9y5uDImly~F@NB-ctHEVVHH>09w9FxSGjNrr6?CJdeGyD6K?v9#Hefr%k1U{ zYybMl{S|IyOD+0~>vsVR_;~)#_u@?#^ZCk8*E7>~Nm9AZ`Hl0_Nq%x-!f_>qfMwe)!M-$t+Qz1HEwu9;Rs!l(uVyv+nO!5w6Eh=A1l?!rU^#Hlr)I`39tayq)uLEgV^W+O~^Y)99IHTj#70;WNMt1iuLzynR>EGBe zB&~5*gAWkY#SNu-6jrYb>x;SKaB*tSl*~CQB|@|}#}B5lG?v{jvRS}wk)xg~*%73t zrMPAp#hDt?meTxcgSi1@(CAC@u^KTfuqC3V`b_jLOxLr5miEY^yX3 z6fI&yO=M~uf-f>G&z{4}ZH~N4GM?d^|!x`cNhveFQ-!%uF7 z-4a04kPtNq#|J#+C+mDdyIsA~LA@9Ycf!;-E_>I*>&`&1F!b5y?9V!bgVxZiCJV*jmD*cgmPy%G&1~buU zhJ?BlJ`~}_r^H4vOX7hsUcA0FC=BZphc{bSKq{kro)pFA@w;<`x|Ia|>OX^js&w#mUt!XF};qPupU?MIpr@`!K?Ve@$WL*hul#5;+!!rAcI_EsSDs^nW8 zdbe8DJU_hL(A!$#qf!!agw~pG^HE@j2ZN+*diDxow8_Tll~?VEXQ=MGuKF#CeH47- zeA-L@3|>ObZ_h`FIZH@qf~yZNsvFt8sb(TfLGXO_Fw#Z8QVjf@2zG`-CDKsf8^E9sKM#M=7CvL#y^^qbSV@!( zi){K7YZMB^Hy#t-r@IMZr-@zZ$0Bn16GjtP4gfg!?H=_x5^W5n!o)^g3wIlIxkjAH z9!maafLJ6A#|AN<)@bKDu9-CIwiRhukbY%&_iF2tiWd)Gn3_JixEX!sR5iCZYJEG> z;MAa7tj658o?EH_$eO0e)AQfGLOVvU0oxptHXTkfvLm&_f?s*Ft^DD_%?m%q{$w5~U3}HRV(NYu*?N-!QlxX?+kVJr_}@ zQ6XU#)3fUY9>33#2ZkmbvSO^n6*?uL-NP6Be4Ia*IQlHIU z8n7m;t}vXi8_N*i9me7S;)$M*@LOHIX8|hkMtn%bb+My`&+hGfJCjfagfO@v0#l{C zt2e$i*Kt$2KUMJ$mox%mo|x1)%yy%KQDV*?fD%P}%SD?M(aw%Z?&`g^P6FeO0Lsyy z>rVypmTuq6tXR5>)MC_|jUpVA%W9;!d&{6gd1f)$Ro!TE0l3x+Ncx5Q5=Z4GU;vE%61enld%!Qi@=I3gT2z;0 z#kql0*!JPKic3DYHruLIw);EOdryb*sSEZb{L0&NviSh0?%`#Va^Vh9%AlBfHuufd z$8bmJd`a9ZhN<_j7>tE3+ZOtE#xYt0Ai^0x8m2$P%SR26WBpL4==e3D-kZTn0@D9p zjb`%5rqPc>G0)4(UJk#GVv35SyN}!8|CreAy9}Fdpg3_Zw5=1nSd3gP3qFJx<_gmH zKlJTxi4mWHq6X*w(np?LIGTK9zKfrd)pQI3%hio5)k~VO&D4eQF#tHCC{g0&VFDhl zZN>57`yXMC+#kpnq(6u?P_ql=^k+njPsKI6KnOExdSB&Em_5=N9hvXA)G-6!tFikv z=E8W;K2hD%VX|}ch~4xBV^)AM*?yYvcGxwNOkiYwm{HgA-L&5_;-aSq&-=|aFJYyG z#fYSAho1QoNL)PWCQo5wWXOY?O*T=&&P{#hwo$S{-=2|Fe)R0-YvwMxgQyqc3gGkm z7@o2`)0_P8iO9z6Etn+R(=`SZK?4Wi2l1`83ZFt|hcm32RR&(HQFy=h^=~apPfCm1 z&kx?W86GccveWXSw|W|rQELg{tn5(q+K?=8U`bqAtWMNv7F(`8M8c`wW(ZL>re29RM*rTfvdJd9+#S@^Evd)IkfKaF^Y&#w_YzC zuzR0w;hG^OCMGU}`zR&BMGi~#V3I1X=>6r0L?V&i{hkw-!95UM6Bgvp@)t*Mc87vD z4AL*t1I|2!0I&7{n{w#rN*lz&yN}I>$0vhTmiu2R$d61vF|MSDdclEk(=8X^LxS?Y zP9z*?puiRL<+S1>DIeWw`Z3&4cCLXmd6vH=-raF!Y!2U*gDw&ro)L4?;c4(zX0oJ9 zgI>sA%vi(n=_MF*-Gjl_Gztu1;Iyt0<5^Y%?-^>tM8vO;pmL^ke1Hd6_PyrI&afDs zV8brK?#fS9f}`=5$;%a532}rp2-L9{pgNM%kaOfcKu0z;*Nt*(5EjAceB=B)408)o zd->z+>QuH}6^FHBV$~T?r}I1Y{l?vwgQ2OM_P{(CnZgzK4Id3Dm6Mf1Nz@gmFq#55 zBp!qv0T;$fUx_YiXa|M6TJ7}jzGhbDe0^(9EmcgH0Cb)>J zj~^Q|b5LDfXb0ZIh$YfjYP_beTnw4DXxJw;n-K;=W^nse z8%3&c<7B()@AseAJFKt(NZDHDolgd81*!blYq1Z?fequmkdanz{EypDL`DcI)K~Ab zeBpelaSkdM5Sw$|#!|bjyMaTJ-Qfz+6leb>7PFw1WGcF-&y7Ay2ITe*e;Vs-UW|XwmePSeR>7DCHG3sCd3z7g$kkJ7Pojk50>;*c*y8v7pMx zJ|M7mZ$vQIpAN4R(G3riJ{S>G!87vQ$@Re{p(0xSZLcTPf?2(F#BPP%Va19@x&$1U zt4!tlS6!&I!tZ+_VqRX)ot>N5hHXK*rNlmfQ0R4=aw7eZD_55&Jqu8po5@gQLdU>B z*eF!!v=moBa>aaXrGHt5SDpy}yU(fy2b>FI`S}2dEB0aQq<gw-HK+P`!{AJspheJkV7KWBTvxZ`YC~@_KN#Fe%vMXk`8dM&9Cw8)AS4+kjSk10X%}54AlR&5iD|U@O{)J1Xb4|0#+;F*XCX|rn^yS1E zGUJL^LS!<@U}Z)~+AEAR6`=d|3u=sR3B_mZ3r$9W1 zkuZO!;CkVZ#TDHXtDN=1 zBvHIGJ{De*rN>u+|OSTSnB%VbMdev1Lkn~j;5?9MU) zs7QCcVSmIf4@}@!5;39H=ftUDj6It1Eer*I58%W)>WlaCTrW@3CqbxRpa$*@Tn~Ag z*b7qYpJN|k*`cc%2`D5d6e&P0YS(JM<;}6;OkFl2J#;cbATJsWg;=lx5$;k5DiO6w z`(woOS<0Z)(qc)y%Z6k|Uju69;^@vT8wh*V-?0Jz$S2k52)t0`OH@4u_qTYTcbnM! ziBT^!-J!Pfl)^uD^QeFN8^2*(8#5U*Eq^fR8GvkO1gK`41r-^S-q8PY#<~4*#_jHv zmTnqoB}*yjApmnmVCT%0_sWqcDVOgs$@Z}740+= zjIekQA2`0<0Z5R?<%f&&mFstIAn}PkYXVmq0kjf*O)}ix&{S%9SY4XE2nH`^2*E6b zMA2^5WC_yrjGRKt!fv|qh=6GWrVq^OSqTJgBa^HpTTo&&Cr-7(JaX)gjqy zP=^hFB>1ws>{P!{9>6!Up%=7hDQ)UG@>HBLGnPq>p5@Zs517!`9G1=&L97r}*XlcHkCF9?lMG2F4Yj59ph)>at4u8&^@xiS*cg646A(5Y z5~8wOojQVq@zom4HUW^%IBE6{<~B?|I&%BtvU@bl?H9}?4%Ox_>Q0G zY7>(_ewn~9{OgDwmxh4IF+=g;vdLXpoq@i-HgNytcR?%wM?>aU zy`lRH3r}F!ls12>7O}9p($ImQ>S4T?$%<^BcORm$2XJD+0p0yA>o35Q_5o!^59Pg& z@Gv{3Gn*bTsFZ?QZfj1EXuKyD#umh{ng9kXs;iz92wJF?8kcd(17>~H@kk%l<_nR? zMC}oOeRqrTU>WRKchA=^?YxvjvQywQ1%7nE&mr@u^lhHt6}|>%$15`tN!xow871vo z+{>6CZeM;DET#vN)$D}+54;BSaGA#?;wfM!0`HGD8D&&*hs~<$6ljS<9;HqzuuDSQ zU!Jdq{v5brCBOy--yN9T@JL0SWzvaXY1_GH3^l$BE1blT3OZkCMMw0-q?jfF&OUvN zhwwR{nt*Rspi5c`G4tsWxY3Y~OPHZJJ3;2LWXNoQ35kQpxO)-7WoUursZ(>`C;U^{ zSBPv8*WTDg4Vt_htUWfg=2lYAKJR}06&R+QACZlwy5K*J)j~4BXEK_4&1?!@j?;-3 zaw7iZlwee{2v$2Z*IRInBjI5pL)fFb_qb@Gz>{;~muS-9 z8EbEn@lvBgo!bo=oN+!qhp(~$rKk%$D;rkXkUB*LagxzZA1y9Y6co+ zn?3mH9c+nYelA!)%inNq6{}P7j}*^wzvJ4VtfswvvdNqj@c}R`zv{J5>Wx{zlQi+> zloWrKu#!@!C9+?m^oIcax9VKcGb_E`|e>S7tV{ag+&g-#E~F?{vE zluM;W4Klr_4m;C1&yj`zs6hLj!gP0c ziF68rw6sBsbhm(ngmgF3AV`;j(%oIsARUs@vFVO8H$Kn%zTbD9vwrY{Sj;`=h zM7>dELm=R`KCW9eqKlaj!X)M(b`#NCoDR4`(oGq9c@rLqA?uRFlCKe?gc2w*yM7LE zo?bYi_CZUFI`(H3T7T6QPs1t&+viU2MUc#uD?a}E^gq*WDT~k})p*)N6G2M;Ko{rk zpA%Tmjv1uNd#zG*!zJxR(FpG?0GXZ_psL^XqF$`Uof`vBdr$Vr2IT!4K71e|B}tuV z7~CKV4}_9EAePT|PcAwHcZ342hXRZ7Y33F^(1RRu$1EX_-T=x9bm1jvEUwQN)bT$P z8Csf-Y~Y1UDX|jjMa}GDsj5!3Ejwmm)`X??fzIS=HRPbrTU5d+4N?~mde<(8GqTYl z&%(Pi8j@JviJl~AFWFytb2xtKHz=M)vOi%u2xt?5Km#7sTu_Vtf3~ToiyS_2J)h_^ z9IQf%UjI#kseguS2;v7>b9gQWH&>nGZ!W~+RueNE%vruGEw;8ta+Iy+vkU#6-v zqy<9@f89W`k$>**hNVqvPzU34eCw^_)A8;Oa&?b<9!#RpZ@`JX#0=XtC;+1XR6Lz) zuW3}j`S&Nb;u`=y0Q995(w=*)+#v#plIl#j1`iX}FAWAD{*+fM-Z_2)vL(=>n7gS5 zVo;;~%CZZWTRs=z^h~`~Lz3}H{>q)u3rDRiM-6l`YG-lbZ}B3f`qpgFa+U+zT`AAek>#vWIa z$NgLe2|0X1RABDiPg7`!J#cNTE{3flTa1udH05W(*jCj|9ypqI(IU= zMuivaUDXOt%b@wy`Bq>{%MH&bGCrEkYs+6=5}J4}5~i(4yf90=34ZpP|K;E3bL$(t zPI@fv5*$A(4YbtjP?5+td)orJvddB6R3^|$FIofbEHoOsv3r;dL;X(znwD4D8>JnDB{_s&2R9`*yp8?M3bpJA#q588Hz2phkI_O~K zdi7+R#)*KiGo&X;ZnOZ6XDYlI*h*U->WS({BfPr`I(IHO@^98U>+mF z4gM9YWvSIYw@1T0$r1W1!;gY(E+hcESK~WiA%4{f`gu^gudrI3$9a!uqrG%(T>Lz; z)JySStDPnv?$9a2ES&qe`iY3nmF&;xDwxpY0kb;`?_Pr8IuZ|2Y_>TM#-4IEss>w8 zK8Uc-)WRJJaS1`&s-ad>^N(sO+u{p?4Gz6FR!&iWT>pw$RY%jR83W!c4vW_z;eO?m z=OsEEQ?~ecdHskLj}M0Z5jg4O5UW%czCsKugbd2A?&?5H17PIWJBYgWB1~ex=X+eS zx~4kH2re~UhV2P13U*n)N~(p_!KhLG!;rVZWIAmOI(|aoS^o;}Rf_kcpewum{B{ z`ek6+nb%vVQ@eT{+7xe74WdN!SBH~*PE5?_P28a^QyBu@q|}ACcwhYr}Z357LsqS>26QrZ9*m|aYP(n%^57CyRLZZI3D&igwKFz_gW?`==o*FHo~ zFj9J7o{oVcy9Wq|KUyTZV0S#9GHLB|(zd(!BhYyF%OUBhD@HD!eZ%leFPfMzz_#Ci z%mnqwZ)C-#{yldfw0k$b8>ui*Om6=S=C_s;Yhn@6w^@i^*jqDU7+<=ga31JTrXcca zS%+2w_0YE^XZ^6BE)*0B=X=Q$<QtqKsLH_3hfjW~xT}*d(GrJ=fDfh%$d(IkrXJ_fDF8F?~ zn@S5)+!*km-#29zus-D@g=(W#a)xUztg9aph1zWlpk53rQg=t+dFEfrh13F5SA{&! z%u|Mm?o{yMwZXf!{-@CY!{A&o1s%70SA#Ry5)?bPS2r2YOn@PS&u9JHAIu8-8qV^I zZhmM0vy94s=nc-J*255uvK(oEdzhlbdu#h|CXS9Is{3v9C=65#3i6kbj#Uc`62|yE zNn9aj1kzKv4`0S1ScG?WbrdG23LOQUcLad=17mfMp1G{BZEuqc@oK6nsrZUDU5Wv- z2I})<=QId(?x*2naZ%yulH$FNFBm7wF6$lX{Sq^m)_g=>OX>TOZlyn#Mll^bor&$Q zol|a9Pqu56Q+?_`FE(VS-xiOSHVdXHSVBdFlcS1@$fIsX2Hil8%82H0B*pZmkg5A? z?z@}zH!tKrq5nWBcU5*g|76chsN0}T0mFqs{tHeJgyMHBiwIGy2Nk7Ug<=HEx()6u zw7J({NY!sni<~QxGtyVsjLrp48l@afPgROU&*e(oqB7!{%T3{M?e?r&k>%T`DFsmgf`=^XyN z7{2hXcj>Uc1b1t%TgMy2xg){HUdI>j=#MMTFC!v}pHTpIej?|`;z&FX@?wvTi_209 zIP>D-;J}I!IZIk3`K}rU-m(_NlwdASDG3RS zl-Dv+QU-@BT_cuNUaK;nNN}@1C6AsiI`bc&DuO2ngr%#Ay*e(TlsqoUg^o7{L-j2= zy(>HnIz>`O9vEn^S-`_Lr!&V^{j2ni`lhWhHd995ZY@mad!5|RG(Zljx81#$Adj|Q zOjfc(D1{;`@Pp|Q%Ha|4XrBAT$CRow?|ls{evRDJLRV%a(%*ING{+7?$l; z?XBmlk-c5lz%%|FaI)-jWlI{Y7HMy`fQU1@7-=Hmt4MK_!sxpy5y8eYjsBN62Z@R+ zU#ISaG6%MPHYJ#D&cM+3dBC1~xoG2G>)NoSBfUASh zvU(jh6CY^B(F7xOPSp+}Zkd07w&(2;Fe^@2dtXc*o%Y;6t2YTTWrFRxwqgv^n+7F( zCSGS0k3f-&C*@OblNolcu-=7-Z)MByj08@A=-jk|Ie&&RzQ-OpUZ2?%ho8mwR}Rur znFIGnHlTG_;O!bvzaE~{Jd%)XOXWuInO#Be_$*Ha;YPsm*tRF=2gNbAAb${PvQZr}2k@k@V zi+yx!d=ZP+xErI3mV+z;$*kf=WWbS$m9^tz^D$+Yd#a|UCd*q#G_m{Mo#6~`x@=#X zH`lUtce4&U-tyeAI7pjUR_?+0QTLaG6kF?}wF-G%h;JBH))5nd7Fam=X-4lk&d`F>MBMipowj^S#z`-OdYDr;a!S2$-}&Mb4N9q&T&dOomnkVt zm|&`q4MBGlX%MQh3=Iy3;bhhMPf@YN@DJu+HFyyekKIw6DBlch4-o#Oazo}dx>D?? zrK5WlXz>$sW50*CYb7K0RC+QtJSIUzBJf8nBMRIH2Wyvu^uX!b_xz$Wm6B(wbqY1a z;red)Pv4qqa6#xuPOfb9=ZyGh`L(?sNfO9Q4)O7tS|6vHw^akv6}m&5&Ei50q5#jg3n(2rG)yabD|wAdkS z>3c@FVyT!V^d}AT8qY{ShxOB1G~EA|mHNU+gxT(J@it&Sn4qG*kj_M12Q@26eMHn#Jr^e-t=pq_2M`Qf}vpQ9jq82WB` zR3gFp>S2aAvIA#QYV7$K3fX4oNeR2-GhLVMOH-FWQJmRLs{>>Gh~nVcK(Niu*Qd4a zqaG%6Y!sq7bj(D7CB{4+L+@YwDa~xW3eJ4`(_0~QbNB10g+ZByLD@;mDT^{eOk!g4 zHQo2z2}{F@)8Q(EoKchI;id&&;00{b1OV>d^|MHObGjMRhrHr(- z218a#8WA#NvtZ1+Bvhux&1iwaMJ(srrdA;F)-jm|1qT5Zsfh}lKh#p#l0M?rdw5X@ z{|caV$2K~SY=!P$N%((YO(dB993J$&2>fd%EYG>12L*5zgM(OEb?)h-1t_)Vz^OfnR=s zpozqHt-S3rCmK{Dc~{vnhERmN9wxvwE&ClT^*j6KG@R|2Q0RNJvM7H6?`*WUr@+g9 zR%_YaCJ-CnPU^aBG(G*jp4i-ghTk38nq9C#SCA6q79rz*q;iSLu)slG4H5OxU_**v zIE1ymGeB|2;>#SPyie{k7|AIwMA6jj#lAmszFaieX(f$QFl-7zsLTZ}c zi-uGL(x5OgvqXC~j_rx(^~WhsIkg#}3^!k4QMs=t)OMJ|WI8JeG~1s`%)6YX1}L!Q zjfWx3Eekfom6d99v(9cCkb5+ty@i?y*my-#w|k~3eMP8k;z{EMUe@6_)z=2U40@yt zY?0I{gM%V;DMyiiwWOqJ?!8zUcfZfIjkZtd>7%R(OD=~am-)t}Cv09@@-up7!DE3N zpE+c2^Ru3vXAI{o7~V7Ce)nxD3WcV0CjXPOZ;z2ja7JfXSF8eSmNBK^vAPN^SOSZz zBhlWbd1j9?3F}cOYtNNhEgE{1+TiPXnm?w}y?K)EdMTf34SBXIU>1jKEQ_w944&iq zR|_y{IQlekWeVFHKHU3QQur#)*7$B-vz!qwpxv-iAQdHeSZ4_iMtMA&MvU04%sW-- zNp!#?+}&Ak68RqORncN%>49`n)ToHrp`KNUc(}O|9WY;E#|O@5@Sw7rqgq)Zvj9v(b1Tynk2ny#_)2x%^ww zX1BRSf|5`{!QU>1TWHYD;10CPoE2aD-MkR=zG0M^=a1IeZ5oI5Oqu#wa-a1;d-V3J z5RqE`U9dPTMm~&h9b4c{7Q&U;LNYn9(ulpb?WFlX`od#4d2~joQNZLaQm4@udtKrP znX>Iw%!|Re;hamU8*ypAaMH!ba|X6XGmhZl6Wk2rM{q-wsHVp|)UO<#0;C15adP;t z-tjvnI^KI$Zpy*ieNXmUM;@^&L>#p+g7>>hrhf_0PxngVg4CPiP@4)K#;BgE zL|qjByKy@QV>Ke1mOC`??19R;sl{q5v2^`<3M}&*+VS{jXHj@DTD3X8P;(4Sw~7;(D~eOXry>5Jy4UsWrTylOz7Q4JIO}V@8FY zBGLfThE!SIrC&JQTWz#_BakHcohrNddagoObPYdyYZnhyaOs_H!pIik=RQ`$(kLj( z0>*a0$1eM`PMbTAu)>f%9W_n-uD(vJw_;*6GRn=AvdKf%8YeEE7jxTANjy6q7}P3hBg`IqlxZ5xmh6Hid??xj z45>VB&#XT`OA$qUPT&Mmk$ncx9glv*-%Y@7Ln7B0ymqA?6{&xoy{<$kBtT zKIl8qJ}B?`wZfj%NO`TLr#qXOD`7w>3kp-gwf)_993Teu;+C_=%6XDbrp!lsGZqfd zhF#|O_S*AOs*TT1UY2f5*L$+Rj6FCxE-^li6k%fLkTTW8{n7VYDLy0EuaQE7-E2mZ zX*seUTq|&9kG->?b5|Hy52j0}#!6I1bm1Y4LX%crR9&MKCH9k`Vr z9qI5wTV@pU#^~nE0*g}+HDAUq?Hw6jUQ>U(){DUqcs-Nj7OfR2HzTv^*r;@cdOy6ag zw4?V+u$o%kJD+sa=6o!hH(?~D`xNPIeAeN`7skmt5nYX=N%v>AHzaZb9}9QS30}sQ zy5Hm8EOVAD0}KP|Ch=CW`xY2jxS(;>mERjw;%{AF)_b#U=h2BH>&V;E;BDSHO2Le5 zI>e~oM}s7;tZ%pPg+aTa0Jg_m<}SKHHJ{0;9K+4yc8J{mV$&YXG{YM`$-_YQqiqEL zIZ4=_t+>%7XxDSrx`32eJ|p1iPEGgw{qsxsi#2?2Y0a0gpgvjH6X^mL?k}5sFqfzA z)3Ftnf{Orq%$XJ#R-RoOwR1{$v?ovw!3~uvbH3&KM5&Cd{AtmREWFEZ>BS>G56?hr z%5ffn88tbPj}%~AanCDiY*w8cPp%g2H`)t;cXZy)zl;s2c4mYc95g^@DUtZCkDtzO zSv54yTi+b`3PEu#!4ymEe2w+aHW3e8JZV{{=Z1m3PX;Egx>HI8pt4K&75vRNT7YFFSnE zcmXfY=6!EGAnYx?uJ;VlP5INcU@U#smJ93ML85w@@o-!1P6-+D)W>nXWs>A!<2cxbeO4SDvL%xY_RA z;C5}-z@SncT3Dwtg2JN5BC-SzP#yPcU@pZLk*zB3=nV5nGj8)|Y`nhpi3KEa14$b} z5)8M2DZ%x0!nvfO*IUE(Yuv| zSOX@E3{IaUIm~fPSbIfm(ZRJdBM=qsvYQ_5XV}Zdjx3CZZs=)8wlrTU)Yz_VhIs#C z$CNYlMi*vh#qXQ7Bo?on%>9dYLZsmjstR9O#66*D6}?bHkyvMQK=*@b((43eVH?Q1 z`APb>hcay1&1jvCl81T$H=9Y zdq|*69R7JX{~a$vW_j!A5t)!skU;ufqnx05ymj36KD_L~=1O9j>ot|ssjF=Y$%XSDc76fsOpTu7pdewW=Z}-5ZH7q+=zhX*JzZXFv zTkxhM3u#}-IiPTTnpIfj$kFNL5&J>dk0$fm4~&oVE{_4&X&9suSqSUN@Fd*(J>P+k z09%ol5j@&qY03kNw60$Ghdj$Ucv4@8EZ8cBP(uSZ}3x84G*~->c=qv{1K2-ub!W50`omlz!_K z(hz_(KOOhvG*)u$ug20Zhqg2afiTs!cG!JO1@`HnW@4xG!=M|2A8jZ7*0_@{Zx{$l zO3ywW{gN1uqF`wd_{J}>%(bB3K;?bC6lbHjb!UC7pyM?*#l7yg!(ESje}2H35%AjV zHXvTw534qkQ<2GvEgj3aSI&m)pY+s&qy;S+06mikAf;h5B@d(G^Xdr-euvFXZsDW6 z#Dmd1?G>sgOfT^OhPu0&ZJ2Jpem;LX1(xwHuG`S=?bUu(jX>(f?+m^6W=UC7p6vV% z#5l&KA9Xy2d})_o2%wnCdERl)ltjqrMGd6JcvxKcY5?z6D_(O%0e=f+KNH-$J={Nsn?Yv&&)zGWmX?LYFH_rad zp53jDY_EWum*xQ5UHqojZ?EyX5xRMQ$_S7HZiEbO9DOawKgTmffMCezj`F^ir4V(~ zIdhW1ruHK*ZbYb?IYfSk=I7I>Lk@_g@{-ZD<5j?rM&7&cYVOXvqkt+imY$XX&GCZ;kXP@jfc2z?$)@8anNkQSlX)#;ea@+o0C z1jMxfR?aV2p4h>ezgm~9b;2yBzb1`sMEX^2qY<*uWjBx1K}e-M*uw;L)wtQD$Fd05 zf}PIQp+*Hr!XE+^CJ@;4>az-V7tb!s0Er_>70UuvyRE=C0IaQuQlSNw8BGWZALJ~q zU=KF~@e>h(WL|Mzdko4MwP+aQ-4z+tO@{2jeqU8NB+j^ErC9n{S(mxRP&{WeQ^rsd zG=kg@TcU1IZSPkf0Z1beOD~n9T$%2;bV^uoI7lt#x1q0%6;)F(H}x_=p}?1^oBA_6 zjJES`N8i((Wo!i8P#GxH1`2(jZ|nF?UXVM>fe?-t+0aijALQ%BXey)B1~lJ|f*`sDZdC|w0? z_!TmwHwxEo@9n@lDRQ=iL`hr9_`B@CzLVdBWciOTs{`Bs&4U1XbbAT|W^>S7`=AIc zKku%d06B^m9KB>L{e{1{^-LY{{rPkSAdO#}ES)*NgY7xin3VVadgHKuFwD;288z;*j0%+kIMD)>s(E8Z?bFL7LfHz=qlyI5 zpsdUq>lQ2O+`wMyno>9T9rT_nr5p!McIkt={oWd}fzfS9`t5-^s`7Z2vl1c{e+yH2 z!=9SBBd2@aIW;zXVBZy(8L@a8ZN!ZrX;_(RE!&X8v)@a6|9z0w+wBP^8|=BLix&Q+ zE>==lUYn{g`yk-CaKA>LZ>FGB+bYm7)0CW5S5C!ofV5&`N7(GVaeA>ZOdZ@o>it4e zF18drv!U^(TMTdyi0-M~?PDcQJrU~5V}n8_E#x2 z_UdJlrhj$l4>-vzaBO!9-J?YLgmLLz!8>dpmFTNB+1K6 zn<(HSDd-w6-23Zi<#JJj&Ga?E0Yt5s_oh`o6Z@-|n78u2BE;(YZs4hQm5=sizG<)p zNn-XK1e@!3rFdWCoavBPlI?9U>VtPGisxy2U6`9{au+8Hp$k08^E{t~mU_SoRnADW z#?h~nm*49~XiOI+5wgGiS*)8c@%A6mjetA5KjfaHnMA@`iM0Z#uf#HCAEjini>!%YAu9L@bxaBy{qX--sI(AdvPq!|Xd} z$7nE9`mU(o6a|q1O`(i{)D%*%KGazbLfV;_XzXBBsznbEXH#H|lRtNMu^_wH;`0Xz znjBDk&~i~3DOAi!*#5iN*NBjOvGBl#rTJ5m-25B5u6Nz@+@%Bxftkzi*0$(T;5gD< zMfaPxEN|9(H^9mj!CuDdrjCXSswO1Tu=c~$47{wg zQ6#R02prhnYr6-dmn9)Y1ci%!00sssgs>P%u4s}P55NoBIHsa45_e_eQ2w@=6+GOpOOp) zmY$Gn&UC!ZXYs7m!dxHz5*nfXah0>A+28IN9zQWgV5#FJ3#jSXiko)ZYo|`L6@qt) zlOk?|qgMD8n`q`zi5^0>gfn~@MMZB2^1fzBp!#s2!qhE;Y52@1O^~K0KB}#}Fs`s3 z0u&mgEyzZ{pfDH5X(C~|(U?<*>N2A2LnRsJtFR*w;X2Ab$DvLs3+2Ntzqk>L@@=)` zUT)d1r6RA{>z*C1rekraH%`gn`7WLaEOKmE2>JYXJiIh>%S!T8rzp>rBMk zyXQE_C`#s;(vm!X_`6lY^3$ZZa2`qo~@KBPdvPWISgw=zi5W$(L0u?Iss z$+}WA2mR{o&@d+3SX8CeDp2|L-AQfg&z_j%T@Z}6-{=V24NCxTuAW1(2m77nU!bIt zlsLX4l$wST*wMu9@ElYrgpZ|b4euyaJzF%icMswI=Bm4ny|0~R?J98XoYL$U^rvjx3dI-N^rl&uf54Y{{>-dHm z%Q^~8A8H(<3fH@!YRyrUlPZrxZ`{>XqZ8hKGyc?UZOv?TA*G}Zd;?v1oY`i6!U6#l zZUn{6CvX;3BDUg81H)Y(_ZtusK^d;~{#-ussLDb~PbDDv#}kXVY2KYPw}#)((zigt z0YaqLK|}+AqN<$C?N7mZ?~PLu(OfdJ1$PlES7_VDxWGtZgv_uw3Etl`9(5LSxFvp8b-6Ow;vE9s-thc|8NQ!d zLEE^)&fyqjA=t3WUq$d{wf#H(;q5h~Go*ey04aQ9W6hwgafU=Vzp)4Mj?Z*Cxu2u8 zruhR8Mo>C7gKRw^twq&tie2>oMHE^JJJJWyYdeKOX=W?8p52qZ3ouQ;w1SG_Llqp-A2Y+a=c<+R8aa z#L5H>X;7YE{F)<|n@13rGLH%=AzTRJ$HjDHGLtrDw*Hz3`gyQZHsIqPbD|3Ewg8hSiUO_y<~f| zJ-at4@Y2C-g>kWtSY1aLKEd=5d4|H$qJ5~TkGz`Mp^Hw`Wp0vod?HCnwB;`AAE5^A zxp*yhORUnQ1vRp%u(DxfoUnkk$T~IYaTD&t}?jk%D=n8aA z60&;op}YuHtY-(E21nIqy3g?q!>6_m?D@d0xb9Q8ildNvvKvSZr0X%yFvNLG1k*Yu z6MkV58yB;Ipd$M#TJ@|n@M0;;@W7@6zFSOM^&5Qo`!pn1e_MG6>1}Q^)8!%Z;!Z$i zHVjf)3HT;1UI-;e*7TW4rD1`t_NT?0P1u1iuA=3!B=vYvV6#AdQa95sA*kF)d%-V6rgmstl(5GwM*8bASjZO^bT=aNK)o6~(X7(nGMR=^(GdNb1GN@I&^M?#cK4jBJAI9pPibY`PNHImY&njfYhBoeTma=s8C>vcmT zgH^P3XtgCxX{MMFAnSege%^I80hLo&Q%+{S%0kpkK0W%t-RYU`0p`*hW)0?At*5L7zqXd<%kFlF~N$kzAfKJ?4 zJ(`MK^VTx&>xqPaTE>8_c1cmA`7>XXG!`aMN;DWSWT2kK;zUw`QG4}EGl}0_tETP+ zeVCYN3&x!Do1kD3bVhwOrCRf7pQ|AYOm$ z5j(Rb2-xK%;>8wfb=@bAmX|?lLE4y%>MZd-iSYKy*wQ_I1Y<&aQfs--2BM8Z`|zPl zJt%S9Zx?RUfX6+q80yIZ(vJNGC-EpzRsdUGb0+yQ-a#6YM2zt;nyhnk2Z-$U{DCBb zMw+mqdX_w^My=qX4-OcLJVVl4i|5={%*p9xjLiL{+?7ax9z%Q;j%LDBmJ|R*&{mMb z_@-ibm^8KUnX%!nNB7tApxgQ}=ZEl}f#qW$T5`GYZ8-QDMcui%T=g<7o)xHCtkvf0 z>_ojzT7Pqq_=DfVM@!4?r?ol~u1kF)Cy`Im7JZ$H+#Rchn2yMTa;2tvQ@OCEy{y3abCsN>|KbirOAWk!&BjOAOX;Z z9olA1s3x>!LM~+*#v*QNJhnazq=;bNUu^OS-_L&774jg^6&3Tiqw&kak;ppOxMcIE za%(~f??s904dL#vJWS9gHuxowWAN(3@+%w!)p(-iE=I2~DEd=haKrA=R%l<_lQ_s= z^Z;C@TsIr2i!=sg(6OQ#TAQroj05Ca+OLA+I-6%M|oD==6QhF*>2Nitu}-*-M9WjX@VI* z&9`fh%csQaX88V?%(*#jec09-j10cpaptA40yIOOwHjxbMuQi5(9}e$S*Z6daNq%k zN1&M&NKAE)ip9rk*sBd${+(cD4?6ci1Oj*xzcw6t7%&w7jSXJz8`~eVIcQvg9*>+z z4Hf93_AE{TlqrxCyu#UXPu+NV9(Azs*w%0{E9bz?bMpuY1!RB*!;p{Eup&uGCn@;a zrU6S`PndFsj}-wHK)#>-RfeGZB=|d>XCbGs7KTjzocxQTKKlI;@_iuaWw@+=<+sl* zBek(5uD=ri^X?Pv1|uDo+-*irR)EzGS!9k`@{;Jhr8*}gfFBO4i-}EhUKTgYZyFz5*KfT&ZuqtYWk6F6R`oYIXeB_ z3Cf@Wbs3u(KH9N^{TzV(eF8k7mx>%Avw7Gg573oS;($fMG2M98pNfDIl)EE$e6 z;5xaywM?1tyP3C!HfsRGg&8#UR`OA*n|_l*aD7_>7s=2~4gcs4vA=lKPDyISvLw>- zdANQz@(v&1?c3+pWO>PM8zaZ=F<*tFbGaxXR?d3snxCHCnY@D7qJfr3Tj{L0J-?mC z00Y6FH3^*f;Byo|pKK4IeJx&UXkQN6rvw53DeVS)pg#mpLLwdz zk?p!3kJ+DR7l86Hkmh8!A~xUdnV%BVuf4?{RTfs}01ia>=plw;V@NAx{#z1$r3`4I zp$oo-DEcr$F&|y_$>z@8=zS-&E1mxBazn|qZdl^2E9%LU+pOpJI^Kv1p4|nk{Rd4T zQdcGFhqN>vVW|~sM)3t7>t`f*pM|6O!5DvrWiTcxG{X;?z-lj=8teezD0t9PVNRqT zDB5H*th*QdF-iDLqOFJ0jXU5!a&{&yOA?3?L(6*bAloS01$Q1Qce9r@GuUD?;4Nf~ zOE;)F*K4y5`o_@cNhrR({{mA*njTM_DOcF{6BH!p=SeB=Rt&B%ujyc3V4vJv3h=q9)K{{(^Oc-e; zKC8+!{QW)zX^yZ$paUY^7LXxs^;0?P2{?y^P-(c$`D27xzFQm>sjW^P*n|xc)He$* zZ!O7R(7yyW9hG*CWq5wq0aP+F{k_gobjs6(W~%D(hA-s72s~NXO8ZTv&c?^f8=za| z2ONrzKc>8$dwul#r#UtUG@r?ZZXdyKdN$G0owNB~sP5^o$>fnJaXFQPqjDyO<0?rDN)KD8!L z^ZcUXC;Qd9DpH0tIOi3h0a$$Y)DzZeSurS=m0Ml9!9=zoNUiK0Zp4>o35yowg{Z}S z`R2{A8MN_$mGk|Q-g!?bK=&F8P){90RX=o3{6X1k8HT69=3^C+Hb2ojHn+Z$Yb}?LHMCnrQ>b zodkcL6yM@}Fu;%yw}9K6t+vytH??;^)0n8)Wypn^aAocQ3ZL zfeGg&-+2ktCojAmAT6w7BENsVIYfJ6pWT0#Pa zWk(vXWDpd*ek46zkCdD+S@XsY4SC@yPu?6^9A`!7a5b;Y)M7z;GSkSf$ct0;-i;)Z zT}*9Yt2|6u_=;8ojS&g}aUbVYY_X@#9}9K0`H_tV-Jpv3dj6SG{qgUo_5*~5)R!nI zt&!-$3njmSh9q)fX!sARjg%wNL6rxL{_zilShDFpjz9}G1$TL%wNLeNrxq>>z2QKX z>l!m)N&mba2Ao)>$NdBoVUC|MdYtZoH3wVLdn~}$Hq}g?lWwg_Cn#H7uXXYXYUoD~0JP4BH(`XihJ*_%7kRNB6*JK!aL36m}`EKZK8`|0!?r zcGiwB2}44Vsx@lSDI2H)2~JtfBI3m<&0^-%O=prg>@Q=k0(JLS@dV-RQ)Q^48+DQTi%@>zx8HP- zt`j$6rts!aHR-iERNVTY!xix_-H0|)C>RbbI-%RdaQ?e=s2sNCA6|tc1733f1@!w> zh;ucqa~D_-RX}-Euzi13mQ|?DKf^y$P47q#0 z>|HT1l~_WcqkDggGg|tZn^o(FLBRt}o?ERkjSeYmq5UJ8NkLTklib7MDjb|h?j;2| zj@`Z-Q@{J@U(A0?ek~gs@`M_E&2L%yx5}hYDNN*SEqN}*Ylldoy<~NZ5=|GTY1IoV z%~x;pfr-ZAqE>f|m6NHkZ1O$KY%r}9YYI9Z6I{e;zs_QO*%rl``15FYR|y2 zl168TX=?99jA$1?Z!=DGSk*dKilM@rnN&rfu==u3O6peCe03*8+A~bRc$5INk_w>u zC+Kxb@V_hFfBlf9`)E`k=^QBEnf*iZGeR%4M(ZGoA&yT0yB#F_XJX+Sv}F*_X5N7p z&=^eMKsFk7V~_oCN2sDods*}X&d9LFXfjC%t-4&`?@B`uC^xO$;A2`*y9&DvWc6+( zq2Koe+_-W;su1{K!1wDL-sVF$e@*ZV>t^VEn1HyI)X+#nW`*j5Sv@Ql0lHiyBxw2W z-3C2vaK#Jc>S1NVLRG=71Qv@ZpxK8?|Fm>!9z%SIofh=L8FsCF8`f?E$^l=3Q#A2# z#7s|(VqiQ53veCxU(po#Hmx&wnS)pu{8z-Gz?7nYkAond0wPB~XMl+8vgnN#+)LQW zU$1G%UjNd=2foV)K&F$2tAys8L-1x*v*$~OV*i{z(SP?;YH6ZCj-d6mr`J_D zs)r3S^j}vJAWx(ta%NsXsmn3=-Y5ZXlF#Y5?~MdadkW@kL0>HdE3@B1@4-a65J&xJ zX*MruEQBwKl{47le;c>cVy~p)FQH#=QUB9)iE+F108W+d+cR|p>oFKACp^`v=FBW* zY(HxS4E_%S5M930OjDXv6~_Fw^6u}3IyGY2(o8pN3NUR(^IuFlCW81XvgBYLP~s^m zF|!M{p8YNKH?JgG%LvnhOi}3(la9MP;$iwbbj8wjD))=X76U|)ba zfXglHzgb{$^Qs=^!Ai3`Pn}^f&F%0;Tk+7{N6U83qqAYV3b{XGum#4^T1XMHJfh2% z>dL0A)n1UC{wT*?z@tv%zNtGBLInL0-FB13Ped>iL(?Sz+^0A2XCKvH%l@R%85J4y zINC_OtFX)h4chA-*?ybNm>c-q{+&w$`ER;>w7C!t(kuS#z&kKq=CCzzLJSghJYBJ8 zIPMvR8lcskHC~%ZnwS3l^^UnS?iV)Fr7M-6 zYW)WFUniYgw$$^5HU}rnu>*_NUq5^wYSkjpA*hEGkuCLN^Vat)9z$NA`1ymu$JZ`+ z8U6uLj_ldr(Chne=+$y7`@e-=rDRrcg0Iq|uW9Ck^E znNQGz1w9ES*!g`7rJl2WA1zH|=8t)0&7!;2M2;C6#%j@ zrQjF|D#-lP825284JaGDqLjeXBvaJ}#Jzt1eppfO2TFHxuMKC=B6><^KCL?fX6gRp zfe$zg9{wSrt^AwNXv*|NGk|8w!r-9B)~966ABw}Qh{SO&uA@ToTTnr-*~V22h~J!P zrCvMzJ@g0-dVtti5YigrJN$^wk^^ssB&5c%5e3g%H>OjS_*(AVZ*&!Sp9hLyLX~sA zxiD5+U0|l+T?rTfiNyzbS+JQdtMI!E8Nhb5D8lo>4+osz@;?~NywQyw8X~eW=uVF= ztz*F;V3xx3LnUK z@`#K;HGN+dPV@hfLfFzr6BzWRpcwB@S55)*>qWi%M+lGu6#|q_JPc(1T^{rQEU)L^ zH5_Ezr~)t>0p14} z*rt{9HvoW{vU-K$!1&f~kH0DX!P~M4w&rU{a3RXTS!{!=O!yqXEA8*u;4 z!C!nDIR6ehIlSo0`{1Pb;w34xA_22b8=EO1NR(lkO?7b9u`~bNRN4c;Z!lV9EO7~@ z4588&T;=Tm9pq(!Mta~n04^yL1nU+71$Yi zplGjk$TyY0Fh$C54{ExbCupxH-g5nkRE3^7bReZULTN(}Y-tAOYw#KrjJm%c2&MMC zZK~{X*|j26%k$teD!QFh7{f7B|A(*dfXA}`{=e~{%sd$xQK4*Ep)xCl%AQ%JkR7sD z2_cjb$(HP$y%n-Tnb~{qz5nOBrJnEi`TqXb>wcBj?RH+?D1ectDN&gZ($R6qAt zlRhgWe;&Z{GIfZU_cp0zO>ELdvgSWq9{ypvAAj@iBbZ|10dSr~7E?RtfpUlv!Oq~l zLx>!^y3G9tQ2)VY2ZeXJAKdo`E_;vo-W1KRYr>qHG8DwK^K0-| z9kDCR9=&OZ_8*k`?PEUllB+ZxSEsk|=);bPm;b%w7m(THy@=2PZfqOvzmd=-H~#vA zR+;PT0PdGB+s4g$P{3pB#l-L()_hO=)xQkn)U2vZuizf+W6y1e9p02d{m)?;lI`X41mL!Ctmz`{boEG6UF_|u6iGBR)SoG(5&is;`CPqhL^0h> zwzec7u}1fq!_t3W@d?5_)^V$GDZodx*n}*1zt3ja5b!4XFA1T1-81*=so{CTFhgt; zt?|F`)IgOb16jst{0N6A3EmbE4}Gh_JKdB&p%n+l&kh!+G1E&S=7kahR->hnBYODi zH$4oJ{1frp8ZdYf8>qjk*_1M5v?YrWQ%!41nQ%sg=L4oP*xFYoozzeGwIy$f)(PvG zUmD~cx}yDq_kL&*1Iy27$RZK0h3>MvpYA8vgO3O-c4VB{`PXd~K_Vnb(s&r0mia#a zD5A>Bya`Do%l*To5gqE=6#g-ppN}9_%wEt{Vo>GGxq3F(XZP)Nxd%F@2Zew`fcUwD zB06(dWTSdfJSOzQsg^Y7vLz#8NWKQjhZ>#_&Xv}zk$h3MQ~-DOG89gaEGUn$2hIHv z5_L)g6J;)j^U3{us|>*}G}y)57++Vd$gw{gNl>u?BcQ8e^8_@0$-|H5Q=@zL~YO1tjUrqAg=H{30p0#4Q* zB+sX7Dy1#xvv>SG{H9~&xzcq1_S^umfkq#P!B7{L#Wb1jr4sAn)V8~Jj$#yH80HFn z9Ly{0`HD?6TzXhTABHD1STjUs#d%Poap6U2RlTe%9J31aIb>KpWlY+M8Ee%jm*bCF65Px=ryt@APshXDwqaSHHmb-EqTJ&br@5 z@KbC)x8tp8PKvXlOj(=uZWjj1z*k5=T-@^}C~0{1a^&W#!DyUseoo&UXzGY83a8~nb<(-fA9GTfEgE^NMF`XA?G|;WD~_m5CVs*v+sODvUO5p({8KsCw5KkuS^SmdbGm7Fu=?cboF3SGT&9oVN~Eaac+GvQl<*O5UO$I_ zZO^a->Ss)tP z`E@TNegkIknXP31AW^I7lx581R(8j{c?=0ET5W2nR*|)t%5uLy2)uPm{5WEpnU2=? z<-6mEzb29ScHcTszJZg6I8jULs+Xr?1_za8OqorC)SOieiL14$?Mj9sXEUYcGjsWl zu+z)fvFI75=U;g)_0Fv(oc}~faFN!t$;_i!tN!iNJmf9tp`ol^?~a<1B}DSz3Sm_9 zYov&V`sAQm9!vRnQk7|uNcyn^(3m`-N1>GO&VR8{3hOazv?~#!5?HX&H24c;pzJ(P zMmDZx%H%LUtd(5ti~4l!v1x{du0P-FRTgeruPP^se8fTogI&r#Xk_St4+YB`KMeWz zfUf)=&`W#1ge{eWTX#zzGsr6T!HflSC5j7c ziYlU8i9;!4{TXa;*9c3~eq-p$<~dkcStk2*vt=Bm$#8FTN z9-5H~dc()t=Tf<-UzsTz|1c{8`+y@;@yPNj3@b3H(97#eq(gC+**FU7y@ymv}pDTRY7 z2ZJ?!^Xnb3vcWuizLecMy2JoM<3@W&q3MCu9<=eLT-*rop28x0WJgYj3?ahz2vJno z0pif3YQga4;Y=}~7DRBab(COapzx>{Knd@92io2Vim#^3w#|E_tEhgaQ%PJ9M>_|1 z_Jy((i*0rU^Xk-j{u}dJRua0ta$v_&-_yo(ok`x3eB>w0#J9ZrV6wCD0|DwZi_`G` z7xexyL_O=FAadfpT*GQ+-Ag9~Z|+U(9j|Sf*_IDb7SnnzTB8QR!wRxwd}xR!e%5k>ay&UB|@@s zmws7motZhuw^%8PjL=X{PaNt-n-M#c9#F1-XGR(#UbjHi?t2)oLI6VYJ@>)8jw5$4!TMi1@@>j<284(+<^DgyOH2-cvSAz(??8yXhdt{!wKp{%UGquAX9< zXd9XkjOwi?1_>H8+F?CWOPV2jxy+4^Oqdn-5gpLsyN`jkK$Ix;*ae%Ei)$q?y*Ok& z=B1cJxoBIB`(!KIorJ?dBI(U=lT4DE024JyTX=Fn$!?zyi8)4tB?C{>- z7&H6M#~}%vrG+TIyR<&SwwGpgv+ar%8t2MLQPIr7RWGs?imC0nxOC!4sk_G@e8W*8 z|CKWH^QHa+P4VBa1(C#mrPaPI4W;2d3&y1?VcZ3hJ@<1(dw(G2`1 zpgU#if@y_&E)0Kg4qsymU;$14Hvd8Y{tM|87#$9gF0EWb%cTo2(^W$7={@(ts3d`M z?Eu|IV$+?!LxRw(#qG~%Nq&!(!C{yycw(K>QcL@KYiv^0dj z8BIJlV_ooz@@=Xl2Y>VxbTJUkdP?_Jq-_GtWGsF-C~DZYBXBw0XYMQRL;>jK+9Q>d z!6dTePgvUc{mTCdbyhzv{EPJI@ICmw$_Uwu*i1JZ6V$zAV4`T%M|QZ%rW8T%`{Rzt zp89XH2O%BE9vS5(-`c8X+wxtM>v&hZ+Spx@6m@vq3ZW`<^-XKQBG;bj_SEGO5k*GJ zuFy3YchBl$v~w$kI!RWq8vQS*OZyFVI6#vXL+$TcOxGA&^#$X-==}e}TP>aPqa`%F zp($PFKPernAfN7~OWD$g0lgqCEJ}8Ka!$|Q$LLCYhf6Cnm z(2z$6vFqJG?)r<>p-|@UG(+2lW6G7&Y=4Flu6}HDnO{Jbt|fI#cOxm_iXq}zgr++I z;Djhu%J}XmW{24xi&-Gjl)J96Gs44%%=CNecU|HJj(1TaTjFjSy;#aw?pQ7#4Lf#s zs|Io;+$IKw(KzsZ#6LOI8NKdc%{QD`e|yG7V8|(0NK|D#imC;v0($|=fimyiB3-vV zMBqs!Kepvjt{)ZgPS)S$TU6sq}6fsXPlJ-pV9#OeP9aC%1oXWm^kz;h1efWS@=$fF+}?KzQMYl6wfmBhDd#I0ph`|0Eu@zlaAaq%B3Q zX>NxzK!Fc|c;$sb#${V9pk656WGz-!o~@u{_rRLea{l-)`e#TmgLiwijV6)SdYojS)r(eC)Tu?2ARSYGHzkM#2d|;beiTmcO zZc*Ea2NN3f8Jz!-+SR7>f%v^7&JbjNvM#AGFLw?@21p*cMh-` z%@Mh$`lfLLs!AaV?K>gBB}$;e*`}Vzw5n20@gGR#E{AhXEKtfD?`VRyAlYW}+F3Dw_|-q9LD``@&+SmBgV@@Oq}J+X0V zDcU0hgR4K+(>S!ytlFZ%je)}uazIgq+3%?q2~l!sVW0yjOD(<8{LSMwiIvo@0*~7( zl``ICM-MMJ3IVrg9@$rY&=~UVH-?0>@6K_c3Z}lkq5oNIlhNIF> zD$JpWzvza3aKz8PmNh)1%@MpclPBN`C{}uh*(=FxWse%SFrTOyNsJMnM>AmwX3JZU zss-pqK6?p?-I>d?FeRob<1#Fkqr!DrzG_BI*1+9GJ*E7G*b;!Z<-+vi^<+Qt5ng`dKS@uw^^&7t z5R)-b(yp}Mx)T#MsGWr$sO~sprpVfKDyOpFce=+a5bb_I6*GQO#m>RJEBCa4Dw4^E z?Y+ycr3=udc-0LdpQY%RCUHm=RjnWY3|(`_;mcgR1nXeniAK};0x&g6L!HqLNGIx}P5c9G@6F1R%6E#(e=vq<#LBpA|0r7hE}eV-GY*hS>rO0`Xn^m|tGw>9?%l4~@{k9{7!BUKp--m}|BkiY|OE-sos7{}5 z_vPFsDx33Acw`QU(Y%$~mCOX~e<2~!Ki_faUF zq)<9+*ZEq+rhz-RtVzWOL^{k1_SsED{494l;B+o4FW2tLH|u22?~Rq_EL8-9Ffl$3 zKnl-gVmfgQUSG4>2nh)v|M+;veqg6Rfzxpa-+suQV%68o@tVk1oeVM^@7_`NIc@aW zZ;gdKg*%@b*7}`x6at>4SsKG`tD&Iqdtmb-#z!$ zgB#@_+tfpH!2T7!#o?SUP|Fh+|DJ5Z~yv^eX=? zsk5}#kPJEz0PpZ&`PjsHUg}q%y$hCpD7=TKb$xr5J}TgcHuopBr^6y5_Nto?Dq#@S`Rf{6--E{!q261tQK$)|>ZagZ3<$EJ~Im^(9Ujqg^$rxz<5QM)C|0 z3#~QBA|`JE+A|!NPXXc2ybRob(0I(m1|wQ+&0f=CeYRC!c;%foTNEV~)%QM|fwX&O zrNLu75C{T-ZZTNRq~%Y~7cX?Qw7h0`Y;CO^wbiKqNxzx4eP-Mf6ePA_O@a8;doC&J zcEC{8Cz4%$xR@hWss{SB(xE;*@=twQL}7iC-og-DH_vABns-_;BD%Qhwbff9grS(% zq;cCGhwp2(kWPD;$YMhFif&D~NQo8}{Q;s(KrFbr6)H8=2}KZ*SRk=#Y-nC_8X@Sa zr@>NTYY+xu(YMXbg0lU!3?mB*`GBC?=bzS|wwd&xHZ`z>>uT{bEf~V-gSC}99~khV zu1?ZvZ$`UwbJOEd+AVv@-n~fxBpf%U{i!q2Yb|~{8Y6ME>QQKTB z-btCHl9mb|lQLRH50NQ!0P^<9NV!;^A2W-ta^Q#-D{|iq7PN?hB#1R{9&hW~H8joj z)(lh98}{C5#OS*xA#bWsufe(KRIle3T+O$0X3XU$3>~ej}!0O2<8TG zQFxaslwJ!#$XH8iHChR$v~*@=WPMudSoz}6_TYKijnegN$&hl|mY;K?Kw{!AAzbfO zq3l65gy7ZZpG>7?2S`rJ}JxWN(jMZc6Yze9!fA()EAHlgmf z8UN&i-uH*=>AcPeg7afUL#O3r#!8WUyslsGe*pY_v;E5IS@)9*hZO0XhP<=-VfOXe z+jsVV#K+SRANhdC&KJ#aoW5dxJD_b~AcNCot3isPivSmeYW&;lMcM8cX9`5j0IS(9 zuD++8nm`~-OmR@Eccz0=^){T+_xr3f@}g^b|fcaBo+RWG<^ z^5qddO3UTTFWvoWMxoQe4U5~&UrM~wi^YXnS2Pa=sc%L@zXYjpnqVkI(3(}ipY$r` z`$*B1+zXzCPvf!JYvea_PNCg4QV9ALcWff*I{(70mfy(~fO5>Wh9AY4u+@R4)x2I6*exuWh z1Col4V6vv$*klx1eB)R5U)u9)#2q3eWOpk8cnQ}0i`+hK#p9i1^A7oF?w{FUS%3R# zOi=_BEWOyhO?L~Bx8acfU4(JP=AAq@ushBVw2V=bLeuJv^?YylxCA&7^;~bE0`d++ zk_N}j`+;4(_tby~K88>mI_aOR9LcYDt###LT501UWX(NCgta<<{vf(8`+Gj{*-VRi zAI~7c_0vctY@uR@%&$CUEgprslO>%~AK@&Ki}ThCJ2hySo*%-HYmZ@js>R5F3gb#1 z*9_s8Hh#xA^i|Y1mnA;f7QLnxc>@uCMfHEzK+JK#*2rpVYF0<+4mK$coab^o5&CUn z3^lK!b`L^oD|lyrc)(?MgwDv=IMo**Gv(Qc0L$GuxuLe402*E4ynTAm!}7{bfpBX5 zSh+hXfx1ZRzRB(faW8g6y3^T$*s=Zs)J3$I1dk2KBX|xf$OmMmD}7 zb+;+h-*?v!vm+CJQPA|w7mD4bgihqwJLhFHLy!pc_#Y8y{Fey83nGBdQ$$?wwS!sy!(FP?kWUd;b?Ob@tte;IXEHtzQKG``Aal%xe;(YFA z$3Sv_%bGXIn#r%cna^ccOUa`4m7v+h^hy(LsM7&ah~wSK&b~n9kkv$0q}Vz-cBo4# zM=C&I?2d;*XeZBy%Cv)SSj?aI_j5(wXog-K$9!Yi=F;zCY)6L@T_Kkd{wTkVsM7vS zPzfc+M-cijJwjIp1_{VpqDPci$mV+TW!~ME2(YTerkI2b$qZ~C)zb7$q)h4B3(yg(UhOIovAUPIJ}RlWa+FdD?TC@aq*O~w6D1(s&I zi+Vz3nSUqA9Eyd*!!s<63nolgkRnn(Mhi!9BkUFFH3MeveLf>dgtrSV9CgK0J~|(1 zz(oD0pa@yD1)}|-tK*y6s-kXw{5(8yaX0@A;F(tl-X9*|Ad|IX4@E+D}V$cgd7 zATH^VtNy7vXtP21fp7TqdH0z}umUTUW#w1F><;4)P8~E^d)3>B5275UWxtLF)F%*| z`){Wnbb+$N<^HK32~W3IA4|%!k^gwhes6Sbxts&&)a8uM6rgHM}}b0IfM#Wb1?A`Mnmr##O-O3>Z{ZHL&zEN# zJ<`B4 zutrp-O`9INNX)}8C=)B7Z!~0dCYxj`yEK7#SFm#%r-exF={g^E)W@smr%!sXH zCH0P+kYGoN!h^Q8@N4N_D`cF?85&JQ>a%E(5UJ0$3|E>Zh6(94;?U2K-E+w;DWm;e zpViZrdvYILpY63yaICbQ@i~m^FOI~~it=hqDytGO_S@&_C8wvR> zEJMk-FSw)&2X&fBr1WOBy9&r7BC-91olHSph%j!-dbd1rH%Xrgo!Go$>9vvjt#7?^ z6ko$BlOaBz$5ga5f-E*1Y-C4V%hkmP;(~4U137pFkRJPs)SqV)bk?_=wEnCo1S~OD zwVg{bpm`^G>-bm=tx{=;PNTJ5$&6M6Jza* znkw*`8j!vnw$<8`#~HUVj8u;VL3PHs-OUZ9CR#4N(q{#Ke~v&|2-M?)nN zRu`#62BD=v@5ve&mY^;~>_fe6enRuezjvTTy1q;&DwXfbD89vCn5fHM5N93{B@2|R zSQyqgI*#f=sQVKh@);6L8zs;c;kqX@@wK#X6?o`w;$5&|3h13jz80`a{&!tsu>qlo zwIfTPieaf#vG+GJm}zQv7b#4X8>HI1;}5n6e+JKB4yHmtfc6+O#f%TW-LkV0NF@TT z?eMwG9)*OMXMaM%NYTF3vxLTb&0rxdM1XSq z-B60=IAQ0x-Mn`iX|V~*71&Rdn!>6HucecFu)+>C-y>_nK1#1p<>4R;ps(mO8~0zC z7TyWkvO{e=@Hai5wP`U$J(Ee#2MO)nzSa$ubDVt)lx7^%%K$_W87BvnY_GOjbOx-< zwO53NY`5Wg{&K;th^BdN4)^o=#c0ctp+6)U_4M_>BvL?0vqi!I$N{5 zP($uz70l!6xw0w-EEyrh&{YHOobfw700m0)N@Mx@1q(w)U5>6|F_`P+Er-LL-gF#0 znA;Nx)f|BPn171y4}ce-J3ST`An}3WyUq`?Sz`Fs=5Pz4!q+TxMPwhZPawsDlug0T zdO9lfsVU>D`4RP6cD=E>8=XYyj6L)dj;kNJ_!sPDX?Nh$L{)miXJ>`v^Cs3f_|E{~BewEZ(7@>d_y_a2EmL@e0HP4xRQcZ`# z=cNniq@5D@-I(5?v}8xw30|rBXs9Bp9w}~$HebFGsIvHg8f6u|yGr}NXt}_i0)Pnp zN&{O!?m9P;L)l$GiCRJ@9Y}V`rqHLMGc10C-?OkqL(t>clsAmj^_H+*U0#` z*tI6-96KsJ!aHFj(R>=xy^$HY2b+OuoY2{(9&auW_q~CF1ky7|WomL3df;v$D&W$G z`YHZbY8mA!UN(-1`MBzl{lJBm%T9DN4ZA5;v?j}8?uF|PWSbz73!1{Xo_9ET3aQvgM#(?N3s(^EtYo`TtAfo+> z{_C4VWgAS~i_Rcr@F#r(TWf@idE*M5v;&tM$o*wnpQ^At zjF^$)M2U_ZlGLC>l1hY3`4SFzPQ{(4S29*Bi-9Wf>tVgIB-Pv?v2%*)pa7(8OzL52 zEQ=kkN-=y`cHg`w;+LoV>70V+^By+XA}Cnl)J;#E7+}`}Ewr)T=UuK<_FEpY&|8J? zD1ONkb7)U~)L6blFp z1D=R*FOU^K76G5mYRIX-Eu2&o!2|jPK3Siw+rLtoKO=lw=vsq-PgBx(O)#~C#+ua6 z&^a&7Gtl~tK6PS`-5B^#7@gTax(S0Oh?&5dL4QzrxQI3Fm6Hj&udim9-1;g{vJwx3 zG%DpIZCqJR(R6kvSk52jDN&fjrJ?QaPH-*yw#=^HA*u01Z6uIRzV)KH|>RN*+ z+ifc>w}m=og{{Bcgvz4v6Ql^29Iy5W1RAT-b461U?thnr5~C)G{6?58N41o4YG0|j zWXk*t3>BX^NBsiBl>Y=n!_iaMptA^1q5v8SBWj*ZgL|EW;q^!EU5kFZ#2}>hmPnn!C!w4132OD04hHwBV2S1g^ooXSP<>C&O_ z{fjy!1qnmdW)~{^W$xfiaI{n+Wt8@3h)zv))N`|IhNmDRm-TFhj8koI0i z7x`|;V(M#dS~Keq=I5ONG2;X9Lpm4vV0V6BHE{Xi)BJ~0z=LhqBYv2M=7)k#vQQnN zM0^<-#j+=Q#y+Zt$>}ZE{hv~^Yy2<9#WBRT(&?c50oyg_bML-LPp{fs~QI$i;x_{|D)6)^sED*eW3)ys#oHu*QW@Sq=*#uN`r zV{+BswwNGakH#YVVa9YNw-{Kk9VL>$Ycj5^i$FGO*tm5KYnpDK9ZleaqwM6|3sqo# zB}_s5&6Fbh7^!l|&Z&d=K4}VZLHrU;5VVxsdfc9(rh7@pma)aU}dAFkv{#I z_WH+7^LxpSQG&ELwg=*%B0aja-IoZVUba+Oe!b0Kk^T0m9A_g{ucmx+AsSQ*-|XPA8|x4!Ju8$ z#wUTB_#*;`V#m)0vRC0pGP-BbduK7E^G71!3aO;24X}I^tL}cgNC0%y;J@f>kkaDN zcBSY+MgJHD5Gp`-H{E<_QXmoYMGwNR`Mo8eh%!P666EC914B|Se1r9{qJnZ)RLjvy8Rkeg%n%VQfkElhE6WXq3#OJ zNf*~Nu5E1!(gm06M^_^;A^9yN%}|7xA#1(E&k|&tfWS4nsAsoBr`U9wM{h62S*@;N z6#u&BA1bK$<^fS(GaGh}c6JIEQ zE%cW1g98H8E}pgX?Wt=6uXUjt-F%?h+pO!bPDSq5mmr07xuEIb74^eT84v8@`WfxMyHn0H3;10BL|1{wt5uH2vC7Q zQ55K-`21)dB}B^IpVvQh)m&St+GHU+^jWLT0JTvUyk8k$qu@k2z&9`G48)Mlpa2&d$MMXkc(FK#`TxzqPV2J!L}+U?uxKW|_!D%{J?bx0X`fxPR>49QFa(s`elxm`%u4iq%bk7M2A7MF zHsU~EMTdtGsg$uPTpYV>C^|DVvLQsdNK3&^K&Y?XZH>{)|q!KDq@UXXb` zzHqPJ*CnHm4~615u8Nd34PngevaAUR&$M<%LI zy_yV<58AX@=i}+V_LUa%KK_aI6PMIpD0|D2N|V@Jzpia!LY8^s!_&uU-eq})?cN4z z?6GIuGtKV5;S0)Se8GM~JWo=ul*e*m2e+x@%Wb`?%WMP9E8+TY9p)FpDQyGlA{@#+ z!#t7wqx>&F54nCGLZQNi96B~i)YEEWZ}^xmlx_@n>qp(Nu;6~kP5E#?Jpi`t7U^ov zNeuw$vPuYGIIfjJ0j)=$OYBy3vNU$H&6~pbPIccCXZ2$J_BTl%2gC6$TU3`5!PhyI zl?irY&;i}`lW#APhb=a?+=ZKpC%$8ld=Wn>onCPBx)m;0zVlx!y7Gw$V%IyL^{7Ap z8Ji(4LybTd^w5pnwjvTzA#2G zmmu;<AR%byi5Ij%seqc9G3rkZEV%78{7wMivBpV8FaD@4}5+uDlc+5lei<921E{(#r0JwjdHY(kz<=tzGEO zswuRb5EnWPC+d00)^_qtEH#Y!5d@%#ENH*9y-I74tN!Zr74n6$9gZTK1;2Wy40n6) z>({M(8Lwk;`eEZd#JIdxJ>mTgGb$ZZ&r$WIi_?2tPd8QySjMJi6CtlZmcjcb;{NUZ z$7q|894qU*CWMWQeAJNVRh@`ieJIUz(%}=$-q4xmQ(t;dSbb0XGd8|?-roXhw9{(v zg=6Lp&psz`MpFNzkYxuiYdMMBai5fl@NA|c^p@IIGk+UI@4hy4EXXkCi| zZ%icYuy4I1x@}fWMG$kgC@@v4lxk;V!IME!mWdR(&ZQLXJv=ThuFS|?ocW@Yuj|@l zBF~eMXkL>NY~jQ=2*FJ4!*%?mQRPlWoJw?jhJWtM{=qJ>^+La|T8^GrMyIEJ5zP3V zWcEnnjJR-@oD(E{XB*S##=RY`=QBHW>j`koj!s;Lg?RBZ2t%R94Os}}XuTsHz%Y<@ z`@Gw2^37h;j81$xQtDY;)YGR=GkfOB-oAN*5m-l>*#ULCMQ8?z{q%|S+jQIsM!1xF z7d73PPB!ygI9Jpyi6V+x9G=T4&FpgZ^uWZ*9({R{`Hb1Hjd#qLW=sC0i|z%mS|44w ztxT&D9x?Gvb#Ozq=-u{yZ+%T84!2V&{eecAC4-}`l# zhE7i#Dk>>ejx1HcYtH&Jyx<4Z{J4)FaR9`yaEY!P8yi=sOTT5Sgw`y$BgwZ)_-%SR zk!gOhTsd^vJ!pv*!!Rn;;0Ou|iY_Q9Xw6HEq7l{88y_o`g+UInk>d+T3Z5+X-lu>s zXS4Sr%a8u#7igvU8rPQY{TN!Zo!mWoUzdZ?tyAa%{xo#8o;>BMj``a^# zsj1hagq_gthWBiB)v7U4h?-S7wIW>5(tLeZZFiWW-1Tk({#x77T>Spu6b~9mB<$=2 z;p9DY9t~s9d*)0R1uk78hfZ6Hs?VRXI@F7-n1y#{dAf`*X{V&5xb>NOcFtgoM_`J;0*z*Fixq@=SV$p{E|6 zN;}h$_IZ7-hs(I@;?@1lk;%Sd{&V#7g|cVCkaQ4pdiqh8Df-70yiS}v={Db2tlO1& zKlG9b)g|MumQiQ$V8qd2t{y%d^?iR4ZS*Fh93gSAU}!IJSGBbjsCh-{#-UjhT~tJ+ zS!8v5E$H_}AN`OX3ZcSonSn#2z_qC8a$847m0>RO(#xrQkX2!rd+y2=>Wu*xk*caH z%#NMK&&2R}bzJ}25&X!ca_xfxnJ8fcSl<>@*!S=9#(hP6I&^ZRLl%Y4%kD2+% zv|aZ2U`A)Huas?#Q_uDlQ$Wu^4YcdKcck6pdExPL5|0aRnKOx_kfivvzP@U8>W7?~ znim-F=G=v2n0LbkFnp>l+A(+o7{o|#Xb09kd|metAgOF8A3`90eO{bX{WfD&UR&;;(Bl$W_Nkn zf=1X;K!;B3#F=B_6~ZVT?stxzEEopnLSD)#?%l=`?=P7|w;DX}sIRN5tfg}yh?hTJ zsznShw@Q?|I7Xqye6dMu?2Q)ntgNh4e1?iG&Hx&gJXF4?cnwYzJ)QBO6|x_MmSUc} zQNo0Bq)&#w9Cw5@xpn8x4MRhgHp8ch3DVcCq#uBF${qXqko)1IM?>kw^Hnj|JO`I- zch^(W(B)1KPg=$9W>f!k3~|TSxSKIQrdrbO@e`cB zObTr(H|*{0Td<@msxyNTuzdS=xZ)XBSFRBS5skp$m18(4Xv4d8_pU^SRw?pV5B2oU zF*6_6%h?L^c-=K;4Sx_@P;km&W4;<}3i5COWdQJK38yaGiQ%EQ&yHm*ypD%Odn(+VR z_|TH=8Se*fqHm?K)KO>7ox@a5yQCB0?Ttrs+3Hw(d%L*+O=h<-{7Za(`|fdV1|%EP>?byDXC1B0!+nS zd6w(~*Z&VfUVbd%kHO!>~L#`@bDD!E0&%cv*g@y=W6-J%u zHzXu*BJIX0uP%S3$KXuPR%JtI;qvlwrd|^jDFwyOoXvc%VE7dqqAS+3uHrGGV6{k> zB*L&FuGo0A8kEd-+B0W#`ozYbW78@*m7<(h<+8s*SI%6|5rU6PL<5G0n`yiBAhbzu zb*7UT+C_=y%Xa4V7- zo~2v+0Fe5Yj7)hfetbiH{p*kr&#{IOT|oRyRkl0p?99w!l9Ia~bl!d(6da5_+nIJRKN^GAFtfs1S3Hz^`E|@< za&3F&?Z?R_+f3bBq7vxm?<#f39}tdZLj0JCg~c%2k3kX%X8Rj12NbL->0go*62NC_$m;)Nh7FV&r# z^$>>R33nr^Qq^)0<5W~sMzX7_sF^55!}l&u6%L)6BMcSZd^Hj($4R zfnfyws3$o~SLG{&+0{Q)B+A7-$7z9Ob;TFz%;`23+d1Vc0pKeh@Im4k*BS;VEHaR{ zpIa{dk8O0*68FPgxnD zL~m>0Hd3)CmTa#kCMH7GZIJDM`+V{6r)t>GHXJA;TnVgaXwClpECmIHXLxwH1a2_g zLY>JQnRG3uH6>We&tn#vts1E4%*+!PWFo(SA1&?-9MGnz=N6@QDyyh?W|VE?(K#+w zK5`%2RRy+8&Y^LvxX}hrDuJ%YD6(Lct;_n$ zTyJx=0&==*cg?Ed(s8T4`yA0mQIB1wM<#<-h2CxV1WE+Dj2{}IkP6dy&zclopEAZ%U+u!1efqt zR=2Tgt|8g_e4kHP7!|_sfxJD)iq6X;e}1L{o(c}!+pJ^>NBxrlNoGGp`^C=keKH7> z01~9tIF^{McOx#}?=5%&T$TXGZDBut!ghDlkvQQ8WhRk1mut;KZsfEHKPq)DZiZ=P zc*3hv2f38z9dH4kwfC1P3i?dJN#Vsr>MgPH-jd^kM4TA7!Si*vEo@QX3r|meNCzhf z-rdG2)Nqc_zj57a=03tE5Mig-iwY1HR$>eC35}3FY2f)g;$mWzO1?8|_D`T)es?{;{D!LPDG#F4 zKSwLY;91WQR0hgN+Fz7w)Enjw!G>ih$cT^t_9H>2 zc4?#*)Xg;VYo0{at{7OG@8FJd>F@aaZLxrbSBmT}5z}0?J)xXJhJ?EPy`3f4dq4pC z(KUp_DK58M{x5IV)TZ^vg^vSTL-r0o!wdX(1NaX;7Z=I#*e8i zB0Aw$pFN3j=Mq{CTH>WCuHU%9_tSc^7u=EnsAlzuzexX?v2FQBj~>~AYL|Z5aa>+WNy!#= z`<(y~N!KjiCLDSKxLx#=RXIbeFZJPE10*XK^}o@8g`H((ZCKc$n_lvINu|3<&>E4P zoQ#K0&Q<~YC>^wI<3J8XXb$)%JzVD(9tIG0Bo!4&5i!?l?k5>O1;>}?*!Z%_$~ZpN z&w*M*30RI3AiJkw9o$&8|0_hx(L)%9k)pwv`uUkn^*!Bpw>=;%T2`{H$S6PVGH zyJFcrFE5xf3BY9p`Nzh*Ewa?VwekI_eRb)irLN#x{h?sp42Cx;Z1&%my>%YXgRL4 zG+b@aoy{oWOXZGBYm4gchUe&l&_fH7$}_aS720lCU4F1P z1ewq&L;+!DUfQ_~i?j&)oOKXgrMJ*BZVtQzeyLh$@u(sTddRtef(~>6*FkIs2nb|~ zQ#Mh1I~%$%5x1nJSEx}BbJeq+jMSVz@HgP4TAl4a36V|A(edgkJiXj!NP^r48x+(U z;Q2Z^-MjNHgB2L5>bZCjWtOSSP9o;=@+E2zl2F9v7Pp|=wMritbEMO{S_y1hRw3=U zb^SW&D?r1A{!%0{=S`EHHm?@5a#zMiNQ33fRv9f-KV7>(4c^`N{f%%YlMbY`dIx~UBU`VxL10&?;Wr7Z+S`6pU z$KH8=iIAvKGT?=f!}=f)rWa?9InUHR1+*o4fPsR(_9igU&3SKIEJl`hn*JCjpok~zm*U;}`AMCf)^3#K>dtJ= zOCKLMiMVGV6Eb;27Ilh8kD%Xvh8-5SeqkR{N`x3*emXts;4(en$)o>GXX<6$T}Gj^r)B>#`A^MI$i zfB(Ps-doa=G-#2M)zU&GQMM*38mP!vdi4qNv`SQ)9bxMJNN4+R^8wk2W>vRh$oNDFfuaQ-szs2ni?YpMxEA=Fd2Tl z_h=g#?fTNeF|hqvuCZo5Cr|c%;q|)_CguK!$XnY;RBCO<9#Lo`m|k{j7G;HA*({UU0)O_3G&EsL-P#aY;6R$+ zsML@{+Rq0phEft`<}*ks?m&Y^D(s)B)#eu9gAOmJ?wqof4{|_Ott}T*h*gxTvbikD zy9C7}0Rcs&P0OD@f3CRQiLzjf=g&Vq#hqqOhKCQY8KPI6b12F^ldBLZy!we_dd-hFG4 z3JRL0r?=$o!y`>r<$@&ZtIO(2Lk=~qt#}bXP6HWC)%!uo+Pb8duZ$dGpU0gv8$iS> zDJcF<;Ymyua<(Q#RxM`8Z9Y3qGzoa^%S5RS*!b9}PX% zo=Ez3{Y3cCtFx3@sKI^j+o>-3`{WpL-(R|a)*89^dUGmt(t5GScaffV?3Q`0myv<;P9dP0ODV$|s@-iM zX8VHo+`T)6l1F_pIjvg??Z%C}GAw2X%?pGZIjL&jmEv)pnc4G;hbUCZrE9?)dQDV= zm4Yu`Y<>U!{SLl=SFo6!EAl$%{Qi;AcW2zsMJ$#vOa5vtH_7$N{LUQ(whN|`A~k;3 zT1iUW@WGJMwJu_D|Bv1J_b-0iBD2O;3&Qda?e_(CiD~cd64~a#c~`4hT!yHaaZqkD zdEoY|6W6Yd{`q#O&Yf@X(&n3)?fdd&t@XH5`c%7m%a)xl{{DT1{5f*nC%L_Q_tFE_ z3odi?^2gnQU_0GrLn`ft9q_A(oAke{A*)j86nsD;DP?T(inzj(wG(XO3T>kc1T}b| z&p1V!h{um-{}D(;sT*&>t544Z>FMbmWOYO4DQ(H0+5B~6vJPQoeY_k=kM+&#@Z)V+ z8ylGcleS{#19<({RAaj#R!I*$sdfNRS{L#wzzq8XV=1Gy`dPUR6XoroK@Q2KrFJZU zW`Z$F{`qZ=KnGlM_?|_HuF8srXC6}NCM=b3){5Qw_466FAWYmggO>iP{KF5Z5k@b2 z*g0j^Yd#Hp%bg8i7-}nmL$9BC?c24xm-L+jDl8MN^5g`iwWTK4)U!dPVV$2}nyufk zp|9qs)=fLgYwo#zeQZiYpY|=`g!;qnO)ef9Z*lOfmb*x*toNDv`qL~d`fYgcnNA2nECW{4CcY~U8 zJ_2iz_auMB*l}3bzs$)GlPVi7TwEPzMOXV#%;hg_zaiv z(XuH_;ONq=TlPoqL)7cg{LPRiJ>!NdC=5_nk6V1HsD^d57KEYNNB50uqEM7?TBv|d zi>pAg(2uLWy^5j$H!(4Z?%1+fuTHIhS6KhvrmtAl&(0(~7evL~_EBj)$)&tI_pE4H z607I#>hZ1mnJOL~ICyZgM~@zPa;k8E#?Gaw2ZKoe9nN+_z(o5y>H-zGpA`2B4ra2E zQP*AV`)q?G30W9!GP-yBe~s-IDU?_j?U+m+F|OIF0)B~h=k=jz;t;2J}A z(yY#$D(>Z@W$kT-h<)8T_8R5bp*G+vN;&mj6GFL=*rE$7p;xb& zMa;Wo+*`*}6DqJ-o`!qvNWuT^32qH6u5t7DUf~a$Ou)&Mk$v?#I6U=Gf2Z|{fVm>a zPG&PgvUDT!r$(5lZ_>fE!SVylw{O{O?AWo}@}8DCZ(>x+7g)IZCzg>ZNimmDslcjjHTaEoC6CAT5x+LydyaFIWy^^$y8<7-$Ec zPG8|%n%QLd@ym-Fhs?VBqw2#tED0@JB?|NkiWscMoT8n<1yw9>N|kY}e1FZ-EpH!o zVM`5AQF-$H3?KdE>Z{phhYugl{J1}^eam~xTYQk?64=NXI5;>&n~qWM-G1h*S$BxM zvTLd~X)28zxkF)A2dg>DmML(VqBd8p0C?MxpSM^KKLb7ezUh(2=SM9*t4K^$Xj z@9yyS2o4TDeDr8Hm&o;s4!#*O117K<0(8tjVAL9$+~Sb z`u6JagVoj5?Uau`+98se_=4P9?(hMUz%<$lJR33k89s$IKxTi05g4oA9vRSSD11oFkPn(}jR zCqw@d+&XkT1Jfi}a$?^-q?*QEO@F*uAAgPUH6NeHwj(K_kxyd_W}%3X!<(Ucdp}y% z*6oh@#Z>$+!G-4vEA?z`2NF(MKtFzX!wR-@O)QEt@>klo2%)<6bMYZj7s2Ks;$R1$ z`uxwsc}|sEl9UF92^K$S1P}!uK782vW@L8RU*);s>zEfsnN7rj-v8*VfjYHUm~Cv7 z|1dc$>n06kIJHToLr&Jmw_G~*6d|EM8R25C1Fr}Ja7yCmTnBUlzQ$lX<>-O~!qtH> zBlsE+)YjJ4FIIP?41bWlVJ*}o9_s#=FAqPXxw$Lkz;3;eHZ{&rXTv06bN~hSHg>$T zv#Q9yX#D)W5YV(x3j`c1+XPZg*Jy1yxBK)(5hfD?3{whm3ch~*n!rBFU1j+3;Wmr( zVO&DCmm6Lf_iAe`cc-GhC2A~F#FY4Zn^4g2EeMZOY z*Yyfe80hmAB3j+AhR>kFt%Owx_S4+7_V%?9={|}};m*H;lb8mnDfI5q%4ft)qW=7A zRb`Nh`fq?? zub6Xe!~BX4&qv{HfQg(;YGxHlQuEU7seB!L2pcoaz{UbVqiBgISZyS0`NSo4H@B>CT214u>E_18 zH%+%0ZBzH91cj-U(6??eJWokU$s1rpS^4D5#O*{a&HB<10!`~0XUQou|3=6y)8_BU zuh^+DD|y4NiqDiTxNmZDa|dc`YnR4X#m2^pdIL^LVA!}qj}L?Rnxr=>wN!cc_M?*P zmfSXj>fP}E?>khlNWrHy&&5w{w^h2air>^%o%GsLm=k=}&j@TccW&z8fl~wXD{NI| zZAbU!2aiKFZdx!=Fy7np(w;nd5LfKkjB9ceXFOzO0R)vR+UX11j1F4-7V!7f01x_06)Q@M`(`&6}xO$6FFI zgN&_S9{Mq_AN;>ZjvRG1ZYB|p-I9{;&m6U; zX8*Dk%aQf}d*IYv9NI``a@Mfi{^jKR9R<+r;yMc6 zVq5K+jzH|UZ{Mym+AmF0od}T+AI>D!$O8w3yzN<6(n?5?@Ht9uHRUZrvL03p*;{(z zzL3rekKs$W(&-Xax>>}H$-xa=@*R>bo)r9}dAt344;)ziWf4Z3iM?cy*lxb=eC++r zQ`)c6;L>1L;LQXp2lJ^X+*Ke~2>2GQT zF{lgxtEW$&enCQf#rA4fQ_a`7y8rxngoi!wl) z6_F_&pIz|uyaQNB$w+&*o}FQxZKpbU6}i2Ae{Hz{4Hr^|sN(L}kVU_>c-XsrO9cgm z<`FGq*k6OJwtT(6qDG2NFAqKXNDvh8QCk?A$mgH2W{oo72GMX2fl;vOw;8oG2N`si zFeaMr5R7>9SFd+l!)Yg3ecqUeK?K-`q@XfsU?)Yuk%)g7xNrsV`pZS}LHlhUWq!2Yi zB&%@%mKIi4x^~(Ba?OtN5Ed-=UD;)nY+?=FEBT`3su92qy*;&)R`-}ZsCB{LPVU?s z?+me6&%pOQbroxmgo<1_HFJ!Abn8o8>?uKZ*CPv7W4PGC|0(SQ?KY`Mr!u3?hJfa& z7n0RWcR|V`dv1Q_<|e(SF74j7g}I#E?|-}ipjG4H3p1mn;q+=Na2^C}FOfh6boBbQ z1ttV0T5jU{=AUm>bwtL6U7Dh`a8U4wyKId?P1k;Hx98fmYl_Y6p#GR0I1jyn2>o!` z^%eP7VMKZx`kp+w2hh_pW9QDDE!uDG(!2>Q-eHAHi#I^c5p_{U6^~9<@tdl6^oOi> zOBacBYK5?U#NWTK^NR}5Xb^Wogk7SfZLFD)G(Za7yLY#tK;x1$gXP>389z+yEL+=> zX2ln_fmYb8W;c$${8dX8x4uf}uch4~nuj!A|aI-vdHV~e%j`XfhjAvY; zys!)^?SuZeip<}blI-&7MW1ep`mN0>zLeZcZ|k!z-U6}5z>)ftf*4FlWJ~_=vq(ek zVBj0jo-NE81zl_EL?H0rE22kj(gTuTG9IL%k+8UUc`~^%*=q#i?!pgkIIGLzYqB)( z=9?jzvd~A{WeyNRtc0$|l({E(&w&F6=FL5{r*z-f;iDwCEo~<(B|Fpx2<$m^svsPo zY~d+%Ve-STBL9;;49{v7EIY^MxVvnu+9pG42V4+BLm0JRWqDko@Ot28qe5#DFau-f ziuBVv*}#Jb4r~HjIdkSrU48xb6y-JhtROFLE75#__nIdKa=XwFykG|egrGs zrTpmJy!AGj#G+5;{ir?yx#h_{WcbSh6oWy9QwaaeR&11@jT0jmZ@a(0m->%irY0uA z=g+?l>noTp$NrelDBC838QA$_aP8cxFAtyG>^Hyp$=Efq9|bka*Za@Xqk{ib$_dLi zORa#}6*ZGHbX+IMO<4bS&*z^Df#F!d7LDx5Jr9O?L!lCNP!_{d0|zRB-^guE8U!pU z#l(_Xb)?dD5?H0%wYd@%u`gxPZBMui{BTWVkn0uw2efX|%uYt=0Cnh_IMM(S81(=V zu;fxS(FN%jE7`S9E-`ccmHx*ZaFvegi(kq-ZXO1Wpg+p&ebo^qq5yOY$naY3bO31>ysyaI+ z-yxQb>J8ekw6qLqWLM#P(Rnpg=SLer$Mz?0NHbtEVc&v8*s539-=liI=o?7RKh9zr z?fsrPrxHg?SlJxk9cUlHfq^pAA6)z|5zTGDO1Tb*YqDFKn2@c!n9qn}HI?8`akmcr zUsQpFZrlX|cyj~&Z0MXqHH7Y)wIKQo!*VZ?UPq?85e|O|6ZLwn6K^hB&+!5mA#EsJM3Bu{ z?No)0%q*PsZ-;0b{Xm*DgDWJL;ql{bsc+iU#|@`5;egJaJ$oL`U{fx*V8~yp>Z`jn zy5I}RBgm;ZnMRG7nQq$#2q(w082bfBJ)XzdZ-=5B?`Fj{#WI1c^1ZJvLYA;)3ex-i z`<516s2hp-Zv;OScF-b)SnBVsWG;r3RYf+0F4n>rEz?MS+} zwRL*_sTGc5;1(Vek*ZBMiTiU@s11*Nhlsl=V?*!|uz_@rjs7(HA2;XKz0m3* zEIX166-qJ-t#H7E6?e9^;L~BgqVyki>r2y-J@=3L`+h~u_;PiqB=Rd>2cYW3OPAU_ z%P3ik=+UzA*GMEc@v|OA>>fgFQ8KP3qs&6Cu-!$gOK|wz=-j>BI&9=~{8U8AgZ*(*iibcUI*ny>$#pYu zRW2KQs8RJKBRf8Bsbfd@?2pBiph9%bSNb9&t{bMc38`) zD?@PM;@4@C+*oVQuKW>k^z7M#7(s5`*+A4IY~R5?F>sbqhzD&5zpzT+)aZlCAd~ra z)%Q)B5?^w@Ws0@2agKqSwg_8iQM)AFGQs|56=o4Psd12QXbE^gHN71oNB^v5A4t-j zCgE$G<;2;L|2`ZIMIu#Mu^^6ub$003(qM0{+w^}dIFm5sN^R_5Wx0;6AR>g*1S1ss z{py0T;@ZJ;@}3=@hkiJimy^@HL*Q4pB#42k#8(Pmet$SQ-bMa%=&{*DDB`g6d|n6K zB>Z$iK?TglX>2atu(E;1>` z|GIx)$l?`F7Km38S4WO6ssOa@ZfrVr$+zN)VcECLZQ zoBR)hA(%#t6I9nFtx~GE{Jk-{o5echzM^$tJ6;Jvs^WOG@BY_osPHX6)|2=&tNtEW zidq4A!57A$N3<6IfN1*ws(7ZFTrw81faJ4V?{oT|E`zVFe>WS2tG?(vwC;9*I&1*H z+lt%K@GBEg#FoOkE=(COy?%S7sL2dKXajQIx`abh@<>=Pea4KXWWJfRXWxB&e?NM% za1-J#7GG7QJH@$Y4-jqxht^B8q9iraTovAF0a4dy+hR@mL!Uiiu%NBCsmKYl$QzI~ zaXk6<$@+$xJ7WkFsfZ|@d-m*kEsuUe49Ty(J%qsqT2$4$r3^U<*-KVy)X_&$hUTZ^ z?LzH@E@}jfF>7Dxitg(yMWm z(47Qx_h=bLO-g9t)ns6GYf|Ie2>_1X>W*?6t{ZR#G;7&j*U?dhCJ%qbRa|>f z^^_K;=l~T1O~OlD6NUQ;V`dww6N69wtQ%HUeN*Rh`1JFkp(Ph|8i&y);Ej7mufZ+J zYW1^gO+fQ?Wplmu0nLz0SYe>YE7mFUEQ-$4qaH=gIM5qO_l2Fg5I?E*#0qZwX4tv8 z@Q~r?f0&*=o!|2CvZ2lae%pZf=@BqJI$}1&ra&p^YU)K2exgFKr@&pEY-|--?^a=) z!YsU8Tp2-WV8lR?ZA%{`o4tFF9wo^1Q&Kz|Z@z9JT^EwqEH6MF2-7dE#pKS2&sg)e zpE!MbUukKnw+r0?Utt<0YviAFpKS7b6XqVF`NAvQc4PRj^;*<#Zx0e8exdCxGkdq+ z_x{uY5`&CIeAl^}Uz_jh)c+t#pVILIUIT(NOXkv_;PBV7u_YX? zA&KxvoQe$#Q5O-6+w|_3eGFQ<1c|)E>BrBmFQ21H1GGT%7)bp;Zmj!ai?6Hh`g@lC zuK6K)LHupcILPt?y~i{a-Yvvr|0@futPbK+v&ehK;ttXCnL^GN_K!8{{^=t9USk^d z>cSfxd==En7xZ^tU}Us@@zVS`C?-Neg$40-IGt=dIxsA`z@fT1VPoRMQWh?c1u z{M7Va3lI3@RUOa_C}M;V-QodD8yj_6awhboYK6<6U{h2^p3plex0gQ`LRSn_7l;GK z`mcRGMtr&5tkWx|UiXQNSCDF0U4Goc_7Q2vrd~#~)kCwyU~X7BBU6VU)LBsTA~YQrALCcMP0A`^UG8)0v|&} z-JeB<)Wsd{KX@Q0_fiauaNORZbNmElWXF4FE)_kM|32=ZGEUMlRy^9G@mG-cvrtcK z8FDS?ONfh-yvtweQ0?Y%+f3v$;_y0((wtxY-8ahjU+8he>*r~YCx0(fR2bO2^$zcu z1gK4NaMb z@`q;dBu>V!Ig~p1T*d_$xVc>8up~u;)91U*)tqkrcCYREhE~? ze2F+?c;UgJ{*kaAMQ#-l^%wzyHfjD!YFNw>B*%`G(!!c2PMv|1{j8dXY3W^!TMY(% ztNu7<_iM)#_VG&#B_@=8OB-bx=UDX3)5amXKk6&||^>Wz#S7lsr9x7LE;>OQS4O_|FetCaM)Weds7Xf(*Z++*eu6eGyd~BbI z0Z*nJYALxwywrBan{6~f&SdPV)i=0X+wKHXhZ-9h1v4+7&VuQi-NhwH00VL9y8Gz$ zXx}nroz_ailbX{@P2%hbr$`vhGAFMF|7~|{UMAGnyJatRj<=wVU-kU+_mtmB_6Bo)Y|;L)CAlcC zljNiRtm8^GX(bn!_$gry2SntELxD`KJr;603jT>~I+bE5ccvU_W7B`SOa~d83z&-O zK?;L^uMj6MP&(G&a@rF4#gEWf5tD70&64<@^ywsBe$G#44ZRc-V@w?sDA4tYQ~I7C zv?*S)+2+ppCBkvz-rv}+zna6brdz#n=%timqARTjV4Mo%X`5J>DYmqixun3Y@EkR-gw^tmd3640=pM|GC$xD^N7zJF4_H@f!Z zcfM}KRA!=JaXSwvH>4$J>-kZi2e`7H?9uFYWH3``U-Rvg*AypP($%g_Q_ zsryr4DEmolEc$vY!W#zb9moW`SJghhvBn>EFBHS9HJ~ug=#sJHq$4v+`OYR-Yz!jP zeu8FWz|gnU_D-9!ciP?R2_D+T>Sv=Djf_9si!VlPOJB$7T3O9a*RCd%4=<>6Wjk4{ z)nBkkKl?ehO#^qA&o3?gEL~68_mg}m-BnAX78zSvT@?;gSuMAimzzKIW34RU?(UTD za9ewyjI7K@UQOV6W2eT))>KBuJE^bYsYuw8olRCGy zBY}wz@IVru0T(Ud)wXudgzgGFIJwlWAs8C%VOgHvTo*!$X@74XP&SqTwF;6lZ_zmBgQ}$0buuM0oq@c~;^^Cvh=a^>aQfJ_b^K zk6yjJK7YSau(~|w(ow^S@4EZ+RX4==oJl8B4PT**mJE|MYa&==ie+iH{i*LSUB7Mw zAe{#i&H-4HD7Jjo**zsiM@B~J7EIkD!d0vgAeTok?$M)18Ej%M=K5T|^pr!aBn5tb z&5N7!5i=778Z`*eSV*KhR#;J5r+g1bp(}(+~zrT@EnYAMN)ULquM#Czf zMNNefHTCdFl9iRs$;=ECrtrdZW0&u%uc`2h%Lgd^$tWge&St0d-<#*tcw#TB%kHmX zW3M7JZLU}nb#m+}{g<+{hO4NY$Syl}{>6tD25F<)^Ia~oDU=i5SO3**t}(lhQ+dR* z#_iqLO|%(oNGo%`QJTPvorcRRGtLt7mFue{BbQ{JnbXJu=$xD6R#X47cybHxqOFpJ zv?NM;#8r2Y#yA`2NV&P$OU2{I&JdO$ zZQirk0j`N!y`Fx{Z6pKr+A8&zDRw;R8n33O{p_A1v0 zj&W&@A1Ww~#KXYM?W&?&#}TqVk3WC^ux)n}n~kpyC`*28v`gWdID2?X(nCUN@P22Y zeJ?i!8JRC?29wvZ#j;^mIKHSFvT7>&WcI9~0q5~}mO@eQ#rzXA3OHQ9C zax@*#d34fkpT*vUJsQG{PG(6YwxO^a75aCnR$pG?P zb;aMkvt?v4KLk&jz5AYRwk(K`OEIKt8(_ z0%nnruNn^zg>dibCdXw^79gTyzsurWu1;FjhktRn7#H$#a8h;H>82HxGP{ro9_py-B8a?Y|eN~n9lDw0idvM52!D#H zZY6O<$(0uEdX^I57b0$H-7OHwHvawO)3pQZnFQACvT&m3tGmQpxo;PdOUf3KE4Nsu;vfg^1;^9-n z;qk+g#DylZ%;RI3gu7e-)cDtO_MirJr9p-CqkCIMEl=wiYBUV|56Cot(cUJwv;dFV z0%Ff~m>;FYfYxn34jDQA(e-w}wKqgR`F!Aajr4CTPx-=DbZ;4+ds%ltAk#h;;x*3_ zEMt1WBvUk;K^!Q|WE z9o@dqt|)Ui$*~nOGCI4Ob<&(Cb_4PAvPjK@Gm)BCJUMd-2HS$-WdS7(;W~Ar?zImi zCSA?!_HjdN$pm3>fow(*-oqfy9iSu&xS0#2rJP_sLV5*dJ}PtQ9m$D_ZeSFZ z7olbGgjR|yECt|pRi{XF(lXLAXGAX0lsDCsce>G!A$2uwE?CM};X~f+E^b_g&hJ~; z;i7N31kHO`-8;bW0+pV<tWjW? z0FW#)7ti=F%)XO6ii=R0)!@IyX`F886V(ugbJz=T*LfV$H^Kw#lJNfYDl_L#GImRu z)Hy(A*0jmS9c7XQpc0STb(k02etu0=vwy0P&EZut-dcwWEA#k=7g}0ItR5fEs-1F( z4h!h+IeiJO)iv>=}G;o{I~vPAT%Qd$5!$&%m8REMVP( zKBrafedRjNaP%mCbrK@;bB)HIk5f$C%eiVfDZur)J!5=M%5QqrM634MQUSJb#T%QO zo0oQJ@-gVoEw|}^{LwOgF`R>DR?&9wJi#R#ol)i9Xi5B(C%*U~7vVF?d|l5GDB-#F z%?$s1^*}W$l*h=aYxdt?;`U_p(ML*T;Q&K&`f!qR@Ae`xGbUr+)T!GA4q_!yKxjKf zf`;s1T28Awon<9a=Z-z~w|XA7GUx9_8~s~mGK<43gB$!(cK5xk;6Ok8yB3}2WPi9U zV#y8C+Mpw&D|7LAH8O8$o~Aq(`$&F8I=<1P=9ETbDi_Y5AHwxhy?ua^2V<0`l0Y)NarYU+m5zz_w5rf_bE>vHFTx*4wgjrT&#r+IhQ_aOeEX?eb#*Nh zUDYI)%+j?0N=BJ(7C?*ZoKb(4jr&1NR~`CmTQJ_pcyw=LdRv2LG`+Ta5Ly5aZx4MA zAVOS?5LfD)XZ@w~qvlR9JUMzY?UN%vt|h|;QuWFsi8msq8B+te+-);d^s68Srxhjr zNjti>YO1QV{w^mS%vR`HoN&b?T<`)Q)%TTx5h(4IQn!<}5wc11sx(8d9$$Fn)AckJ zE(Nt0vqPeR8OP#Oy0<+gACkV}C@`bQk#Qrw*=0ZTVi5@*m8?Bk+n4r8mBf!@xfGj| ztk^WXil?SLV1HQW))%i|e;SkT*-$A?`7DU(_~$Zs>O01nGo2%Ism!zOT_m%RWo~OP z0|_R)IXI~r>S;a{hp)o!dqFhW8Ld3CZWp47^{X~bWON6La)D-wGvecua+McfpEjeP z%P|34vqxS2^A6AAQNKzV8Ze*W$Wx)~qct-^=1&iqFLw6k_ou=>yryG19ANifSgTt2 z-7QI+$EWWu(jIF8vO3*7-#7DQ*u0a9&~Cjxz5d?7;zdn3-=tZ~leOPAX=PMuymbF%%DGT<60*m zwOO2Fz}2K$Gp)wzY~Zx-dC0iNPa9LR`>Z^}LR;`^7#ARj3oKBG*{w}cFjwLTQ93Tl zWAm-DsjNgaw>y_V$suVp`KiHNq*+S3Fu!=1`oRL@)VrSy|@L#5m+$zNH3_nPsQy z>P6(1S}AOeb`Jj~zm>E(st-JwHWE6v1- zWp1?vD_ydt_bSVr+Ua;>%+6_^1p z*oYReYeC+FtphX(~s7OdQrD>;-?RkhSZ^^8!u!q`1Ery{Wd#yy#EkLE?kIA&|9 zI2lNCc82-jS@61-uUo$5y84MYjSE*HZ+l?wEz?yQ}r8pW7G-?da%O%#+4_2G>q*JqJlfG0+ zVR~&7_@tp{HwBB6wUdzRWYrzds4y(Xr}vmCCgHJ}Z%D-^f0a3xL;0G(nqkg9ubMX&`{Kh@NWxC=Sy!|=4_7YnjVt%f?LLu^3jy0*l)?>d$%tqqlq#4-oDUlIybBHRuQX|x>Rwi1gWLtr4kWbYA z*?=-1`-y|@t4o6p>HNY>JDVvFIZ$cN5SOJC`8i!?$PGM~VOlz-Nq3ol5>qo0GiJFE zy4NklIz2XQ!Og?nWn|u}!H$s=$_`KMm?h2_rNNK198Mp?bn9EvYoc4>XLw$j|9Uix>Mgd^}GEyu9yK~mb4JRVgzyyz8uzGWYvem6c(JRvb6h~A^sLp9Ig0_m7xfGJ%){ku&z+W%O$I zGKR`beA^jvcOJs!Qw*yy&Sgf_{USSws+rEqD**L(PjZ_*Q}6swiaTHSWG~-bc{$1m z=#j!@Is=l(tm=(pyGbuf@^B;$$_Xmz*`Y@9-Y1+jidGk|h_M0bD=jye*naMidIT2 z0(8E0O*C)^{);Mx6+E{6RH-sS!Nu%a$LE~HZimHC0GOynP%iJjH*A~(r6+xuwF}Q3 z&LP%I4^Gq*UtYvtGLQq4w8~$^M^kX?*0sOZ=?|t6f9E9EQaR!1hNEGLPSRZXQ4CEx zPvU_bkn#hYG|v_qDs&au0wTMrc9!DN`ZVib!Ia2KvOWUo_S{c_D1PQ-J^4f77cb7l zNb;yqa9cNSn9S9xF@BjKsgc*X9hlV4h%rinIElBQa#2n=wKd~IHpSfffn%t4xS<3= zo>HAn!mEsewW3t;&-*6_c^74Ts4BY>{p2Zw8>%tJ3p$X{yD2EO={%W#@^-J83SAj- zw2-xhReSf8LwvznTMAf!t_e5ephme;LFJb2(tp*nH(|}k#|Ng2CqvIsQXgyOv)gGO zTLq5W+@R34O(TPg@a2Vb{xp0uuN%Vce+O>wIJQGzRAFU(-J3%-=mzg2=7buBrLK%C zgpbs(7<6Rj{(4aDDjK@jKk=`6Q(Z(Cyj?p%Je@UiY3h4lleUl19+3yw zT|V~ItzJ6j5hlm|{KAQJTs)8$L9M_wJ3uG9ET+s-N0fjDv zB!pfq?91`(Vws`Xy?x7enn0Ileu5vThR-L@ML=vZCu;7v6@O=PXI&ojS!su`!RFm5 zFNurZNzZ8-xPX#1lloqw{!C#Fkyqgk`4wKp$;AflN#7oiNWi;}y4Pd>eIzET@#OjH zHn(NqSXq5kq(quHl~ekscRiSCv~FD#dZI-1d$==ki%SZoY2|VkPU96ZnD>u==b9*< z&KlzNdxOEfl?wI`4-b?byKK)N4T+m~`>Y};=T58sIRJsda5S)U=I9s_)xb58jOT4B zMbu1pvAyHEERHiN&10*QpXo92nN*)$g zK%V8ounuKygc@D42u#Z_a?UoOhS}_X|INBJMG3j8s)~PVz6F#36u&PVHqki0}%|_!J-MG)V%K2p_i^#cV)9oYmCS5TG+5O>@zffcoJ4Rn(|LCS+Ky z23O!#(x+QhhDq4d;a(?M3OXGz*AazjcBl}fmW}=4bB=%X#X8Ca7T~?n#ugS=<|!#i z>ab99Aac;f^XI35KOi6-7-aKO+cyU%k3c^Eu(Kbkr~*#aPGY#U!79lb5--=ptcP-1 z=^`AHf2>fpOfY5>h|tJ#66$*S(>bGZZFl!m^h;Y_zqqe=-p)VQ+QQ`VwDZpQzB?V6 z^)Y#!cT@8iyCV{@681zK2CGU(LlL`;R6Oj9Gw);#B7+VT@~|L|X^N@u+( ztdM+B5+Goesi*w#fq0nPy2G_r10`tpw$+nwDeaz3!fALhA;AoD;Q}%UPSGpEsV$^$ zZZWk?`|+C)HSu~`sgMGn?SzGC2GCMTi^B>YJQy-i|Iczq-kA8DX*FNczNh**Iw?Ob+;$YK!|h6 zHc%JRn#PAS8=Tqd7y0Wu<>4T1=TO}{-)_=srqTXSS0yx%T|cq|CznVWZhl5ku&Ylj z5=8FGvzzr0J}8+;dyke~FK~sg5DqG4T)eBSVIHId?9X%Nr@c&o^E)kn=ugc^CXm#O zUcY~d`jY@gy~h%COav{$&oRRIaDIharOT6vA4-ocFg!v@39ZGv(i=t9cF_g;?y>gS zyD~ccb6I4*5GYp5;($R>z5q96D6qX4SSpHTw)+n&^%*krCP=}c-Mj0*6O%`^`!ViW zMSEG*nY7u*Flofh*>3^sB9B2wNK`j&Y6lpmImCV|87$uAop=VAltuHg+KUB33ulmG?h)Ou5U4O}Jg#)@g)%Iy=Wff}_G)T3az}H zXT2Kfyvd?-i;*bRWF!NThQxkFD#P? zu~D+QW0%}-TeLn+88vOG%En9{3ed%T@X&+1g@*7k?Ifnt{6rZFq3T{aeYCMxE?eug*C$aYHrE zh>KxidIGk=)a=*JZvN$-H0tY)@GA=a+2;|V-wtBp3a0lUJa+72aqW*6J&CO3MiwtH zGrPHi-eC_`G&|!TLkyoG9Jd%PJFaA=+K=l+0xcT0RYL!_Q^R=-gr6fGtM_ zzU5Noswig8JmY*@-sauo;|X+_*YL@(jaK@>f1TL{U?e4H5;wjObsB^=bC0%)F~~0UKBB>g)K>Je%>f7#B~r%siiM59D3QlFTsA&&)jh{ev$CA7KY0 z0AI)+23bPnkOYrYh2tk_&mu!+i|Z(L1acf0Mxfri5D#xzIXNZ{De>n5sz%sf3VXw+ z{qykdKZR%wg88JUNaZi7{(#2jGvwCo5cMfmggI7`egkA*&3JIUK>VdznZ&NT_x84o zaRdR1vHBt_Cow)Q{zETpd}PP~5nr#(#S$WOo zxj|@h*d;t)o)&^J>~f5~?*0PzS=LsEd+)!mUqOtQEL!;9d0;bqANpv9sQrZ=$+(quO#!x40&M{fuf$m6t2s%efv_( z)!-bv1kEF~An~PK7~eTvy;A0y``YE>xn|xbj;424o*&iBu65Al3W~lMTGEO(z3~w; ziNd2LQKjdP8a|~c7?Dye5t3xuraHu4#)o%W+am$NAcRgLb5W@yS_p&7LlN}^tbH{j z7A-<7c&u>qD5`e;1w}LD=q#Q{1OfPHH86o!)e1|l@K%dq!Rc21>Em^V{?0!4%NQ%N zT8Eu{^{^`S7H>~}{jmi`ifh1_-v7cA>NMm7)k>Gl$*szSADBQx`6W>xXGE__EV_ys zlx>BuCj8y{6=J3QdHnd%=xK0f!hAqLE< zxwL6R_ojB4ml53r&Axg(EGj*H>DzL#=H>|+PNrgp7+O&I~R=W?%u zx*Y%k#l%;5FfAEkA(qp`$$*0|YG4d{sZLK=7tHXEMF@s?Yuf@7rVl#uBDMwSm|5@8DoADD^FWs2om+R`q+wGGk!GeP$>39h4&&-3 z<=vdr14OZ$9Ef=B`|I=PzIG9=1Gnh;F8fj3B1WF;C}!!XYzd>-7s7o(*@&vi~8Nd*~v?0(w2_Ff0k3;_TNA93~O6cL+8_WWfjP?_Z~#3TIB zNKf?Z#W3Yi&vF@tlMfZR9q42UhE<*W*vu?AcD{sISwvmlG{E6GU z}i_XZ{ryzxaq_feZrmoF{*Xo&F;c(DrTnRUl31%Tic!dKoS`;i$XMkH=*LE zJ3za!dI37*7$-}m?qjtcMDDuJAnq3-5Wtma=Ty?CYRScjr(#W`9pPnBAY1Z`%=d41 z7}{^XNQEi9v#=KZX)TjpIh%~0hf*a8AM-pB`t-yNVfX?rFqWqrqIS*KbVnK086vF1 z7gM=jf_Q;QHH%*1Bcy<34-ZQd;ygMn#V;y|KS04WxF80f+mPi3X|9i#99dRinJwP$ zS+n+i#!eH>0b**I#O;BxuY22_0^JEmsA!W>N%^_{4>2X<-IT(h9)`IFru!0RQlT;- zv;ipysKO!Fw=cE#B+IpTT3}vhOwV=U4gqdTEXt<}9o;@Y4c(6gRKJ-Wr3h^^%QU=3 z_8g~WcyDxaP4IAv&T|<<3HUtEWvqN2j^!(%{Bk&rSmUKKI~u!_X3t)|aMW(nynN}> zd;~FjW$D$Eyy|viJ#Fu(G4pp_s*dQ)07xcL{+PwD4;4KS!gwe;ZfLq1*)!R+WlVP< z+hC4ZGlsHRNVkIkP1Iv{oRT2fdDLAWZONVttO-VwndK|u2dqrbO~c>D?Qfz=<*p-& z0;n=cuMvH#<2;=Rx=J$mb7?BG#;=;qDOAp7p?(h76G=1z`O!kA&1xybY(A{|B5yJd zxa3&5x|klDvvWux!lx(J_s%H;C77hk^g&%HKwr4RO1a$cVfwa`CF)GqfjuNEH-;xc zGrlq`7ceTUljwOzW-9dZJnQsJs4{xKd}rc6VKQ1!nco|hQ`^(?z+Ap;D`8F$%+%`f z0^3R99ikjh*mUkoO0m=Pw2GQ?f`ZzAiVD(g5Cbqps~U+ghY}!Mu&g8~Kmk}HShPlo zHmQ_`qR&_hzvG(vIHq;r9r++Yrr`cqxR6mIkQiXaqZFqzNL+F;S~dNgjy3VoSJfd$ ziN2xe+Y9a(yP7t&oA#@{rz&p%cY8x6{%zL?>l9Jf3uuQ!7m^H8R67+$5UKskt2;u! zXJ~@07}tVSb|?miojr}=JLS7j+KfUDJU+Y8ivyIkFW0~PUFTk_d~~PU4iD?QgZ3*; zIo{T8es@!g&i75s*Mu9HJGq-T#;W9oFY+7i5FTtiZQ6+8J-hd4KJBna@xayBcYQqN z@#IJ2*7JJ4`rfY;-qvikJilrlxf?RBq#m~>EAe$gK(A}U)oQ5%OX+s)Cd#O;*HGXRcs%K1`E2- zpdaqI3-O6XwJ>j{(TWveWTa7G0B&4jkyUNoGdGOZT|JW(hE1AeP82;(tWSM2tewxq zK1_HLsKKy&(SP=j_j3qEuAAub6v$YRENDBE1@N`9I(txVVzA&`D1SRNQN7(}0Q0^c zu(+$CYJ}E+Em8ys(FQUkk$pKYRh-|!1ouX zC}CoODO5Zab}8T5%Tn|_!P*}!!pfqX#Po7Cx;TWO2`OUAZ;0%*EUtyj?M_(71);~r zfZThtj_Fj7-+VRR{DOXqYMfL^4u0lSg60)0qNEaCvR9Mq%ZH~vPuv(OtnJVfYPOMk zn@Wk?mB-bdW@Vn7Moq|J6}ix&QbC(`q0O3SuV?l;lp4ZnRvJ8baLFbXd9q-tiAreW z{6UJ5>n_sBwfo+YwJ^+c{sGO>xYQK1%JC2Lh35;6Zx!Zunwt_CMa3cvG&4Mn@4iE>mVca%5xVh=a27@%)D(*US=T!uUN`y6Fd)wdQ_IXe6r2 zyOjw!UMc58HlC2DxDq$u?I%9S0MP)%SvA6n4^?!%u!&5d8Dj--Okz;HLIPF% zsk0ihiNL*-4ZsVchEz$AiU~hGQv@%Ktv>Q`6!O%0(G2-mwAGN1hHR0iavkrlY-{N< zWhoVn5LvaaKbsc(-)?Jl%Wxk2Y2~lf(%=6Yjfy?Q`x5IyZP7pQBL!sYx z4%6eVu=z_IiUuN$u2rZ#+A#I+oI!6!OxuVGVJ2 zmeK6l`#5zW5Q8IJ{*d@Nd>Xl>QHzfG`Wi(%M`z+DKxKvJx}BcXOeSYYm_2v}3r(78 z`kNC=MS-R{682rN`f_ zDY!meNE`au$839#Z69*w!cr{o7wH96DXG$|^$NScWLQI8yS(tKmyzi!767P3PwS4@ zf3Gn=Tr8p%r5`ndxp2oC9ou=AV?}eBE!wl8^tFEftRm$LVhlfRgt=qScI?jBOxUSV|~ebMLgebphS+N2F%9yRlNj#u+X5qkgSeiqGSE(SoW+S&%OKC$-|P*^{X zSh;MO==)rFkQgeqzTL}PoAG}vqyVH}hgYIq!5h{UC3*bgDAuxwZ1C{69w$g-p)+4L zyJ^Z(`oFm$WsZfTFW||WDfYvC&z`+XZ`H6NL*QjDj8uIfUCLa0t}Z&&tO+Ir*%;1? z+ToJRe8XHSc-cNEbN}~P*^@Mi7*ko#IbPw+)Y_3NIDeM|#h>$W_c6m~c>1YvGnhes zdbI5}amU4yd$)c_;&g&&8g5+rS7wOcW6hu$snIRvBd=w~h?Pf^H{rbds- z7-0hN`xszv;i%PfzvvU9TAFcuDH)51boMEwGv52h+m^siK91-F^G{)Bb$ltlH)hM4 z5vU`tGd)tNWJ65BYM|QjPjO}u6cLC&x%E0iuu3Qk^Z!>)6+Zw;q;8O^A2|$#*|C)G zsLKzAi*RlH-y=sY%-#V$%#En1%Bj#DfNr$Xp16NkQ#Hq0giw3xGF;6a1u7s|ni?7& z6z3R0(4Z?0lCIVFP*u62Lyj3Dvy4D!E|e$YW6k>1K6gf4dxxO}G;VR`AGCC_A{1IQQN6f9^ez7BYs0r5 zNr!oiM}uq8<_`fv!{xxTlxc0~iG@q1HYlV1hv+}6K-DZU?jVibY<*>c{v*xfon`k= z`!>?B8I&tox||j(5rb46UrboOY}sLF8r#;koIppF3k%8tkoGtu!<4-)EIj{@z4wf3 zD&5+L866uI#;#O5(p7p#QEAeflz@nI0qHehg;4~gODLiDA~g^M6a)s4ULqx+v=BlO zAt0gs*GA{ec%Jk1{r2XEfDn?s@3P8uU2EMyrXiG24>SVke(T}XH?#J7hs+H0d-!NS ztaMqK|4X$t7+@$2m6kS!pb8`w2#X5-3qD|-!h(hCbD1e2WI>;P?Vx2)lT1Y#*r`MH zjmZhKb!M;oC;Y(pL4)gRGL4d-@5l`o2>_~=9uz7@#Kahw1)ke-^}L|`MT!7&bwGYn zIQ|K5a$-*d;8pmM7hH(+G&LpVFAA0OhrPH@gm2*t%`dmX9YEi_UQtr)*_alcQs_k5 zH?wru2MfnsXM=rVM{fR~huScd`5!%U!6wMU0^4W5dc(2j2#F|Rv9LXcwXmx&3A(VJ2=0z zQ-ih=S~S(Dj$H3os8i4iIL6b7$P1undj!fD4u(YlTmXX!W>Kj6)pk1Tr&bsQ6}y=X z=`m!%dxHDQozmebmJ9$`wkl;VfNTWz!JSLDk1;NQI<>IL0iX@U{d&!or#Fd#x(qso zBG90AFVq?70^oQj0`aj(s)rPwYU{DHppDzCO8_xwL`s%%QLvAYGZA|r55qplL8=!4 zK-96e&VqK#;TTNT;^Lws1vUFJP z7JUX1dyJ`23To#RIz9-qXKN*MowpdpP|{gH_r_*)!AW>rkOU%-=&<)fAee4JI4%lN z6A2TY!+2pj@!}njti-+nV*n`7bV0zn;s z*HH5WI|aT0Fbh6|SgfxU%2&=aD}@5R@w{*p#Tx}Iao>rC-fZz+NGwsgKOy9w<7*)V zdkbMqq2WvyT!nol^x@IRJ7su1H@_xua#g#`F{)5nk9wm4!ML>q{12+8K42V4Ry0ch@U7W|;} z#ch^qBuJW&8bTeYccV&!2pK791eVHmZYWn5+Tnm1jz_2p1OSKF1*ua6&L2R-9_xs@ zyCHQoPw&yuS&MfGm|@Z&Ec+p!xOAc$RQp?b4&)MG!ueuyMxDhF>E|Q}Y}G)|y*qjP zgBmc*K+hzDKmo$oI10eY4?RCZ(heL*Q)_E4G?d6h>TtnGgZq+qwRv}P^plrV!_aox z#u*m0FXXA`-p1Sl_kf^eaDwL1W5+<6qzc{XAj0N0XlrinLN623A#?#bfsW`9%w_f# zz1$ROI;Y8E0J$F2n)g24wz(&TWC11v34WM1>@vd`ELy9G+K zGa++AKz~MQpSK9&WRq8Z;mch1Zx)BPSw6Ul^!I@-AzL*>;EOS0E?Eek3jqWKVo0Nq zUPK-Umw|N00ICM5!vcO7Ae|L|HoAvmr$E79R(2bhk1h~cA`B}6Z$6SqFpPs7A(&MX zvHu8P|7v(nRihz8G)=26?(N|s`QQCN|2|G4Q6OIEKw8isEvFzs%Iif9Ap!XWka|#5 zorr*fP-q$Z_2g#*P=yuT&f&gPmJ|%q!du^qUV{zMb+Bp~I8!os?N zjDm`;2JaZ4RytN4vRNdZL=a0_o9{;QNq%!n08GJON1Ou_iNw)xuC2UdI#h?Ifvz3S zilsmyXBR?2fuu&;Ch5S`wuOQ5dwNxQzrmwv6loUK;yLsi-oP&FbXNi7N@29T~HD*A(%txpV9;$kZ(&$l@8?_ zS6dUmegw-A|+9G7NGi>$@-F%}d(43cCSfzWn4Db$M z9TZR)B2B7}1~;5%i8GvbEDs&rsUPO21@vWwud+jNzoulg1iGM ztt#1;%IYBSzKpC0Px7Z+J*1p;AT{5iL6v0|`0 z>mlHUsX~gaTo7ail>a~5_#hR~2;>0!P7_HtKWISi<^@3-vT)4IGJ!@y1RF?l4N9g) z5C@VL;=ZDi)OBAE^RhR8o7uLeE1_{Ndb9GtrCV0>oh6LIv%(0D57}zYu}u59JkRfm z)gbY1)sTqAeh5OgWPQ=6drmqm?Z0(DMU?N*V=bc)q^*yMGTDD?0iX&?z=SSv%W&%o zFoMIFcvu_x(joGjh`Dai|kup45( zeD`74!)?eXDMqdS!fz#Ib={Yo7b3@oAY1RE$H10oLYxPRVk8CLsvCmOO_9*;-}+g- zAxMMppnvy$nD>iSqsu@c@oRK+Ca|0!)~0!wGeFMdN*)pT?s{K!m!zUr1M6Y~Fm9^FZ@Cnc>8bqW>XoL_Ch#Gm(UwZ8-C=ujcvt>$tl;b5V@vEMHcyp{RL6JBh?|YJG zeOQ1z8`4>hI?t}#jJ*8Rn;CnYlIt%h35?gg*y}Vq3M%QoSK?3{AJ5Ra8>S9OuQTw? z)+W)(19I;7M4zwN^7x@o>29yF`C4nDh!~HD`m;Yl<%}0l_DIz8Tv!erh@RdO27KLm8yfG-O*p?OiL#JC3qqo%1Sn!9` zzroC{zk^RAl{q?@%4yD;XE}~pSTju>`=KZ6c1-_6>a0Sccg5dN{P))fo}$xo#}jx9 zE>vt?O$H_tvnTO?To^2|<&bD7;-E`d5(kYXaQGq|Lb|f4B_~3rLD(g7HiFeC7T3m zxzKDarn(PVa-Vs$lf8D^>TSW-^^<-yM#MC6d zI@4iUj+V;T)rpyLF5UgFmks@JOq|>6o^A~u(z6}59u2l`Q2)5%4WrTR^s{1d>*>^& z`SUV@l$G>*I52A`tw>SA99m_WO_;LeXY?59Oz1apF+XLEFTmw~kGfI*@_)aLrig5& z|3>5+JE(woQJ&jiYu}p_4gCh4#q?z*-PW@IvANP2!103(fQb|7%l+52Zhf0!@lR76 zI{89(i8MGQU<(xZudP6r5dULn|N9kw{C|q$XIp^(@#}~BXCUAPqX5H6-Tkwy`+q!7 zhR0z{J`XsqFyhXsgI@@u8o<# z@R0s?81a8z$JP%|iT-nc|L0i`J^$|S^M8H?KgdY`|NnGrT>d{Crs>8lhw{IMAKEP{ z9E-Z=f~}eF#ZXe?e;CC7`AnIa_Q}qBZfbLX^!+cN@Z)c&AWZn5e~tW5P4!1X#Q%N* zHC(dqr;Rvwh+eMPXTg=Fsb%$QE6Gu`r_#ww(5m}$8QP(?eTr(}{Pk-HJ)C|gF*RHf zBVo5pwmW=O#QU6avyQ#IHN*BcDWyywV+}76l>7Ssz1CQdzn5Hzk=@*yfp#|Ch{R6tMEKC3VMSvCNin3;1>TfNmvRe#NR8`x4yym)> zMf0UVgbV?6bk7(0>viXUs|&=eu=bw;Sh3N%0Pw{oyPpsvsGqz45>;Ec)ukoCsOexO zQ{cW&fP=Hi(PKsnYV9lCM1 zK+z`WSioCW+*?**W85#ZYW<&ekfDA)^*0_=$<_QyN!huU*kqYY=M=K;#v=_M9?q@Y z-#E0sRwY+FcgHU9CF#MZ9{dgd)bz9rt5HAefh$V327s)C6$Kwc|u{+V!M-4Q>3dzp-ES8r}OoQUA(GCd>9|rkZ-f!-68g%u2p<9V} z>3;TvMrryYt&?xZvL{hA@BmvQK@Q8)?#CmFUzo7zu2K%ThkN>atcpJ`tAvb_P99;{ zI8NV+m{++f_=W1$UFeYpcQ2XfGSa*e7dx0Yt85Nt3BjWU9hdjIZiZZLYv(Q z4lF@4x8Q6@2DH`7As0<(=@GI#(tHG6!!d$Z&etk`*@iMMf5%Bv62LH4w{hVHI2s+B zMg{F&@kVv&HI}CocOQWbpHQ>DpT85N>8j=a_=|t+vm&O=DD1~Nh_OU$5Vg($>y)36 z(YY>^txdM_>AT;9_4c};+C5H%YP|;&^nNw^Z|~;L`rLUNzqVlSJimhk%^(}^ic)QI zXxxkeR&!Odn6;r>%H9;&nLam}`LaZn8U6DWg0Xba0GXF%U^34jzl)8{_o~Orv!lJ^ z6GtsOi-Wtf4dnJu`RdfzDAZmmb{!R}6d|+BY)srqb&<-Kt3KfrL5u5A*5M);{yHtJ zKc^+3)#i0b8vEw3BqO@#e7fXX?YOh`h^DVpF6mt()0K?s19{P?8Jac)m$CeYiHkU# z=d!mYUhJx^zEJk_r6~!cO?X`kp|0@K=})xq$h{TI6gxN;oCnDJ{#3xX3EB{LG`;J@$aomEY4!M}bGdag^L#F}Vc zN!?&?;<~8lD(<3V1lAR8msG#LVy|E5DEWoo;!nSmR(SH%m#MzoC)|-QY(^YpvEBvp zoAIJ@^X_ju3xGRkcREh4jCMLg@6VM>`Ba)}SGA;JCS>KFHS_BlO8wi4Z(T(A9={sQ zdfl1a;kRe|a=Z0*+^D0ux9W0ob1JxUWn6`r3+?ILK|RZ-p`TUViGTa}jI#SWi)~*) zMJJ|m^}y5$A&vYl@oYrzcTI0bmz7KVCkHfLV`joVf1z<7W_|iQ4w6CL#iY>(C>{yA z&N*mb)E6Ey5tSSI+@$07Em$rQtT7X+A+;tz(Q)$bw`_djf)Sz0=Ju5-hjypRGNDAF zg|7%Mm{HjJR8-gL=4HDII{>1)K#{29V-lj1JCGDX5whtjjq5Mwk@I(4Xto}}S!D|G z#E^5QRi*}tvc-z6GG7W*8}gDIMs*g;FYbS;#kgBO)Nz6SNcPC=EKpnJ9?ujPqOgWh z@e@(aF8cK`r? z!tdMOfD#8Aki|M?!ky=UT~Z7!yb_b)73GQ_tGOz+nqz0(c_~>x-vCT2|BcO|KE};= zlGX!R9wk%zYt??3vsB^qw$4IrrX%#aYv{Ohwgct5Siajydv`8=@ya;|b#ILBu6!*X zB(|eIY__0<8Ckd}SQqtlc8VP?ohh&EHW$IRcg(jftJtm~uhMg3oWpZ$xmqiEC)vzc zF3l<{BSovNT1uI#*DmXuVXaTMG43T}qE>ZqzU;p?0UcfX(Z9_v`zhwB_gHro>OET}*bgNR)!OAeFvh9OEYga6)v>d2B@Pr7$%UpPm4fSGg?}7W z=znA8pVo%SUdoN35Tq+rOyy8%#oghKqhrU9*>n{4crGWmQ9CS*Ife5RI!Q%to2Oe0 z&MDJeD_sI^5#7fhpytEgH_Z;J7zI~!D%iogYk4c<^VKffJZ_rT8@4&e+Lxn)E^vP~>os{2c31L!-`03E z&9RYFQh44rj(|(A->mlP##!isN0GIJN{}ySS7@KZJTSEP_j&Y7brK(xbyk6~;!(gX zw)&n^Xu$||mYnsoo_2}KRPyh2tT?o*BX){XopCNf$(}X6u9?7!%fC@{fkyf+ZT0=x zc32<>#wToslEytJB5=iE({3b>vk%M?edWPZcG+lJWGDxiChoaWf{V~q(=vh1Y5I82CyLvzrhb1b z*l|VWF|b1eE{k)FB~PH7xhO5rW!YAEviW>=u~XXyJ>`}rGU`2*O1svl{X)KZ4u+=4 z3uG}!N2e2p<4@OK;*?Y!^*hP9yuj3x<#FM^T(gR}`Q0CjrTdD3)6!%I>4Npt>Ugb1 z&}C)K0oU1{w>P-74($#fsG9q1B1v0*jvrkPCR450v`F{CZ8YQNFA8-x*nE> zPI;UZs_7#oVkB~?C#T$)92_iZ%d+JrM4R3ah zjN1h7#jfyn2}BfGVC$>D`^~P^n(BY0D%Oq6v%N8e4T*>2H6UEui65QF795jX&(0xM zY$nhLlwY9Z>=dskvSUN1c2*y!Uq!7vtHWGDN9yDcC%F&4GC9#-Y@0sh?v@rbc2vhR zwqd5zRye3zw6fyxub|WCT_&ooKW+c_eED-(x>D}OY~B+2HD9AmHN1_eGkw}tnHz<+ zs|O~gI;vH>Z8W8|E;-*dIX4VweTjLkj^yHE^NZ`IcuG2<(fn*fS=#HgOll0yriLw3kan3vIZqdfxW?$0xq4>E^9^091G?u5+VGjO4!Uy@%-x)ZMtwilcf;e_V4&u|y#Evbg>wqTieg<{kgZ^Xy=sYd2uDSWjw$J7 z!+zVr$mH>CVaxRimZrW+)sWk_GC7P&YO z`R2>D1&nfh#=!;flV>sn!| z-JgxjH?cD-c0Q~k_Efn9Kxd1I338wAP$$IIYTU=9$9*7_2to}{W=37ew$0{7iF}>b zWL>y-A}v#L^#D0KYPd7@nUX54oCj5Tm9JV9#e)%4HxKhGyuBO2Bs2bOvDWUYhwhCW z`=1^OtW&Wx>=HN{Sm*p*zDg>t`a0h_1vT@D8d&q?&l;qmYz8PFodw?2n&6se93daf ze`JA8HQu}*UMOA?Mi`Y!yoBkLQ45w>$btoL{%lpe#6C5hZ7tW+h4T6V^ma(nP4~$l z8O3FdguZiFjS;j>kskJ{@cA~na1X~;sq*5csi#S&IOApwQ)tR)Ge0P}7ni3@05IE`}4)NPTMz>X}A~l^ECw(k{ zG71LceY#Qa#*^}fna#%Fwk}iOwFpMxkFTsSO)XsQ4Nlda+4ks!{RWQU^VQBV@>IC4 zxX9h3iPrMd(#> zNRrlHq|farGVLhFqB!r_Q)IJ?=twkUyfS%T(E3$;cY6{N)C67DwOMVI?0<7Uy#{mV zVvZHRRF7U?TzLE}A!e>c!|K#+@@tm<;UqwJs={kXi+U(&MAKf7&Y)ZJ>_r*xC7 zz&iWt>P) z&CjQ*zADtxlZmpak+6OTD=9-KZeYCo^Ka;JYFTqBMAr;b8`luibF*g4Xfu^gSk_Ok z;P1#7_k~`1R0MW3cw=3Arr+yxc>Ny#vzUqp(Y)2)*OCuE-lGEWWrkML97iQ9j(KTn zQ@#%*Qhjb}n=8{wl|WY65GQP}>?>+oXt?XJL)K*c_%SYvHmSaowNn;BIC18S6 zYv*LPA7y?|n3k5|{Oap+j~$*P&lG^J>@S+TlRBFET>^>6VgKG2WeMu)0RrG(4`%Y0cxWFzlu`+3`R5}d}p!yiheG!SoytLSDZU~#ZR+B^@?C{B!u{}TC(|IOiQL8#fCjgS1 zXE*BfaUT9vMg^rcdvY6ch-J#DslTPtG_4oQwd78n5QFMfG-U$+`K^z2|3z8X7!&|F zfM&H5xCq$-f8;plzl~T5Tk@hNpQddI{VWyTn1g8oUk?;U5;ijJI0zr+kNUtc`bJCB?C+VetuktV4 z1RdR;`4~@RI;}4+OwMGhV#X(VGJlJm-}e4hS{v$ZjChGG|1*dsv93Nc1F*TN5~yHjd;o4-efSUfTDp=Ai9RsiwW} z_e>)+Wjmzs%x>Q54w+I0Z>~ofJ3oR6G=jF~MONKrvOUQ*yA(Pf$(DP5C+jY@JgPmq z)RC(xzE0~^Z)&=4Xy=7xMX9cZsWCW{+i<3vd|-D{CSyA7+I!$4!(4^7#iuH`0lc zzna1cl*pdDfCoWt)L*grYAAX>dV@BdO)r;pG-w`Po0FSYS&^>A%|p@-CFcCfM~bG1 zgfW>uSkS25Ehqyt2}wq}o9(NNzfiP8A1A&MvLDElxpW;Nhh%nub1RLN#7H@*M7SUb zWG>?{6Qf@y7+!LSg;B@LzP(3;UusVAu?iPkuOB0Beh+Z`#DX{f@JBhSRg>}X(SU~i zJo;5Sh|4|3W8kD=$}AO{kNBYYzBqxk2GaQ)2Dv26Jb9bs+F~>NQK2!AM#?5{CzgO4M9n>U|7L>MFiYl=^WbfrfE!W!PG(3C zt!tLp5Z?|Cmq2r+7j4b3G1P1rF2KG2X z=Zf;eWCd4RW|;}F`Zl$ZEVOD8yVVLtIfB)qNWTYXL3sLz)z1U~&bY0#jf=8$PM>@9 zithOti0t#%SNS9(S4bu z_3g6zFnw{vz8>Dm)$+XcC+OJ)iRIXnDWb_9K5_;dUbxbk{%rTR%WcHcI5>sttZ(PZ zbU5cJf+lzv3Q8v`D2v$YDp|Js{X-P0l(}AH>0E#MCand<$@{U(!^wQ#Up60L>n^=) z)ORI&6P*XPzo;9Ll-elVI{^@emE*o|YQJJ|#<#qk5YF}8DX2z$I2znW>45I0w}&e? z=#L&9Ll=7Fm1Y?hrK@(nKO4xolC8GCxe!kM*<85t$3C*WY0=GY?PhbO4T~hebT~NM zJ8vv3j=aE?%)K-5C>s1`X4V)pt{&NtTeD^eMO}4=pQ~=Od=;H27(pX5RAoL}Fh^TI z8GF-zTEFl?-pKP0=|Xlp#l^=RMFlM;jy$^-H?`aE2)#>Ic?s&qj&gWTd={sord_GP zKG}`f`*A)^jU3x8VZ!?R#c4~AmfD}2`cdrjtFO6eA5~xrK)R80wm&XXB^++6Ok4L4 zIv2;@eNyKfE+r^`UT7Tb{#x!(n%yE52X-GZZlL+TQ(k}c=Y%fLL?M&?~sk_FLla4z+!dlq+yZ8 z=IQlCVYkeb3>k~oSZ?tz%xLT7p(j|Hy+1LpKg%&9>KI4!Oyqf`85TJItwhzIiBZ$Y zOFuGIS0}HxRdSbrr@L(vb%`yDU_a|FKZh>Ui;VE*4IW7C&!UewdK@|>aE83;?(EHD zsaPh<%Q|(I2-^;%Vu{|kv|A^{@0<=-@s~B$!^<<_W;^%hS4Oxd1e=_|cr+>CP5RC| zx*h(GC<*>~tUt3SAvWm52X3=tGNwpdGOQf?OUATjzS!l=uhGGv_j=Kpl#z@abzgqAQcjDiUeCG~XHrsJlAV}&gC}>U-|g!g zK>yQ}8Tl$IS05xiUK`|X5a0GbUh=h+&}L4Tz*}?qD5f|7yKJ?~>GwKn?rbl1`~4`r z%aU#k+{<=5Nb_h@!<#9_6HBD;#Sdn=0~vLQwlhs`y;C9S#yS=vvCf0gFnaJWF_i0j z!#KAI=fI3?==-Mztp=TJDvW&AStMy6n>}ZHgnK$8CH6TCv{vHnSFi$h-D=vAM=cgV z(E?fo1f8aSU;ipjPz=TB_b)ay#A}*5*>uvfUTCyP+oaKdTf@@U-(XI!57}^T+{f}( zKZrlZC!#sp@KVh#NHy|2i=AcnUSFfsm^she*}dMczkT2CPbu+Oo#owbY2;xFY}|TW z_!sWFXDj-6%Xr>s-?aKbW=^cIQh~eSDnl=&M_7mAYhS5qBYa=ka~kfAT&_FZ8M>F0 z_V(tieM!$aekydK*kgcqX3*l#VmS| zm&8tSFB-&8z7<7(JJDX!RCqV;GFqcEAv*8VjF8!CVfajQ=c{j_K2vsg;+$(T!cQys z@(jeJzUVxYi>f|PE;G|pyhoFQoZz#_JzSrFY18j0=ON~mw|c1Fow9NG%>oi|V&Bwc z6fjaAZ#T%|1ah~+=(t0LRQbl6zWbaX+j?%0+B(Mem}+b5r5Vh6Ubkg=lsxh0^<-oY z1>JfDcu>wN<_vGJAcc>UvQ1LK3aN({+Y*c_zxHQ^|0d8AVMd!MiL$zmT0Sk^@VG4| zH6P_tP_*`2Zm!x$!zp_<#`p1;Axp+gqmE86=hu=a) zT~EP8RrM$ISk4=Q~mx2Rrr@t~uHjeKp?LRPI28)rHNBhC=QhVIj)QaxVoSv2^>*uNIi{ zulj*8CAR~rzoCgs4eZ4ePn5+5+f!sNL9g-C0iT0>wAFQTz*2Kyi>mJEMS_`qoTwrw zh})tD*EiUmDV5Euwk@|jiX^hQxm38k8^641jv5!;iGmAamS%RXnD`i*I%UF^QoypOcO~|9vD*xt?83)ppFN>kR|;ugkgal-G|M@bZcxAJQSU2Jfltq`)BhpT2CBhq)j)?0Xg0bORb`ViH{2+Zwc z36#gY2cu+C=>To2=?coDqa56sy!g{!C4;ml=me&Ba>myvZ7nxp*x!+s;AmlvWawB6 z2lTv~e@}j6v-nkCJN#(#E1t%vhUG0wKKnp43NZ!|87$dI* zRPI1oZgW4h4}?*N55KfyGoOl#8Kl>tIz3{Jg}Ev=$06A-D=SH0{DS&oJkd^5 z`$p!swMJp>rARY9djGT5mUh~*rbP#Ogr^>$!-kBjZ*UEJ7_;3U%xis6*49g57(aib zsc|V)o8nLrfu+omg5c7F7=&Un3q#K!Mrva|hLBO1Q#N2KR(nvAT0>t_x+^ruYmgU( zb52ZCmPj0ELka>MOseeOdKSd{I6PaLmX0l+?TMJPUXNtKTrs)7%s_9&q>oArAYn5{ zlGb=Bh~4(h_^+D&lu!ocFjR*n+(2 zrM6{3*Bg2Tj!s2h&M%4PU1NtEtmhvWIe&go$%w%Zk1grVyI(VKo9*d^aBOv5)w=U} zvJ0*FxtLq4bDZNh5vhtvS>AL_*(N2ZdAt(eT*xh0Qf!69op(ls0P~O;aGmQ~Lb0v? z{y0%;^82U8Z)Iby;=AyWu{<1Y`hK}q(00T<&ZS?gCHk^n`Nm?N+!Rwx$+wvjjs`h~ z{GM_r^TmrNJg(bDv_?zH;s)2{JLgM-9uBSE3A+$~>4i20n|vuhTuRN$Ev6 zsZAYi>eM;!E^)e9jJEK0DJVf|^2{Zd>!^sE`}4`JXeiBszu3e6vik*YSoc!*O6L5X z3{A*n)VxlKocc`w_Rwk`8o%(Ky=3llh#iFqqwh?5qG|=}I*>_pae?XFfq%vsEVa{5 zapV((V-Mmp3N~QXZF{2NSXfE>7WxdLPg-z{ovz5C85agf-?@hMvU_ZVCkrf$JN7s8 zc68n#T@+5YcrPN~YK}qT#gO-Yeb;L`fl;G=D$1Xlw=nK`RJ*_5ck)j$vBxJjTSQ+; zu;K=Jw`6Rzux4^QSbITA_j52ZI?4Wo41D3Oa?w6X`sz8oYLK3#_mgAZR?P*iyEvfGq<(1 zA@uJz&Kc=+KgiLDleUaB`jl`hap_d1SFn02 zLL_L#c6_&=^IUS9^|z-5`VW?@44r;bA3uij#2-Jjdnd-O>`4eYCv>0gyZ3w1BP@6L2TLenMw`qpCq$T|XzAF9y3* zfBl>VCQZUF{Z=yRH-X$478*H!uDiueDk-vbQCMKl+r*?SJB} z6`%!W+Z*>qoZPyYt{zCM-AMj)ldB~HUCz=RX`daRGfMe(CC}_ z+cdU@_+7SWNUjufv$PPbq6?TIa z8|as``BMx_WSg{6dT9{cQWpdA({q+BR%uCyD3sY0PFYzg7sc?KzJTNzxju6z~n=n_1bwm^iaM zI26!ubf3$TyU{rdT;EvRd?R^sIRC&?vDz1I2W7gV#Kdc@=XNh~J}kjOFhg9O#4gs0 zQ#>Ub25fpM@pADUfz8Lb21qltvp>h>$Cm`JpX=)bH%D^d8NYjr-pPmQyhF+iRUnL& zS3V(p4KDBiLD;J8k!*j-G8{J6m*^(_DMc8p-5ravfpQqJtUv>|Lo)m_t}qe9#C1#D z7k-b8`^xc`bPhKJceZ0rb8>vtRX2&JUvScfp|4e~>^7r==FHQSUI?fjz`>e{v5-GV zkoG~SMDU$sx3;rY1wyh<_k3-8_4+zNASbP`8BVZ2#j1KSyq>x@$T-rB!$eO^=Y219 z1GKNIP;rx53{NV!olkmRe-}?lOm`W1eNc?Ds0pz)?Tykew>-)CH)W4?f|m(-b42w+ zg#0;ReSkiMZh_L}F_q9u%-lmu7nKMJZA)w@0OXn$B1xr72OzA`W{}u;ys`LA5xn7f z6iE%8TskutQ^)}6Z_;B4+o7t8YSt+S=x=Xbm$h7CHISDwnr32Y+42W^9aD$IyI?BE z97V?;d@|#2QMEn8I?eO~hM`?}kT?BUYfLH(<`jGNTU3apfEDW8p`OES>v}spe2_#e zDpkP*Pq!i~lTumgMS=RB{w^h7YK>()`3{yk0)P5pjg?N1ET zn9!e6e)}B2b7HF7#y+NdcWZq zUt!ZwPhwjq>bi847Xj~xkL5ZRuq+n<*LLD5gkE`7_a6z_59i9o`-qzDR?gthk>>T) zskQN+tj|8)Y%)i~hRg`XDAF&CJD>BNI|TWTru%)F@cm>bkJpiN#iu15L{t}^scYLy4Vdh13U)>m=`kz9Ilh7!ryh9w&R(us=h zUHT6(+?_v)&ZXimUlf(qn26D#yK{-M`{^RzCOSZ;5+_A=N1GN2*%4D457#?atRT=r zkX*QDWTC`_qLMrC z=7!FToZ*RHSHwjknI{-Zd^2Siko+-WaaRm4fY5IEJyiK6e=9>tPA$WVJ|HZR%FHHO zOw;@=hCH>;|LE?gomt8s4%551rm@_9GJAAvrD9*R#O8D~jWB1FA3Gf{joVJ1S(fgW zWROmdEL{p0EZ?|~qPLY#WoL8PLEOreKrbhOTH57x0zDf?;6CPGiek;mP%LZ=a zUytucT<;>O;E6$v`!5s?Hisuq(CyUnF$zkHmEax!4vUZIx_LOu{yy0iPEbfhkgS}> zmM^oE?Wci(3_at0lV*u2G#{xX`juZ^xFvnrl z_SC(t8Y-pv&HXH1OB7w4bjI1h7E~|Tqr%W@<;2#MQQ)~^aC15aUUI>!Xy5e--h3s% z1sh4!MfV{P3_my*s$+YOu`2{pfTRM}{DsR<8ttn>Ag#KGlG=1`SUEJ!`_)GlW3lg( zQgczl!z+IR2pUsZZ&ySv&SWPiEm$;@TA(dXN0O6;4IoE_1NS+C2MR2zeY{9psX#Dd zBL8`OzWASbKsn({;jVYbW8hZS_*FGgE;pJm(o#z~8#AP>OqSlo56qg>9?k-8r4Gwm z5NvoZ^$1h52iW9ARZ-f{>VtBWMD66S;CF8z;T3FUtu;Oh!bU!ZUT)ZYWxwsGn@1zt zVj>*9!8_7=W*96X_dcC@Hpe4D4) zc*TqBVg7vmw=@se2Zncjkr%UbcDUNvdr-~BD`9|@yII53DI3-%HDkHWu_6{g&}r(t zvA~LlPavyKn4|aVELzNPxO+x|&A3(wSDaapc>^LGAZ4cZ$sGx-mvX5bzye1(>oa*0 z$pz0J$|NvpsAuIH7RcAyWZh%*yH(rXj!{H%LqDzXGzs32H;dER_ESr{uv!`XgKP}H z%mtlPomBglgk454GWF#)OT6M$x`%wP_Ke<~%;`#-3fdK~CE~bvvjxIGyDExF<+pn4 z7HNt=oQP{#p@iMZE8ud9H`OYQyqp4!JlAanai#32T7d03pPLH2&Eqs*$NGAv!V$Gb8yn6 zDr5HB!B{)5m-TlMJc`=+LYu>;KKNu_BYThnqkt(c`Wq>wiq*=7!IChvy^uS*IROLP zXPIE%n^%e2CG6U8XL{8PUVFJcerTaOpp zl$`WxT#$f3rn!7Mm^p2Y+X*#~4UbNc07lSa$6iuo#bq0_DnY{=rn#IRW!!MSCA2?j z9e1Ac%jWH!J{bl*38YpkVb`4OVkaX%W7t?Qe|EMp|2^b9c2x@|kPB)#nX7~l&6R#0 z8~^wILqGbDOLXJD@O0<9JQTU-g2M(hr2pk}3yDnJ~+$5ig(KumPd%`a*UO>gT(lNwmqr$7_p!|B3<7hANJ(*FkWE6LJ zY@`UnswqlS)^IR^SrlN;_1I{$C(l9zSXj3Z(RJgt8DMh;b3O?%c3!8fGSU{+FHQ{V zb!lmYuVFqwkC#(?!sCpgi&IT*)E=&5E+_viGHYNZ;@0$7L=1<=4JS`R>&i^_K&2H= zD~8`)Ablo(jMfx1zaDpsf5pO+<^3xo{rOUz%H_m<@yGf&hk^9^IK*i?SKZv7O^%{t zY%VOGdv}mH|2Yf@f3>KRt_y*L&}U8nbp|solo1iSB;Lm=)d+3s(j>YxE1e_G=B-$3 zldWZ-J!Iy7=KyXTPEp7-6U1F(l*8Cx_LM~NX?Y0$cB%3Uwan#KzK-RF7tRU5E(RDVTf{%#iB8ATv4Y(OpBI>f_l_?)kf(nhr0G3p;O)wq;k2PLk9@zpW#t_sB{S8r{2Wyw}B#I<_|-QB2)B8F>!vOmrD7 zRMIlS&;<32^i!CkP)Sowh2erB(08tqmd5I}bQFyFN$M|n2pvkH(sZlVaOabAOkti+ zE4FPHQE?ySMUzLI#+^S=*RgAg)W8-6VYj95W^%Wn5^wvkQMJ~M9UiD5TcUt+oS4;H zKZ1@GrB%`@zvJL;j`%;D^TwDI=0gBZUmq3D>jm)GiWI+HQff#*xkp=f^7tkV1rfEu z=0Pd;RFZ>SdcZSfF88?;11>*IzjYX$mo- z&~CX(7EdnS<=ph)Fywryo;;r&Dx6zF_bv|k#~?&R_`({c&A!LU*GE&=7%!84OXBJf zHxA6e0p>iy+#DJtw8a{7j4|LxLtE7iU*}VJjWgy={nDoim-YeW!UHisSeIgmpvVo? zWy%OAHGaHR8|9J;o&<0~$<-+)+Nf!p6*$lr4Pnb2hXFAk;&yAA4;*lv^0h11m0oLg zX__}9e-61>xv{j05%CEJ=>GC3b=O(o#DJtq*S3)Y6}DUuV=((6&#Ahjv^!lwJVp;9 zd1Hks$#ln+#0Ed5uw%WHg@>tcrkD&g@*H}}nt}`AVP-|t^-6c{X2dW}G5C5L$}wUz zdVzAlj{r?#F%iz2pSZuaJ@#Z+iTis!T8}Nm#qN_T`h@PzG(acx!&I5GPinF|i?VO+ zwZA_t9{Z6#?YvCkldd#d8{b7&^&O6%Z5Zv<4rtTOlQyVct6ldbt?viFTZw72!)1bD zjxu^sTID->W&EC3msnmN{u;rTd}e(t3YS?>E|_Zy;wD`Y=7?tUw03^O-jcIL0D}BD zBcMw_kUHi-n_sr?4)&ejg)9aDU2{z<^;qMCsOQjglx8HP1waU}q4WC_`tYW~2u`dW zVOAK8uC_YdfV(b{v-E_I<5RI~g28WepUcMI3<%ovHzIp_i^Wm9Pj{}+`t0X)WHrCpJXEu|onJMd2ka`5bvb6-fjqK6vh471 z3LjTsnLMN5A{c?W?DE)1uUyPb8G;qNO1Eq#hNO;&q%R}!^6wBJKsgPOC$mtO4%ej`)>>@ zyDeI7bdGF8kH5IVIeS5(1cx!`r%K}nv%#Gz=$RJv2>@>gT#jnFXq6r5Bh`YErPNBm z?>C^1fLpE&?zQ`rg(W%$*T2lsyc%Ck2>vyy(JIP$I8FW`|U==acuUSFFLwDkdZT{ zrFZ7CP?r{WywR(s`PNJOQ0}eV>uYkp$x6p#<9h3B!|DUO-GJGK%;e4QIW8lv!=O?fxk|Ur>dl$D1 zyrSQQ9qoEX`UvQ^|FY+S^xDLOEZ% zfB`oZQ+<2$YE1~z-lNtiH2HDL=n!}Ci@ggPti}e~dF*p*9!>L|BadGSe*sn~kku#~ zSap=+Ml@nHY(Euh0+-qWfL*cavJZF_g-}Bk|C{?jN~4enla*F*)_>}$x~w(;AA_;? zqs`metRVT#+w8m{hVF0lG^L73u8&Z_t7iiuq!4QD7j9*#oLV2ung~YGs_AJ#(~`N_ z;%l$$3Q}?fiT@<7IW?OvF25@+KHOQNi;z7q-Z@PU6Ey#z1`I_I!U03duYiO*VAA_; z#rXATO?_}UD`?vE9CO*@v0txNQ_T3vz3pSKUTCa0Nc*6M#X2ui_?+3X)IW-zXliMMT4* z%%Hc}jxQ*bj%TP?Po_)gsI^&j9;!Ak>eEu!PbnO^)*|MT1Tw}mA+JZCm+eykce>S< z7b)J#Aa%_+T9171kTe#qB~-&MS>!nEiC!}<8n6{JS-cc?$@4b$qyL84`McJA>#w%$ z&+g0$7H{fScSlRRL8E49To3Z1#YP}^cF^kP=5z&)Dd`hV22Dms4YgZos`l1+6pcBv zvgDXx(m<_4O`h|);i5fG8wcJZuX5MX=SvcQ)?eBeaqOsN<1;5}pbf9Ki-H;1C@~wAG63Q!qP!1HYkkoQ&t<)lx9=!7f|t zMJ};PBn=ab4B71Zhs#*b%R zJd zN_R+#l*G^}(#_D_Jq$VIz`FW zLml8B_nJnDP*+c)Ld9t7)`QoRf9BNwa%O#Qr#+8{0}&a;U$q56Vg!O$Du5So#b$T8 z3$;v!IAJhtnRC2h&ny=8Rvwl0rdc?203!;a?O?frxqjPDoyMmNuwo5ArK86}nhi_k%L$|o+ED7L;81l=muu<>bZ-2IhLOZteACOY5;6*b`8 zRXS(}Tn7h|9(xo81Y83<&S+<3x2?eXQI|#}Fc#K&P4ouq@81A%jbHHHT2Zs4fdRb4YZVWUPJ^L+U+%Z+!@$0xV(o4PT>!-w zcJ^|yNfyQYL5Z{Pw>TUqHNOm2E4KJK5!hp0F8`Dys5~q*d14224b!vixX39wK`5D%M+eS<+ z?VZjEz=7+5YBBOhvH44s*U^fCV zzN^5hkqHIDNlo+oxCOa7*OM>>%2cilMOABs68zNtq?PE5FTMB80eYWU0nQGcd{M2A z-M~u3;K40WkHOK1w^fut*I!M0bD#Uoi7D2L3{J8Xj)=Gh)UX7zX*-vIsY%?`382!X zN35T^t47{l^M%>v8Y>7(=T{YIfpbS+fp0Gx?5Be>^U?XhuLr983J(6+naHaQ>R&;D zO~{IGwjP*OS+%Kt_x=-rD4>(~O9fq~znYPga}_GUb_SUH2jAG2`2%PTU?c)w;r#(` za9(D@dGEnDs89u|bS$^0T(I~v|LZ$@NQ)?NNC*LcJ#efAPGW#l4~oZx)V)B1@1NZ2 zwRk{3mL2-TJ@AfJZiqy-LtmMaz+C^QZw$-|Zn={nkv#Tb9}rs`g?Sl;I!g)s_viMy z;Pm74^ZrD%9p&P+F(iDhNJTP|Ih&V=WmaHHg+hB>K!_#}AX_iSEF8m`*EG=rDdHQUb_fw~1y4%n$ z4l7qoR>rw9*Lp&pf<2ON1?6fPuUf!q$;X~~OtStFVMt*nxu&JgKHzgpUcczLZOXO($B$<|Q?S5=C%=j;Up*ea?}}!QD#jz4Z%6>HI-(7TZ9o z-)#M3MF5RIEjV)sJUOmIvP?kW01FI26~!3=xVceVl0>*E;y2!8gO|A7?juUfnN;I+541#8T;dEJb!CMO)rAY3jbGkD_( ztb4p=J}^;n4Lx8bil>%S9Qwya9zdInSnHLZ0bn3$-IUAc7yf?GuKX03B6r!`z{9vt z*J?6}Ep-yqtvZyxh)o({#?N04-_3PqNnV^FWbChE%RP!N(z<(AUTAshx$&PE!OLX7 z9QyN|=SWoN_2;8}wf4jXvjIB%VDZ!I#16s8r95melc5eJIT~Z$OGb?n6N7YeN#guV z42zVG_yYgg6?gr;H5C^DV#dANmB*$rekRN9{5)`MLp9C7q!b>BFIhuMW=nk_x4v9SMF$8^)s+^ zSN{``t1h|G>h*bFY-29>JUi!8O2IFSaPvvb=N#vUxkt&cn%9X1X5A5aE*+S|IKAh9d zGORvhzrT1+uXXjbTKmT?(|oNMhl#`J4#?aN@_1dmM*6bu#iwWig8b)eHS~s!f|dEa zMYng^pZ9a|ARmD&ogjTD2wlf%cKi3Ru%F8ucFDk;1WbG?dH%iy4EU6?jk;|cf7}4v zSID?7ESWG;^nk1SMg({@JSRsEgp!IWIiL1wlIy(RT)I=i6B%kW=){!o(5It2aTy@h z={*pKbH4p5`b!KZ{Znwjahm_is(w_C^Zp+CkN37+8b9>H z(c~bKCxeeFBCLlAeRG@nJmwW*9HMxe{sYgEhnW=j3-JAvBtCBwzcS#s2Ty zqi2Ip-rKUT_mX1+o-;A6%40zV(nJ3`Vb6Tn(ZY3xtB^TdSR-M?Bu%*1EEeQm3Y|iA zr^#dQ`d$9Mf4Rq9UFh#CJoxscXY?l9Vosl9N}O;jd^&7S@ko|nFW>y<<3`C2R5LUO zpZ+|AOR{M3hMQJ>lM~`wAU_jvi*{e?VYOauJ zUNxlVTG^4p!5M&ug{~cvGObd_p-;djGRTJo!n0RfokJ_<;7Em@6<%W0Z9)g_xQ8Cs z7UHv`?Mz2^DkrQXqCkWPd*eR3e|PCK$1lFu3;H8#zHO2QdS@ORHgUJWbTgT}bSmpv za~=>(WPXP7%9$wpRyn9dI`+*5Mi7g?`#VTa4)~z@f5XNdL=76etq!7Lg80Z$uQnh5 z=Y2ff-@ZWhhH%S7@AKGou#k_w1CGlNu8?cU7FOg23ufPLsg%1a^=8y!eqkXCFpr}k zjXoC6@9?o!QY$B)b{3n@pm-L&{WBWB2xdno|xnft5q#r*dp}qgXs4l(mR>`e?kiG{z86i z!@Adaa+MH4$FvVBa^<7--k*>%Eau3#d4in0tE=We&X1Jh-(kL~GQGJ5)Sa{(XOsOs zGO9l`&nba?q98arUG9Hi*`w>1f`DzrM}yn`1*;pi3d?v8&?(}6Okynw zsdI@Nn*p+Lhet(4nPp>d?Xf7ZNsZkXN_nyT5-WDO-+OTNB~Ee6gJiahf`XN$tp8~S z^Gm>kC&iRMIcJU%2xZoV3_W0;^r)^D`x3ts@tqX(vU^Dd#BT;6R4;ld_EZZklJ^|p zW;W1-e80+2_B66H2@MEJAGD%#jP1wo^78pbGIQ+r&#Gk}m9b8uX-|ZPvBUVoC5m~K z3_J*uApvRv<7m5pqLydh-vq{%Ox1Nm6ik|{8?C0m_ionLl!ir&&YVs0{NcAS!$nCbo?5=f9 zvu-GGH+=gB;%0L#|7ZJaS$|RNS5z^;Ozv9yqUq1)D#6$(Z-br<{+|FYa)OYl{{r9w z#ZT>w{$ByEj^jdnV0$&K0xWhQ4!4WViHsw~Pv84hG-n9cHq1P?vdRVdnJyUBE5M;7 z52$XWhW}CBfXo2#nE|nah@3IS|D{*KVKm+A5b=$<%dK?VKdE*Al-$qD*nsq z;&sm#^&d*Hrhq@dv3p3M`GA;NFM zKMQ^;bX|IG&O8}zsqzq z*DnnhT&b)3^jilE{I?E>h#15wLI1lBXasb?_vbpGg32(jF!IofCk}9SctXO&g*f`% zjMpU)dHL5D%FJ6^3UDfC1@HUJ|A5<}%zDiNlwT|b2SQF$wJ4vUC}6>I4U^3R{eUUP@ldwCDv`IGOHPl3T( zs^ih0bWZ$yHF~_C68M-E0+4r_>2L{cq-XDRzr-o&_^X5V>N~o3J;Tb`5B8iZx`osR zYDBg8xt~5|-FrTv+g%hn_^sLr**6y2)jO&0af~^#1p(L%ZEQ9Tv1h*G!_yn|+d8UI zvLl+X@%>22`~Guhv7gKK#|nL-)YP>R$?V6{|Wp;t?%Y>R|<93 zz(P~L_il2^e7$3PIa@_tO5p&-As`S7a&3x?eEeOXjrwzN`WTeDsVo&Ta3@9KBy8Yn zl1lzik=+v5#Mnz{G6X+Qjkm39aPGIj8ONA|k0%wxu2$D-ysNrTufr3Lay-v*|ep)<6z%Lj)NdBB>wg&yi=3%tcHj!NjfwnBm}z% z5i$$hbiZwQFRlg_ka!*xYQ*;PS|54Y_B-%i>JS%5uTfuLkBA1g&UF8ZDsIukt;J#r zL4|{O*Xl$Nu|GXOANB8Qq%K`B($q_b=@Csb&jlg@6ObjFjD|+^e2EmQ2kJ=;9hM~( zm*c^57&|*3a6kAhp}n~B2ayny2=r@Z$HbGvf%>$si4{WU-O}v=EB+&cMpn}*aTB}f zf=(N9aPt6S0P@EX(pnxGhGr);X*ofOn=WBtVIZTz*vj?ajubIvGP%S$cQiElcNB| zcy_DO`kcg3`KnA{fx3o=hs_i~?`&3c2&x}ucakP-9)r}|JkZH_;~iS{!G9*CDEtqs zkaWgR7=02+S5u3vFlWe;rwF7=>No!o2lD6HNmTMgyl&c*3PIM}eUbB7 z1ApS9n0`}Zc~*WtJE@M%V_ZDEN-B_zeEMBDmY+pGD|oa6y3bNA1a!|$9{4}Qf>bmY z%AHpw#0V0GboTeFmg=zBX8~0nBK_pCssUt^w%8nDwx>UcyaTcdz{#VGjSkZ3Pfh_i zTxYL(2(n&gvV%psDE2Yu!+}5&zZ)0DfB??c&5&tnEedzX#nP(k^TF|6@Tnsp+g~MK z{hBHGcbX$7FK-d#{qpEA5OiFBrmsImrf?Zz5)23tgt9(5y9?%1Zw-a&25aAE%3XmS z;YCqk{ShoDGU`<7jOeA zQOK4lz2Fs_jXN|n-MhQ?=8rIo06PvX3=L_uODvWC37I9PxcTm)UHUY=d=tRFK0ZDZ zL4VXqYiVh66SG=h)jaX zjsJt0xt;%^(=AS^okD)CCcrkPkWL<|2voeGZzqu@V9e$ zWzMyd1Eh)nYF7JiX87l>>J0ejG|YT=K97T!(Bo^SLA+bvtm*&x`PV;QlMeHvhM1l( zQP}DYX#+%7I|Ao7ZvXpUF{g9&>fe`uljwqT|9jE(LUR1)q8D|#*Ka2K=Z9peU%JFU zSHM62x5|Id#{b{Bgj?e!m*!Gq%uDY>AWP*kaM8V)1OVK1scj0(MsEBGgUW4a^R_P6 zKC^VW5u_=k?kaxS>MEl-PL?X3a{rME@qo1y4qt9tVL~B1RKbnk$l`FgT5d!} z)3Ji;288dDq+iR6OUzSRB-g-`EQB8DVb!efTWkKVgd)O|`p;t2?y! zRl#jx*r^b?5RQZ!7&scyH?2dab)Kg&Fa>+EkMW@sEUo^IFo%`;T%vDbxmIDmPa_N; z1DVoybf)Hu3fFG%0+}w>t*t*Ew^x2#$m8Us9z32naKAJYe(+MO9S@9`BiCZ5D-q-%5?q_sJi(;RoRQV;fgko#VkQP5%{2p1wGZDjHYxqvZaM|H1; zj1_)P`$~`@QpJpkON^^ufwuRcp^8>El@TpocIujrpc@SSgdRe&!R}1Vc(tGR4ksAL zWY6o-OxUwmCp5C!dp;O68ItasC$D6dHdyu0!$8B>v^krz3^|6lQ_Ui&gmlzW2rERi zwRLoAI#06~+HO|lP?z7&P4^A&gw*k8%Z=#CxS25`(|wCBHA=capTx*r&5ifkU{P*?D07dd4Vxt5I;K)mQvEukF}e0oY28PN4@ z?x*=Jb%UfPrpL^EcgoXb@AB&cj`JtfjZ60xjfs%Z?^x{>teL630_a`YQ8|y@(8J^I z+D>Ir)mf2r=xSZ$ZlLfZj7oRwbV-zv>2bEQYQIk1NQ)vJKnI&PxCX&O3-avIj@zlw z+RlD@5ziR%8PKhiOQH%LchNai)%X3~PVW_^Mj0Knd=IITra0 z2R6Q5g7oVL!Vo^1hy-{}_e9uUxf?i*uaZT0rtpE?-Kr`7=#%PkW(8HSN)}5IkCR5w z_~{xLI+12lExVCh`dAjk3z1eNlC*#`WOO)9$^|=10t5Gx6T;|cxYTEdr7SxNDR`G3 zpZ;CsRL#rmTN93M`*Noi$7@GrM60Fk3q_IbaubWtK}g5Mjwq+?%D&NjC~x}cofYty zRt%^WjIcH_-6KiC&84>S#-(YYLA}YO^aYIO zVhx&eKS)m;kEGa8j`)yiAsc&?ukAG238GqG zA%Yx`&=j#@Xa@v&2&VLtg{OeTm|kMapxmq1qBfCC?VLsycl`-r_qa+=mRM*`#Fa~U zf;7wmU+{SB;wp_QDBYDez1_FzLRPp8PB@$iB2 zN`Oy_ZXO~D@R1zq8pdp{(#2(6RC|WWStU37(Qy{{Qx%3mA(IHBr5Bn*n+QwC&@z6*$r_B!B2+MWxjnK%QSlJogaw`UXl~InDy%JR%2a9YEBEi^sxtyj@nU<{7g3nczX~sjm zx3UrAyD9hzB3uJaX?Q)CrYY63Ze$Gd#?;KnBF48v!;R?I+6hfoHlsLeQAC;vm4dku zIMO3++uq$KQ<*T@)Tr7AsZrGrJPfY2co!i?*L7p6kE>&<*Q(9fX`Jn$AzEjPyMa29 zGr8>D)AL2jG{cLR>>+$oYMh_73|Krm6IwG!4XePu)DQt^L|&8cU<5d%9<9l`g%u_Y z&L*xOj|ZC=PML6``)TCQ<{vS-@7s5+6h{W-CZ8-g8NcfGt#M5LngN^N#t#r?m59^- z?7d}@p#^nhXMot)noDHuK?O=6i)hp{&61*$?}FElMH3n&^jlU>Cd~4wq(`(mek2ka z?X^U4c`j&k*^b1itR<~Fx+*-kSPAKv#ME@SA0`0IpBmP;3dn@b`{7%;dg3H&$(w&MMct4>jVyhi#V%^r)A#awut|+jk$?vX0eHd42xV}x( zai9j3tuAkGXj}S-9<_|5KAqyvzCl(X2}?tsNJr3 z*{Q)r;*aVE@fC#bODj8SX~vfG%lhVZgBCOsYS&)0POr)tqfU@HlvN@s-A9M`)^29s z0(;pq@pL|6?QnW4FiUm^k@_udGjY_qW-M6R+bq$@C_mKZdgv+7G*H8qc^RSrPx+e0 zQFcGDp32PHMLIHgJoLRsQrpRW8rjThfP~(Zhuita3)6ukCnx#}XFmrfg77yT)Qzn> zz1L2SBKKheMsAO!m9q$posVvtSz}7@1N6gUBZw|dSE&$7ntP~_jiB4|C)z0wU3Ys%PaV-i3+5-2g_ z3qOSt_YRNkHk)8^?}ce(OssfX3nUdmAe^|xL8zh~;(GYiOLZ<^zxE?Ux5IaXg&w4i z=_fS$Bz2$E1eT1-)=I((@mzdBr?byf#v7u&Sg5%2z)B6y0w~Y2lXeO3(NbR3!svFma(gNj$ z^;mSjo6E5=*l<1TOv4;UyCVLMYaFvJSi9*omJLk%`q{}?pO(9g!PIpB4p+gNJlUOE zfr!!PekseZ#UayjUmrE~jh}?#2k0hz>11#DA_(KM@Fz^O%r{<{2{t-g^1K(9v>$|T zc@$|ab>&$o3r*Esyu7n47SwTEphw^re@=(PU}KwE{w$uR@pgz9;aAmTPQJtO-C(_# zFIh__#``Fvl^%y?brlh32{i1~Dy#p9t9VYZf+~W@emA=|rlGXzJ`%t}kZ`h)5N10- z_ka*c3RR;ONDq!*O6XBtWPqic8ZLcO#Az?H-LAK&jD-=bZBQ9d?S#F0u;D7dXi`&4 z`b1Fs)T@QfPJZVGMU$-=MOq_fX#i4*t?izC5@Fr)xzIkGHc%xK+?%_!h8u@y$ndZw zm%s3Mzlht;0Mti<_T8rKRXA74 zSuqD+p{SviR9dl>flS*oGr@NsvC9}NzugTM7A|r-6S8v?jYl6@YRM8BEtcpj0Fnvo zK7pwqnqu5Rj;&2q z88sE(D(m^Tgi3dNz2H;?*u{_=(d-=efPF?E>F|i6qJZ8kS(<7aUubCpY8ruI6pa_^ z_N3d0Bc9@BCPuMpoEonBj0H|z5({SZlQH)QGqXM{0Z{o0J%PTC<;-y80#;-572Q)T zTDVdhyLe?PA_WL1ogjo>T(|eyRN_eGR8lySR(h&MrEp~%mIw&%f#d*?h+qQ0ra=$- zENf=1f`djdPl4c^LJv*D<^Ed5mPd{J`Nb4=c#^cQ-fJgrmR zGX_f;RiZF98EnTPs!TWZul*rLC|FXlK8UT5Z0ayrIqL=;BUZJeJejTSYMarnpt`yI zirE*+kFH$q36ySAHTS{ZS9s`GnlAEvpn)0E)8Lo#EWNL#-$*nx3s5J#MqWB}aCVaN zsIS`yG@ZTWPDg09txur3XW~3PK}R_3l}dz=`bL(`7Rsu`88~9>%zoy$O?e%I znV9SVx!T#@>I4>%gAmH9nM{NhfB%5Hdo@5Ce)o1*N1OLrXgE+;$R@T%Ez4JtX85`< zPY58cFTnC;p^e(rL}Nbr*NP4_Jqb;NZ;BC2?e1s-Zf#SjY-c`d5YkTrWK}Vo5(4*3M4$HthA&-b|n$lyJEXxpCR83HIs5A+4V+ zD?QaepIu9iv_a8`PqLHoTe{HE;Zk54 z@VoDHgVAif=1kUb&}zqiHE}sJ;YA0cTvh1UTt|GbW%#6|H1J zW$o75dC;uuhtGpN7kK;3@*O$In&W`vGJJ!AQ;ZpqdN=_>nBm4Pb>g@UvEc!5z*(^7 z#`UIER61P*z%jqOg|YSVLa_!K1W5)$f?&z%>`l<*R$ib7&RloPV)kU>X>WTa*v#M? z0|1%WsZlKqei$Bh+6{6t^4zDPEf}-f91thw5Gq8nRy3cHX=X*2ZJv`jy?Bk;=)a_jsa(1;PNLV!&&uS;9vz`5|>@ zO1U?T?2j!tT~-LK*0n`!JU~Aq*fM4diX+hrEgGJ}7Iseb(%(G7Vk*|9IEz;#vsB4- z74RVOR;!q}q7_DHXHN#H^}3+z5Xufi(Jo>N=1-TOJN7jA+CYjcwpP6L<0n!OoFgkmnvypt@+KkDew17JzZ%q$;l2 zazdspxgQ=C%juq`u28thgr-#sgr(|xIH?m%jV~DXxLvr1>GN2)oKSe2ZbK+KBW6Z( zvZ=j{0MtpMsyWUCw!`V)_KY*w&I<4qYdIOe0acx=^z=0e&FK^6$cZ4l-M57uvkgft z9l5p9_Jd(5u`??g#;e;jvhc}uM*+w*3(#N<-Iw+a-G|~rQzFB2J0|xkfzph#Vbn!p z6mwtS(}V==&c`Uo$P{>v0g0ZHOlk{UkU9ekZYN{J0<^R4Cs!D;8S8p0Xd`?ehP|?D zEd|(jOPj2WLn*{^`r%A2g^C#swb zuUTxA4&7_4j;TDZh2}V3u@L~SA5N!zSohKFB(1dm&aB+Js|p~8%6KgO-j;ksR9VCB z6V@$KlDa+*tk+0jI95;u(6)|_3u8s*1AZQ3VIhoPz{Wuy@q4p3VE;Fl!4J@V$W{Ur zSUHJ+7;+Jt z`LWF{Yk1zzjgXkQG1w5f?;(Ko473*5Hg6?Ng)L@~l7p9MYq9|Jm#@4B=qnt&!N^s+ z)Lm4CLg;p0scupe#QGJ)0WM%|S>PIbAZ_ju$LUzVro0xnj-^DMUf%6+e5jim6WAQJ zG}`)7+qHe#o#tLVPB3lH(5`@$v;Kq_2j!A;rFAU358My@hLEnC@S0YsW^vvJ(!lK& ziLJlT(urRa$}E`MF^&f5si@>_*Y7Um%RjpUadwx8OH5jfjG~opL(3SWPEo+!P+R%M zwSRSn-Ds;mIKwhO2)QgjW=eM_lZBj|4EF)9Ibxyv$aoPy`!KI?_`}v@i%CTP8QD8(A9&9!T zTM@$p)(`S_Z|DzBJO@Kc*P>{fPlrk?Ar(V&IwvCz@kE9fRI4+qUAcEE&-g6t_UWX7 zmy)vrDF`I4JwF-5CXaz>59r107y}JD>CtQ{&e~%_z%li+4K&@r-4aH5ywB;<8mE#Q zY|jtCnhguq*zc;V(xDL&yq-wG$J50s=i5+1c|bRDIdj1B4y|-9j0vm2%UG424@Vvj z<(#XWJXltbd5?E0WI;W8Ss@jEH~~B>-C^Oz#muRwti>BBc5(c;d0`fQGN~6Wc)ZjGN`FJDc0z~e9&ipsR>o!x z<**9@Ps}m& zm?l*zA*qbdaROdo{%l{UOsMKNAF|PtAF31v7lZy5Sb%Tk`3f*FZR;waJKKXrw#>Ja z$>%#W&N@K1XQ3&xHgGc>V+LA~1IJB3B-w+wSdBRldF2XsoteFR>%iu2yGrg2M~)lV zUx8^;h`dcP){d7a>*kQ*|H*Y|bbLZHu2N818A;JTodJ~wUK3j;fFA>Q;=AhRxp+$g zHIp>`-Og<1_Imp5RB{QOE~@IsDca+}f5{G4l3Qm~#0 zKknG(wkv1oN$`Hh@3R0qEOjfXzeWLksJwJL^pYQVYRj@+00~ZQ4H0GgTFY1*z9dCJ zma+%8>vJe8%nZh=@gX*aLAr{>(m@f)Io+rY>tt%xk|E$g%{F5d4xJyq_7TTG@uKtU zxxvOS*H;69UB;VM?OF^5DwPQjZS>8X7^Vic16#%=NEcqSzG4ngdzkD0eJrMsL;kcHl9!8v(rm$YL-=p4$i}e<$gD<9zxJE6ssUpQd6cu$Nc|j$T9& z0hp?2nnr_X>80Dc%+(3B68_-ZQzy`iDlCenHREPiFtBmExXs*y1!JHnSkkL0%H!BZ zHlUf*zS@y1J)!_?SW&JBt)lamwUlJ1HXi`9Mzi& z03PUlol%KtOBe&GOQO@S?X`*DrMmU(`r8khU4f<;O+$5L|$Ra#OI#Re%vs3 z%pR`IQGW$UpnyHQliTfKWviQ7UgY)|q7VY#+oKe+r*)brrFWy@|BT~nyxrn0VFuTd z4xL$iULq$y{}!ojw|rT-yj*KSZw*mEGTXLlwO4VpRaA2e zf!b9y*ZgdvJ>CWEtZ4ZYLM6ci?c0|g9bjqn8xYR zBiB<*{)xKC-XVKJ|5E=NT&K*#qDe}skCMFQfuNJ&Qjk^oHY2GCVm^96beel)r>6$& zd?dB%dv4odzjSo58on~)Z*OPT)X|gHjFb*_&5f-0KIQhs<>Vai*z9?kSF>GMwGf!! zSp<~AqvNBB6MBA!QKVxYrr+T(a3Ax^c!{KD=HQtfgXnI?(9)y)gF$ZcDZZ6`!|qHc zHKx6Oaw3YcW_>6mo~ilFI_nXgP~9Jp_*>1vAr*jB+PbuW@xoiOhS9Ew#)a_dt6E3^ z|MLZ@WIBxF%nTeo^P{G&)rP{i-C)~MhsdepqOhZQa6rgPJEPEH$Iu+El~-nOQUaZb z&t|LTL&Rk&XR9eHy`FZq+qr(&1BZnLyPp)5?@Z!x)_#vJf8WT$nr~?4@&M^bFRh%b z&dJx41O^-sh;LHHAKmV#8;_hrRlMi+4U+fdmEXqwR!4cGO|D(D(fiDBtAJ z*QLFw!@?5;%0)g_!YA{@-BlY|_D5{wAF*61U~i%8sK7-cLgqfa$@!eWeY6aOz}MCx zFcF!-f5gtg7efoN`}#NN-^0=XDaS+G@-6)Gfbtxjek_o{=`>y~m{dNq7x;D*yaO4&v zlev*z1@Y&f#gQxPx6$;(^n^fpCPG$HX3PrlQ5xm%yQjCg;9} zE8Iq!i}k+yNs;A$0*p{rwQV#K++jZ**@%pTtct~noSimaoT$hc}f6;d8!zCR`0mYb~F0+e7CXmW4u|nN-CpQz= z+w-oMfRl!AGENyXJA2FDyRVc8+BQ$@!x{>_Piu7S22}dYF%vty3TSs%4LA(Zv<3i< z&!Y`(+lJmQ2&{cXa~U90~brM)4W0J(HDByX+K41 zQ${lk`|4u6MG(z*MpvgFLs{yPMnRqYjCStq4ZfnMF(;jS`fY1^d!bc(NuG*{XE~Xf z${Z`#69e$j8|^5}iWJZc*L(nWG)&p249zjNbG9>QAI`Gsw|16J#gK{|ubzmCZbiDU zmFk2o_9VB)Y+k-C`9N~PLVrCxVGoF~F71Ukwc38>ezW6jK|pgSXxt)fOzbe^O-$Qf zBOzmUJ-BNG4r2m=>=)>Jvn`xz$i+n5&B1ui0X``AMeHd2pmOovxGl)`*qd}3>~iRz z7)Ene4~vhg-sC5vTt=zO@nv>)vCi*{I$L?}U|=BIrK{p7lD_tDS$Q)`m)?UeMQ|1x@K%Gm?pS*FqmKUxE%-&A@7RH68)6y@Vl zVWko25hbw^J374)VC}x5=$MW!_V~VTD17uVZVlRYACF)m$zzIQ8|Q4Zvd{?e?vp;K5fq!0 zAZ8NPzdej|a&wnP6?LPkgDG1S&vtm1)yb~mT5GPH9BQ}V^{p9G7xLaX)(ZA}-K z88Ii0m0@wvTIRDO(Q4*>;mzls>v*TH9Cgb&buK%4ChlvC^^d15?9j$1Af$xOeo7f2X) zm`XeuXRoG*Z4D*09=Wa+p9vFEi^S@~P&D%z*H@jbrs@7P0Ssh$EHCxEE@#fZ* zWOy!7cg@k=$wR~20`iMRo+AEmJQ^6B`fPvR&OPbY&GXXBe^+#hRP4#Ga>}pEW;s$o zs_iJ(j2vQrCzB;K74?MM5`IT;b6YU!)!rhNO@$7oi;#Eub4cvRpJjJUAAR}Us?ZN2 zMhJg@;r3i@=;La|(-%2BwQIxc1XX8^Qbd*YZ1DAPErqT?v zpeFq91kHq+5xr@L?|1}Vf9pjZhciLK?AQM7yT#efVh8&}T|^(kr*mw7_5<_MD|c%W z+FPy9^?7ddR;+TT$<961;P8g0jQiOkk@JNeQvA9i(UeLNWH(w>HRmMZ=V=7z?tPnSo?7!US> zQBqPGQ_Z?;Q3m4HB;x(wc5EDekq{{z-RY-aQ~zQ3c2VYxo72R!oz7pE%Y>=8AO z9SoW$8=N-+gTJF}LXr%g5J##e=M}uaV*1hjYZmkeh@906{m98Ef*b6KQW!5ju%)GM zdQ2CACXaeX`iUwLvvY6Qy&J5_sh{+ZCX`(TQUJu37^7$ zmO&-ORG1>}IjIWRN~I^EWbgiaZYy2}wzvLJq^W9HD*M}SvrHHe_A+;jlAfA>roKDr z+K%I)&{JaX*dya`_5Qzk1u?22`1g*U`WN7wd7!Ryb5mft5EhRPPK9}APsgJEzW?YZ zQ+i=HWx37=8z1T!T;V1|lb5|!nlIpQC{DMw-M)Ua+AVi1ZS*6ZIwDqST6*c;Rhs%D z{K>t?w&E-su!V(n!I|rA_X&V7Nd-mU0aPe6mdYa$+fsXsK^cF|7 zpdFK*+hXZK7pwmlRR(wJq2bp*zx=p0j2xe$6$qP{`MJI4GF)YlKjq?@i`y-HD0{L} zfVxl5+-b2D!_{iUN!uhj2dKd-Sl{KEK_S-(NhBjsR52hG3Zd$;27xwT84)li9e z#-^gDzwg6nlW{S*V2?qm1ktUx08RQri9Y{DdL><)!;tFB6Qy^Z8)CBBc(Jj8fKpy@ zYERo%UV>aJr^HPliS7C8XVa5=bexp9)g+u*yEzW&w=Q3@qSEElQYJ8PBZd6UDHp6n z#`rA2-&_3Vu!c_x!;cM{lyW%c=g#;bfo$*%2!O4dWY%KhmfWvX?|-?fmK4RNox(6{ z`i5WKYJ9^<&rFzM$chkv^WWjxD+FLCPoZ~hFLI4S7XxMB0Neq04%5$x#0LcmF zRaw%3W%m(KgmsWGn^VO0_Kyas8{p|s^NOo7J(2owpF@N^OM2_ll@Yef1ervS_T)`X zIe!dKUX5$G^ju>J}ZOF6F#56vr36J|h#8Gq4Uu6xsSeN2G+Uxu-NC_Un zb5N*k@cuUXRCSO$8PR_umyQ_J)gmYV3tr7KWt+xpAS*uM~wx7v6+*{_Kpr-3LE(YuKcOxPn=nU6d zG%sf&t{6zs$7&1q4EBD?vu@8aX5h>sc1<7=kZ1VOu=_DMQ2a?WwQbD$y3x%iH|fI2 z(jy*ywJ`N=M?Nik96bN?MSzSVfqAWK8LHMV<>TlPUCwA3l`dUTx62GJmFM;}RoTu5 z<}a#8Z0PyGC-=PN=o*Tp4K{yv*ZNKg?cHNCC$H*)qldr0S)C$}%=)I+cUa^v^(KKwhX z{h^Bv!Bdm8BNpJmq|%+lVVy8$7SSxYBqiH+9FV2P#1b!vV`rk^EzTxoCV6A>*zysN z)s&Cls;j+&_FsV1%$0@cg2>)$=OqM#giDW!$Y*y1bgX%0+AY32wkxcQyub=`0Cl~C zy;p;!ytLAv*cagFiQo8^L$0*C#_?z;;DmQ!xa{T56DoRo8Rg*L6bpUNETDd&S^ ze04UC$9XDoC`vY(BB_pTstR${;DM%=96v1iZ{}U)TKi9z+vdIY`2M>PU<7KA8E&~ww$wTJ2doTFn6)4$+g4l;FAHQ*V^v!|-AjpT3Afm>E;@Ia; zK2Us4jQ1)IzbqHWjeC8mOU&53&xNt=H-evkUWO=V5o?;!qt~y?GEdu4vs)84>wNhVd{6#GQYl3xA+@9#$iYjl{c?TzEy5)QA}^)V~ zn<}KLGu{MipZ1dvOPc=q(Sd+B#EHl`zgCL_6&g!0KwVT)QV9ldSbUOyCwSsb3M5dK zNwVxQo{xh6*OrHAv_J3OXT7PR!$}1KQG2Nqy?A`rz>Qw-cv`l&c+cv`t&E|W57~>a zencz%CG-aYC2JF2Qht4jurFW`!l?E@0C0wBbhCZ_^fRMZ>>e74v-LAgX$FK8o~@#t zZHm(@nn#n=k5jkg%SumN)L2FI;bfgnHJabJP)l;ukLyBGI9arhA5~NwFXqzC z2mQ#EB@sKT{UM+bb^U$g?HAQ9cm$M;Pn9%pYXu-P6a9Z4+@byX^e#Q~4Uj_ghrB;K zjlIaC&SL&A&E=2!Ke&IBL&Fo*CwB__!T?j&CkG<|cbd;R<_v$&?T*&Vehid3P{R`m z6=rI;ZGK>W5DN1gXYn_o74D1)zS%V8K$8;BKxT`$K>VSqP{59_xkc$V??QjMGy%7^j zRo819H7n*%-jVfKmCQC6S!|&+qEhk-7F4HKX~7zkSP(K`zZjDd`)48OVf*zuYee0npSM6X+Qo^afM}0YLc^Pd~5G$4M_3 zEDcFP?@i1Ha3!rVjPrtqCTvMJA*4;Wdfv03& zbxjeoc$1d?Ct9=p{+|pYdY_^u5Am6KEhT^^V;!8?UBwngrc1{KY>mGoWWHM>hYs8a z=!CW>#vVE?hCe3Vzp4tDNpSymJX642=RrKnxJ$#muho~ozr7ps>@OOk_i-fp-Zxw- zdwVQ*1@cFRgRCkEUTxhjsusy-i^9`KjR&9AoJl4Xe0+!#I&1LIYW7YGaHdh^|18Rf zeGJszPySMV`|aQ8SZI;pb4`R-TP*JSxP}+Niuyne{h%ZfXS`X1R9uHz@4v3xY&05h z_E;%Y@fOqHpUZ^=Hs7C~4tZU5G)cN#vP;nW{;S$wcV`PVRc@xJrxv8&=&n7y8JT-X z{^&-hO`_8OL)TkJRn@KW!gNTtk`fAnpn!Cjbcl%3-7VeS4N^)Y0uq}pX{4mPTWQ#A zy7}hzocEq{zHzT(kHJ5%T`}kL{OVZ?VawRH8({FgIbRe2$qiN)9$m~zxYD|+X^v3! zf_1DT!MSu{&)1@T%=PQLWpl_!WuKOb2_CV^>&MBE{?2(CNZSh^dpz)n`oNL676YFD zrU1E)Vg)!-rkWfRa{0qZuUhICY{6-)IA+alJFV}qu!)(aJi7NdMQ%E~OKCp4;a41BsCr?WDpAF<&ps-j-2+>V zG1GIM9{g$WX;f6u;;N*g1Sfzu71wP9C9~DY@}z@Z%G6C%+kKdbdc(5RVxk5Pe0O(a zKYHh#5xQ%YAlIh|k1>$u1QgAouW~AD5k2v!0>x>2dir_vS4}t%e`E>!0e<9Dh4His z73$l9mQTW^OB3n9r|~c=F`ViecRlvriv*O7Gs~$lQ1$@;+k3&(;ZOg>ZHFgT|LB5J zW|$W$Gw)k7OF84X^^n~AHxj~Q02b?X1!%$@MgxVjlqe#sHioYihz5Kc(+-bI?oYjd+@aLVg+x0q0p`GPEr_XIt%IfFK z6miI8+p7KAC67qAXBOnOiJ}BVVNw=Hu@vC?uDbxklBH^7IM)Vuna5P;6=EduG$ z_U8Y!4)x7i;xiy6ap7F-kH0WF{YK?2uE~fQfDt+#JhRm+EcQ57TJ)U`vd`7Euc7x+ znQpM+l~Kjnia9L2Oy%+#3*b4Lj^D7noVb4UMG{$j)K8z@uWyqrw{NoWwC_AY`S=uk z5zt=*VZ+k^QINKJo*){3dx}boK_Eo{uo*yLBzIQ}TDk&$sBrY3+eqzuHElKHy~31v z(It@@C-8t{whfs-_p#xhZX357$ctP6GtL|KakrewA;GTCr>LLnwUYQ=4pV@ci{0az zk59$$4XmL*!P7Y^uJvhKT&Ho|E(Cb!Y1ytyTv`Yv2_~JRnTo%kY9Pq4B^tCZJbIXs z#Lckzsn0l-&qNU3I$VV{ww+^`B4)e~nG_aD0pvvHkL};@{>&->`zfN=RWbIi)J^}RtcihAV zIn(!;{x|eRTe0swPWK;U@q_#~p*#Q{7IWM55Dhm)BpmJdo$o4der$kaQXY|;2htJ0 zV0Aq--RY0sW~*S|42Md-`8k^KK=9^boaq_lj1O=o3APm4g>2~YM~d)k2xwaH{oKFb zY@et`*Zv8{m&)gi^!un?FA_ipC3W($Oc4qxlXvAGuDMqUq_PXE37w-^!=hs-N8E>c z?yl{hU?71C2b(yCt~3v;w-dGP0+$c2rgUAEg8#8wkH+`m^z-^xD^C?WxqPMm)qBCj zcEiH+e~20C_tHEplO$)KO}Aa7DH;ONf+7m2S0&*W*S&q7Y^T2J@Ln0kJrV``A40rp zA0o2uVHzc~O`L^?o!dYK3n}$H^!0Ve6*!{|nWu8IA4vH=hbDO1F@NFI)$JECobE1^ zRW=jGnCVWEPtt;wnm%Oex{^pjad;vR#jU>>*xtisocgohQ3YX@(#eD7vfqMT@va8P zZ6oRYmNW~|uRc{~pwS%Z)#oKtb4pysrF$|N}>2t z3+=iWx)gjM&m7>x22SVmpa5)bJ`13v{d+%yfql|)=pp;fk@CIR-5ot*@UjwP1f2^F zZ$xYJp-US?x}ZN8G|9Ds)_X=5DqeE?3z!0c3CV0)q09H{B%n8vkC6X^cmDniG^jnv zZfeAJ2_4_0OjF~}UPI;nLB%p<*ST3-ckx`ZekHIfeD+^!q>-ua~ zWrItpgfIE&)AR}>_}^MIQ_#OGvS&Lhnbn@Hh(LUU02X+@^~p${`JXI+x1AV%#rX*P z%?aLp$5N*hq6s}%jnkLcuu$I{vM!T%$a4@8$7W>d2-ry^bnUQW9eKz-3<12<`VjJ7 zUz3tqY%fth<_nN z4E_J;u?FpbJ{tLo5yZToxX2(QqM|;Ta|uNT@U08fdjHw6I)Sk?wq`eulR3M9=ebO| z0L%_I-B3?$?mxJ?F*|aDV=<!y2gln^wm}fOnn>AV-Wh zmbeR2ThBJ&`FbBHs!ijRnEwKE$J|Zlv&>DWqwim)u?Ff6lL532&=1&#*XcwQRN*I| zPO4u|YW%-gI(GLp%A z4Z!7|3CH+0olw{A^Svsp0WWxqRGdHW7*{rbhmS%js2RX`ce#@-lIKSRg}(dZvXb_G zN!k-3i~WC=oJ7|;fr{pvz|P=&`-TPK&0q>A6=q^FNUNhAqOQm0xZl~ZnF3X4ue}5$ ztH#-7HOW<^2dHm(jJ+Y8OO%k3u2$V!g!yyIBhS{`3W4+wwcL1iUMG{;Y%AW4Ec``IYpC7GXS>p0-GJDqhV1ph_Rq`Ayjm>HWK^W&HdP(b!CTjKRbP=>V|L zuOe@JvkBU*KAO=D<_FSPvQGo)?$ObzQ)7T|>$DZ1 z&?Y@TO|aFEygJBwRr#efLtA0XQ0$X7TDPA?BJ9Jpl(Fi7o%ADS(sC*J7h9dS<7J ztTVN{Dy@rXp)4u=!8+(MhCMBvfp0aEoWe;sCf~$ax+GP%XIMe*W)0&Bi8OwawUBTrk zQ0XO@Z=X7HB%JH*l55>8xk{$u_QBXCGNrZl(~v~v@{Iwh%NwPO!;Jp zYz@(c*YX{7ehxBdCpX;_m}F&n0=ZryepdxyDOcM#G;e<5fKC}DTh z5O47`xAzY3S4PH@tJyMS>57y78sQ%vMQU=|95!@Icf&-5^R48y-@Q`hj^kR7SOuNV zae(ODM7?Rx4+Eg5+~%AXBj8?1ucdyGdS;)o5g*93KrH!-`hEXsy-eC@S5J!F2*CMp zHKqr+8gWoap?8@H*iZdXpO|EBXt0Om8%YayoR$yAHPcEKz;G$*gIx?~`TAF8^y@!c z#?^W!JYX_!@70syw)!I0z)LB4d1Vx!v?}VMQ46kA_feD}iWYa>#>PRn`$(=C$6n7B zc{;=n9FGO(^>;HrHguelez&XF5OsGbx*adK_LM|Ey}+hdpW5#gH}_-31^vC>YkWx{AYlYxgi8jzY{uo$ zc76n2dqv|!Y~x>Q$;cm3r$0Eh3Gd1(mzV*XCENi2qjaBv$ENRF6Lb+7lqSaJ0AC?x z@N>kuo*SVv`c(Q=(`?HP&Vs`&T`SZ9p?CSc-{-5RS?i!B6!Yx2&9PqEUtoAyaOHdu z^yN-&y@luw_jJeVb>#M&iteu0Oc_>qm{M^15bX zs66r3m$GgUL75ucTp~jNO1I(K%p0F`Ds&be0V<-NsGZX)@IMRQZ}hyZ^|YIyPm+7- z;)Hg0+mn@Jy~*)`#idQPg$nhHNcXx(3*Pp#rE^OiTOM_rxtL*fn}Ej`L;4r%FbtqV zD&Nj$kFU)IJC?)NQ%2r+8OD}hU#OQ{uDHWh;E~-|he;?~+*&Z$5solLTv%>cV+iSU zomyO6v*m6ULQH%zqHs8wjIbMe3ePMh{dov^vfSDA;@Sm0dWN zoiB$96pTB6$Bz-ot(3tv+?g1(nU!(SvQM1IE7)Z;je#`{e3Yk!EyU2nniM(h;$ z2U|KN+<56B{JWZr2AfXb(W@|fp?g}ppo0$(iGb3{lszQQoR0+DM=4e9fozC!F=l(v zg7sI1Zn2_Iou-c2fSiIVSMPMiP&oPhtt0SCfgk(`Ff`rigubu!$5&ErH$CU{EUb@I zx5H(UfU+`BMD)@`8yDEAumKimPS;G`xH70H%I~dzPGB}PKgdOqr;C3KWU1KOL+S@By2zM9Lcwx@pM@*$#47ZD0FSXzSW{NQTC}Ua>*kOAM zp;z6ckRH?gCqQ`eLE&L-X>yH!lth@vp$gZ&HO8N=OodWQ2gmkQ<;-Hdt$S4QhCoah zsTu5chM!VKh6hZNuY<(?5*z@{;r3Z#x3L792<#zdfFB@;CByUkRgUj14F!GfZhzQA zOQ+ki$1|@QH1j*W>vSX2-ZHMdxOXSgE15+K&2xRM1RwyA5e@>S+!iI%#?Ydm2y)|*f|Lr{ud_Zt>+xtfC>Lwp#FFW z(Hw7TM^)r;!T`o>qo836Et((Oyx6%iE$N>RPA_?7E*{##^zR(PzgCe>Jph6$i^}gW zDP8$>KoNRb>zcbV;{Y`ChZ#C|6xqGTs!@GIqPFV%#$)ZLJjs1uQ+c^5G>ZCEijx3o z0bu{Ny9w*JD2nLR@w0XvHXe1mIsd$_KVUJ*DkZNLOU{F4cL&HZt|V|~sqcnZ-44rk2}lTBb~b^J{(k67__K9K zz;N|e^Xo*UB&tSt$pOOt(87kt`7I{&ChB>>V+>yVuQQuQ1OO1~-EIjBx@|g-Sc-c_ z=QDJ#mQg@{p7a?VZqeKY*JvFzml^DBiATl?jp$2B1sMRD7U=!r%t`Im6}iCI#9@6s zkjahtYy50O`sl7$dP^s$_e)_|xld#@c97cBV0DTeS0ybovUIj-zhPgtvt{WAz~e36 z80--7IVXzxC9|DWocfBJ!q@Xgx>rfL9g^iiKw5mR9UVrLHJYOL@eO3SIPtaIKiE7B znDGDp7cv6^81N7V zqOUyUaD98tkPg-fU5FwzLKoI*s8A*oDXu{mIH&BknPw9;)f@}+5&q})!hdcpT=Dyg zvI0u^4xBm_k*3o)Qv<9d1FACZU5tv2h^%-TrygrCpqu^ z<4lbN)NL0w9Gmx|h;Pmc@bt%KtRd6+D!mo@LtSorq30{Z!!0pWwL`*=4`Qf#a`N&) zs3r<=`HOCG`)!?GBs#QlX;-%nECH5xZP(=- zx$*d!FeyhG@zW|2;FZ{KnLh)n&e>-XfGihnW2ZNNJE3KKQ+CYDd7RRhJ$*802E+=- z6AOIeD94I>Vby_je#F1xo!K`Js%cAcht25s23;XGGxavH!_Ghmt`|c3qeYh z@V`h%XSzC31O=ppl1!NzJQ!0G8sW`e&JW=5D+S{K{|xRd8A+R+?U+A}pab?s zUQ1Eet|p+@3!eR4fDIi|Xoq2p8})VY0hbFa^nv_!R9&s}ij2dfr?C|KHEs&NpB|u= zr3z62duqR~DiK@{YbO-=8lz-oJD)?vK{%skmIUux%0ha@Z^K0Gd7-=DvMBzlt~ReW z2UC&NhF4t^vspK?-%R;jGy9Lz#|Fyv*Jf84;}$z-0!vxFFOjKRpri{f`$&aqzupc5 zQ;l~+kDm%ypUnA36JAwa5#FJZqc+>~tk?*k5j*sA9TV7&Tzh5at3K=AG+o`WDm&El z>)waK4_PycN%tJC>XsQ_z4$lo+C*gvVr^aNt{GV%=RqRH;cw0448nk1&j?Ih)H!?m{nV4lc{OAn7xkXJC zuwtHI&QBq^;&Hz*2S7I|44da-ksh083${3)vQq5q&UECUCmqqRqA@>Hup%9%8SH7R z8wH>qh)AJE55At@=7w)XAWgzuSFfmZBv!^gV~!~Bz*o-~fPIR%&bOi;G~wbcxeL9@ z*?BunWIF;6k+D0Av#Av{$F$&FL-=jBvKo}*Wdr>=?J9R|KzC1r@$&aX#Qh)bKyUaZ#shWT|&|2^#dP! zsA4nFut5I91J&P;D46q%@_EqXFx_d2#0>v_Qzici`OQquY3hum5MZt5)R%l$M(8%I zSp{K0+a?ccx{iPsN`%zkdbqGm>DiuEF84^&#p_>vt$;%j!2f6?yrF|Y3ml8E{B4|n z*2t5j8Xj)+8s66NrC`4Iokd*692TO(%D(MFb@Ss;@;hzkt|#@TclrN5D*Rma(<=?{ zw=8nMidX*RARs(Xb!2~S^7X{i?qeGEdtGiUV5}5=x7y>>)zyiu{7xR#r-XM9^140| zlIWkxY4jk0$naM93~;km0}tb`42Hl`2YQS51|WUB8MgX<)9jQA4-=zdd>XecxDpf0 z#`#Y`xB|&4wsg4uahN@Cy6*Bda9*zQ$lR%r{Xp=~=vO_ZO#U52j5%p-?b$lJvlpAX z-O{L8Q+^FHs9)@6-Nr$^USxy+y;AA;wE+4vRn7`(z1ay3+byV<^#!=g9?BZoe83{l zK*;pF`QEx~)hRc-5L;()>C0D!t{(lRlF01oOFIN)1mH@3czJI48!8e|cF=;Ven!nE z?rBCwxNaCwpJ%t=cKWARLH+?#Bs}Q(ByuD7Yc5*A;}f{k>aZe1@J*R~oFx+E9^*6O z1NNdm*?q*or@eboOq(Q2OO^0ke7X6zzCv1DAavoQB4mw|(Y>F}{{D22KmPrrnTk!r zd+)C@iV;24;YQgz1u{vXez}-F=)mT8x@6iguHtJlE%^^9!DtJ!ag+bI(B;&Q#j|A( zQk?KzNz2`Mzs7vqN0mga@^a@ZP@beC`@Lfd7`kbhJA8QL%1V6y2LVQK4jz(bDU1F{PoY1khdupb0d zf{$tu6zSE!GFz43sGt0qL|mR19mo`ZR%j?S>8x02bGT%J$xGmb`IRCaoQ;zjuWI6Y za}223+4Uv-*;eFo)MC5@M+DbsH)!(2Qen4h&X{kCvsF-*x8?t049iwT`V~puR|DCB z3&WnGg8TQ<C6*xr4(A@^vjIH6fFt1(>IN2fGyypIx&2d*vfw z+|8r?vkUl(jX9outQ>QIvc88WBWd|)ZAYx+@>M~2LcCzVRbobVU*{>PnAeokyRTGG z9zIT#HNoKAqCF=V*x^VUoizzbr9-2#?e9~NateYXOs5fltaY$P@KGw3aZTIfjodxl$A%4qPV6n!pwuqzvNw0jCN=%Rb-`xD~k5PRsX%$yPG~zTFzA7K1Q? zfF6iBslBBFnTHx2Mf-*;?qEbel^XIKOJ~tt*(Zb`bqZ1=x;^{2=%2ObA4jyC5sax4 z#y_MXnqZg3tN=$O9$C@3>-JWj!rDbZT;l4*-D>1x48RCfrY8VKh{?&`;{CroLR;mYB=|()Ie%_kJ#z~=X+l^5nn~@4+o4fQakLYD{`c2V zdVkd%cpy?7$&0e)vr6H+B@wHeNB>z4e*eW+l0YCu&}kZ{-n2cxcSr`Oa9!oG)0_&( z;)p$xWcoV4u+DjiicrjBD{Ua_E{aG`Agblb*x1FL2ETZ-+2a7SdPz?hI2gY{* za~mWMqezkD zgBrO%bxiRuZkx$;^%}9Q>h`eBSwWBtBs%zPxwQ7luv=y2bw5 zq&HbX8O68f@uE)?#q%&R>-!Ql^Reqy2DbmJF#snim!N%}K)+UV)N)VMS%>I>JPSM} zoKVROq_51YkLpEgk2$#rEc!4XrX+?(GR8`1${`ujbrMj2!`q3)mPQI44-fgK{=u`! zSEFRw0)&0{tBx-8V+-tnuK~UyhcWEM!MY(Gye6HHeE|(%W@)>M(~l>I*7EC<9=-C6 z6?oX){6aP=^0iy)5;Pq|W;j#{b5x(P+d82ETQUx$6@LePD165(lb=9~gJdot0c1VQ z2EoI7wgYahm9w`&`8CZ%rTZBY4(>0tJV%yH0v?B9yqtEBGxWm#Bu4xS7$g`xTi$3Z zz!L@UH(znOYH|IWSk?!!F$is0&$WUN=MG;SqjvBwlB0+WW1sC?M9&{)RxM7}?H)3;#bn52YLdXS93+=B5JT zW&~xzrJ$B2vft5jA@R5baR`iBjguok4Q++!+jHfQzO-n4AH;NYbp<_nCOKwBJ`E z?J_(0`KV*j1G0Kt9s}YH63GDQtsW7~SlB>CK>{t@fn9?9TS<|fxQSx(G9R#L#Lyop z$D!P>wZ!S@XEdOQ`E3fKNc=Mf0xv`6zZ?I|pGI6^9EIJYn{%^szKT(? zA)qEMuAcDQf(d}{Wdwki#o-#%!~$U=BIf;NMj|l{=i%!gWHWaH-bOF~4}gGoj1bl@ zi~hsvwm&zEx#@lH34lR+b=@*sTxQFuZaov?thc$I8*XF|raf1to?ykLC8Io&Cm(lL zv)|(dwu%^8wgvQ+!FhW}mD9VwD2t4(Mh?@)9D`vD>xJalqzxZICa<(c0cd}#z^3i& zw(&HmI7|PMh{mJ<2^NR#U;R=pilY~LLSTSK6(!6s4yLQ%$!N@mypTPZ!;eK?`)U-W z6e!y}5G@C($PNbF8?PBV9R44^SO13i+erX zOt&inXMSYKf}e2{`;$5mq0`2-I9UI&^{Gs1*XDHAHFGLzn&IwMY~to&9x_AQU{!0r zSwH8>o(!ThQ=5xRkNtNC*>WOVs`HX0^8;7+?kJ1ruwO(Z6xh(&&5rtAW=fk*4e_n_ z7$AWr;C2BkC^zX5Ze3gmAF3TYBA=MzLHE{H)FHB@2oQ;Zf8W-;5_6I$b0S^p5hr1h zv80pG4!72-izia%3xKfHS*&rP86Chv1zJ?I3zWI|d)M&Yg&oM-ua}Tyy(vbco=fO6 zjzhUjk3M;v4WjKHa!9#!*utByY9GQw5GOPH5-v+IWq-4Lu1c>h2?3=E*-LkFku~YO zKmL0N|30lSb6!wfECNRMol1C?A)c?uYNZEdJzF zvGn3rH`61zA|epI4MTTBW@r_Qo<2J;P&l*8@WeXI zTfSM`gaLEO@4%=KO!r(H9h7K%YXsJH$5Wseha z_C5XoxpVNJ<_v$2jZ3(POpl%7h0rMrd}y!l60m}=V=EL@aJj3wGss?I#q$E_O#`IM zsOBBmjlz7`0Plj>BRR+@T;5(IhD#zft0QM91*+6Mcn}OXtGa-bRBvv`w&%jg_G6ni zt0(LFc}MTEZ*&Ys&&9&?-%u7mY5c1wtbj1@&-J6u^U?Cvw|7^9Nw-;+7MV-D%w6 zYS)Szzl!G5C%O5lYYv|1sJDQTgkfNnAVkm31=4r@DL=#3>GAkm`Hxr!vZcY83iB%u zDk!%W0)m_fZMb7jbuOu5sSRktxLdATV4&~n8>y4?m*yB2k_SJ#B+nk80W<1WI-B&3ktVSq^?BQEk8(MNV#9xT8Ufe3m2plbgG6YkouP zMP^*N5*X#=^HWkvl!y?icyPnHsoBx3mF&OoZrE!iMRSbYj!uwprgeL^Gip=^Mu0NT zn;YgrOw}g2={Q-Rt)zQbRZy{ZyJW3{*eM3io2;ukrijAAvYuZ!^ZHkMXzL2VeFUu` z^`$8>c&D=;m|BUCPY45}UBnS~cMMw~3Wc~d+u%UE*J}BSw*Gf+_=gG~mYpw}%|jOE zw(#LXBQR#t+b2gcammlG2+!<*oXjboG-Oi0`;!B}WoXW9{I35$OlpwVoQHMqwygu= zL(mekEWRS(I>QJ5axA$He(Y zDE}?_5{z^Tp+I2=0iZSKCZGTU)d+AsM=z6RjS?XcW)BWGs>+EwzjsaLr+FdgkI%2p7V4*6+^u=c9T>#xc8ehYEWC~w6C^g(WAF*NuBVI;5EN|v#)|^{Z zbJ_wLI1i7Z%fZ`;-C#6>Z~-S>nylll7?{{!9cFun6o4;BkjvzeQD+2_Phjjq@X}X= z^3ah111Y(5u(N=L$K6{m{Zr=WFby)rAWf1PtAwG}_=Hxln;J&|(*wgd-z_QNQ!>D< zQk!^{PX3gE7}t$EGc} zCoxItkLjO`Lirg5T$+!!<-mAXTvBoryeokTji<~=?4tx=Xb6d!dR_5--6M*+{dSTR zC22!#r`BT%ttei%>YrwaGb1WoV{ZvPZRj*St$9_QD5o;1Aw9qu29uX$MnGJ>wH4Oz zXmUO0*8{67E**cj3PEwHEU@0DPn_Z8cm|Bcfd~$4`&TrO1th^}8ECt4pYS7rKJ?Iu zKd3Ot%)}8ie#5m?>*t?&h)mu&3vW6*Im7eQ!J4HVU~qui>2d!$!Qs+oF>DUKf5d|0 zI-Lxd${^lBC;XJT@O1ko0+=FBrV+ykzYmb5-%O8yv=T$>F0k`}9dUR^W|gh}EI!$5 ztr^G~(V$udQ8qTTs)J}D#HFDwLhycYSASjd8R28fod;1bpQZW1OC!qe;9!!VpdR)IPS*Qm> zo47Uc-OUUh2$p=ChTUb}H*kJisR)JMs^8FtjQ{C?r;eL+(3@fy8Go)dF#7f_3xscJ z4Vu=i2iFOARo3Wb)mhbBZ>YgXZ1HwQQmNogO~i9;ndt=PJzGDT=60qm(whAxlO*Nc zYMG7t{bS z=MCzqCu{Ya<%s|=y9Z=aHt{Qq=VMD29bag?gF*HeTmU+NVnO8%3y%;5qk&FyBVxDb zbD+UHbWR?}5w*Gg0Dbr92=BtCnvac)_*y5g=C7J(#dOFoUCgzC*WPJ6{s+^1;O}v5sPs!k0h8{Q6Rns_=w1mo! zF4Vk3t`x2bsh?#{-r>BP(iV+r~Z}8pu(7otQ zrWA9w-?Bk?tKrJV?!$u7p&VXcOdMouYW=dp;M!;D+6mQ}2O|r1e!AxG*>gN`qahiD zmaEGA<%J=5dZE%pCZGE&NbAMJAGnm8w@+C3?Dw$WO->1XGI_K{G=9-3B0~3+8JkA= zJWIjM*x3B>kCy8ZQ^E#uRqgTi7f#I!NXVOOi4VD%&O$PtoW7=qg5Qm2 zk~ieC?(-bKTt7?2hmvK9ooaH*J*}*t;1MJFe@H-Tpn7uCh~?p?DeTDJNZ%R{H%l;f z_E!TQl}P{RQcpMQ1E;+c{^H>`$tDjXG($_^W2cWi|Nb z&#Bu-+cPcJrK>$?9z;hvGJG?g@nVgT|5XI36G{U*_i90#buu`J?S$Qsn4{yf@>w%4 zMgVTLo6^U2tZt0X&F*KuTUqjvWl4G&TGpj6cMhUTI6UM2HSa^H`D1x%w;DTY-+$K0 z%oi6`4F5n9JiH3g+hc8iTj`CgM&$kz6O!k8KcWqgjUG+Z6%5`{Xe9GFvt@S4zfVj~ zil^*#{?pA|P zPRwGcQn~ZF!I$fIYVVEPw?ep(pC*Pkbr`lR)&~N;Mjk9gtwj+* z2EWU5WsB1)rUeh%lq(i7eCyp`cm5mhev0=mWC>$%T?AixBVm~;=j%b3D5>@Yys5=5 ziYh)lCnim*bec33)@sT8UHyE3r!cGhgE8~>??r{*zm?a+`1icL2>YjP>6rynvqaca z_{O`}4|W#H96Uy|oXrT=>jEC%d|H^DvQ#j)vf#!{{NR-(*V}H~%{3NBiKd~i z5*TvSi$)43)$iXHeabbejpXHIosT3haC`OfS6tZ!!rb-K8$=#TY+NpT-+iQ1Mo z9GCQOB|#C}`k24~=G?mv#VWYZXB+MxLVb>KG#XIV8Grn$9GI<_Ool8g`RAZ$Bv*Ve z9w5ast$LK$7Zuo?S>cM;G1*nrX*0;ew6>_6TxHsCTZ{;)eE#;0#^4U?4~-TVidxH> zB)KPfp*YqKu@0xVbzEtDmz2CNA-{IWi{}SPv1S`kL5^R}WzSB?@7o2MH;T^SlG1Qk z`z_|lacyJs*OGwPlv0D+GjBBwcOUT08t(?L_OEB$S9mKG#;>j0t!woyJ+h2EUvrcz z4}UomWRL|LpWI5+ay%Bj#&_On~i=u9s?$} zDsoG0J8S*r)wd-)~y+$Yh(FXg4(FMhKxRtInv|IS55QH)3Kl+WtQ1Rb%N`Jp8ApOsf8E zx84(3I^MRFEbv(C{&OgQ(mO7%CTkq!mZSGew@!T0%6i%5g(yZ6$d%9O`#GK_ri~rR zu`w_l|Ec@p4m-bJ8@BWH>O6eZtkhqHWkxj5sX9ToU}{?c-;cH}EUoD8%d`|Wx|f5& z8u+ui>D$vPtSzVOFBYYTDK79Z8Yi8XY*^mcD|Gtqj5_5lhwVdQqeum#W@!G`SGpeS zHefXP94!8DwDJ8qnGpEJbNclC2gPFy5tGx?*JsqieK1Z3b%Ab37zPe_pq+64p2fOK z(cE245<|GCh@Gy_j)9XVtMBbF<3lYCf`czL^PuDEM;beeZ9TVBc0Jgk>EyAht(i$q zPLq(e;G?!>E2Yk>uTGzcqAZ^8jwK%m=Y*-Ly>pfI1hw;T-@fpVOr+<08of5rAg>N- zh{kz(cYRq0iae)lx8wo!0v;Yu%7!d>Z?Eo3Mpr-Y*sljg6+-lnxSSBkYZ~9$G^Z!D zQ{3Ogl`J5Ai>47%`+(1%k=*D3&_PkjQqvcb-Bw1%W&Pua&7MJ=rWc~0rc0?!?FxF9 zr-^FhOq?GVwW)M?W|d~=e*YX?Rj>x#4SrqPZEpRTrOwY=TB*W)n%0FzaA~KR&%Vvtt#8S*wEId!`e`I!oh%Fsvg*7NXsV_2_(oW zP!}WQ`%5!BN84k<^Fr_`S3KWlMT+OF z!$9ngSAZ`U@C6HPi<0W51bJBta7Er~z4z@rwwhq3&f-bL%tLd%I)3gHTONndjZ4$} z=eJWa)Lqx#-}es6!uBK|)^-w(O))~ztQ#to{EYb-Nc${4*Qm(+I-yY9Gi^SBO+zei zt-$chr6ht5zK1%LrJF~4VO0)Zr;Dbwd;F5NE&iGDw2O-+0!j`Rd^D^&iW*W+THROQ z02ln{SDYOAzRH2FAmSFynzYmTv-r>PtWSOZ&wj+{ct^RnpUQ4Q+^=p^!9Qfpk*3k= zO~X~=EcnWi&SxGz2X%e5~@p88u(mMn)(omk-@hYprNkdLUAM0%kYwa#UWP z9f&>mIMfxsvUDzvgnS+_RZ5TC)b^qiq7w)Ep<{3oGg>oxqPjV^@^jRlAe**aZNtdu zq#E-RPR7`c_D4ir;*51g^F?=Z_(2DW8Si@qXjt>UR;TL;-D9-h2^-$X;oD}n8?S9` zW8K^?KH1~*s+5nlv3>~~oU-M}giEKHHu} z2nt1c`LdSLCD3rdQ>t41a)UdkV8i<1;qithsPic72Zq-WTvQ?Sjqb zgCM%3brLkpgsG z>M|zVqp#wT&lB}jetye9pK-h(2K_t;^_tU5<@tFOg;L!8V8(BjzdJ5NF()pS-xfv7 zh03t!&Pe_IS#hD^3th@?D06iN}(VmmevwRfvx+m@kL~i^&$4)Ph zRkw5>@S=4wC@~LNE*~j!Rvc_k@SESQtjQ2(=eIR@&6w|mb=B)mqcFOZAbCyO{1+Y; zX$?)ctEw%F<;b4R)WPB%*6~GUMfq(i3Ks4rUmi3*h}6=X{9Y`ZAejx}ar_p&zEZ(S zOq^6b(_d=1FGhN7c$AacQcx9`JisS?TKw`Z?QUfqe=7| zIyg*|C9mW|iyo8FylSPZ@vbwx3YN}0e|h~&<2>rmthDL+-qy&r5$0m&?5KZQqIA`6 z8|ILnoMnYhGA++?+a?Q7rOhyb;GzyWdXh;YCwboxYxdWYjtyo>&{h^iY$%s94{}8% z|8hT4fH&RG6YVZM_PNJ69NB9L3z?iU`;1KolZ6T=E$?oAjiX$+M_Rqgk)N2DssQl% z4z~P4MsCp3Sv=uCFycl{HsR;Uop&lfKo@08hR!sJeFJCB(!A~mgum9)>FD}=9_vk zC?P*P2ZIOFy^3RBJZJus2$`;Q-xSlYf` zy;*eY+_p1(i}PKgO<6D5IKy&_A5kczw|*M9l<&(BKJTTyy*wSJZe9(5bd>J86t@zh z|xSuJ!SPsEaKk)Tc4MIaGPfKCp{cAa`k|t+rWuWVgEM^amLx zCh>6|VPq2o**e{+axe~c&F`UrC-5qe3w@dexP2rXA5%8L-qB(0kJ*MJ-;=;9 zhj8_C7FLLUe_5acHD?rp9)y%i8K4^H`Lm-Pz2zs2kn*l|GbQRlbi0<2x&i^x$l#L_ z?J3*E{)9ifvU0;6#vFR%m!5kHlnE&OrO7{_JN+ zzaJf!NeE()MY+DNJoL9%@E!gD>IHwsnQ!w}xxRSdjV8*7@g~W~-xH|cqNr?-8vrF( zP2B9OJ!-(hIilw;IX(P*R!>-6c|IbWT(Z!%)LBqiR%YYXYMi#)T1qH7ZdDk^5OCxD z;bf)xxtBiqop5I1yYCb&p?^r;EnN4?Qwn%jba8`LCJY`7tMEvmq{1d?5`NZo!WRJ& zj6-kU>nR(J`-8y$h1Pb=+*)I83^M2jtaU&cP~|MXt1!W&8;d}GPg zGpzk}$y~Db=1`YX$UFYeZf)MA`9t=4nis;C_(q|49iXIuR+fR6a@12NQtNT^K2xU+ z>I37&J#(?KlvlK&0un>h{jA)S&AhyZ`;+9|miGYrTHguIY&%C$Igq^&W=}zE_FUQf zJ+g65yR0&B-0B{wD1ci%v$179Y^Ue;iHtf6s|`q0M;q7jG75d>cZz&&H_!XQgCCV3 z>qVDUZm{rB4BSJKzz6@L05ay~G{9$ji7NF(yZ)1PO*tSniCX%P&(T;cbreh0D^~7` z23mdP3dCi(EJ-RA937NRoN$bu}GbNZo? zcPgy071VmLWvB8>_)4f7nc@q zew2ItBZi7{r5}>1BXci2D)Q^JP&`Nf&~K^fdiwq4{ugM~aU1LlN4Xmpx_}VgvPiH< z(+#mLo6=h2HTO3a8SI1QFJ#odajiGkM8)0&BOg|h;xo*+o==mOs&`rqi*$S)m9X2{ z@%(%03qCv1uWO+~zwVpsf{e`8td+juH|m0WuxvZy4@nV#2%lrH4aF=jGOTg~1%}Vz zX$;S@R!3PNV@cT#>wkeZyb}-BRxcOCS$EVu5f{>`Bz%%cscseRx!k*YE+BL~GYW+r zy~T62e50Nu7cQP-jvK*EP9Zbci}-QQ3ma%og|xTsAMd>DfAH4{hN{ZmFO8nW;4%t8 z+BTX3SWb)0+g5G<$pY|LLNauvrA%rj_ci!XC#EJ{-J{*C4ys|(py4ig+@ezM?_b!i zD_aU7HYj0u9=@jiu*~wkp>Aw zSLFD+is%f19bz3K=F;aLh2!FkUG;{HtrZXMBgxM5Ei7Lqldjg=Thp}uks2r7T!EMf(H+`7MEJt`;)ScIfXN_U5Xg0wW! z-QA4}ih^{vw19NCfHa#Bkl2K<>F$na?sNY481FmA(GQLTY}U2bT)&$0TID5}X4@xw z%+Wy(fP={_{a@yiAlLY|8~W+WP6Vry?*uh*lzhGM8qqToV#92_8K!u)Hh>w+;Z6%D zJ>NUoh)CGxkFXTA;n;^U%O{-d34N-Te@o|ApD~mOhEjL4VPjLzM*KcPz=k&(d2(^w z7Pz;HH(W*coVcSz+WP(bLRFi2fE89aM`|7uQN3H9qO6-<6(90FaCXOcLu{ubS*t23 zlsYZBB4Xn^XEQC->KvW&HvIuOYN+<1*A^h&NV1RL4WMru3m?XwBF~Z&7^At$TlI@W zBW!eAPd1O&5}ddbZDG){@!PDDnorosL%X>--i1l-IC1hs+0Ep8-<7SZu165-T+2%d z#CTvF_z|SciVdM(Hq)LdiD%K*u<-+>m7N&rM|}E5YccQ5ZBk&Q{L5aK?v_ioF6mu^ zT2zh3^nf&-Ph$D+ND`Y8(HRa`tjb&z}C)QcRFm4R9#s!U4_;9mzF^NU+;uN z0<_LV+Ln$x)lELFJuxF;=$;hqzuD;j=@W)Z!R6G7x#hNmnUacf`uROo<}k`z;+Xu& z@G3H~$(4wt!jPiUm#hZ?SC{J@aWjJ#C6#m>kbw7ye8n+o+RF| znB;i5zK*-{fwq`c!>iz(RFwnzD%L+g{7NeHxuHI-rk3}EtyE29WN$oUzJDL>!}|a^ zUw*tXv%Vaxo&D;Ggv_p_Or)|>7^ofQVzPNoO)$aSxSI^WH?HlSkl?R!nE3F)R{X(L zjt~7ZD=VwK4%_y5r+dqfUNlb0Yejch*%X?<>is}g#AI-+-daN6T(X_IwwKzZD_`^T z3l}>!cI);p2uOsHY^E8U;9GcZ9ap{%|0N~Y+fshJMXtuT({i>WXZH-{%RJ_`<(fCc zKElowd$d!F0HQSIdfdt7FGfgAGzu`s#%t?_2TqcuO9E(&;G`U(kb%!w9`2r{VQ%do zEQKdLt=ASu>nHhHuR=SHr6e8_2He8gDnC&Zm460u3gd-V_V8Uid_V2!<6ecp*5>)h zI~SYheQ(RnR1=y_)Jr!z%s@$I<%*4U>QhUY;j?n$rAqSg+@i9mTu&{@%|e8*$MviU zIewU~-xvP35G>yt$T}RMT?S1rg&TKS@uG$WSZ%0#WpTJYzm8ch zSZY2+{|*VxB&H*$@|xd5=;lobPR?fqfI$9zyus z;bV3!Jvd_kp4=OdRe>b|!TuFTt~I-R&8Ke0P3KlMxJ%}6hyd(f)?fVN|9;(smK*wK@f)NKuj@5k@bHdbcI2;+!N;H>Pj0QM-7Kk@9P7NtJ%aQQfC=TP zrDa5%Fz7YS1$TAi#t3@?;~J0cS3r8B6DGg@dV?OMkbu04Phu-x$W%CDb~IdiyMwuG zihBVy%%g{r#fPf3>9g&cU}1t4D0S?PP=ODWZ{0Yszqaql$0&-i_agPKS06mS<|L_f z6|_eO`(9$|!uA=g_4?5Er)fN-Qa10+Aoa-{1GlQ`tpo!6dw*A37wNg)XsS;Yw2S*= z?#8I57@d7?GUcd1rVX3Ma_Lcmrv+PpyfSZr7u^)JqVaU(_`nk=!IODx!!+M zg!>uOGaAm8ljpm52L9O1-IMvpYPm$`m+ydd_~4i;t`mzup~&*=LT$XytaFH zHksdc#@)U}YHncm_J)pEW!!r&cNT-4B^*+eN?2%QFJ_8Qcd{tWxZvF@LG*K0N8ZA) znYVw{#f?rrHf%LyE4}P6yErNg;jv?CNJn50P5a|EYJMEEoQX0aaO}(wPZARaw|hb} zsk7etgOQjsnO%&c6-Ga(uv8SaMrn<3j_Nh4%`8H80_cQ}-5K+^*3AzMl`R6PL-?h4(UKj({cxMA zOR#yi4!2jfj{iNLcMELooabgvv+gdfB0vl+Uj6&US;q$AHQpCTq5`KKmcS!xGI(@y zlzwN6U-xm_e_JOw=?1r<6&CBzcV+1mjsR|--BK>Ohg*&=^Kyj(zV+IDUHj9D!);O3 z@U^b*kUKwXhMO?7EW)ioKvOv+E(zb|)etwW^wXhud=ma)Lc)Zxy$;?HeT+|F1wT z7nOn&+`qSmbNCO7SAW`FOw)8r$W{soVFf+-9gHJY~2KLRnidE6jYri@{SK&d2EA3&2@4 z2@xDsmZ@7e<0LYcBc7f_EU9RdS@$GxW+F~$vW!a>`y~>ta&Xrqqd%Ts3Y`bPAX?Rf zd(+3qW?sOBL_-Yu>z{p=(2dPlP`End`!JLdNBI_A+^ z>~A@F+$EPwkHoN-LkfsLT;D9)R;3a5!0&1BPrY^u+1#r_I@Mu*b9=I+gUG=ke-jzK zDX$1{?7Ul!uQjhPtK_34A{1D-aqLzCP(Z;{v^(np_@0_o5e&PUYkjoOr&05jtad5_ z^=Igh-O-#|a$#O0O7z{0)Ott?@Hq4JE;z%_L|fL{uAWSA=`iB&MN^{naS(Z6#Q@~L zf*m8UJ3g(E&pZj{WFT4H$O6f0z;Rpq3p37p&(O5JIbZDGQq#%EqP9L-FA`39vLC$i zJNPk_Qd_4ixsaQ|i2^p|22+;BpQue7ej%Sd9+w=|4bmH% z_mJ+|{)sGUUM-=d|cVr_`G zkoH;LUl8Kpy8P)7BJ?gobW!DjQcmQ{PZ%?+^8-)c0mXBewIo4_PYpjk>KN_+8CTV< z65&-SJ+9xIL$H7BvC%z_kK3VQq^Qvgd9l+idW&f9F(a>H)2HV@e*mIAPj6TQ+(s@(+I* z-3v(efVy?{?SdHMx9N~HA*Y%n4|>@}2ozr5Z@St5(t25SUy#V9HAa-A#OrS<2|~uIUzQdMIbVL0^3tKkDu?hgkTU?ekgXrPRYH zy9EcwNRiZ^?cGTxUFUU!+C?Km|emW*wD4IPyyQJ$M zl1&`am(U5A(ALRmK}F41Dx8vJZST&~pzlR}`tdQ!0n8be)rw6jD#3(+F7sTA)$fJe z+PigNZt(?SB*_NWCz0{j2Yc72p%r--+>OBWdaZ1scC1&ZT+@oFzOB$JYmQd*)De`o z)-S7sA4$>Qjy=Z9Y}UC0&3vD@6nvUxwkx6Ra=W%Uk@q> zGEf#70di_1SE;Q2UA_@RygX3%XzRKJtqKm@X<70Go7u+b2-CHF^MC@)DK+K>_T_z^DTjIW} zTh-2Q#obsQ^P3O@h(Ww)8%D?5h@qWublDN`D!3CZs9j^W)nwMHYM#nG{HE5*bP9-qN#5;2jNc{s# z4YF5+tjo7uFg*kZW}bXkW)w|zRLAz7N~VqCUf$Ny6#h1{{22$gg4gBA&yhUD?hWhV z6Z)Cn@R%6?Cf&y@EO%8^M>ec`<+I;X;VeGu=-dT$OHmaiFLDAmC+G=X=CUxK++$>bpmhUVX)?j@Yqdr-nM7KOh zk+>E5oMe6fHP#IB=c7U+U?|+Mk()bDf8F-R9YwJJ!`IVJNiuBi=ORO?z=^tW&S^6dD z%dfG{s~<()Gjs<%>*2kjSvSG6{U;hCSs;3FT{8s;j3%D|ck?kCpyM;G+4nQn?Y7PV zGKui-`S}!_6#hpUWebgZt(o)r)O{6MO@{iwIT*CctA~M%R!pK#wU#I{=c(MpTNEFk7xZ7@q<@|ODqBtntTs? zMw6X^rDVwvHh}=~ms9>%08(Sct;MNRh=_%fhuOFJ@LZi0*bHmP+X7e`m@ip?ejWU8>oL5l0s$Iu zq0y;~R}R^V3hKR(wgHyfNl*Kfox9QPZ-0|)NMR-Bd+#YY$`A0@LJ|TMW4u1WDF6cY z04*Kxc`@XRAq*T$wnir&k-FC@exQdw8 zA}|qO=uQ0PrPvmloYW3K?{|s4ge)SXA}0jVKA6=1T-bg8CJi;+LlbdKJ7#Jq=@SPG z_;G3B3Sf_R(lY=sx&LaA5@R!KT8O};o#`A=s>T4xoQSw2Ic>drH$Lf0Q?oO`B;Kv zswdA1p}9HYfM?J0e_C>-51ZS%IJI@R1pU224rq5X_U}9T!!Hz{_E6QYO5JAmea0s$ zD+-b!H-NFXJdop{ZWvdxgLOZgy`!VI#gJ`gVfJNcBQxLw({ihn?*^7E^17y;eNcUA z&4_l3Do0Q&YcId0a03s9u-|m#>H12%I-3JlMemR-UFxhMShJ} zmn@Mqb%117M<;#8_l&jpFKVDRxmH#GT||8Z-{ipgMB|`#Lb=D79b%nxb9f4Bnvx)n ze9H3hQYD}PdA(+T|NHkJ)D-+76U|StumV}*q?a7k>jWo22!NZQP-~iIX2IOlRChR6 zIU_rp-Qo{JTM)6ozrVN(x71y02{>z%l$l{+5pr5uf(=5+*N>0y5-s^_Pge+8l#WS* zrGNafb#8%}Ht&0lao7`p6M3Z+VCA@aRH4?F{}env*f8WufOrp$AU^@S zoTebz-32P@uL&ietdUh)XUCr{lQ$q_TPL77U|z#@=lNZ~gzT!SSOOi-FkXiR7QN={ zY3XolYsj7u)Fi8C&Z4b3^NSxF)?J(s#?kgZ(StuT3EoT+=64nd`HFCTPY=@|#R%Rp zgf1eI;{sD5w9J6g&8F#?_W#`f98}<@5#+UbnSxhu=%V1wRV8!6K96}B0~Hq^R3(PL zfc=T%><$;m5|Tck>D=AvNJu@L5^zcjMk`y9jofcGdtN)HU`RUc+zvEjNw~FhmU273%?@;fY;L_S90l6wm;#02>_;*PcEr5fJNeE=; zb$8Faf&vkGBQFqtH_x^DW5#jh?(uhhSxkCR^R}GJ2$M$DkqVKID(%BN%hL(vQd1kz z?JiE|&!Kqz@jNJG;>Dk+*;j2&O3%vrGU36FkMmuD*oK{hQ`>a=1K2u_lVxho zZ{3XY9FF1Nky_okN_1#=4>2U&-r;Y)BU45+6pRAiHvqVwP0*OX#@pq4$r!Y z?E~AEv9Wtbil0qf!j0lq;`#2hQq2+ID=MjrnZ2c=Ry05=)8UbAFjevJ{YA|O7go~L zFyUwcuP<6S$hm&skN-0RH-26P_0JJvZh~wk;Yx*^U&CX!l<5h=!Ko+3`c@qebx)4D zD&;ggn2L^pP~5-mZ_w;-dmIw}mQyK}lEbZM2_ek1bTjt$ZC1q39XP?ffskU8L*PO` zJs(L-R)26nuDNVtkiqtEPRnt-pT>{hn7J94_SKzIBL&*4k6xk!ADlU4M1%Dy+P1>D zU|7X=gf9FH3@^cwO=|o_0D%*5D^$omKxyCPTGt=_rpA#(^3u`CInBh7wibshJL_Ql zTcp$%Gj#as=C{sHddery>RXEeKG#2O0aGgD2xB_?Pe(~lj>9byK`X&Jaj>Thc1*qZ zCq&X(M1kD_(ADMA`2M4S8gzM^Kh}#rI-VBnlN|ho?{)%mac=Z$$`lT=xi4KohlD{< zncZ0V3M&Ovxw`6Q!;$j?`af1S8v1I1qv-R#3{4R22RP zi>>X8Z1x>jG!seX>F{K@`O7nm=1Z(swS~@h_;;`Z^T|Nam(;WSf>>13mT!gNyb zCDkEf`LhH5#w$mB@KyIq494dDv;6@Lt49S(nxrf>m>KkA7II zK8w0TI_jTIFwyr*-gn#9!OUA2;D=hR6Rwns<>PkAScnC1LTbYB_fjfE{oNG#LoxQ( z$FhED|JP>{tU&wF@A}E40==c?vwp%M?oR+GtXHZb4+pIZ#;-4@xc3DnCD1%^A2hBE zo3HD`2k_#ukN-gS4fa1|SD(9Zi&x12!%l1Q{epwLv5|%5Tq3o@#6dsCO8kA>dn-mE zrfM+m7X~@aK=pi)XrmqMR?8{7P%uWr5|bx2zo&;%)(K2*?H*7*rF<8s*v4YEcNd~Z ze9CiAyV>Ag0rMe{BD)uCQwdq^CvG2K0qssCS=-*-rlh8ed~$xu;?QLZI)R4Pqw@5x zZ5Xrn7123E6DG>gs6){s@iFDA9M0~fWCguEyu5!`=OIXd3D3!-D^?H8wDGl5P7Q`c z9^&?KMtFZ@L1tD#uklsl=iYk%+P5&=o?^Sj!V_kIOt4!YVW-IXcD@u*;7f2JorA1R z(0iO?dGpKS|78I%;0|WdV1S7}fcCbwcOFAC0x;%SHe*880lg$L;u>YsA#*Wo(s!UP z-N0BiAnfO1G6_npiUfK1)%(BL13(ZC_8~K%pJs)Hbp*(Q5ddAW?z94}P9HHtuPoT?splT-u(Bs@ zFH==q8qP9`zWWu9SW@vaB3BL?>YuuqE*K_~APF{fG&K5hZg2mXu1J~NV_xmeHF=LSvan$hx!&PW-q39zpIk)5Pn>Y<4@EbqHrA<&mx7 zTt>*~+-p_Nqr)X0@`rR?z42KAq;mqJhPQb7-4_knJ0R`{eq)Tw59kJ3GC(j<6qU#UHe4omt&>AcYs- zdbEvd&A^`tc1fQuTqL*pLBN)PNDPvigV%-F?q~`C2ZL4Q_BY4g7zun7A?|Saw}Fd> zNsR$tR_nDca-&7_^=oSGM!PgQFc$DsZ}dZOGmC}Q#It_#*3Pcpo55+sW1kc*;Yx7s z{|%5EEvQeD5zV#tk$K#$50apU6X988NcHg*<0`jjy&%{obRd8|M{&M()_yW=HPH#t zC<#Od_4FHQXzYM!fIa|B$n zbF_sSMN9sx%8-eP@h1*Ga7H8HO_ap6w!Ad4@CUN6C#3P8_1WSiWcy$uz5ytH&)rp+ zjz8%`TN3CDHrg+xVLeyYhN{%!LTL1wuHM-oXxaOcZtHwvM-!ZYFqTcO{S}{Io0D6j z^Uu@*FcXVySQb&MY(o z5dv$_)`dGeP9;56)sd&(+CHdOET5Y1DNKao_t4w{-~%Jos`&V4#@llS^+uZI+Vjf znMb7hx{Nvluph>hB`VIN=6fM%cK;E+-@xe4i}Y-?gBP6C@EQ<_D*hu}TpZn@{;Tlf zj_9d@nf%~GrI8`~y`dX_EGI?n)lsfeP*03x)*V1(1oHUzxlZ$k^J2(0|N;s_3d0ud9{@^G}_;zOvcJkR~DJ6!V;rj$(!*)Zv*g+?cIaN9HS|K z80{{H>_AgnRhsr2v!sUWXY-J3X6WPf@IhjN6{cracb3Z4yl;N9b8v31He(oTM=!lA;6SL#7AhZX!bn^KjmQyMjID1 zlXqSRkhHWCENFjl7Gwf{<@=Uy0_qUrG?HC(ED8CT{hGqwu7c(2t>076CYyYv& z&9&fhJR6!1 z){>Jt{)BLpYKo6v=<{g{?b|?#p=ZL+KjAh!7JzF8yFANd)r7>-rqJs-f({5Hg%b3L ziOK_L7N^_Eb=-(XR#l$t+dAmYet!bKU20KPVm=gfSF5kk=JBtGg} z0hhk5kN0ID4M^y93+1&tT10>>9{Z4k&++Rb{m*nu8K!jGb7jN z{z*viX~D5J=!Iz71Um^tXvh#eF!0@fIC9pkPnT!@V~Q|d+PGI!`Z0gy&lLj?kle?2 z0||v^3oQ7eh-lKs2cWqEQt?;;l1H11)9Qj^*6*7Z4z;6=(3rsEC<8Gv)3tbvb`Sj% zx;Zz$K%<*?eL4jD+lE`eG+rzYs$8~}wGAm9Ih{Fb?qV+vKMemBdbR%e)F#KuWvcGb(8xBZy33@b!3{|{H6y#b2he10%mT*-vVSa* zk;Mkn^3LRZkSe^Uj!pXDKTGug{+`KBO>OGApvXQpU0%UQ0vCs{iqY#!?JFb#cF`_E z>TS@o`>fhpx?9BHo*Xo@L@dst+Xfzp;Vcly2}xg0$y+f+*2Z@lF7$tK0B2K`gwH#; zB!JHFZG&+N;Bva~Jc-w`i>F@gqtFfvXUN~*cpPVI^D`V=d}a-FpHo;^l9ZE762i;E ze6{08zVF08Ow<`yip&jo zx#-v&5@Ym3??{v4V4T+uQN@{58RfQ{q5{cbw=Cs zyZVq3k9{^(u9;c_4xWX67+}T&`Wo*F3BXamhh+b7t-Wun-G_1ED6Yz)%^a9;Co{tW z)Hv8rJsO3LKi`ce(-zQYm3FYjjk$qi%Lgy1myvX@7`6+@T~UWp(%I7~9^P9rbL!>U zV1KRA-T4W7_E)n*b8p0ArP}A(#I8o+i}<9|kT4D1pZe7YwQ3rwa+2;`&_M!*(I**x z+@hr>kGFe2>q+?*hr8%<0bv&+>;_M>zUqOtWrgz%dPgDdgnnUI5ya-f2G4Iu4q7aL z5K#E9A!$kpeZ(I@bT2IgfxGJ;h^hO?Bv5xT>r@6wso)P)X=~Moq0b;;M1Ef3VzQl! z&V%O$TpFWb$5>RX6}$iF=zoka$WyO%mBYC7qm)F>+2QWysMpfFNHxmLj553X zZu;ib9(eb`vC|t{VHBv>#b>7uX&W0zMJ!saZiXo2m<}#7GqT5&^=N_-*#~)2bT7QN zyStG06q%KEME^cm7%iW$4Ss#T=edjEM*mrxsE&9d`e|#zp^w$ge&j;?c60umSfdWa z23|mT0m@Cc&Ggy32PUX^QNC|>dRd8eBGC9{Fp~P<$H`m@I4xxYa&A;9OgeqyI{8nz zr~3tFTH*oMm?spBF;i1=UsPC)A%OdBx2q-_pLNuZKjct@=F9!)$$z`855NW9nmaqS z{Ij8?GP1QFZv|DpViX_xS?7vdKmpC1GMziE7e5~Hm*OReag~Y1LHFUPYfC^A$nN>i z04F5c*Lu76cQ4_-q+4EYex5F4TVOyI3EZXmr)O$5uGFhDv4(DyP^sYE}CV%mD56OwEO zLq%7GyCi!`7=Wv^PD3#H&!oR4)5Zb-9Y{~p;=A7Wqb3Uagch9Kb{Fi9w)A?F_}jB& z5@LmYgmxDYErWxlwF$KU<(SLfgLZnbHs*OeACoBH7QVOV$YnFt^IbmO^?ZM1d92jL zg?pO@BJj|;1GzU!3jV^$5N4LhA~=D+OVvMN9)7)}ccfJekDim>UluK*#>cflEBo6@C+2&X zDfOo~A-U40JQqpfQyw(8uHS#W_q6_Y+Pj;dAL8M)7UWrIRj%T97Zlb6)aqW1dipv{ zTh!(j)z$kV?wf*fhAVEDpY?9VvBz;dKRniq8Ocm$H<8UsS*%W(CM9bo*RJpuz#>{y z7-VyJTd&7oZA>JhBdnvOBHx-Ktl6}X#5xjxL$lEAf`*mKHx^}QIqnuZlH%bv<|1P{ z;?7au^fPy}a(8S5q?_LK8ZmFrmwChTz_VZ2JyTU+f8W-hiVoubnjPy&U_DkBwU!E3 zU^}uL#wbO`w=I^blhTh^&*rreejZq{x#<)g;$}CxLo$DS{iY}j$jsLDu6!DlDFU} z6aG?N42Qfxt)=%`dnba~eEWi(+vjvn)WOYm!x0=S#$-9ekT>{Z*O&O#?C}^miK^>|+IO`p%YM1wizwyXzX!wX_CgSwcHO-$F5_@$4eXY4D%O z6^{M2QJ+Y;2F_^4A{2s|0aPMB$@xd#qjv9v>LL0U?$(pS`^paniu6=%e?9vVp5MjJ4D)bq{ddtB?s`DF60^4eQk4l%?D`(+*h<#-WVPl`&&Py zC-EPgBmpVc%5TniR-@zNx(4a+I*ox+4_c;PSw<{y#5vI2N`i`Qe>YbC;lNM()k+hG z?GMUJs$Ad3cO^;7pjanFdSXlwgYy^Ha24 zJe3TTN`dR07;IE!1EIfd>R#@jcKvmj4RoiqOZp64U*E-@;E#$$70-2FFC4NcFPY(p zKLLS(ehHuC9rzdqgi*3`^(ui$`PeYbz@TUAYDwxFn5>4&t!ga$h@SX)`UvrNmWbzf zf?ic-_iA$>1s}qjjO<5#$=B@+iNcB%2xHQSW0EDZC(VZ2Pw89)jp?62O2%x|XO1JT zdZE_!#>wf{?~>N9M0`KjJVbzkZ7eh^;v%RjzkT-c8r#_hEp2|Q(_uo`0IP}g=C?0v z_qtpz-TmL6KTYtem=-j9_zEC$K-B*_r2j8T&5&c$UjnbDo;n>d#|p^5eZ<7TghG-# z-}879CS1Qmc_@=In#FnwcQ{U?#=3$tN1228L7Z~T$jbHC=y z#RYuW4n<+cLdfeX5ItRAnK*j;i;A#GIHkO@R(Ew^r@%!Q{~E7jfNF|c?Dx<8TsK4> z-NC5X6zV=8V(E+R5?;5Wd`1N+upu$9kVQ7ba5ThMY-nY$DYHuo+tZ47{h+jL1MdXBW4S7I1A3}bX@GJ7ps&)EEM08X)1Q?}66zFuk zjxf)>Y_#pZ_lMWk@`nhW(29qUQPI-}HaCk@SWk9USRrGDy!iTrP8mc$-D&FX&eMAD z^kK`vh$EiInL0V<@<W{v-@ei8TFmp-JzfbaK~!2RX=Km9;s|Va@kgqv><+a! z*x5T)&HG5HhEsllSv41DGm`K^^{^WE<8RVZP=OL_T|e%|g!9_32&7!>@2fNl%TPZ6 zTy!>{&V1~CUg4}o|LCh2jJO$V*rZ)#(w-?dAzKJLX18-F7aA(ar<9nLG^cL#u;Hy` z^y>r4wbK*sCUCgqDQ`e=bL0`H`NP9W7xLAS!t8>AV0M$(u7T+8oFsV~97Z7nUH zTkmWiymoX=Rt)Ymo$c>*5I9X}RMeys-q2?uvU73ABYq?6CS4ZWm9?uYF9!R_jtnBQ}39t*L4 z%+@}oH+c~5y1o7J`thlnlMrkwbl&{(oRpm5GHY^I4=Rd2vnDlMU?#mf3;6`(6bzI_ zTJ7cvig|kL(Smh{%XENvdp0N}f(}U^{zX20EEY})>v?QG8B$m2$!Rpd5TX}6VE7a; zz75v~<$^0op}_v#5a#5dbYHKSkYP7SmK4d!DHtvoTdb}eqcAU@&}YA++BLcR=-#L7 zguK>%B2&%7aSD!DvQ$RVW*aWwrVh{+Tvs^Ahnl?!*Xj*&4JTe!ViHO)HHt`)1) zoo1fr9Qg^Sp8L2;!$azJ#E|4don;!5UG5|6_VY3Y8n(0(lSnkYCOHs6KIi9L7drzn zDx7Dm;(5csblOvdj&NxVP{$~GIdIudiI~b?$$HfNWHd0tKwH{o8 zg*t*I2)khU9??rNXYEq272-1VsR%ET1S$#nf7mc~Gds$qS_$sXn_{-_Hs#(^1XuC! zR~*+}fk?>lQIcxT;E+lo7oa zA%<&FuA5B}@=95Tar?`&q2UB!VI6Y0jarYV?=OV%wIA~G^4j-BvVw^MY4i;@Q&=*O013Yin~Izds(b zeb;#?X|&t^GE?E+5={-_KN$VLzjaAve4}B$!*;K~P6vM0+wjEb);t>@zlexy(CuB6 z>zkyKZLr@)3ay^z_B(;&o+;E3zo1Bpp3Ub$GfnyNfeg%_Y5X3GuKF;+rSg<#B=39f zOa$}DV5tZ9LSM}TkJ?@vfhX;{EcVcH>~mrJZ2DEn36;?* zc?R7ZqH}ooyW9MQtQkH^3HqZcHO_%9hI5?}+Xn{?A5G|Qa?4r2 zV9oB;qTJ4%-*nAX=!~7YrZLnAuJQJiN#0n^B8$2fvc<$&NnW{SN0}G`0S29Ck19EA z(<@Y3+UZB&V{^&N4<8ttQA^B56#f}x98G~)nq{?a!+r0w8s0})RBUw;tn!_C;RH>m zB)GKp3vztsPWHGCb6&>IOLhZXHwTA*AWO0p@qf(|L~n)kHMWj*hEW~%!TSmZ?-|W2 zpY5ZV;d2yOo`;mCUkhQ3ZFL9{d{>(w6_e=y_)PQo=!nInmqcG*pO(&?q!b1V0au%H zMPZ})+J_srV+GvUr z=ntg5x8}0=8dOFV1|{3YgB8hiU9AO%yF8!XXy?k50kl z_)4PMOEne<(ODFe<$cdclxwuUgKH*r)DvWXJK(ql^AZ&MBEyTXOxR=?kd@JU(@4u@ zDg3xLgi^w@dPBToO{gb>RrC?N=VRYAid&obekqkSTwIkl*YMK^I^uR?T$gb#El1Am zd{x?W_mNxMdyn}_JO~o?(wk=$yhq(anMbV4={y(2;6{X(2~tdoeux!vf zMf257pNg7E=pq|j-e2Rl$8dps)Y*!npUdr)E!KI6rf&~)mH*HzedTOrq7f^7A9$8; zy04%V!K8*8ehl=i2pbFD-;1CBF< z;}S_oZ{prEJo?rEFO&(+j_E;-$O(0PTk2r)VgN<8qnAFC@dN_SRD31=%7OTiST|-S zKM+nZmuTG(tG@BdaIp(_c9NBch`H|%rYauRjdOSLyG#+U6O&Y*n;Xj}^c6~MChNiA znzUC(U;jfv|NC1+V+q?GhcpD$;^Q-tTJ#C3^~FKw2#{wOmGE|A#1M?XwO1Dxg@yb| zMQX51+v~+&OW0hh<-wT^LgJQKZ4Nk-w%yRUd|APm9_pkXmj;Z6usCBHAY8buR&g-?I z?;kuM54D$TGhhZ}J{KO(EQzG7do$&M9YEtQ^w674e^SiPfpV7nr{YWM6%`c%muI`9V`Jr` zD$2^OhtocNyM4laQq1+gKp^^_yOMaA^LV0HrC_WFW?iUbhrkdu#jv%n{4B>W7npR1CWll)y}sQ zoZCA7Jh>{svuT~|;V9(Ief=fYhknAZT<&|OVj*N!2j_MtduCJatBlF+E6*-24jUtq ztawA=73=8SL6s8>^KiR3UPOGtz{kTY)^Ej%X4GgSwyA5nI6pNrWoZb>0UtuoN#YLwoWp=qx`v^^mN1!b>=p-xH{Qo{%KQ}<0J2L^C zmQ#2XwSS6BK`vmi8+s$}M;7(JEP!X7fcj)*T}c3J=B=)jix8v23qa(T^o0%Q_vjmt z(jxj;hwO}~7psMBYX$F;M6Vx^=s zTz1L>@hSB9%is3h4#!t+Nj&o`cBVgmmAT%~NNncB2c*kSFyE8R#c#lL?q@N(XZc3E z?5z|tg|}}Js;fS8M{&3Fy`!N^wCp2#JPK0j?JGP!twmYFYa<^78O|x4?Hzwkim-eo z$rO2CnO!?<=62v*!Cj6-Ar>GHQyAl~9P%+gNT&9;BjuD zo}-ZZn6f|QijGa*Z6Pq#U~&;b@4Bl1GzfNdMKPDHJetBT@2Yud29>i}>+i{_l0!Mui0&Xv1+j4RZj{gcP261?Iqm9=yvFZd3G?`Q{3>=aU!NUoP+WL$PKTeB3TWh6#C5D~% zL7R4X&?gJi{7P@B(iU0@>SoFnZoE9WX?1W3Vbvr1$q*%#%6bP&3np!e`QMCp2pJ&% zcS?Tvc7Ie&pM|j6mN%|j^*76fh-e4ssDL%(Vya@#(Ud1u+1@7dxn?!>sAVyaTU~RB!>os8%6*$v#P`}uxPv;j4i8P>qq(}d z#Z49rnDW@oUf*muWdeB}l^!CLmXU!%3H?TL9OC|%jnt-*f)HY2 z?YW(0P@n7Krxs45(V#1^Tm#)mO09Uhs19N?kLQl2e~W!LUGC9Qw^KgGPaqYg-mBVd zj6YacI(R|X>R<6pPK1d}_rn!5Ci}pUobgSwYfQc8x%Tm;(1pELHmB3OHnBjF=gvQ-0#Q94QOo~u zoDv#Hf(_t{-U_}1lBiRF{4=w?MJ&M)Tv{+$5>sM7um=hG-gIlPn9C>T+@XA1>0(dF zWSc+~zfU9Htw#}maw1EQRi8XynvqfWd$!oh%Ce~3>~wXkw>P@~+dE^@QwD(bEn}U< zh{IindsB=z2~2464xF6qnkR=nw*sl6hRYwuL>YuXTe9dkm+0(xVbL0*(`g8>`1wJa za^9f4g39|&LvxjfNEOM<50G3!*N+x#0wV~ECi42k@w&)~fcAMYix+^n-d|N)&GJo- zIg(p%M<<-}4h99>Hs*i4$*d~-FgGOaOv2OFS%vN`3mf0of<|}qRSu4hJ6Fb# z$b0w5K9rX3r#u2(07m{6S%~*5i^(>${K!5 z>L%*w#VWo_!etdTbO*@p$U-ahZ#OdXm0*IGw1h1Nyuh$Fcx-I^KFQy}QpMeEUE!4I z)-0OY`$q@4ubA=f*wvPY>cvzWJ!R0BwlXB?1R#s_!w zX&B7#Vs~5H!a?9UWo8+Zh^2pl(grK(OeWCVuJeP2z1?1Z>94oEAI7oE^05pdr{9(7 zON?%A4%4xQOURQZhcX&F*g2!ug~4M<2%Vwu^(EcMR_~>z+-};j0mvZ0S_&Kf~y|3y85DVLKlC{e&A4BR9Q&BWE&wT@AKjBOCf?<^C1EG?xFOAqt=rKIB3 zm_&X^`lPWaqdV}m23V4v0h}58)L?_K!{GL&(N63P`nj>ib%1TsT`pYW z+)V~mujiAxmIT5Oku?c?<=5ckdNzs|EtloPP|*nV38XSwsNKy`qWkB(Zrj#n{=|fW zvZHUM4b#eQMuDwc)hgq8*0pcYAUHYvahn2j_h@$P>m#l!;_YpHr)O#RRXi&{!ZiTB z2<-ee#-L?GnbIjZ4MTHcIr>HRLYMPHxxr)f7_`hlV|H`XON+w8!9}K>17X$s#3`ft z!POG^S;-N0w*c;{3Cb}$Vn<1~<`p({L5RzU;BkU9HP-FhV>M3n$O(}ONhvpuH2=(}}PDRTdTIPQ7ta2hD`AbBnl`w0|M$c+jLrG zF-r?H?ajC6*&~W-nr%?})b`Jv*&KR=y`AFSd$sCBpnMh>HwdFHRETqF7?`S&dyxZ-lvQjmDWbR ziM$M2HIYk8rUC*2);o_bOCYj8VAyQRqA>l9il*^%5I;_lJtA5q!HGl5!37T{Va>6& z#NUWO+d{w(pxOARsU`1W`WxD)tsp~$9eN)&msh8(0x5%OCvcPf7;6{FQDU<<6;5_HWr9&bEyzJqun%Rk zVK`SPV=eCXHo+=fXEE5jA#&2GS^tl!w*acLZQF%G5RfkE1_4nT=@O&`5dl#U5RmTf z?hX;@mM)Q!Zjl!01|_6H>G;p}zTe*e{$`$eMrMq)?)$pVI^xtED|aKYxQ%|w?I^it zY$1*X)*G#NExtYAS}fHqQeE2G>3gBH#_h943zJ^$Ail9;kS zCcE>?dzw~Lk7FwJn?%3v{Zr>Ka z9H#I`@l19&VcJdS&-fOq3PjE4^IZ*g@H;I%8D>wYYXp6^fYA8!4kH5B`5U$E2RFjK zIlJnE!8|Mp7FV`2TisHzwl2#pw;|5cEJFBkB+Kotw`>wo_PUO<9b$&Rs&*65t5}qM zt(mdyX>V?fTQIC=u(FYDT&EP@mH7TC{|?upuS}6DJ_TW0MW$zLBTEeZ$RcH;TF`=i zR#xGAu@IHRJsW|z*pCbsx=o&s(t#^f?LzF$R4|l3r3;k-0C=lu&GE>8)8|6?2^cR3eL&w$pD9}Bz^KEKz9&y@dYSJ` zL{4YLL{qlq_OxZ(gzhp%wMZRQC$HyN1X@3S4lh>0iyxtkeE;EPP91*f5;z)`OQL|9 zGaw*+JV>(jZ-X|vhlgb;?%{Ju5gNQQ%@EB8v>ZgxR(sI8HSD1COIHY}YCaDeovK2j z@RjtTMU#=UKiuvnnFXaBD~Nx98!99=yS@!6iM` zgM#^qlU4e5ca>s($JkCQsp^C(@J$641pM10wJYN$hT*( zJ1h%nk$+L;e~{4sN$c%|crK(g=@Jp^G#$mBV%cL+sDV%ay}{&$;7=^=OlB}QY3oEBh}bv*M}6q z<*N~t%N^gGuD|lN90`A&Z*Y*d*>pX}_$2VEE+YGNnekLx%iq1Rh>T@=Fo!n>LK9@& z7T}I&qnv9|r!7t+#+V|WuaAY2o)Y4`W_S9B1IDfP`&th{3h`McUm0Qdz-O!#BE5Lm zH>p4bJ}*o{Wb@V4-jpnxyOTiRh@|1qnofK6*p3{QC?w)MJosx1Ic5D|oAHQ}lt`eN zHhmn-@ZK*FEMFciE`DBVWQRV*Q{vtFU4+x6yIf!$2G^~^vhm8jeY1#VQLxGqJsY9Q z@c8RzOwDig(>xgOj}oB!k<{FwC*p6qLHIKvuKtEW{D2KTru)12A}G=xdf4I)iwkE0 zmD_l4bKm;$!XkdV16=gya!7gg6i}msOI5Bb$M$Z0xY2o3YZF_1M~>}@HPeUdZf zvS~dg=G~*}zvyp$>oRbsnws{1W;wv$OBU8`@hE=;fyvEmngL3_XGceNnhxvr1mMPT zFzX^=c+q0`8OH_iK34XUJdZlMs*jf>5s{HbyK@bf>k>C+kJ7#y@l+Z1;zm=8^n5WJ z`^am8o51p{XSQ+6{1)o6rk#tEF_55LJsNV=ibsA~~e!tOB zB}FV6%Dn2ml(tOxTA`J(#z1*YC+E(iy}EdUPe9Z0z~@|II*2srUOt)fCyqN()gRyx zb;WukzrU1DXDu;W*#ppMeCv~1J?g`V2B^u*3X$`i?1k&eDr*-vA z3W%Bz?}LvtSp83XSz7&kO2M1+EoW3uSpEwUIEuqv1FZqMkUf*f$!_8s&&52D-D#9)R>9wb z5F+HrL?9UL*4FGO!htjCdcLz6>>wI zPEWo-S-996@%W12;NVa)TUGUyhP0X@+Uv4t5i~i0)PR9+LniC(tDlbwKvmGxWXX*; z?}G;?da}GiZ0=1m-2BlvvC~-=y(;T|8uLg0*sW=O&e%d!(ir(zvv4f(U#tCBmgIAJ zeQ%eHaNaWV0W1-Chc|?<*K4{?iUEGc7fXi|MD}TGcHwb*+Ruy_#lg?t%hM^Q>PY)< zj+r6&r3~LI8UMRWt^&K#fvV$nl^)ZVaEJ2yclIo3%a)NS64ex3P4T7AWtKOVd`ExQ z2mb}?z~4U$Q8dt!UyCDHP4%eJX~HJQT8csb8w&)NyDwvS*LTAPX_7{kkTUeX`o@8y z$Hbz<>e9gnWfSNFxqA&2QpBP_{N=o{_xVY-DpcLyOP)zQ1sEdPA!&wK%0W!wmI6;4 zc%t8wX?0F6+8Bb{7J{lY+KINb^V_b>Ip@Ln#5j?ID8hiIdiJhloxTs^pX#OgIBvPH zAX^4R$9C-S=wBCB7pJ}Ji-1Fp|6VWEEI~N6+BDa{2WwIR56-|K)WgGsh)wc^sKu5G z@|W*=SdqbloSQrJxk_E)x`2%b_<;xNJWGo&%tTqhsq+9vgh56?yUT z8OnCAWRjdJ-an1sUM)o;R+LE%D1qVR%-xT9+`Qk$HGn8Q_^G0{r)L7~rAa|*O5T6G z&Hp}w zux#>yF^K8C|8+^xBul(-F7JjrD7QAuBvJ?LshS0Y6HcWc)ei3>(m=-T_IC4D1QyYo zG%8@UVF1dIeJN!SvE$zCm$Sk%?cX!>J~?3+n3zU&c)PUSshpy0lmq%uhXUCxJFh8z zxMfCCL)j|AE4sHQAwI6N($CAf&tMRMU-4^$HwCmV8@>y3kL=V#Za#q%Y}5V}3IYwv zLef)LybA~c(%6E9-G$QoGHBwRWd#MYG%Lp^FXxv*R?UAXh%l}zluWE)wI-x zP9%1vODt&c)nHOu#d|Q2`KadAhH3&RIU%pfvWzu#^^N)gc)HXeN82{s>ltv@ghkGH z@exfxds}s@ThgDD)Y)&@VNh2?PJbn|woZz?Hpd%A`a#^cwW99q#5bKCY~WactNeFc zw7S~^4ndkyeMehMb7fs)!;ybs45;>^!7A{#v>o#gHY zrWVO*aRH$z27FtW*LT7c^YegQHXZl?PVSEim#&%)cZGO>Y(KoAf+Fdo9L+}sE!(W^ zD2*7KcYipm93B7+c){~Ht{w)2e zm>ft#(Y(;V>MeZw{=G!LKHmKJB&94W1rcAnmQ1IE?P~e?=eSu@M9}`-;}u!7^a=+w zO*raoZ@Bo+he=drSTb(ZT9@kT@%3oK-$OyA-*~o#k=_5+7T-T$Gpgx6A74gFNoxYA z?%*Xnc811b0LhM}qDRHfhMSi#YL zCms>SOvAsC|CzAP1_lT781`VRWQr@l%Xf+jjR+6dsxr9E&CQKSc?}NhJjMfL00=dn zTKecBct!nkdd`2iK}rj~o#8e$HQpYek?|(9>BC^1dVoLj5%&>G%pdX| zT3v7dIO&UA_1ZKdsNDZ<-W}T0b58@&3DELBimAXKC>VZ&C}-GvU|%Skpv&l`Rsg6r zhehorycj)T(J&Yk^u&Ks8X9|Sx;~O>dZAv@3$q~gE4t=G?73c>9#w1Hplz!%k-mHFuiySy!VSFT9!w= zA=^zIBo`3o?En6Clf|PP$#>v~Hw*??s5OdxjhSV}H*sU>_H*BhDQb6xp2DQqScSU( z1>SJJp8AudnEQ8=J~;fDlbJq8H!+c;Y4Jge8Ok(swp0Nw=h~lvoQvHKqj=Zly1l-k z+b$oz(L~WpuFB!^U+B!vPTK)dv`e>s07}U?cpOgbXaUHH#lHpZ-Z>$j43CEh%xFpG zL~3MC2NB|dEB6zlnZW0H-dD}jUeM7Oez>#n2O~itr&cpy^r>Kc)NAFFN`08Hu%l54 zi3ojm9dd+dbu|C!jq!<4s%_9=xcRi37tE1s@xf-+HUhM{E9*N7&x<_edIIy*nImat zii#Wn@6R*-_4;y|Y-umR5S_&_NRRH$)A*mKoYY3Snq#fcIQ2~;tj=WaF)@7tH5VtP z1qp`a^3F+Ksk;BNyp^N46; z;q;8Il1*abq~zvw#e4%OeFj=AZVjn?ST6xYUdwe+9XC_>fwtw(6hAM~WX8TtJOM(2=HjJfbbHi6vBt_a04Tu=2u}^gU&Z70; zL%==_><;N)osL=DQzk$#gEQ}cM4F&&$;rXa0&NLQxT>no-o2+pJ#{}?nrqx7Aemb> z6st1jk6~{jl?%CvUTlPsIQaO1NZ68O2qpk521KlJ$qDujPHR;`C*OnfC0DmN`2>A0 zK0O2b=NH5a`z<^QgaGIP4d_R+Ha?ri(Q2O2TC90?r!f?4P&5wL7#Im1J39~5J3O?g z+eT1UK3_QH7pG7}clY!UnKd*BAAP$_CZTYGeANdDl8&}r_F&KE+RLdX(=OdDm8>4vv zma$kKQ$J!Aj(?QdBCYW1HrN{iB{$(ix}38LawpnxK73MRk#^l-P&yTca-AWid{1$H zG_d)Uw;Q3xlQMcj2KcOTrcIej%qMDOYbeC;3ABbTNlHw6!KDNVF64bCx1esA zGlN!7s*cX*(yIyM&LJ!fBcipzn>N&9WND4P(XRRcs*2C8=#9`EO$uZh{u;T?!K$SD z5?5Bn&e^noi{)9ylaiY2YR%kH+rwW9E32#1wN{i7q@2b;RysWmweI|H$glH<^S;pJ z2E?!M`uckQdHzR_5=%=TI&V+4L*{VZDy!R=Z*(C|FG7l<=fE;~#`tZquCv$u?WPn> z{fWx9<&DTS>>wQKdM-^alqYdK8P&hx=&sV!-5=fVh=;SX>~^Asov*=`_(V34=V+)Z zEmFyJbfgPNclYikY+p)EGh>&5H;BA>dFK@2Mg)eT*H;TRtQ;j-m3fhW2Sn`;u@u0;3C_A5Bbs)T57Nt*CLN7^=n7jn zaB?1%U3P!?x4jURl%drX_+0vV3FYK0^pv-*JnQ$$!^k3oEsoyDGsWqtOJREukNsaR zfI;{)GaFlCIHVk2r%Q|ohquZX^Fr6SfC1NZJEt$SL9YU+iJ-8fm1!b0$WXJ;mul0| z`cy^lc~8kSJgryJ{!N;=|LGK|a zeL)$-G6;7V(nQ-cMT+wBA@H=q27jiA^b#QPD^&gRoz*&Rlf5Lh-D};5_b2kKEG6I=T zP9@W1*~^7s%8ze&iyT`P{=(SgRL;g2Xt=L(7;GzB#Ug$OQnGrVubY@kW@ZGUp?|?h z26zk-Cq>QcsD-DhY8AWlWW?0dMeAydBW)~trR!I}7gb*`JZk4%Bh<5zLj?VhXGcG` zeC}0L<|*w;?^dZ6z5hJ>G(PP7Jh+eN2MlMi) z%>D5Uf)r6K#|qm_7&gsiBPzpYgmFkYSYEsl0pGVe6_Fe5?&019(oC&ym#Q8~6Q0o#YV)UqDuKT%LT?*h3K@HCY=@>wu<9M2BgX222&9qWo^1 z$z_fakege2?umtmV=y#N1f)MCWN&dWq9CLo^ab7@NMXRsP~gFjmU(c)1GgEo>EVEH zG})s6_M0@NTT+0=2K#A-;F+Cy&x%)H7_AZ8{+YYi@%8qr?I9KuFf9jwwua*L_wt$) zOa%B%I6kH;@smYvI0g@G6frp7f`XpL?H55bm5`LCre>-&Sn{wwd}y>aQD!+?8`Ze` zHNqs*>qEA5$ja~E<1!`>G2WP{Ix;!zC0p{FDrKvF!N7-y1cYpUEC;?5k(!#lNBL@I z+-Y6#qD$VYn1K5bBFI8^Cq)CFNg!j}ImU95jc(zP&AgzsPj#rQGgNi_Rr9)NOAlSL zr1dW4$#a7e&Hff$ramUh83Qx!X%KGIuV&mEXg1*)V~P3%!&h-#8U!@xFn={YRS0Z< zuZM30DT$Tu31OB?q#q9*Kp$+Zg1qPX_ulL=4L4;ah@mzJC^{Aw-BoI_s_8JwD8=0Q z9M%Tj&VO~Xy51cz<@+D-=>O3#Twh;o2RGN@$%N#tgN ztcxqI>v`Anfd-G9X*V2Lc4}~?|J;nciuBjIc25-9(VWI^~+m;N*i_*W+UuPVJ~XUlyy84n2hwXKV;qr7Bcq` z5s}5-blGW5jT&UFG*@|2q?%QWKR~bWG2N!_17NKu!yh}qE@c2svqW7{z4%7_1~A(9 ze=VWT8lV}5k$j(FFwv4pLlFuee_^Z3jH~mmgvw((vxuF8u$N#kmu=!IPkB>Z`vRs> z7&vR0KCa2{BKB4z0Ida&@6ShAz&Ud?rBW{$CvaehCiT8RUi3Q0C3xHZ#yNdt=5hxR z^AX;rR1HHN*!K1CqgYgK0cX}&9X913Z0mB4$Oc2Ma*m<0p`)M>R8ZC_JP$kj_Y>ze zwQTO$df&uE@J1oc6S;p#DLr!e{LQ3(ni4iMHE30LVN=>mQj_Y;^?1Oi=I$Ab{$+4^ zM$-x7;En%K^C`=RRDhGl=b^I)2o$hHP)^M}YaDuBff&mgv5#+=1hQnF&mKl}dIMeK z@bu{uSOeHIJe&LKt+mJagqa{g>PXJf{J$o#`7jK;_G8i?Rr^Up6S7tZAvRD%KriAe z&B7EUJT*Jo>hetedvq&F_Q4#4eOM|5%5{DH6D4hK4C ziA{lK5yt+J^=0p5YW$%h=p*}g_x{*lu*g$H$*T0++fQxvnOWOU{?tpqRM1vReG${Q zYXPB~jWVL(7n%=n0NjG^0XUPEUMj)@RM}T<8jT$B+LVDQUX!SzB1M%ga;CP8XaoPn zSFV%5NnWLagXF(I2I{fBG-2^<7Us`1vN4&Ry0h^8KoM}xtPOei^xp=|geLG}6JPp% zYC%rgLUZ1#5e{ePGqrWL&v{m18zJL5=}=ky4kUV~ZXPM=4yI8JS@^(P@eDjzXiuvtV?;LiD2atFnpW@b}#8$1IJ_FHfumwY0u?ivbmG z!5iu;;hIzqzJ;SI9uvBPtF{&#x_f@t>|G}Rh7fKN;Iv+(9fb5 z$s#Tsx#*WotpR~I>`b9( zi}XEv*CpB6$dr_nUuJ3roK4lVwfp}c3jcCPkfRIVqNkK;s`cY*g!Xq%!oePFCraKc z%HmM!Ysf1lq+J@6e=_N}-W%hkqI|Gs2d@|$$Dhc_*lsGmj@SXx6LkLYRA-wqWQ)`Z z323BDZGAOXaR5UHgQN3*yM3DuNdrR;z(rM4R|6H6lvn3o=TT3y<*XK8T(!2*PUsq+P}ZwEmHnV)D5M1vKS`g-u~6;7<9 z3`^Vnyy3nWnpi|A-s=&({dmKj;jI+DH37gU)BzV&#z!>3#~Rpup{Co{Lw!L`JdfqG zIPro}JKOsP#WcA^KXfpzjNnrZ*PA9Yyv`f zbb>J>NNGq=OofH_9eCessvpd+&Fd&v&v`*WRd!TIY*xglpaOH!2XE<-QO_@c10rqx zIIJ6q!XZY+@Nr{+rOWf?1@Pbz*BGB5{bErg%=4L@l(OH9{nti~NfOuLS7O8ec)I-$ zq73D>wnC9Qzw^hvHg337VHjA}V^$E0?qJ<{kCSzuqbTc>%9yC{r))oSEW<;9S0D;~ zKBrP=1IM2QSbApY=Gf?9>LiRA(J!yqN5>%^7NKyTso)(fv2`HYeWU9pb63c1t??&v zeZAnTH=&=@ib3QNhl5;PS`N!JVLaje0ZVYl(u(XytAG;epz%UvU;XP;PaBYV=hzs< z@Uv^Nv9jfVlOjtfXiOcCsd!!XSGD8OKLqgj_~g!RrnA@%?ZL@eA_%M{ z$2HZ};8va5-?#b36H;|>ruS;Tk&(mokGlq(Oz|0sat4ZMI5<{%T7p?NoM^{Uzcx0H z$`)+QWpRrn-jKpbgouVv{$zv>fP6CkN=_H-KT2Bi`UMmBK>47JH4UZ)2qUO>8p;Kz z&oFPap`z`eZ4t6BdA3TrIgre&^VKm7hmwDPUFa2RT}JYM5sVRfgUfOYuxLO~?LnkO zHa{G8@RqBdW$wDf#99Kb7&OYhEF_D)z(6usykik7DyFkZIo+&W*Y_FU8~KZwSW1bJnX@%Lv^3STm9gvPGZdibWyBpdzuW%o#|@j$2{zGpg+3G^}midhNo z!V*AO=0*k+h3ca~L)CHdH?w@kjM> zqekaN&y|=F+TfK$bu}LQ(7KVX+!C1Wx}RX1*tW6)ZxJMkd=v`9-xHm1RovGwFU1f*!YNZhqf30d~HEPj5wK*B^uTKf>8yUatCL z^7us3)|V3;FVxkOpOFj>a^514Wc)wk%l|aa9h{-Su^b}7>>L1UYgN7e7hq=@aO*3V z$<~$-NxIgx@kgD(KLUFJYLr69l8Y~qo=NK1gqQ$4Nhhcvu3Jg~i8ZN+o+{sdjf zO(r9Ve}wP#jL7I{pU#s`%c>52zJ)ksK1Mad{jN?2E=C7F2B`o#sw@RR^F+dSU6Mf4 zEnB@ie{(AY-p;n<(ojcyGV3Y!f4TWj_9MH%HAxx&1Bw*wSF|9Oz|+e&kwGqx50x1y z^lEk=khB#{Gc`GDx#jy^4%uh0%w}?D$?@yR#Au(pc&uU<#t$v||!0P_8F`=aN95?DMaYqV? z7&bJIPJ^24;!iXzL+dM!$M1gK{g7RjdqGiN)0|*OKd8!xzb|&3*|mDm)S}lwDoCaB z^ASG`3CRkoIhRk9BTRQT&aYcasuz!Xz$&cDbche;+#VX#U#t6lBiv-_KfXkziNVsN zz5TOA@hL@WbT7$}tgPa#z=tx$XephWUF9e&TyEPKm)m_IHhm6+sLSLDeSttD$uVS; zN5c6}8tbAi&&xwiAM|xbi(PrMa0glcu%5v6GW&wt%@WNY<$NM!_(_@C@$0lRcMj4h zet}hXr2RSw8!$FGKFi>(5*896nj^2>H?QPBU zs2Nw=??aBru1&}6;#wL~4$>9%<9D!}_RdbayW=K~>DGq-s1cg-m@f60HcU~iCk^HY zc5Soz+ZIgJC#NERb2j4^byfS)MvRAtvmGzxe5l>kr^Q`Ae)z6{6%(_8xG1bjULn-a zsppBKBgM#%3BTeQG|QV^R$x^JeE$AgFl|UTW^!gBtjq%s;f|^sgLl$nT$kOD=%t*Y zQgY0Oin)R7f5=DoiPK0;A?8u8<*DbjZ^*IEZ~KN3_+t)2XY;My+e)V9*3tzFp|#N} z8#v4rK8q;#B};mEW7rYPdYk?<3Xmy(V`qJA{rHX`6((ivUPxvIVNs{SL2ghrH>1Pf ztmXpL$8)oY$37i*^pf1}N_SX_rs&wSPGE#gtbt`qw6eC7w#s@|*lG}iV*16qMLsfW zgPZ(g@9Ob=TmB!0zw(`ttY7;kO5u7j(oJb1FH`3q7f^e)=`jaS3`|DS7vk z6b=rnW0r#olK!R`%}$wlM4srURv$W`wU!#(a#li^SJQ>lB0*Ooed zOV#TvxJ*0=LT40I?Wd4*V7nlM4Hf9gKfOXQmB&o?e&#mRP~Q^2nb}ih#rGXv_v`)b z-AC-W4^b_n$j!d!dMXEg^tmZsyIaSQBBK>fV&Y z$3sQoyFWvT8$!fhw!LUm^aOQUj0bJkv0-Fkv|FnQ70HN@+wt0a>0~*rZ{2}@srs2d zU-Ez#rikjdcl$^GIKCLODk>>8&MT9}!?lB7zTK7{jZHmp6*aAS{q!Rja)qb>7OR^* z(rbDAfM-m^k|hT_lQkh*7v^O(QG{;j{CGB)T` zDjQ!o9TxzRp3Vf;Q1gYhX4l?(OGo%5gcwCyU0?BHdS&FZa4BjqTNH1vfwiJ|3zs0r47-NLfg>p*IiYtMk5fF#MM68EXN|uiX4!sp za?MNUeo@=kvI{PrM+BVLr_Y}bQ{C3fANKX8z2jeBH-CLIgKCGD5Gn%!it3n3IEZN7 z9z2gRl^z_VwlQB)p#YgSb93cU8>1f_9it4UYXIcM_;B{KJaRe(H^D45LYuOC>J5uD~M*L&-{M1zJ zEIH%YI7N-Cu0C0%$*b{=i7c7FbaJ0OsXyrOdrLE^eGzeI!YW;YA{l~~MKobE8)8R( z|6`ocjFre?`1k$pnXf`KQ?x1zANlD$( z73U<#E~N@Iq)Rj(Z>?h`#!#shp}y#Or>{j7c81uO@qhmd6}cNC{g^pUk}q_8HeH4E z>tD1OHtTssVww3z3G&LuSVQY1OC>lWl}Bx1$XM6)l%pIJDx^4b!DiZUZ1=8zlYZp& zT4DFGb_tY1KDG82vKE-E8IGho@4O;k+_n8oa=o#hy@x@{p;M2z|A3`d+Ad4|T3b8b z5{!$B`^fVS6R|GfX$s#RE*;t1N2(l;v4yB`3*@rz*^nt4rwJxSuzTRX1r};yX%TJP z1c3`?$XHlMJZ9mU?V=#@-;2$BGFb=VmWiE?ey!cAg1jf64iz7xdNA867u$ zpeO%9zUx7+(!s!wvynoL0C5xUib`xyM0ZC>coP!}VBNa2;*KYNO6PrC5o6=R8;lyz z?%R5`V{_UoagA&uF(d&NTAQ1jTAz*XtPQ1OrK^O54?fU8@E*MXpVB1orToo29*1sI zD7RrxoB2+IapwzB8ju|vAZ(IIICUctPR6U>drj-Blg9tP%2J7_xIPk=Enu17{WK@p zw(ci@DkU*hufu2h)y4~Z;h4cO#^S`wa@M?>v7(X~1}*IZlrGuzOk-R5$H%-auM#S&=1KgSIi_2iwzT6q*FKcWOV zEb!!hgi~zK`Cx;?cNgFbHZBRGJ*##HbDu-9V*B@hf8Bv~A_nYiD|b>p*{vDScpuLS z9OTMWA9Rc<;S*5s`LZ)(tmyJPR?yv9p=n#Tjm0MlyIuW>4xT3Un}K#bBnSr4dY7Ae zTA~tIc*E(4)l0Sw+XCp$At(hydL8Q}b^1{Jnp(N&J?7-+A&yu#dF&pe?%k;E=o7+# zPkJ|cCU}izwv3;$*2jg|kXNnJ$eYAM6-q8a*A*(<%7MTTCGMej^^x}yYpxXW1E3}mT%{+T?9u*^Fziq4V0Dg%_#1UJ83n$)y=LOcyPLk zas;#VGO|6y7%ba~ydeIdzA}7sPRvFt^dn)_BMsdwr6(~Sv*~(>MbH!YQAE1$F@a!%5-w4Rbkh}a3lDs}$@K-C*bB}Ay0Pl%-dmUBoEUm7>2Kd3b3 zsLGqwy&B%eIEq?wTK&<~wlkG@TvVO-B_?`IdNX@_9^A-bi8C%WqMWLuHapOjV#>Tr zv@Y>K`Bc+PwCDYmLJFC7ShQAO^#QkDjrFX!Q8an<0pkmWNBaKzYG!8lCo1$7*Z0Y@ zRYUI(G3&$d6mgqiSb5$O|WD za@9AJ7$Kd=a~((&1!J2|K2*wMwsSJ8dC(!PmYuwOds9^%xpfsROxu9{j6FFd&Wv-x zNBIA60Xi#T+o$E6Tb77r6K&>mZ8=hLz3RKf-R*ilS5!;}Ji1EwPj2FnDE#XO^I+!d zu1^8wYde@R%@{q@bowFEw{ThgcB-qqV2 zl3`dEx5$t24e5OA^w2AnmSge2_%T=d)bz>|{mYIxqPpy)C%L1~R~3?p^~sl-Dr}#)6rDkpL~38XBQZ?8muYpkCgOX{)*jJe@JLxDT~?<*$Xa<3iJzM@a;)905`QN9nn5ijuHEtf$(0!X^apeEH&()f{l8;{+wJ(Eh); zfMG-Nb*)nVFD{N=dX629h9}3Zq5aFTY%yIp;o;$x7Ly!g*b4>Sq1^@#wJMSNlQ`wz zy8JZ%NB1R`&Kf5?=iD(J;^6*Z&l|9rlaQUIyP_q+y1f*SA5A~FY`d~igys6L49^yD zB0WD&%X%G-t(wk%ko!L=Qw2TmOxPdzaT~fzM7~=2wu!azBjzgusjt-SKtQ2D8 z zq*qGD2<6jL%YViL$=n2E%#l<}nnb8}oD9f(sCKE(iGyM#sjn~l*HzVYqxxSeF5B~( zDM7Prpi2M;ds#2Dbk{9F5`}Ncd1dk+A_`ppV$?eRhO#toXqM!HC7u}~x3&;+V>L6V zm6!loN@HSWIrEa@RYK27oEtAbIR!RNOvLdJ2M})1;GsxbN%L+wJf%txhC0A^HYRNA zDU$(r{AwZ$X70yNtuQT%VU_+VRlX|ycEDU_PbMu-g z`f1#dk_~^*6GZ4&PmhlL1aIeK1f|egIJ?>=R=QQ$d=O%WG0+z+`FsxqllBK z%Egs%O>~(Dax;?N8vf~k9Y%afd@pt$DJZG?X*2e7tS^)VZ{#FFsEMR-(Rls zT?~ThDmwK#p^00ys{`Gl6R9&JbkrX>K87LPWGu`6@1B3|Rp2f0Kx^h8CyGc@bE}ZC z-+#vP#G2bNn*EyHiJ;o|9Yzmoc}8rpx$)$UVZ_Urs+sK#TO!HaSS5PG`Kby{5-j?} z&*(ViJlVJbu_Cd<9U zqaFF$y3Xfx&F0{5?h*Q0ADK8W1hafRkL3_}$z)F#G@^L5OCgDDy&$>>=v|fiovS~; zYb!W9WxqA+B(h(xQ|Ohrbd)@d4kJe=9A(Hj`0nH+V{fdzg2za&L0=z!l1ik6dqW>3aRsU}9$W&tX-Q536Ts(BViMYKipmtSpAkC`=23 zwt)XlDXGC3$d?t_OOm3$1}?sbZ1-Rpw)CoJaqZT~Efw1qao> zZL*1HqC=V$$0YBPqW?mn$J8rF?=XPTB^UB`H5$eBBA|kdG!^6aE%|!}F++9GZCtd- zbI3?NMWyl97sh7h<sB?QR;11#Bt#DJw>JP!93%OuZ28uIn=2xfULB0Qwj=Z+z(Qd zP-1CqTQ*&_v!ykE+;f(oiSgK}VRNbkB)z7d?rrFSPBF2{K&XE~*qs`!>Yhc5P^4yJ5s{zLh#sB%rfp zvSZa;zo6f7ZiN^rAY%$$h$2CDe-;a671=G;a49$dx<0>ITz4{Q!Kbn-*Dqxk)6mOD zH%u$=-cdDVwlh_>Z!+EKf6PNi5APlAr=%9k8CTy$oWpeWjvl?-o);18k8?F~Tv-T` z`X4*GVt5ks@crZ299tYFjm!4aeHr5@$40_9n0&pMYArUkwEL!1`l!zr5f|IF%A z>#-8+%dVLMm9yzt*w_CZy|CGNicq0)r@{@fi!8`}5d-;2tIWCL0u?GGxukY04Yp+U zTYC9TdR`lY>^rQi9KB7Y3N$h3j*d>*Il*G}-FmCDh6WKjY<42qWuHIMz;D99!rU*! zQAxsrDbU7G4_e7pTfr)~6Z$$Lv;dSZWa$}!d8ek4^3ih8!CpQx50&Y#*n1)U;z9k@ z#tlr)rLvKTW!!X(R*V&Ch1A6{-{(f#d3eCNE^ugzz`^$_cwR!682So}ny=;y&Y$&r z`1=HThIs8}rq(jbRz!q3jH2SkP*KD54?V|CclpQ_Vw4i~;!vO2b(FCI-&zioLQyo|Qr_REgAD&x%yFcK@v^ARA{DHX<#aw|> zRQ_H)t6dpM*;T%+El`KRB`7=`vsAkd*SXOo=rsc)BR8mn9=`s9k-)6%FV95ri-fRi zWzx_(4VN0ib`Gkzg(djmxiGY+Zp}#s6h?RhA8FAgb3_hSIKDY z`uc`G$v}OEi;3ll;SNLt#X<1o!vQ-=!>rRb(!F!itmRbAghzLtI znLe10{zZLzz$UGt%YBHc^!)X!fXf{Fi7nlT@nEP9M)C!Y@B0S#t{Oab=egcr;SI&a z3Z7qXGsH4k*0B~h|C8drt@mOw6i4LP(9xZ+!(Xg%k15n6xpTQK{O$vNK6g#_H`%Im zVq(-vA0D>PH8^CCxwWuH|98GbCd!e7#iEkd7RM+j`nsZL^#GriHU=a?h6gL%WMb}r z*-h`+H9jZ)UE5|amqY`_Qk_@RsCjj6;TJ;cboWzOVEE5Wt5j3{=g&@qce>g?o3G_> zb0m4}*r*f`UBqNKQDT?ID#ZyI3kG+~(u)bvK;!~rL*FM4zw$G`<4S3xl-deL^`rsl zVCl%Ma@GUwaB3UZ(UIKxVtpWZc$I-^gcfpyAyU-1ac43XEPZ{2Q=>Rq;N+=NfJG8< zmm}rMvv2(ft!Xc@ca@%1a?3eW>7^nU!=v`qvpcuD-?5gIl83I%^TW4Xcff=W?#_f) zo&mv+_?6hW6yN2(1NrC}SuI;6b@4;W7L?}OUW?|FC)Jp@sUipE+ovLlFVEdrQc`AZ zxhpV)&SY5G%A{Ut;6U!N7jfCJR1v30{SOF(f5q=uu7j~cBln#6^QXw$GS6MN0yLln zBFF7rQdT6|cXK@-Jv;Ro77YiEb-cJo$f_-?sGtX1`1j&U`+4V$s09s1BgQ1f{SE(z z0~!2pM_L&DgeAHHL$c}zyuuFwVcGbyO-=4KDuzsd$EM%RcsxO_{YWuKt|CbspQbN?1tH&wl%pV_;e0#;5rA8ct zZfum3Q;hpSTyNIm5hwF4aoSf2y$tZ0Zq6K)GmBrtZ+UC?-1T<@ZFW=f#MbxLU86jv zrQ}``y0*obG#{{|`@d64j1>;RcJPU_f>hf=rhio5`T(Q)uxcS!M{g3NANUo~R&^q( zTCbP+E43L2rI-`{HnwtK{J0ArI_5$5-6W0N;!hb)9$K-u@u|`G2r&tWvyhWIH=jq* zY-ISlWyXm%La zNkeB*WveF>i00qB`Z-!DvsK+!VfBZYwKSMDgnws4{~SgWfq~=iI~)-8Hs>J{Fzz4Q z@U`~_^f^P|yogTg*jv0rvQI`;45DJozhz&93UM0w+FR>*pNgqRc41$TBgwzwJ33ET z{j+Q>lzZ|8kKyRCy&4)0x|axZ<0rnPh86sc2z*P@MxQoBMI|&?o^}6gbJx7t#WBqN z-rmr9Bx;s)U!5$*03Vo`oSa;h?Lsr^J>4$1q+&L8$z4^)LYJMOZh1jcY~wYc1=K{+zo?HTp+9T(4n(n)X* zD|C&^lM;HtFcnAo+Ifcs0^EB$@>BE9A(|MwuR4OZ+Yf=K8=WcI5_{>5kptA3hN)UB z?TU*#K~e~qDT0K(0~`wQiHIDqUd#P=%;s8LT}?DwXCwFS-JaJeJw3fvttB}qJ;Q$f zRF}_R2nix|DTh*@&-`u*IVSE$c1^&wJAdcccx>2ty})8K|3g4Sfl-cra%{De4!JNS z@t$vw97llWj)!z%$bxAHB;V3r^%Bqh1X}hkhkhD=l~oUE*Fn{u^&@<85{yKtAuX*A zquQ*`UvU{<5C#9TSg~)IY$>obeB1p_1BA-N5Zc_SU7wBt)j$|1fOiH+*7mS6hXXPz z%vCYtS)PPQb0k)l>oh6t!3r>agYQuXKr&f)}R%2p#{7i^UjZdjUp+;w}X%HV)|vW zR01T^(E#%Zfa~$jN{oM3FceR-Tu#|*c*zRMbZ=yF!p_{EZkLA1H<&IMKU>nb^!_S- z;n-5tiKfA7^i{y2=<6Cz@AF>_mzRAZ%O!v50}5TjkbERWPr4ZR_-YKc7CK849*Du! ztlaS1Jl!P1Yae#mXJ#z%B$lTp2>@&D#(7S9=3DmDZ7&C~aB$mhJl3A44YNyHc$j2H zLqom9*?NB8{13&!afn`S#IlqaODo0>^&4q^Uo@o4pKPLM-*k-h*t#0hF2$*(>62E1 zek^&wR*6ARFTttO*WKQY}$CTscACqTU z0cDRugpqJ2fZ*$=4NL_)el zL@5ag=|;dn5L5)EyHmPUxP4k6tfx({6k?mEx&e&c)Zz5h6LASd>3 z@3rQdbFQ`j7*}~lJY4nz=3@}+^V5;ICpu$qzS$q7KNozA>VM`CuvRFD=m z>ABgkY|LAAlQOJWzu_X6ULAc-P7Vy>DlZDS%=Z(HhoDe-Mot!?V3P~%>J$Dp{a*Cb z^gSmgDS)s*Ea4UwH>dGyaj|om=uyWL(}h6rA(j=h;hnmKt85ykM6pr$V~hdDUJ7{? zIbH$@+S)~&OM@;?h_2K0c9IHPvtjy}Co$Ih2e+WO#bpX0SajS*P4o(cpNJ;&lv8Al zfGr}NE;KMg7%Dmhho$5d9Ag*M;CJoe>)o*LM{|Bi2HBdi9JH z6cn}xVPRoSFn=vHgp5rtyI=PD^@flCB7OPC4i0=mw&P3tlTY$BjF$Vd+}4WQ-p9w^ ze`mI#cHxW`swlq5D6C(QV9jkdmz?Q`mK_aX=H+nJ!IhhaBpEDn#9#Uwp{7Zf75i;M z3UFO2>NcpFNVsbbNUJLfycbvip#@~cLI>TuFw+%>O8~)oECz&ki+{QAQU8pw))HCHZMO&*yIXm_vTFd`f3xA!Vx@xjDE)BF~pOc(xKWQivFaKGJ5tX8adJjbnLe)*u8$6u!suB_y2bpHM4HL3X@Nv`fZe42Z z^((Wc#v0VJ1kCXN*rzcdzs`LV0YU>d9LB)V6>y6v7L6m_{N>fQhQ9GxSc~bpdEhC2 zVI^=IAe-HvwHE=Z?bnIjDQ$>DG-kDa{wyn*q%TQl$Jh&e%GL}{PjiB17IW$UZiYZe z@^E}N<_!d{V%9A<)257ZVR;-zsDr69df%f1vv2u!hIti#?y(c=yQ$NkZT0n6*!5Ab zNqW(q`Ea&keXhA}Qej_73I zrqpvZhs&CDx7dSwoRkJoaG85~Ub9+$>|`Z=hHke!MVCt}-A|LYlKu)c&`iu?u_J>g z_S)f+efR@a2t{B!_-Rml1#j1kAPsoE;j;<8a23_Y5AIsTl?m@ni#1V4pYuiRPm^#9 z3$i~A5|yKCuO<+9F;1=Xt-0X-;MI`@#JwuTL$y9&h=8)iEnb2LqUjsdU5q^e9lOfs zagSfnw$%rs$Uc*J7(&BN_UWSo|E3L{>Vri`Q8)YjXEi6s9X%H9a$RS{R}bU)eJ;aj z&#Q?OlSZBv3WQQ;$FEM0kJ!?m{ic={0A>+1L&!je^T+FVK3B1I>YOJ4j60oWlWr_b z(Zi|SZJot(P&2q68fHkCChB}fs*ykS_T1Bh&ugyJ24I+3*@pPLyRQyBP(%b=z7e$G zGDu6SxG30k_^sW#K}pX>?(iT+cLZUlLNDzZeb+jmV@4mgww&MVOQUCRZ^j0L?8G&x zSDAwH0u~oF^F4w(E;`hLOiAvSnKS|-@UsD-X{K|XkU-Xj>w=*f_UAuQ0J<59hGfR! zr&J*SHE2FaXhX{84j97a5{FGpXK(!dFFzqyXnl+~%gTdcEiNdMIsLi;MhE6NIhe?X zMgyG-@ZlGYaF{U!W#su|Lvw(h=B`iKAMnJ=a8g#I8R2An)ELwU+3rq!Jk^Vy*9T~@ zeho#oARS1)^xr5f3P-)9z=Df6f&xmi_cW87OEyHKFK$(=?bKChtE4VGbzr}GBRZ{3 z;yzK3UuSOr~?&@`VrQEnUy~Ho)!U zx?ykc@Fj3-mYDD=ekE*5S|%Iczl||i?c9B(_DrLtLWd_te|Bj`QqEay&!fEM=Pg6E z!Wm8sXvnx~##?mIJF)WXN@xZ#8A<5y?ZEE{`Kppi65+-&<0&+q9Sv^-YazZ$JyJCc%NEOLdTCH>$tej?VY2bRn@Y$Fm zzlJURW~1`XrI0pcgX%oSphQkCK&6hkJ!&IS5Qn8d@a@o;y*{t2({*p}7lksXD1)k+Cf0sv)s{ zx!|w?8BO%Inpi;%j4A*URSjD7bQ@|CVo2!I_P!Y|477DaDQZ`He9`Bm@hQg4awZ7@ z9!}^JPUE~2|I3yn3?%&KHZ!pFQX=^09_zO{llY0G9QVt#Dd;Ey%|3Ov;o6w6X;ceZscB9{)B`D#84N0*u z#^FO~FjBmULsJUWURP_)^ffGFe!RZG7P%WFguksu0N0%Dp@hMeW|H*G zynGC-FH%yPl_U||oiVW@&HaOL_QzY(!2}ch(Vu#7`UrAZ-k~)&7|*c9n=Ug! zAk7ihUmx^7^y<LEUj~Y$Xv1lGSnKozY4_y$*ssIS=AT;wIs)KqB1*Rs zHtJpNGA(GO9BIC)(qqS}_bv{}zs_X8YSj01JVh8fcF5pPmcWHUm)4dUjYL8yBAQ@I zWI+G|OW4PBhWRbEG-rgv#XKgdjsV#1E`TW%YRBF)P zYtUyeiZpAkZ85Qf^`xIQ@?CvvoO3EnRz=1X)+<0EjNEKreto0x+s_k8Gn%$Oi-C9% z54Wg0OsWuQ!GtWLlS1)k>2_!a#>+>hc+G4gJ1hT^ug9X%+NkeE55Ex9$-KQ7+|3CM zJ%<i1dl}_ z7d)}Buv;iZyjCRggchvs#DytB!7+L6C<%6F%_yTXO#qR06^v=@SC=_Z_aB9tc#rpj z@{o!cOe;$d+0)HBIWccpUh4o=(NywNy&|DI@4%nhmc^Z7<1^)(tv~ea3a8E(c;+aG zJoMRR$3vj`#btdops}%$?hL%mcigv)Yf!Cpn&q|(DQ>fPzm3=4X8jxHo7dwK6T|Xf zl}~O7Ge!kLE;!xImTnm>wSxa|cm2kVcb65(;BzGy1+v`iNp((*s+;uR6R(q3ic4|u zg1az{h8oC*eb?4Xw`it=P`7)a7vWNL3jLA>X#9 zFZdFIJub69<~=dpj7$B8ZOhZc*!ti{c9xZt zg0>}NYTCLHyo0tayt?PYKX|G58IdaeW9MK0jKb8R*sdzmelYx3m0iGCl7LorIMG&C z_;~EsF>{@jpZozz4XhdAC)#!Df!AKx+*7m28vEsz-4PHL7d8%;KOCRP=5F|+I3YOg zf4B(Fc^3dBQ02PKEZ@?UIG0L7)V}Krwfy*EAU*E~T<0GZVq4`MpzWwO=4Bdh3U zk2Py_1IQqMgW~AmD7+wb=FJzLV9j4{M?ov4k5AL()?8q*f$?uJO5pM;=}E&noyp4X zootZkW#$vQDEIcw9txr{Cgi%kZcuJrCt-g3lJ=NBcrid9WYxqM9d0Fu~>oV0&DB$xjs4{+*Isa1V$Mlup?J5~-x{8@P$_etT zlM$0HUE}Fcu+bZ3BB8w^6@_*qNq>7d-8JSJ%)9`2bqXonW&`#^($q%tw^Q%+Az1mx*K5Ba70o%G=-!z#L8`klMfx zg*l5Nafwl|pP+1Sn0b9e$=u(& zkh(ci2?8{z#4OBW7lG@9M4;epgfCV_Fpa9%+IfD1lZzTUA>Mb4v-Rn!a#A+E$l0oR zM;h0m0D_h7NlQt5;$1u{UaA}Q74&dM*f|Onb>zvc_csEFA<$kS`g?;z=xWn!<{cgu z><~T@-RH{czq8ynLKI+Vt_0Er?71klnJ<3-saf9kFE_T-$iZv_ruF2t0og*TFa3t? zr(DpD;OQMPFgzYuD760VFC8!8k;-$CvJJ`3D)geNJf=l|HN^amA#3s*W#Y>7M*swGe>e{^wA9_0ED*iTW8h>FeQqe$tYA}a6O=an2 zbu?h;GyoGC`vvr+(PjWwoj}6pKP_*yUOSO** z`^-2p`|J&!r`1#u0kNiV@BVe@O6sJ%y&d+a$l$~We<`g8K6k;AXA0HN zp1jjTJ7cG4i7}BWW;GznPxc_sU>xIc^IWk*n-4TF6{~Pv_K7!c3zT=bK3zf;_07(@ zp&Q-y9(+2Wlt4odJ9`0Eki5a}))%W24(~b4Q(Sb!?`a{XG%OL{I{QP&p8u_Z1z-=z z_)s@rhGv(gK4kL<{pn7Vo&CzG$za)6*4FDzom)PSac=ow&M*g3z#aqzFaU*a??rH9 zq#B@wo0^==^w)yP!AMeJPRpIYK3rrZ4P$;NT1*DQ+o-M_Le2B;Z!>kP92+wO?V{z+pRuoF;!$dA2h`3$zS$#&Cr*L=z-D>(l9dV%I#$lD)MTn2d4MB99=>YtyNk zRHbIiyhSV)2fkX4U|kdS!gz+6Z}HKBRTAvBcWAlc47Jf-{!0bCBPAm7Z4x*@4a63C z7Laki?o?q5%1wSH**ExvNP+kRxdKi&_#oJY)((k4Z}??yNYwiwv_;eHTo8~PV^JBI zE6@K}P1k~U-h%3oBo~EExeUw%AG4e|GM{bj2VtbH z_`4?KrALohODI4!8%9m=fwPn}uHa*ZH&0y0{JvOhMeUqA9_YweokJG+>kNne;;Ct{ zH%qc3l=WH8ZDen#E)E8=S`nH7^BhdWdKi7b|nPzv3y8>o{Q0g9V_+uUt!P+45-!b@AfmdG^sm?c~mX`+=SG@&MssK z`1L!|R-~NYS9(mA+@qd7dl>>ZaWjiad{K&Z0-i4R(_w#PJ>oQ0mymG9ffOK_Qt0vA4%x4adD5W45MPx{Z1NGV)Yc3=kkad@ z^4{={x#LN@lQcfoHS^kV6T{jCx$^}4QF@z&Z$>y$Nq$+DKu0K%aI^M+F>jD*V^vO0 zZfh<^^~UYnFtsG-{X<+K;*S+OSvQ-Z%oF_k_bz{AIdp9y_k4Zcg~j2cdM*jvk>J=E zJ(Sat7r+#e7;x%+D0uxp{#_L43npr)-a6^u4&N9@1^X8g;YoT0adR z4K#&I7=(w5(BjRCpva>FrY&V1tHhHkb4m$;)%^C|=1u>ykB{P$H#pb2Q^d@j?j3nC zLPpUa|NYE*)ZtCBeiWEfbJiX~B58pFHGS{t+m88PwytS4--%$;ny+qIZ|o+^aFVdF z%YYlE?a&7Vm>KERdX0!^(d#ZLvU`80Sx1oABo?9wOCpGLuoA)$Pae(_t1 zN7~02+@AcDO5i7+@VpJWGEwutghr3vC9s;uv|sdVzg*jJId{M%pOg@ch#~&aISO!K zE4uwUDc-TSfC~j^z|Y*a-rQ2PL_c#g-thMJsZnRf*g(qHQD~dVTwC>5Z!TaE@zGi-4WT|4Jk z3Dsj6BC0+V+S$ZzTqa4EjTbPw0qrSE5nngi;?>_jy!YgH){~<<^)H|p72sF8M7=(+ zLo~ODSloFp%E|!7z!PUMvG}E7aZ^r8Q(!U+m_T+F$+#%#%tx_9G%r^wUyW72kfZq* zDbcjJbH~z+R8dD5RV&T@-Luga1e_ z*-=bq*z=HFXkFrWg(T|@a#fy`M;Fa868S7+%OcADd>1cX(B<+zgXp@89=Hs@k17In zw!bQ>09Co<1a6dP?o&IU`>xFnCxi(rTNjU7gWjj;$>wxdJgpkx(-~~6%ouke z{f+1s|F-!XPS#?k#*wWU zwIBoK1-$vRW62fwWxo<)niX$*wuVk6o~D`;R)cfUP%hjNtdcj^C>VP?fvehxc!39= z>CgLX2TO~&&3*@}0Z_!E1dvb2KageenYG>nw(pm|v2MJ!&Jd`ZwwQUs%yAFu>@E@3 z_J#+3rCi!mBnZe_6^me^L{wh`$H$YG?GiK<(91AoJsRPyg!rX8->)O_ZbP?cpl3Yr zQ(2@HK9}9s?Pw!VAcC(D8hc4sY7>2w;vaNMZJKFzRqBT6f)}q=5`cfi!;b{1lr_^e zB`KYT>1biL+7_V~@gZ6*TQf=;UVB~n2=$$k1h zq{Xl$9l7}4HH+NNvc6_8d)>aOr*|L#=&e@!>O^R0(7zLZtn)XVnjQd0F9~zr&_34t zm>1zM&QZ)~C-Y*k)xi;!-H)9)@($CMVyHhPkifOH#@{%f;eaNmh1R%At%PCct;;GY z9VTAZz{0MUdk6WoA!PHn-@+7_I+Y9yktCkiGz3r*1@)R6*6N0b5VU+LUY7hGehp6N zJ5d2vLnvfLJo)O|TuNF(A>Gc_JutGL`ob3;&;p%ErKVl9A3wRgD31ES#ja@s1I2e0 zukMzY8BGj&|30`A{(?EKQ8m9A_`$-^=!-J2a#F8q15oGG;#ZwEJ8IZ2nZwPuO@C`$~uU^MRXFuMlmk zZBY-KKB8#+gbKK(cld;@NC&M)R?E%_mKbrl^79ldOl;~?U3CXEAi^Atu35h5p3Hs#|B3eGZ2MnHq){`SJMgd^*Ou?R1$vs!76>V z{Ezxgwsn3ta-RObjZ&^(3ZbCIbPRx%1EVS;VNxrNCsE2hs7AM*axjzY>X1!NAYc>} z^y@44T zl^uP)fQbzm9V3A}E(*6>h9n?gdS49!o>hVo3dkho<{ST5lPVqwmX{&bW$~aleA5~0 z@qU(ERFm1iuBG7WM4;Auy^d&Jzg~{^UVd0OP<$-UMLL_);#90ZI48B|G#TD8{|2#I zpo#PPU}R9ZG_5YDB^L=Mz?pN&9d;|3OFa{9E1Qd9QxfOoKuI|qJ>*zzu2!jSuu%|8 z6DOC;)JXsF^@2uX7qX0#g1<94I;4bZTZ?HC8neL+ANW;c5JYNnV^X|uQ+y+PfEI6n z4HVJgv8R#iJ@S%jl)nfGz1KuPB|Ha!ncg#>rm2@>V$jlSd*x-+6~bmrbL3;_+wI~t zC5W^aNv^H?nuvGkC2`h+cH3=8FjiZ@nzd&Sf2i7c&aD?dbauq|eKlS_2PE$AK%=ILe}SzcdeUYhEES_g+_Z`aR zqSfgQn};TZtF;aoj?l%{K6nBpS_#B+HF~^m4H5@Z2b~imBMB!-p560Qc{69YX9{S( zMIy%@U#Z)vOc-%5Fv#tu%ZwJxiXC{gu)W}!s;<6qyg&^&?CGQ1kGcI^1%(m`MCdDO zIKIKWj_;#l$9Wprh~wo4sPWeW0ppGbsbB$hI;zb_STgjwD!l;;iAUO(8!4c!(lLb( z-E1+Rnr|g)s6TtDVB{3mFH{+jC2=^{Vf>ZVEsx_9@sx{+iHXG7Au`E%Pzy62v(#$H zP$+TUDVdn%@5o5k8OsqIglpAUj}0tZ?t~$(F#8jq>p3|Is7f8(gb$r)SC`PiTc0#M ztSM7464GGt*Ak|Jdi~k^w2{L2@b-7nmhIjLT&66p6BWtrNdCYHWsf%*yM;P0L0A_O z2Mo6`SY|-wz(>2Q?YeBb&>g?TPZ>7y`+JL%+u`yHT<)KJJ;TcKnX68M#D(qV&6Xv- z{YbBfg6i_P@1q{ad4mRv1(AWe*(R%BW4)>lt{0vzUMdgp0Di*4g5kI-^`RPwtbTOm zsj7c1>Nb`(SGx)~Du@!hx^YuCTRdn7$8gLAElpJue?^S({%Tv@og9LuBTar+9M?&E z8sW*kSLKiIskEdG6+1Uwjhv&}>FxGrjND!|>RF8!HzJLaIuYY`-_Q$NXGqTOoEKW! z`!>KSCZ?0x-}i^s!{cOJh@J)yHbdlxPDAnwyR z+x5a}aOF`YGVWT>kQu{ksJdm1obWxbE%aC4YR7h8#6Lb2 z$qV3LQq7z671TJjr4AeEm$~P+f;Q>jgQ@2v^hJ{4jrrP%FDA(`t{QAgC z)BZZ02G32AUv|w-{y4byP5V0u{Qmw8X9bqU<41lS_k$zoQG95xnhLl6hdd)cNqNI8 zG*37~8%6QWFg+3^RL|9Npc7|y#9^7WkJL!Vwtx8VLoEeDjoekMG0W1>kB6ENvVxq~ zS{Eoi>K$vfGkfoNj$SU9C<__>wq?+@7%5w5)PB0G?Tzvv@Tk4X<^Dc{oqAWSTXNC5 zOwWynvZR+~$0OmfwQh-jS1wg%zXdKjfg@B-*_gdtuRfxfrkzFVE8NAh_rZPKzU7la z{U=EUHS(I;PVJvQqH)$Q=Zkl8)N&JqjYx&9rBlpB zkF$zKf>ZI!&1U=6$VZxjW}lVLS+iJMw|#~6T6RN>;G(nby6fh4b}$eTRbG#dP-1lV z4-Y3GSBM@8BxW)0S=E%V6NASbX_*rL`;KkZid?Jd%}V;sjpbW~->mN^>$N2CNwEaP zhM7jKI6Y3Q=N;>Q)g_8=T1jtbW5rhZ?bMC0kIbT!77oep=dXsy$d*TDxfK0UxEFa$ z20nocQ%B?4gr;}8#85? zg47Tm-&Kvoh{oBg-z`@rC}&*Cr|RTCgkAS1Hk^a0*}pqG=i1p)=Tba3n0a_q;CM6B zXxdfUqaC|nm5}ET3Mi<-eE$)jnM$r?X|g@`4J6;v9yn~GEy!BKEA+!!tc{yOv9&8U z=>OUds=_u~@mC(?miBzs!rmzFS9yAR7d*=i^E(?WE9pgF-fe88kT4nDgk&-_*2q0% zZ8F*UU`b-v3GA#hD`7(dKFTY>VFZNFeY<7p%h!hgjMK#QSV+DQoV(|d{Hd~^ZM&b2 zhwTQ};m`Ug&d_%%*(M#Iv#jgw8Qz}NHKjClSQ{e0D`=NGOZrGnfjKoa<2~?R3UqNs zg5eU!&W4OJ$7XDW+xaXkxPEqct^LmT7&-t+Njq5$?db9>tAw>AF{+oKWf8juMN z1epD)c`A`FG4XKf3TLd`oBB@r?cW^vXqB&ibd>)o?5~Pp#o92W;uOg$xpB?fNU@r>6AFj_DVfqo_D`{&ni-^rTlew1 zO^&~C#f=HxE&p-wq&p3pl@ytwS!ups6gkm1I`c!ny4_rV{fGOwdy`3X#E&Zxa3)(C zN5*|7g!rG5D&38fFI=trzf-;44_5>y?Mt|7m1hxD9frZtk@p<%VKXojz zT<~U9d^|Nz^&0XCm5T+r_(<^?HSWvh0admx?Qea4@s6;O>B3vAXZv;r?GLzz?y%l9xQ|`)JhX9m z_ok2g2@%iaPg3L^J2BzJaO;<)!Bgj_il!=z;&bT6cbATqBVO1eW2uN~q_%mC;G$00 z_zno}7Xt3u$3^?eFh}FO;@m}pJLQqN{dFyi0{owvHbVPM-Pu*$=&o?nn718ec_wFh z?ebDS03DKb3hXHN_x26@zbi-_u8M1l`L+g&kV!WLn!g)7!o^Cv7ZJsgt^8-!Jf*`Bha$`_}<7-iy!6K zLGuCr()n3@0x1HpWu$n)l%n~QUqjnRYPr2d(BR$uS!i-y$dhPpYdi5g6S1N__cVBu zPH$?nwaY@zE9TkIfYY7qHz_G~%$rYr^Fnk^qLNy_VbZ`5++55-$*l-TC8JTk0%G9d z0K-sp0;zNcP7jKYWZY@>?&4bLn0KbtsW&O!WMBYmYv)>9z1Rf01HHP>Y z;Qg_Am^r^Z>cOMAMF?;v_@;!p6%RCClKNJiu$I|s%et9ghxv8!I^-tB;`HRP8tR~b z*id8RODe#t+RRe>pktWqPxi*{BnF<>r2|lsw2`u2-EfV%$cI<{6EpUe5fd}hO^5^3%;E`_2x>caRWeB-An3>IGvyktuf5n@m|ucT z1aN-%sKuZxsrvPO{4k-!WYq)uFV)!xR>3O&X6-SkByKdxSM1=%IWD{Bx-EAdxUVN) z&VxouCf=WfXyLg8WgG$zN!H8@5ZnL;Iy6){VR(N zEi**>FQENe{K7qsjaK4KRZS@n+3YlViNs3=G+bni)Xe_WkqoOxe|nv%O}s1>ZSHC&cjkU zzwQ|s@IKuWA&zM`{xxRED$*0B;PkcFXpHsU#pCMIh&i*b1LpDkQZa33TLP14d$xMd zEsoHR?=bG-T<7OYo28#GYq(fXy=oGGa2pfKH zgn8dej?Wp2FS_aB+V0MtDF9`_7ZK%C@L{CSZkWWvb1Utin2=X88$#`|`{_oF82>U9 zA|apdSj!=+IG!bh)RxZv=RG1jgB?alZ|8~^7|q5DGJ32~K+BnhjZJob=DE9tpe3vm z>=s>00kYOFv*)omgq#Xs&UlA1QrBi=W4&SBGv)ijApi%r$QgN6s8FkM1Fdwp!-8s#~ za5<2Wdv}pKF?wy=nUs|JO==XCm;6;`JmxwroJstNTBZy4l>7-iC!eCxp%Da3sKv_> zid}%|5=2-dd~3IG5Vgld&rA1UkM;EqM0Ac6Y1la8f=H3z(}{tK{Bdxy2aP zve$)$>Lrn{vv`o(OU#5kv&+nuii#LqK9tDq{q{gLaxr@9{G*8(Vs6Gv7aCPR4DJ)b zc;e;tk_>h7aK%eLQo8pP{C`;XTJvbvC-qF9ZCd<^3(ynFf7$sq$t&B4ot?h>w&B36 zs4e~gp;tZT7)s(gxvd@f?x4ESn6EV1`#{x>DOqQN&BpG|(jDI;23CFve@Mcny=xYS z$8HQ!tIu#wyexW_LkRF0`?@ILEyrKa2SC$7dCR%fWNB*Fju|PFK#gG}#H8jN=tm%$ zC-3VU6#7DUYu!aW$O&Kc<<#Nbdu$N2f}iry5Po56Ngt}Na@%`(yBeC&E$^V;LW(lq@}ck*tg~}C1Z};qqI~++K`~U-z6n)-Mf}|`JxTR ztB|BZd78g75w+doOdGRHaaIek)UOYAp?w16jFR4-bFW%1Sdn7b>6G0Gfg|Q3MqQkM z9~<7DX*>6N@Vu-~QgvaN&UrCuh&ZzrtEz$`E~7d-F0wmjM_^YBE^MdF#2m!tp|&vK zN`@;OnV+&5N3u&dJsm2+iL~NN1Xkz<_@L9i<%OR6{EUGVxcHT{{i(faVJJv#qd1&1!3@YZAv2xK3B2 z^w`#y*X7m7l@*&%QzsF*lnyghCUE?jQrY-;;Hv2COtsu%(KR*(`FwRX&6l!HkLViS zvI*B7G#}^xLzA?cecghDJN-RuaA(-5aHoCa5{S}Z>{U@!<(-g}99~Cn=8EN0`sspJ z^Q)~x`I*}lt501<>wQOTI_DH}S5!Y-6vR+chpl&LcY$a1N^xJe*}mEl_!IFYtec?c z+kmxpX|MxL`!@EYf3l&7==)?<_#ANYPxg3OZDx$$2)3U5Gt6rc&yHE9nadh$UK!H; z5xq{8^2q%;_k3*W$7j#_$NLm56!uUTS+K;CBcTaR&hKJy#8tMte3ZU=o6KzMZZ}Nd zR4iTDs6t&Wv(;>%3UY*dVK4)IV;?F&>oVBt0EK|UKJ~4zlxS_!gwJU`D^bLEf*+2F z{hjLfr?dP(VZ&gQFKlgmR)0j~QH{)L1P$0`J00s?sNV8A(Zp;tI;=S zattKt4aQlwqEkp!VS^9te->9$Hm1IIb{tuVuE-{(U~CM#hFQZD}lF z&(C8nEcyt9Bg{IAtk?6uj15y%R1_ad63f{sl%)PmgH>USjzBtbUV8y6o(Iw}XJflTPVM1I?d)iPHGvzBb zWi93}KzZ%?da@sVWRhmz9WjP5TMe9F6Mg^K$3YZ)Rj*&W%$o4U&@(kbc?HhLE&jx$ z{ZxJ*W8c);PFBXF2~{O}Pq+KmdC@y(B%8h&Skg>BXfni^jRb?r(5k`oPRpm#N0`kb z8gPrgo%?Ir&!pC`l&@cM_mUVWcRsvzTQWh$mnpk`0`l#(^9{Dl$R_G|BZ-RmKsOV<5|ee( zPrOMjGvha0Hl^j@WHKb{z?K4IUDKQc4FsLce#<}x#IJqRnCP8BE`yQN1o}VRH>=b9 zGkEczQxoFAzW9A#MSrDPIop{6xbLgiX4<9Cr;1XM-)}i-l^Smp9nclLb&HwJJUV*( z!V#)hs5VqweE70r@BSTNYO*ZN7_bb9AYftTOybMIgU@%ZGXBr9&=x(w%do}AX1(JU ze}#qA)HUFS=Ub4d$m5}!EY_Mh}rT64i&NX;l_uaNV2H(8Z{FmtYQo$b8 zl}t=`*F5uzYTbFnzK6i+`9+>b3tjp^7i}>Kk$yN4CAlxk5;E9WFP7Mkb&2wxF|umov<0h6^uvQjo>& z&p9a|aW>xzbhlP<@Br2kdRAbHxhPB90e@n=kOKkq17Q9km?^6Y0fmMGrPqPNnM6=_ zru=y)uUj&x?;Ypj}{RU{WHM=_teO}YPgjX?*v+q zb+4HTg2arEca!Vpd(nQjlwQq$*1``=y`ZfKILj??l-q#p_C)1EviciE%+8*YVfA~< zmlG2H;ITx1Iz3E`shiXV|52pY)7IocqoZRthrz$7q`S_i7Lwfwe|@F6!Q|3C=df%h zmP7&(dK%03-KEo@UD4CTG$>5O#MuNd& z!l8K*r6=!VywT{_$DQvB{!6=2!p8tpEFmS9um& zOYfLjR?*MdFKyyt)gm9C1~<)!nDEFs>qvr`ZP;2U8wE@&2S?$8xqN1u5!Jshk)*|K zvo5joXKZaUH9{k|!B{YDz!HXbE&Zeww7hH7M_RdFWg~BC=(wxN0{rB7q>_TazTlGN z5jlJaFWmbVXfdD03klh!VkN^F;vRCy*E=csgvcqcI3Z=S&Y%mLRiNmAI!xOUqECTl zaA8y&c`9b5{0~!}f)+%I+&b9%T98f>)a1~2Xy$pX*b+rw!)03T1drfDvYxW1S$4ms z`TgrK1rJi>D6;q(w6?=H8WpcLx3$SGDG5tS(S=3@&e*FZ?ZNG7ZDSokqse8dCw~%w z(V<|(6)Z39?;TOlONB7B^&=jNh-svaZ1zCPCnMijSZ4rs3;!0?Ya0!wCF$xpcW4OX zBLe$_Jt_sx1lIk5mCfyS%c|U|W6(Jwe#1cnQakF>J&?uEk$5|I_B(|M#(6 zK@q}zRQKRMD(DDM!8Ueb5|INWgpr#M%15msT&WG&cUmYH_7@v(5J0^K&9sux>7rjg z29?&=(tN*tz?*}G+@@qWwHt?{)f!G7whw-cz?4H5hrFEF$bvI3m_`Z(Wq5f@cwrQF zt;#`ZUZOZXk;h3Mp!S}>eTpw$2zzVX#xji|LE=@?WUqRKe;g2dAJ(Jr!c`ao`mpsr z5=mKtSE~XVjo8`LKa!8!6^8}QPDOYIpfNke&?UY~F0-(eO^!_C6sc_FAIxj8nhI#2)+QR}F#s;YmssjTk|1cs*0b|ae9ght%d_4&-?6Lzlr zIdgB8(!Dlw3#;s$tcaA9N#sewV@0o;PN-i=Xz69(GDor!b*)l!9ufoDsc(tO|HQlI zgn6V5=6+U5+#eR3OJWvlVSb{|hVH)SzklH~)EoN@-I*mNV;0tOrku>OXN*zqGLNjp z0lSI5*O7s|auPN7R6|RCqSpw>&(~ty4r(({#(ewTr2qs|8g3o%vp00-)s9k?0F{+ zfc_89a}r1VM>*dLGIOhO^EKoFxfO%=ACM-;?c@hW{lQU@GOjo=+2F5>=*nN4^!0D5 zQ~*Hm^ohEPmI>kWyL-F)dsNeZmwz@Z24dF3(Te~M<8S51#>$pAF8I`ad-mb6<8N(7 zj6MBhZRtsJMD*>*`#yY1+hH8+Jkl0uF(@XAre(m1+`w&&{H-`Xf>Fz}{G|;H zi!5@QG5}zQ?3cuaHVB9)EJD>lEmA|n4q(V+nwTH)LI$AD+Wu{G&aNzt0M%gF^Lh|4 z^5er)-yOM>*9k~z0U;mAf&0%e=M-K zn8dzLeeVhh`@xepOi*e9Lb}6zJ2vmt2iai|_^j`xEk_O@TyMp01GHdlljbI7p!6E}wZ zPZa+Q64&x+0@r7OWZ;`uXlZjKM`1hpUieH#B?mSJ4C93MhWCT3fS596dG!MS&BT~qU`nnO~ zt@RnI`O(^F>SHwvY{^=0p7M^{oGgXk!d<1uLx3?-bi9basB8j+JZz&V&d9)4(-e;* z?i9~;9n7}R+?m@MIx$fng{PU~iPrh5+G)76qL8O07Bww{pj-eC=;=YGjs$8pvF3c4 z3h=w+q`QrbUTQ2%S%-4L?Vu^y%`+k4eZrqm1+!v@fg5nIjU;XV6NtfE%Nv&@_Q&hp z&r52bX)v|y$5!AHke{^k@+<;dn%?^@BgTYg2I~3SV2iiv}!OT5~HYDGxe4!sZVeZGV<Lv3=Z0`Z{7xl}; zrw}a*J<7CYUdG}6F(pV20kuNdC|v80oIHmQP)2|M%JJ)O?_(G!@=CEXqEUY3$(HKt5DW*kW`92j*m|M`#OH9+%ONTe zUEJed2dQqThy!yA3#W(s$KQd=U|Jt7rP0i{<2q$?qaW4&e+kxs6|5ci%&9Mb3O zeo$$A$#z|m?zjL`=${P@?(`PGjAu1GVVDY@++9*A`cDp>NdW`HXfq!G-)Xe!K@~dO zs$?vRii|!4RFJ_!%{Q`}Ni5uOo3ZbH*Q*fBjTEFK~ojkkE>9d_%kD7aQPBZ8LTr8wB%<@7n2^j?=e_yt} zR~;1dr>JzZ!~P}&1b5L>(KnyUTy%X8jKDuwvANFxG@j41iQ&ME?D9Bi$x|-Z^BIGe zW24|A7_iLDsEW+Bt&Q97<`K%%4wfyX>q9*KU)=kjzcnwDOfSlVij}${b>CGL@xUl` zO$rnKKg?$8Y*e4$mx5wWB}&O3M}UWA#E6ebu<@uJ?TbuRt>vG)al2r5cR%{6=AXzh zUj+v(3SnLM3(yDn?b&zc7B#1zZ>P}c2aIlO*rVcsQqg-)E2tp1>p;7 z1H_1`-@-XKMHzgzTvZZOQZf@}^4iA^+6g`CkQ6R4oYA5tJXUKHiiZQU=~WRE=6fg* zc~>aPk|%LOMF-92<9I&Rv-%TbNTDow;7IaM^sK`HTgoTuHs?6ouRqFCPBZwUn3~!; zItL?e#|t}s@Mt2A3pQpSs`$cd;of8J{L_6q)vc&DJs5@LeBS}aL2OsoOT)K4 zvwPO&vx-KzK|EHxtK?)wcTEb#@9TNmLPC*x^!?HZ_V`Xv(1_=Cu<{R6R8MrXRbSf* z3QA#gMkSC9Fj*}iq9i&Byo10U!I=2szdrB3l}z)Nma-ydHZx6fSR`oZx9Y#pS5Jsi z0m?4fe8K|*zEq5g)eOKD;&xLCtZh2pEC*-*hpg{_r@DRrC!x%zgzTcI?7gW(%SiPk zWs}XZHzyU!4B2~cG9sJotPqE6=U5rX-s69rr=Ics{m<+1I?vJZ`F!sCzUKRS-`71K ztV#>_CQfjAr+u`m3ILkM%yYLt!G0aP9FZUGZsauZ$k-;2?icU2r*sE6A?J9j*1j-c zt+5Hrbk^`(OlIaY`P#>Ejxr;6RSIl9-en9jQBX|N*xebf4AQ3n(%EHHQVF- z%eTAI2Sj_>rdziQKYw?Md~I$-W{jpwyUgDrMh%UAu{t*b`d1^FiWl&2hPI>CJ6{5- zYbX8$O1d!V<~$@JfNrGZ7X6s^05EWFQ$jN_UGZ-&Ku{xU@cS3+(dwn+Lu1+}IYy)@ z-v?OGl*ew-vs$jG7kb0*T=w^Nu(u_hHA*AxVG|cS;~( zoni6neEe&vjqjX{7RDQ`vH3zmrMP%qb;XQ zlPpLf1vVZ-N;M}mFf;^J74d5KF2cJ%BqYQ`v=?o<=H5Pl>X!6bZLlpWRXgmU1m3s( zy78diwmP_eJd?zsonF5AJwP4elWFbl+IoJ<+~Xh-+KZC4k>dwT*4NAe10}-op)Po^ zSvcFqd96X_%_}Lko`JD?khvgtA6sb|M|%rtm6{!vdz_};k01QvRXBO~BpRW!y++*I zg%z0+-YMiUZ^i>f;2r>VNB`yc9>)z+bIVDrO4Tj(7kR@LRUPbL z^IF%ujNrtmH{h>7SNUdWd-#>f?u@)(1gyU5AYnU7)b{Lbp8C|PmoTV?mg-f=Rwf)` zGFSfSk~WfoBQYDgTi_%c(hZI_Ccw4YB90oy2*fwkY=iLCsnd%wD z$-wF%>%9EGUgYK}ewUu$B4=$6r{kEdd8k$MjnwnEXm$Xh?x@Z z; z*u9G~?5~t^Xo_9hG{6M&&)Gk815B$i@qaDFn6Dp&k>WE}2&$01Xw6Y+T(T5CuoYSn zGPLC2fbCAttL`(&k8FJaSp99L(1ytiwVz{&ncc@MjJN@QGrV^KN8PpUjtaV$CSl?R zSf++`efhzXHm}uq0&`kh9w-&-D@^vPdU2Bb$?c6_EVYcR6ImI^@}IWElH+geLD?jHyveWMkyUD7}d^s z0iJqfeY^x<+;Xa|fc;H&ScutO)cEv|VUKx%)rF^zNGYW^NucEssW$Dbo>1-+264hP zp)xT?CjjQ4<0QJwakLInu#AMub2h!E!xE`L;?>@8~GH1n^zz<0I_tju_## zc_SdD-aLh`!_VM6QS;mhe!A?g{8waG<7j9YTR&LOFCI*-+zr|;V2XI4s(CxH7Q4zI zyF@{}PchpwIPz0S4Gx8cIuVf9c#nF%4K}`3scEc{>+-UkXlHnw!ZuiFAxLZSG_WjU zH+6MBwCMB1$O{M`D@XP-ZbR9`R$K*A_1fKiv|@sgH!!R-tbH2lj~bk%HorcH(^qUy zu^Xn@;CNh!nM)7C^1Id{92emsltO=+86y^}N6PmLJG@j}VjUpWH z33bo2bTo?_>bNZYy#GP}f4s=JVtnwwvmMr-avoNdokd9P`3!@y%JWeNCWO)u)t&S6hD0HGAl=t5Ic zN>2-@l23GQ%jiO#JyzPRs80icE|#Sqw|!!i)xT*oPA@;tXXoT@d!csnwN|&Ud@;ly zBM#Vvt2{Y2k|>>@3^7i-dE29bx4nE;idigXq4uQNoP;t_uhqksPYXMk!K!@g{i#7_ zh&p0pY~|6}5<=Pff9SN#C{Lr#=@*|=D(Dm7Ve^89^kuU5@mLU~YKYOgcmi=aAL=$w z*-*7?3}A{tXOyC;{LhU164bXzs#%Y(kenxT0p=}m;H`3_*1kT&B`U^yey{B8?TfKi z`Rz*t1P4$xxskV=;3{M`L|nAK>l9(@(fBfR<$>jp^=Sq*9;qz#?svcfn$6Snk7#QA zRCX8w@itG{z8m)97z7aT6}xL5kn8T)ZQqDk*dlEKkFoknmZj9ftfpzFm$keo%i%_-bjA91Uq|-E7aJ+fZ~!HXT69!N zPQSPhs$XetRc5llC~*;@tp}`hVC=4%V_EwubdG3Fy6wzk zy@S=%d@AbjCt}o|0I}(hUz|BFa4C~d? zbwDH^Aa9QnbpLWSNLOfXsKQzt@D#Zv_KP`NbNom9t2_|c#5Mn9h~D|qw9W+9rF^-! z6BtWm(U)ETVPDAT8dNeEg|r+Yf!mdzhD;A@15~?@i;GbmG2+hO#Gv#S^MqS(>O4xb zNkXDx$F3WHPhpVG;u3(dI96~;(aeW7D<}_}Ym~o*+J197Ms0@b^ zxa1Tn?*A8;p4HOGo(20%pgV?6^_axDDxe@J)Um{uuC8ujk^J$k(a4oT+#=@*RIA9? zgv6k;*vCkv^JIHxnhZY;?Q|c@q$Fo|Es4aa@fvpaI$Qd6lYLvT4VJT_^c%n=_MN9HCbGaJ_hd#%iNp!tpM|n1IejrF*t?ya2g zGLYKJ87V19be1>XfuoJH$^ZE6!i-`b)X3^rlAZobx(DdHXer z{Ysy#h5~AJ&|kgGpH)Y$yua}H+Vq>l<6~sOGpU$1bZ<{)KWr7+aA6{Z*Jut%q$Ey9=Ztu%BJ!o9TNWrad{{X=!wulzK;Wjpy+5K^ZLeZ zQ#~>nt?;8`_$yNd2Zjeg1Ws6#c-g6>|KZKx9UcuNMa@|F#ZDu>xuMUdXO0h@%7zUE zv>7{q2@sNqHy%1ry+BV7lRDwB49Xwgl`vWQxf5rcNDWmcduET6?wjqu*JFX5;?Qv= zDSPHhzJO4NAr73qa81hKo1xe@Z6TwfAwg#d@JvjtyEw%|)NZ-!3#v|DZ1A;ya1YtQ z91w%Lb>l7}YSdabC!*s;!&gGk(D)DSHdD6uR05%W#Ub;jY}r})0u<=@zU>(y`o|eg zNB;x|JC^T%u~&X+*pw?bJFEXraq&h@ zkB(z#uJB_B_HSD1W(}{xP7EqK8n>^Qbo^&|-hki%B&!O2l^W!AV|=rp`L(*=r&z&srt&MgZfKDY`uo*&!5uIUMghthn#2dYBNqO?=OF=43&S_;`MP`hoWq3 z-w4WRxMoivl9!?15>zsw*b1~vsNp>1G7M^B|BveY8vK{_6Nheeq2Y6nUlLs?%lz;) zTH@ww0`R~3usIFz`PkCpNJam<_vvk3N7rL85k=r1%Fr;13s-qeTgy-30TAC2D;3@# z?!`ko3^v2&O4)mu3*%3RWxSAHW}605JmWGnR}g^Ghn(*X)pG#(0bQ7JXt!(Dg|FJ( z9YYiT0QC)HGomLF92WkzCz0hLBgA9SB)**MdT5;XO>6bA+oaI_@EVY5p?z#xN%9Yq z5!2$}M-Of)-#Co}T%f0x_QBGgbl@cRnk%9-@0pNBgKfLOne&pY2$WfO0 zed#moaWQu}Yr2_5nitzT-tE)z;pEt{9WTV`%IMBKi^u%~`k&NQbL?C6vk-Q1284&2 zjSr_%e5ifOniSpa8b6YL1F#dClmYRtVEAc=Vc+|+laH?v!cuh@rnIl-$1?+b1UpOV zpXiOkrss>?;(;ID7-uj1Bpw|w{bUWZD_8DyFoOs}!`{Wu9p_-D%^iO~Jtzo4s(F%d z57^k!G%5d|?>N=nGhYIDc_m{`Wp%K$bsU9ptz^)=~+U9B{4o=U6&-I?jc6CbXqm&l~$h zorFr%KSt}ft=ee6r*f%vJYuhTCX^XrX`+v_)7jr_W8dq>=YVWO$M-&gk3D4;T1>H(q zI7Ta(8IwlJ6Dm4piN&4$z1)(L&pa{&R=I^ODQ=q6=VhUWM3{p~(X}2hCx> z?jD`u%K}PJKLOZ@C%O(#8)ua(w~dM+Zvr+LP-f?aECuL*M^k4m0?3h; z8B2gto@I+1X#M&s{Z({H$Lj87l;r2VVBEl@hobXRRrR9EMR&J*ba z_RkB)A%2cM_Na$(I01MtKy-oWQ+*@l8+VrneZ1%MT6Lthr@pUU1`p_EBa;gce} z+RpSzA>f&G(R~+^*Bk`GB4xdZj)`Lu9i}|75Qk|BLU{U-U z_~$}d&LDUNYj@`<%dPi!ff}CtB;a7p+|iK|m`_q07*4JsH||=u34VHwOWXkM{qv~5 z{!dI02%LA{2H1&*gX8e9rt?`R$=XLQjygIq&N?q2V?QjBa!FkOBkn@|*k#|x*y?AU zHem*wJVinJlFA+qHXfyDOkbarOY8 zgjN3K$vY3^&Th|E*qIQocP)}%|02BKCD=VYCXR`87R;F1-VYS_L0=cKjeK^p{Yz<( zgC`C8qdi{F(^%t0O}}x*d5;6tb2N;9BF_=qU{$$KT$c)#;K9<85bYdjyS+zvOcyI! zW;QQ~?-7a4+j&cOT1c&q!-fc_n*sDmSiXD z$5Wr~1!)8A`>w>iwS^(nIikJ8C}itV@+&t(Az}Z!6b*O{b{RDF`om6(!MtMfm8=qVztL|zvwT{R;|%I%|NRflCu=CwiK)INOrUY1>4s#NKctkmMs z(UI7yQ^>1>D%JR*i;izjFS;o3B+wR%JIYbLk+7L6Rv1kTmMIXf3Lz{ z@2$qWRa)uF*=ebtUrO&kqZ~qF5iBV=8IG;`MoMO>RV7Y}Hc!Llbq#hTUOTbg-ME51 zmJGBV^x*eMpsmHm5lMOL{&8xDKQD>a^lnD)hQfZ^j19JpK&AQWLq9x%Q9vdU-2k705R8B9Fz2GgM~u%QPJsAO3#2)_1qajc@&a z8H}v&S1Cq4mgAd{O~qH@iZ>Ygy4fIN6(v&c`JfIauZo#6P2Q_v8L>2%n!)?^&02_R zoM>og@7R$1wK!%QgBlO!I%Ft zO;oROY53TpY?68Lm~VUWc7y)#hp==_L0kFDstKYI?LDJ?_mNyO7>@+heft-L zT3E^Laqhkgj3hcN4-%{r4X!C6w4slaWutgtDNQ|E{DAewr$4E7y2g>gKfkW2Cb(X? zEKhi{IkUz<3>K@6(zf|P9S-OSqa3pm&s);W9j}VJ9q;BjtEJxDCEb}~s?gWd-yjzx z+meUUyeW2GDsW!@*FrQ}m|nL{w4)W1lYJw|>Tof#WxM{|-+Cr4;;p;LYv5u;&Dkdu zHSqMdweA~eI3*X@M)7D6v+g2pv23TfJ`q-~*e8WCW78d`38T8_GdkWo(DozXxk>b6 z$?QKrySSv}j(!esVim_6-SLS0kxe4CCE4etRp1l-j>@~ccdTuI$yFg1pXiCwaE*))qehi++jbK}vEFVJ9h#EA)m%sJ`A?b@uVtN(Z?GLwJ|H@8kiS z*+7V5+E9h9-6koXOo7i@uO3$)V-ZiI5V42lI_y*4ELS->b~vu>Y{FlhE^O1+PaE86 zIE4C9U-ajK^V%-T$#xpYS~4-grt;D=Y-#XT)x_pWZb5Gf%=U`pudxMNck^KCW@QvQ zA4rDv^r|dgAML(gaX1&mTh3h7Aba7`-xh4S*uw9Q$IkoY+@cUxLlGstrRF>~8Pvwr zO~qWX&v)AS$kuM&IPk!#PRGcAKJxJ`lIRxjY6GjCoBbS_|D0PO`lBs^GzW#a7}rH6 zo}QcfKKdQ?(}8-~&>z{S2gBvrW0=-Hmkn56itCzK^*4{U*1j)^$Qu$%E+i)LI-!`K z(~MoG7`l~FoZ|zBnb4cE2WFpNuAUO3m{;^R5KvOl{$PysA2XJX)09r_rHf|o7bvgU zOH7w%9!^gx_4WR7O~z&Y&yAKTI0YywL3-s>tuyATM1_mDTLE0I!ZiikU&j9TV1K>u zQ`631wf~r93aMA({HD->Gs*OF8(&a&EjK^?k zlt5Pa>FG3|*aAa6!au`an2&!#wCZbXmiorIZt-X3@4Xcl7~<-#E%lFcjSdpq8hv%- znOve%Cu=|u4S_`Nv&yJ)!RVEJit(KcP>lu^sqV3Ghp+d2Gjp$`rDg&E??%Gve3HOJ zYHLs_CcTd?9%(^Wdx-7>PET(iN(ITKfDlqpbEJgQ*E`DbR-@(5(}{nMQy82lntbrS ziPZ3s>a#CnGa}4K>!fmlozt;UnaC|3ZFcvIU|~wmCt3Q z_hRUxb{QAxf+o48f~lT7onVIYR9YH>y!?1`I>*_5=o*iN>zQ=%A}{SRjq*Uz@n@2Y zM?ZevV@IJFd#loNOWxkK|I>W1NWT?5U_R3PlRPG?AWKTm+nlH6xtYjE)6w)ZFJ#`g zlxF2*rGDlUfElSV_D|y+u&GziIMYVTvZWD-UYS657wN7$-18;Py272FoEw?^>XcWo z)Tuz;&<`}Uj70_T^sw087T1ek>@zRB&YhMyre&$au9L?!Su=`Q3^z${%SxFKG!$C+ zMLpsOrllZP-{>D3YHC1w#QSxr-_qQC(=a<&5sE#)sO1iqFx7VZjn6hMY))+sI!TF; z3sdcGp=z(obvCAa-Lv%8{YD$Gx+BP2Y%x0~w-w`EU(8$EMy?jCqOZAmCWd5s*6Yeu z9>!4f|FsD=ZbRYn`kKN?Iy$ZUqjDyhImvt>qUquwg|W~*?IXEyh0ESfeK)e6+oermAvm5(lJ8^dc=*ihYSmF05hHY33sp>+cT>onkq(odR%rF28QX+2KY!v1{l1Ct``!BWuvv+GufYdzlJrhjn^<%<@R3%s8pn5$LmRav{Hq~)Q1SuW_+ zP#;72++yFViE|q2U*tm=Wq6FE>HDn&X8B?~dGG6VI z){Y_zomzoMFx8i)O(-^QJuHuHNQig$LUIMO?{llXAfnvQDl#K(Y`f$(2qI}@HsyC@^Sk!#|K(l zaNB3Vzzxl=JD2Z_$iI0dhYTv$2-KEcyEJ~Z$_u#rMbSoAvA~A0aFH6vZUU*bS=u{k zf(;(p4g6%U-%)-2yb~*r@sg9PBIeTm3&-7u>SGWsl!6CZRrh;co*xdhpQ(w zrK=asER6Q^t3iT7g&9g8JbZk;|IM1JkyI>rV31y#QVa5OBs`+q{_R?W!;ED;wb#S}YccJG3b*Y(q+DO? z4Qr@TgNR+*ZO;9%N-oT&xKRhXq+Ag4nrXRZ#KPRv!ttK|)}q<;kM=HoQ+;lagrHdT zHx#Pd@4ni~__pj9Sb$rsQS^tDn3uU~+HNAa$=FLSl-tVarCTn?QW^-TTb6H~o#`|O z9riJaz7%p-1To%uNGV?f1(UtD36Tn(u**Qw*^z z1+9i^%iKYfBv6jqU4F!{@D}4$nCp2o+4eXZVu~mEqm7gvM-1ULpBx6aC0u(n?)i~6 z|6@>p+f~NuS9fG^o}-OcOLl6ocj^R$E>o+_6>mEVqklYYX`1EI8}5>Bq-vAf;t09A zeB3|9|LW9D6Ov>=|m)J4X??y2M}XjkT(B8c`(^+x6SOr z;*t6aUv`#9tInNLrapT!hH@yO&2SeT%6m4Td8Z?;%<9Jo-3Mw~J#`ugz~U-nb{nX= zS6O8DJ3nHBStzVK!6>|Ek@*=b*G5WQ4k9Xb)X{D}S~&auFv~95C&4M+j=i0D+>y?Y zwj(L?0_AWiNb}=sN=A;nW}=qU+dG+UD}6df5$72eE!7`-Y(Ko5JvI8 zINM!wFmtA5;PIaiAc>x#BWp->(G$nk9tq(5E>mTU4>~4)yZP|c9%t9=+In|mskat6X<~jwh#9Z6k1#`7?@+%WCUSKD+3o-( zcDm3=%h1N1+#|s;5nrh!;YH3su|7(sNG|uJ2z|PV)mE~zhSQ9R)K12%H)H5Mqn#mw zpp`FX?NKsmlZiw^!w$>9SHxWJ#kI)HRbvJwY>VQvF%l=|PRe1xw?9#c zr;GmR#^Ej9wH2{fLy4ACBm?D;QuXP1l3<1eqZX6=_WZqd&qHUmD29A7WGj&nUX)^Z zhL*U0_5_uSa zlwBPB#86h`fF4k-ZE^O4GfFCDu4{=o4O-W~Ofch-4pJM}=#)dc1lq9+Vm?EZg^DPc z+?+O}aILAbdaq%rBVW|xi}BfQylrVPIJy^m2k=G4 zVc5Q^)La_e33?t__2B)n{cN7wYA6fX@b0Jm;m+nUz(eXd-d!8*Fg6v_>xtGq zFEsGy>M?SXr%PQ5n>mi+Bl~G~ZMUqz-P2hxCUB2&mRr3}t=5Vb+<9~AM3)eGx>D9` z0T$Zr8mUP0$h+Z{T3GKA8m;y7j>yZauSOoqOvZW3ACDVZa@fU2v;Loap(iINzAiVt zxgiji6OQM8cSmdI^EAZQBQ{Hy-PSo1(sEOc6|uqHF++{A?$d&?$eEi`TfwMyo~zd; zY#m3NJ5d_`-HvvfwBGnsU+?c^6u5blc0Y6aFnd1JXa`l`9^W^k%u=kt39;aa`#!gZ zNB!9ZJ?VC}Yy`Qkh7y6T8{K1{yv#CsOdY$cW zMX-7K)mOkYo1M`?(M0uPKUI5SE?TmBX5D#rK?V7y_HcwrF-8N~A+HS5&QMVC{&;=5 z5*ox@n>p2Obe^a$jFzFFLzhfGxtC7j%0fQnSY3q7t}diT{yVcsF&Sv`(WTvSe00zP zaINFw#oy4=@5eXjpjKFmk|;m=sQ=0nFLc3d#VK&4YWk^@G_z&pq`=WMR;GR4)@Wjr z@wx4}1XMd6sp0ApEVo~`G{JDGf?cfc`NwME>$|l(cn}v*lnNK2DWqz^;`6%yY#6cW zyxcP4HVvW1&YBTc2TXL}uH%Cz4;yJ^S8ptT-lSjqlB5q39hu&Sv*4mfd!Zc(!uGd1 zkqUY}2T=#3+8+M4U6Ha=<5_y_rvcGZ(HxOhp?n?@sS$?s90)xf;z29zryCb zl`koq+W~hVvmqM~s6Ecz}B!*}^afE6a>)yRjQ)H&yo>^S`{E}=3@tC7Hxjofx#PJSc zchv?_mYvJv*^d@}kO8mP${t&XbHyqFtU9B?b_S5aSqqdvR!ua^=)UC94#w}cd;Zjy zQ;s#q&t#m?)QV=gYV{F+59Y7;*3)0ng;5@Fh=iuo>z+3B}GO>{q7lT?eaqzx4y8BxSlsAf%!D`d}IJAO(oR0i9+(~vECYLx4988 zH@I6I4OR((22p9oZU;#0SMItKkWPV1v*|uWPSj$NPTx1_!c7EhpRpwz&2^5zse*un zx5S!VQtZPt=kO#~6vZCz^~)VB_ZTTW?&JS(VDfJ-_Zc9c+Mx!}2St7t7ELE!{I762%GC)N^ zsIXBZ$FKBSP$fnVz+l?g-4n7qy9o7ST~aX$Y@p70B4H}d&-+_3&VaEz5KeOYc*EC( zzJazQ;3K~>-^A>}Wm1#eoS7_Im=SlR&b+TMfM2mm9vAinhr*jmu5P+5mM(mrdL;Ou(iKKg~-6(u!tk>sII8Fa+nRY++ovmY0D&tV!wHQSWbv>U-6-&$K! z7qHI+e%j+&UT1miWzlK{E-H7cmi__M$C~G9jYbX=9z~a}t~< zG6PuxUqpR4M{TuQZe-OX?gs|`=rVXm^pM(^L8o^f0A@i?rEb(=^|wkCAg|N`3)2xl z@YW(|R4DO<`gg_LiAOBe?}A1hmT6er`ATcs0)+%uiAU!CvM zSPjhEUghy;+}PIR!p_E|-&MUKBdTF7gVJq8>{IL-n;tr%UKd{mUbY_N(>QHpJ%hwI+Y1I>EsY|FK4N_yk0# zl{O~J#J5$%WDQXBv4P|(R>aZUOuR~qayFJf0UQqYnRi)K`rQ}KS4FjL`QfK<*jfP= zn*BoB2@%ndN*J91xTbYUEJg1;kvw5AReHW0rusws24#)Rii=zzN9Pbk zPpyNtv5eRcQS=oxe>hXl9Si)%6vR&uG*FORP7CnU7K7BZUTtWbG@yDNL#%~(`_@0; zwQRfC&`B}V%&TCGT9)g?AG(KId+O>wJ*rpgR0pkWeQIT6606~&gVsbb*c?2qU#mxz z+wlg5I`9X;^lVmi{%pP((Mi|gETcZ=ucvXw(xnskG0=?+ z#D0SB`xA07G_*B)$`y!Wb_xloUl$Z-F~@2&H{o&;o9tHhAH>=$PSqD%f=r0{moDH1 zhKgr0<3b*cG5L2_@(x-i_(#^3e=F#%XPnpQiM2WQODkz{-&3Y5H-BYU<^0bBO^k=z zRYk=<`2ltCV1BC!#w?lks%MjcHi;v-`xRH!KOr+kJ^rQVuHH--|#I%hqocwHK3sNlRNrBo4?{CLFTkZL4wgt)ZwI`{N89C-ITR z-7SmDG0XP9o{WT%<9?NE|3oQn&Jex9gXqegQ^FWDrowtRfOLt>=kvi8z5jPg+r6Hf-f zDZ!!YJbfz3M$*b6_%AtX&!~KEo_>se&7t8_U`&ChEOIy3J*)j4Occg|7W-6zhl-#s zRP?lVyk02bV2d z&WcZG{a1r>FFdF*u&OlbX8Dwyi<>H8j))|q?v&TVlhHZ+Tm^pbZebq3c&hb9SdG$h zd(rtEM}#&1U*qp!O!camrj#JGeconM1{cttMGq=P5>je@`+zU@APG}?s{m;)0P-a5r4!ZobI$;i3m8e;1sHuOpd4WnHjzWf1mKeU0SGOlXM!2pZs+JYb|JG@M~O55oGD;E3nK)!f2f{Srv zd2E#dXXpMBPXU`)uAUjA@vAl+hzk!Kh|;GW#fNqboY_ut@`A{oct59pJK0P)Fw|B& z3McjEu^Q3KakUI16v|cSK}cblf;b91uMZ{HFXdYt%Zc#T*dhn3`GTk5~oo)!Hsve^1| z$8Yw`6uJYa}82;pvXS1eF27flW;d_pHuYt6KvfkM#=9yRvsV%#u^wAwd2j&u&P>k_3t@uQRr70Mtp3Q$LLagbG% zC?&@0qe@SlSB8c6XZhDkvgx^eIj6XLCBBLXZ%UJMer)%-O;l!k+h6Ju9EeTPU#K=X z+xhYai07AnHJztgWhw7kuXcQ_dXrOd;eoY?&ilJUeTAD^y{qn)u-OWX%qA?_}hLKcVb?5DHfi}NYM zaD8hHJckRLnHow!#&3RHyUC|ml*Lowcoc7EPMhcx2DkvG)n;DH#o^fVu^lPi*d40i zHkSI5LmmzWdI)mpr_gG>1!Fr*+ z|5fC-gkQtoYj4-oFV(oK^o?&7{WciE?Tz!p`zPNm_v1y5omg8yG&4j^*$AAWqpS+P zib0M0&vWBl=&6pbsaAR4-uPQHULTc4%nxsGOgMOq?w|U=4aTgBTf|U zcCur!t0!*kslj(0dOxAd49@1^IeoFms8o=*t1a3Tt+r8Rv(`=;a%rE&d$;^)WATV> z$gcMHs(eEokoC;W5|yz9uzXPy8wK@v?tvy=rR`=KSsrww%o=2-{mV19u~HAqwamDJ ziZl9x+LVSCpmvL*(c^h*iXEoMCv!^J7g)Q<53n z%!xm;E8bX*Zt9%p;h+k(uID?2(p1U(c^1zm>I&`~Kh~<+>v4m*6)`t@XSkUqM9Kcd z1b6I>`VEzZ;*NJ;_^w2+UzVDbz!P|9_(Q}k;qft#qb=?xr@15E^%U>x?rlkXh2Ga4 zys^e$S+DIkEB5>;ZSRE3>_L7|u4#=`kUCYvyko^uduKzJ|fF z>XcV2+bI52<=f=pLSDsN?V7fORt$ zfoDg;A@TdP9jie6pr>c8G=f&b@N8a5?jga8hgto`1SVu-(c?`%Q!RT;UC?87@vWc#GfAdN+3i=N4=*dKF!O!9*;Esz%{KPwEJuW+oV)GcTmWnF zINTy#u^GD5GeA4IKBFBB!l1Kz?~3+;CTI*#237ECD?`-H-6^cz?mfKLbcrGO(isbP zPYscI8wrp~aaFN$veo=B0^9g~wLw8?=X0G$XW`bc7Ffd{H&`E8av*yx&ZpDHl7UpD z*)P?C+nscf!u_cycucT%H`l+`azrc>)=@lpScBPjDhXLTT~G1kf%O38Dn{wVg^1-B z*tOa;$8jo*fOSOQU-JH?E$y9sRaZ?I5&|S{yeEmVy}R0MPd(Dx=TFwH(LR#e=7cx_ zzpkx$esF{8hv?z^#UnFoxapaf<`ogJhN=2}Dpo}V>$3H}wpZf!skxAa?eqVmZmNRP z7fI4blvC%G$s|u+Pd1~@Tc}YgS$gsN5kS}aDz~klp*&&~lL_%SA1JJj)LzuEO@r$g zNRgs6;0UeSdXf+L@9&m;tJoUeSa#Q-EQI+!h1XLWVWPQhaaJ=2k8kmh;B?8Wkv5X) z92gr3I$x$MDNbG5l)2ee+F*B0H1{peR>x>6{^i4c6^laimngqgEDygVcnLnXNdDNj zz{{)`ekr&ctvdvKL!pZM+hmsMKoYBHNpYknglimA}!M;n@{D1lPD-R2$OAvd!6{Ux! zD`qvx>L0lGXvV4}wqL#T*DoE$2eHJxFc<@3&MVQI@>WH5dAmGd2>VO&4@HIyh@Giv zb)4CFGH@J3QG<{rJIj4)5C*nTW5T)(ng>Gbwzt^F1Lsdy#y+^!EZ9fIF0ktmtKl11 zb`+(N9Rq%3i2->h8_6NRiEV*CQB05EZW`~EHo>wAF+KNCmMAq_Ss+w$d?2*oH&{C5X!fmTbxJQ<;mf=hVLo1($@Z<}bZQer~l49;@fnV0+6C3_|BZKH?AV{s`; zGgvNtog>S2>DQl5;Su)9-ksb~(cdM}ROFBY1D~|GflOKo;0-m_Cc3^IIQ||S^v5ms zSMs*}ib3sHalWy72GiRpq4w<)8nsbYBeY|WY;(Iud&G7$CC&pI!p8p-+eLE-1&8cv zH?^hDX&vidxGhoYH(-CPT(T+e%Ydy*7(eOlrF-78g&JH1y49MIkwYoh{VG$;vEQw3 zqF4b99d;va-IM=1sQR8;y)V#a-Y=6U!BKi&JxWnQg2*cB)zO zba7%EFG;Y_D2YxgR%t_0C_18B^8`MB`B?^T?0aWEcDW{UC1D)s?_8!;4Ww#h@LQ@+9<2NhuYE(om0QhLj_`mc_g;Ldo_AjoD1REe)ok~{a4Z} zl84VrYK|_ST|$6mz4bhvt>=^Po&KLpr?c_NOX^MYQEq4@{S^ zvJ&GAsq*nT2V+*I_SzdoX4H`GRheC`9N#%npIm+`yWsKzF^1viZ*>WOuh66z0VSK! zBhA!3M~eL1SC3=xybnUm7XsiyLky_itV@9aDAzf?W0%}5XgP3)IHX6HM2baL7SeGCkYn<-&?nW={Zg--=Z;) zm)~uPnMWYKAT{BjLNJdYT-Y%Ezyy(sxTi(pKc>jjg`gI#`t{7KPY1a!?3>*>n zXa-Ff=!un|gER=|^K~XQ4!nw2yckwaau!z0N5=D|Q`hkc%U^_uCE1pdIF_v3Jk~rr zLNPfE&>Vwt_9@LASYa{aD(cJ&1Fy7`4p?n&f>-e3D2QLSmrX?MzcxU3P8htk zuet%*$_X#q)91RrM2OOt#*&NG6hEjb7ZlyL-5_w&_L|vp-RuA zEfnOiwSt3ut8W(C66{C`AIW-hXe>Rmq?1O@n>^;)mC-b-Rk1^*q`Ha{yk8u>v!0EZ zXg6zQi3*ezphjplToMsC zvav0VxaIlL*_(exJQ6z$J^7WUGEzgK9h?)y9}}F$$wqzMcO42|YeXwnMQxGGCc4gI zPt$Z=d>}-t@{wd%O~A@xwr#j}B`Lc1uh^4lX7M4W*ByW}0$Yc|(MCzNKNegT5}O4h zncWEcz!365T<2v%JVMID`g+4IB9>1=FinwyKr!G~KdHdX>Kr-w@gz2isM!Z}_sW!9}Jd|c%7o^Ih2`b#jTY`19FIf$OT zS8{*|0AL|X`WKN}06#nehqrT|R+spm=2uQu0ReB_N#1Cswu1ty{F;!i{$quzUBXxm zl9x^>5=imDSC`hF8&>l-!6NyFHhu{$NZnSqs-d82Q5Lhns$mf{Wg?~%Ihu056}^1k z_yKLU4M(m&-Cv<3)7Zh?TaGd}yVDbbe-51_%Pl%@;T#AyYVgR$T$?{!^p%MZpW-i& zB=(%mYuf=ZwNVBfQ3k^|ItlT~`c#fEl<#HdMWB#XShdSD9i~4SpqONEqz1++Fn?1v z83|{Erd8nG*|K0CDvN@=_zT5j9oIGfMZ9$C!P*Of(f52I$ZQ3$j)$S8t&uqTj*(uf zzANMk!~uefe?S2CR>CNp=r z%J=$S*UUXL_UXubx+!oe$L7x9fx}q@t(Ut3JEgRQQ7tS>&@T#19$8YOnq*A&(pKyH zAQdBI8F4{=-+{e;WpO=|KFgO%I-9z{o$g4_=lw-m?&ei6>=?sM#h}mLB7!K5aFxlC zeHFjX7lxF&zcqyN>8o7rPXFVb(AiR=TBX$Rwb#KHrDbYalWqiU)^k7^Nvs%{NZnKP z6Y&wXm#!iJ9xwO2xT>tp>HG49=X(6sQ{t~4!H8aa%d+d?V{t8HlzG5%(#@ zzAvA7wo9DsC})$?e>l~CEdmjqX0+x6mAs&>A?;4!(zL`_dLT2s9{F6~?!mn(PM*<~ zOUYY`z*^HZkMgC~Hrs`!4aEQ~uvOMP+WPR$wunMN5>j6N)tLvU9GKk8PguTbUJL~n z4H_;l{qb(8hoBs?r^OAI{AFZ^&AMIqIfdrBDG}9n|d8(2l~}M&HLEN#wg^d z+>@zDD+b`xqb>{0lsoC*>xV@2j%PQDS#sPpX}H=S`>`y!p`3vU({%BFm2Av z0426G*I&I@4{SYQUHpy~>f2T$dE^cp$jfo=aKube$^zY1<`$?!O>S2FT3da61~^4P=n zQdi6?hMbgb7g$&lnCsW>7%3j@j+kw(XG_*}9OnUsSO+Q=zBgFC4<$%(waUdkMN{K6 z$JqdjRRPcYzwrTkRj3y*9G=8*#x}eRYZ+Zc6AL_Ky%vj0pK?EbLgx5Uw)&MjgZJxX zYMZzukeSN+S7<+$(fCuEL-T7uyzP1=W{6F^{q)&1PN?+T$-pYcFWyophc?nj`UlV6GRQNk3m~R9K4TRdj^aLN({wpFX*9?c@iWqs z1G%dP3L;6RghfZ7wMW8Te|U@ z1Q=ng7p!pT)r*WF?cjz>BX*@bQtu8;RkS87eeSf~x|6_&40h z!=m`Of)kXmAJSV3G2vasqGjj9n^;6HA4)( z5i6F{356gaJeg(NYE119l@evG@1YSV9xV!=#eO{vNYB}BY?wZox2rSV*1Fv`j)P9? zkc0!DSIZ1(ZkozPJrU|HToL==<3Vukqsd6*=#lyrGFI5M2+`p#>g)J=s8iYY5AiCY zf_FwM3T5YkT)@@@Z(92URA?J4#VO|S4=H&~IoACJBfDd4TvIvORuN+NEkkyz_Qz*e z&T&Qg)2Xb*M#IaeEbDsw7CRkSp#s`N#nz@T0+t8rR(Nh=CPNG7&ge|)udK8dF3n|0 z4RjvXaW($LTtKNFicz-JoR!r%#FC}3^fJ)m;gGKKr1Z|-rn`+QzkEey5@XO;5AZx3 zU%6ON?O*^y=Dfu$^I3sbOP7b=c7PddSk|{2jxKmX_1%EfzY26eUwL#NTp)5l20R9D zgFO+-z!miTcCLY#OP|1gDfnaGs^o9LpDy+9bLX49eoOAS18 zrYG$RWT>`i=iYa;#?g@mdt*W{9jzoY(}qr0c_4G>O3*+Wo4xiL(!X^$K8Gp8S~A`K zyod-&*uQr21ci!4Bn? zlt^6S=eHqXf=`^#H8y4~8x3^POcB4hYzqT}zB_jS2DZ?`#{~riM(?ez`uO>&Dl03Z zqPJIxs){Hbc~G0Lda)p%`f4NTSB+grf2bYVaNyvDJE(K2KMyartEw1Cks z=H}+%d~`6Y%YkrsMro;}SB2Ty`nrmxrRAho#oPS+$kEZ!o{0mnSD}ZXW-mN%X=!nE zcUN>b4Gs==9e97XQyJP1v+Z4(YPIPvFmJRVu5ND)5vZrRLbmRF%z?pJ@{X}dDhCJe z-5p9vP0c7S7KP3^s$yzts(<_TXX+j$+HDAnK!87>eIJ`)p}31L_ovx0I9darc*JZW zeFyI1;*w`k172(;`>Q5B)a8}a!Mn?iMb@96GL?D|FbaKzmRw`u%#g`CmpPzAV8+lPI7Q0BU0Uo4ab-J5Utj+dmbzZ} zQygfWWp8h7tYPkaYqRu_)5OaF4})Tz#{F0ZaV4h{}oUELFMekAs<&5#bj`_ZopCCW`v z&4id;`U^Pf!eOCty5IUdYV7Cd*VEhkIYH1=8*Zkhr6n#Q5&1=bzdzf~SJ%SEOZqJ$ z6yO_a8|9fAi4A5*tel(FL%t45*_$17kiU~77vm6c_L4GCtK@tXY5xHwqC zpCD|h_^n5}i^mp#=z!C!xVX&!LyI0(Yn5+WCM4-G8q_cWezVW_<+J@uXLvbsHPia~ zI-TssdcS4#;!qjm>UXo%H8eD=Uhgwq?o{PEHnRokaT$l8dM9N=A_aEES%{_Xghu5T z7Dj{DP&*{;i`vB0G~?a7OB;(rehoI8i$lz}Z{KdAKk(aGdT~qfxz|s+)UMI=^iy^Y zL$k5^1Rjz0`2qcr*Hu;UD~()=LDJJW6xVi8ND}Zou>I!ZyD)G<7`y8Qob~9;*7WqW z|7>RBp`d6sSZGWPj%Y*O-M+iE^z{G-VPu5k$Z3Hm!1?bWkHSLhW)>GCfgLwZd3E;p z`!!5Zc6Pv$7!@}+x3!IpsMx!=fT?L$ED z5CY82&okW91_+~XV9+pJ;T`6*k7N1yg74nFv!+q^=oB5mZ{6FsPbpFGIO~m+s-yKfmv6w$aB^9%Ae@AibE>E$ARCotOexf=NAR8&p{US_|H)#BqRW{ z|NIg+gb9Y{Sz;AewgH=kETN>VtZ}Y4H>0HF-WMVU@_Ppy@=nrWSm-O@>ra5FuA&T0 zP1yhl)C0%&K<|sgiC_!GiodNyi~Ig$m5h&qp|* zf_-oP;Q6GHs<1o2GY@58u}YF|gD1av{-IzhXz6ujiugTWYmaMT6w!Z^M& z+|khi{Q-dXSiLQ*cJ{(Iv^!)+>_{*2nrfMunPKVL78DeOT$tfF`-LT2XaF!H5=voMpjDWjGnh8lw zu6kFP<(XHNhB6(1(R~MWmiHtQ>1}RqgrI3@TCc1JW*a;SLE@AnP$AfgXTH3;290O{ z?SM1RNBhinZB54M92N%61*U(FU{;00 z;RkpR!Vak)I0iL(*$PrAh`cik3-Y=5fJ(+Ih)DiF;Re8oUQb_NL(F;A4(D|9D&Nno z!dSDHK=mw7iAcB*IKIO<5)xsF^X(6%rKh2>9&#C}yjK*kI*qn)SeRCo?!mz`@$vCRAnKqkTtK7-aXQEB7ufaZ-@Cg~ zSy?$gn~jPB5_yp42V55l)sw5+5AqnGUatU$FFJ1#vONW!GyEu6J{W|EbdT(QB z`D+;0X;&@44;H|I5uoh{1^D4%cF+X9ITI|j8z6QB@aO>$s_XiO{|eZ3eLumpwZ8S} zc^$s75KcrPsLvWBITdz36k_wf5t@ven!UW~Z+ItXVq!8LZ)hR2C<3fj*Tf{SrJoNS zOxb`iACgkr&%i5y5vkhQ*#Q+JH_Gc2BYrutyt|5P1jx|gtO!(I0OFum#jsQK#fuli zV`F3WFp9~*X@O_cC!c^xE6|HWGp%)6-`rFMk;#>S*?U8Yc>2^D$bNaLp0P2OMT)rY%WCS6|=fV}*i(LQLzFb$6ClCxM_Ou)06yrCxxq%pmN$vP*l8fp#Jy zY<9PnmwWVdKF8@O2Dnms4J|o@F<258B0qY_mlB_(A% zCmZ?nqFxsJkdl2{lKJ1a>A~P3f=931xsb@Ehc@kXnH+Rs$US zlv{{71EBBQ@S5ZJ8uvB{ToMuzy5{C!RCgQiHkx_2OF-@djahzYCgbksQ-*0v0>7)O zs^*zhi29O-WfuAm!k*uQfDd>gQ)x7ZoFriWJN`Sf**1V4hGc)oXrC4FA5-s|K^CF4SBStk(yZqq_frT z5yDh9;yd)-l4-wXVm1Usi{QQ`5c?WS<6pe^B++liv^fg*XiFC3E2!BRWRaNbiEd0t zOl(Y(@wNtm7!rv`_40L)MJ$*9x&bjWn`nj8alBe*`sNS7nu-C9dYh9Yvr+l>?FC*P zBs3*Z(Ai&OmVD|*qLJxw5 zX0qti2ON;YWuNb?ZlW|kA)!IAZ1`8~)@R}IQ8+mg2m*PNWlP1T;aNBz(*GPp6kPi9 z!a@GRFX=waq8iPq;M)eG!UUhvwwjs$*GKiffk<#bXZy! z6n#NH6U8p`E5I1YL_QTF_U6k0((O0c)zx*N@Ln@d;-~Y`^1mw4ws8}{(4v71IVqV5 zP}l(p%|tUFioGEhh_dv1=s9=JdTpk2dF&x$15mv zuYI|87kv2q!Kqu@^z@#}=g*%-+y*~v?U4~&AOc4L!vVZU#dV}o0_1Bp-yG z0GIXP2d8>ZU|9G*Kqdg-hybW42XuVG4@)tJ+^Hd&OX-cniF4yX25E#13VWlYoQ4(wc93(ED(4Hz*Sa)pfa3y}RCP*!be2-oE8RQU&%O1hUD0CfrE*idw^iFlFVViv>p^87##o0wNCS(2q0> zAx|}E5ON(BDfi*$Tw6P4K%{0qoho-5IsBACyn)2EfJaY!X108GUN!3E3>X{Ow!k~L zox(xF(l8ON#0k=;{SdN=*{s>x>BlO*SxFYY1man%P?Z2@;LIxsHYPx5i3C>Gkm`Y9 z1#TqxtG8k zv18xA6j8$8&hH!=VvCH7>;$o5m57UmhEyG}I=<2>%;mM+#tg!@tPpm0B}vM|0@4|j zvw*}EN^WavYUs-k!p2&!XL9aON?M9T-v2UnEnDd`DB``&%X1wV&HnH~9K=@e>}4Ol zU0}e`0A4@*rWCYqQDeCo)z$J)ll>NOw`XT0f}*hyumx|9FiR1FyaAI{207`AgoNWm zlmjr^_H0zh83+r9eY&-DEi4pW$$GbL{R++R(GlkBfEKfZ0bn*t3Xkm{uLYP8KrJ5X zN+R}CPmUh0Qgg`%SPqkp{IW8Hr^h*DPl9Ra-nu2~N@jo=XMU@1Natt>Rd5ScOvavo zV^>yI0MPV3so~n6Br%%5u-Z$R;PHkWLkF|{43#Bb;|{%ukN*r>R9)Bud>Kq#AY`ka zD{6ppikZ0)`0;hH|F?+8h1&Z$#C}8He}#Ide4~s1vh&&ZI>3)RpX+`D$shN@M8DJN zA9r54{=IPW4QN zs{~)uX1?HrYouS7=XtLFHMVb`t|Yv$VAs``U0aR7jSmCet*X#QS?>A}zTmK1m?y>I zk5c!uk{{12w%w&^D_osEnf=YffBY^~{J(bcr#${cJPxuG!~!z3zUw>uVL^3R=(1H~O%vpNM-+~f0AIIp)? zpa1YB%Bt;;T%5mVbH8v@Y$1p2c1PXrp5WiKvfs24H%;Pf)Cd|g_f8`$1ZVQU==Iz2 z)&)q>jRA!!vNp#iUP11!0m>3qX@_rJ(Yrw_P}-wLkRvXnN{gb8h9UY@*+&uspT3Iw zVUT|_t%8(abj#+JX|4m7-U-)^J@{eDw4Vr=$fbU0vljcCd&IG&__Y7 zlU`G*5aaBYn;V}Iq;#llnoEc5))F-T->edG!y?J$b?+A%BhVM1cq#d z>6w|_@!GP)G#^5A$XW)AobQ6?vqPSv6p(<7gG76|sNY@b;L!z&g9E{Cd9$9zCsegi zoOJol$Gemun(cc<6BC~X8$}K&x#QY_D{m<7HS2x9M?(}$O~AOAn-jiJ^ki#+vz37> zjminCbHswzuW7rxHXwazR7eHYlkJJ3M$QNx4Mn&%4V1_gHVEJX$Z}`wfGKT&HJU>C z$~O*%IPosGK-Qfih23svD!qF4zd_?qvxWVaru`)f*nhd?U&8n=EB{Lt|IKgylEwdN zN!XeAz)dy6`8XvPck;e-`G}`$end1Cz2UbaOm~esiM-a99 z3~5)cK_?PfywVLqwDa$is$}_DjN4+E_1ddqJtY5MXA=FW7(~|eDypwyX>augt^5K> zh9#_FF$&Yk5H&=k5GxjgaM+p!nu<uZzgo~Y^; zj&6s$=tYZi6X8@v8X(9#N(i-f;@|V+{JKe{?Z&CQhMnyvPAs?2V|}%U(XF?r#B)%f z7eBuFLA4(-!cK4#SjU)rToXE=j7doFn!&K=?xz*ak?A52+&Yv%Kp+x_|J>G$iRfBp^q|5EF3sw$3pe^Zkg{JUa_AO?1Y(!mZ`gsa9c#RW9lj{V#u0{_H{clr{CW4gFE1;pZ^G8I!)Q4>SA8C} zJK1W9rujN&FJoC)Y1ZJU4H+mO$3T-$FKK?AQO!fN+0AyIz~7yYS$*-O>|Kv@iHS;B zxlvF^Kejn1`Mk%@mSy=IqT}i4I#J{Vuq@_}RN)aMUN0Iur&v;u>uoki8w~jxLpg z5>v9#`lh8Xcsu6EJjL7)XToR|1#3^h!_srwKrMVrI`TjiP)!2Ix zz0I@r+aE~?N>;SxC|U^XzfMq$4j(kFnEBCnzI{Gte-`Zr3bLajoH9t-=bVFC@c2~d1R#r4QX9LwJ{{2>uACDQ8Yz^$2Oe~}{ zsx*!TOuhd1Lcqq1c!7~N=?N&d?-yd$W2Zj^vNRf ziCLmFYD7>bHi{+FK`C1O=W8>^AN8MOPaR2IRMnu*I?$1LE9XU&I`N}Yuew8^+)me! z#8`EKR(qelsDe^V@}U0oJUY1$myLH$LO;~FflJ{+4Xqd4OTTEQ-qDV~-DSL}=};o4 zz!5yV$bQ35Rj0Evv|sJOG3HN94CGeu4-$`OIHh-U&jz4a#HZ%)7=MYivlG&a!tA`!OeqGH$taTqL#4ddrq31o&QRZ!V)HkzgFz!C->(LE?<}S=_ zV~O`d^NH>qS2jI6PXNRnyWN|*o|Y!PvzxXalZKQ>HdhpB;&ki_B;&xW^UIUzg;+J( zjq5Y>R)X#XdN?3&eo*wsi;mKOwDhf92mI&>={SP^71uS zltOCq_H0|f15jzZS+HcgLB>o{ZNSRM)k;29a&w%{NtQ#>9_kk}a|hEqqL4nftGaJ5 zHd}3a8XPZZO?lN??Ce?Na$>L+oXXe0NF3OD1t$KYC|OK61)VE3A1)(TZYxlwu-&sb zy)f3LuU=xNWvGg)kf|zob#z^zeeTsUV}~oWCY#3JC zI#to+9&~aNN2{xwI{qk3z`EVwJi27ItuISV)>~F4RbM@R&1b{GiO^z(*&Q5yzmaa- zRVJDl%Xj=B4CcA)D9I+{8E9C_8S-0W<8cQscX4zHtDP9^{S%3l^)~VOQ%gf z{k&LmKCaqvuOjA+@1OSqbWG@EVqGrpnP05%zpecEKy<>6nV_EcgpABxakVaqu5@a_t2`E?xLkwL7i|4lIp8^C zd9|&&SL*2v1^wQj2laf8!p43TQBbn*`3YumEnfts$8VoWlW(8S~aB%j{D2 zEz*q&eYFbx5bCs#)LXjR@h|GRy_)D=%?$jFjN3XHy-kzDM!duZudRhq`AQlbUp(Dc zO>RMH03&)AANBeg2-RO2>|MHLynHP=hbisstRpCsV3JCvSiINzaL~Q&ZJGHOM>LPX zszvg}P20syBgIoA#Z%UMfWf9K#-qEli)m{G@=L=~Uc_nejtLh+GO|Nas#L%_(p~{5 zQvU7|RX%vD?wJ-!JEPZVz(Rd?xWc?5_Ev{^#7&|2iNK#=Cj!({>XCziiluh+Tl5w%4Qw1tUpHlMe!H5G$pTq(j$z@*7D@Jqe zfh$bjzVWvtro3b_Z}IRDePHh$KXd_M+(6^y{UV*vNT3aXMKTKa&cU_i<5V{hQe@0{ zHacKxeIIv@hxzt8_KDqwuWmjwXo&~eLVBa}BDg7#dHAuNA+c18OK2YkwmxRibUIJK z{OQ{b#4@_GSL(q4V`>P^uEBwKOmpy^hyxkF>MiaBTpuvI76{+Futl1UfOs0Sh7Fm$&7;*E*}c zyBWNv-m!IecbfVHjlRsO2w#vdl`v&gcz!+uZuub}fP76l9qWumZ04Q|z}e>o&N>is z=lNF2aKMDT+U?JKuP3+lI*3}P0>I(T=8Xc1^s%4j6BScR23{YIM+f*5%FwF=ebp1i zRbC^5ezxU)0iDYM56`q;-$);ffiy$WS=6jCHXZ2GomIE&zj>%K(N>Mna?}T!+?7f> z3KB8%N=6P?PCC5&jv#tfFF7T(>C-}TTDoiu=mAJlBVQyDc%98e*h+jhO;1t~%S#?! zt24s{MU2l#`r2<8Vct^+nAwTM&Ae7&i@h?O&FhV6k`RbqV}iL-&Un8aygY_R!TUKk zuIgk+8_J{K_}TRxwM5$Oxd7z~aKsBqy`-x>r++E_*2Q~oveD-i(&>x`v9dtk-s`i# z-dl=_=ww7fE-;BXbgA=aXGak!Ra600)lK4t7FXL2Pa^XkltDQ%p-Yk%WU2a)){mot z-R}ff=y$w(rDZJhUQq)e{Chv+cXm?TbhXw{U->>+CbODA#|FpUZ{+s20CVZkjGCPM z^p%}UDQUJM9C#^k;>043Y;Rj1kk@Qyu+ABqtf`LOlNGynE_#9D8{$71)7BTV`&-&* zL0ul_@Qe6|^I~7T3vPa?_kdX~Urgi&oIpp%f%dWToW*EkG7iv9N@2r^sdBZ}Q%KD-YqO8m(|bvIXNHqbJ!C6><6(CQ zo^UbsK&Z+)fLUg<>Z&@1hXNDq?+Iz3d)v0!W+A=BO@w&`?OK6Yog?Lf_qI0G){2@m zPegF{7uF~Qn?F=R{NxC?Hv(P|TDAaCVTz#&n@~jbkk*OC6udI4n48XHAAU|PxMR6; zu$Jeo{-sP{tRvTgjr*{?Y>DQn85!`^;pOaGE^T(Dl5q}EiYYNg54X3purDGt3I$}L zr74N!_(4t1(alBqyJHlZr_OmC4XUB2Jc~1n4*VeECWMnj_7#4eZn%9K~2xjwMU+Hy2HOHVSfC$BM_j;-J z3D?N7C77r4fI5Wk;w5Pwm?tJ)V`|NhF%t{Fz|SCMyS|jgE`ye1&>^?Ex?Nhg7nF;3 zYsb&(ZP)g4;d*o3g=9qw?*Nb_5%zjrm3E9kxH>X#c-*~B!o?kDh3f*rTu;=20hXBN zm6P^$^c1jy1c~}YhoEUq-zfP6rU`~PZh&8_d+CUTI40FqOm2wwUGbBe`+4qC7GQ7z zb6@DT`!;ac8SBE<1drzF+fvy8&oW=!0;Ve!XA4WLAI^JlkaXzWzPNl2FTV71|TR1-pIV#^NQVm)YiwnVpPZ@$v$ zvDtJd@p)EOx#z-)%M9CiHpTe)evBALe8=mF6;W*+PFxvk&)Sy@Aqzl0^M&XaG~?5(F=fg13EGYrWosHU*Oy#Y-o10ChqW1CCtm@p)qe*7QH4}K z?tpfg3}292*PNaggDlE}yk!kSUx9J3_q(a6*Aox{`Z*biYNEphi|Mde``sD2ve`9U zy}V~E;bVLstjK!(LOoMoy*JOiSmvQZYq9ra5xXoJL<0rY4pMtrDMC>xcm$ESZ~m8r zzc@;+3^y-UI5aYYk;z^%$G=~jb2Z*9@bOzao%RdNzbaj>FxU32pRxPU+AE*ilww*> zDd1^(E^S04hovSO-xN7d$p2&doR}h9lHfn>9YC4{CTVcYJVDTf5TEHTzz|vrIS-(i zr}*8QJaJDKq<1`{A{Rl}YwNHv)${0C2~S1HJRH$@8gFsmv*}<$hXB-=)l8R~p{F@OAmUR4=}lX36(;tY1Ck#s(*b@R z3;DI2rw3AcX<{Wgb1&DswSn;h>HsV@jiN}YyhrYsj+YY7lasAfB8)CSde)wH@iME@ zY|rEk&Vki+z{nFO&9>Wa=X$g!l=$tR_rj|^WvJM0u?m6$?&M@t+ch}3nc6Ma?J9Y= zqcIu;TjHp>Wq|H>+uSRyihv1*7dUXq8idVUuhj-d+x0J5kM7~pgLpSdJL*}}4G~8U zxqWxN$LT5T%@2W2C(sUY+#?*p6?aC`jmsum^jmr#bOR0%0bHcO=dns9YMjRwYHF>T>?e|1yVl7;lv6e&C*+q+%o*0T4StOjv4cT)qV!SN~u3DKoztx z@v*YrVRM=RjzOB(U}|lvztjN1)n`ynCzRy{I~14&c!O|z7%WUO(|i}-;cDjVR$#hG z^0|#W)svnq8^f>}B*ui!pwB+9205eR?#)uJVAne%DPQ@$CLV7BEGq|ts{NdWg-c7h z`&)tlTJ+wX(%HDO_BE`_vG0iHp+%5t?dut&Pum-4%eLm`7{J$uV}>MyFSsO{y$U)6 z^AC5<6iYQQL(TWsID9qxx`eN(@Yu9`Gy3*a8n`Pl^fT$Zsvdn@t3%oWahn^DVkM_y zTPI+l%=3A^4y~1bQ!A&W1k(5+tVGJLT>*5XZg$WV5FkI)-Ys9wp%jpDdga8n^8=)d zm&(oH4VQb@t@%A+WJiFEpmiidGt2O+w6DGSy=zbLtuGaN%xF zB_@#l!>o!xkv3qd!eeS?;6AsSY9_ywo4;kku{^aL{Y$|9J&Wc@Gwr>(9>M1NTrlDsp{0W)AekFLAQE!%I%MYb0xUG~_N92y0kpVjABO*0O=}#t z19&%T=N_RMoKrXZvZh=Nx*yqp1mJ9|UKpp5kp;jjAyZ6dtun0Ey3=^EV+w)?@>A~S zwPEE_jsR}R#Q657Dl1z_Q;|ApUK*qUfW^S-9GLzLnG^C~DBi1id0BJgIN_qJ>-Yz8 zCE2sv&4AY~Yqo1~t33`(uv>kX+C9>#=P3%a5*eso$;x9l!@l30d_@sc^vV`miwApkSANy#&7J%GKk0nwgP z#qtF6qn)A{lC({WRDF6cXW3AI@J+!!v-0BI1Uk7)NIuU}`hCFCqRm7d8-kSq3W zj%QskPI#m5BM!4VF*7&Q-eA`+^x_C7$j{9iQ9{zU;m+c00iI+jJ&17tLSetSH2pyD z04$<;@N2uVOu2{};36wyqaInEt|!wD4~Bi851_2vaBX*u8eLkGXMqP;w?sq=In zfz{3f%kaV%3m(xaP07)+qclM|i?B4bUOS?FR8P)pW&yeHx~+!8&+j97zRd@OUtrHj zQq(AS^QQ$zq`Z%1UCtn`XV@P~+ik=gV&yIEu?s2sE%6WH!1z=^w@>e{8;iVGRNIf2 zaw|8qpK)CT?*qmAPJT@Tu)_@!D!bM}IlzX&&|yXZo1$Ee;i@2Nsl|%v@f9yN)zfD` z*!X%}1dMG=LLwy+;>3XFK!Cbuj9r_9z!d6DtE4N2-*B|}eKp8q(~zFD1EId4eHJEA!um@(L zRioRwyj%~c-HlkNC7O^a&2L%ZZjM#i4EJvz-~mR`sCJ5c}putjh_I zvbeS{WxYUR?Y#EuujvBPy6~iYEeG4(o6eEj+|5?F;x)fx%#bm+0fmYUpas-Gsseov zNDr`O4b(UPl;GeP!7c@YzJUdge+1;_j{;}%$Ha$f^D0C(Fw7-&sLUY|;Cd$@fHwG| zHU1zIFmv-9eJGO-lx2wJ2Kgp)x`*dicIg_qd;>AHE`mbZt5LhcyFbBdbLZ#Jw0IJp za;x#@vrD6M$CMZESo?~D95#-dBfjA1X2E5UQ&jUzdGXZO@OOdyJXW(UtsVGvUCvln z_Wl|Ke=R<~D4}XjlCiPXuduOOy7@lUGfswps2tV=f~>XH1v!}O^xDT2&q5g&*jWBN zX-31ZM7X(v?-F(?=;0CKqlsZc4M>w5?mNLkH9p5OgT)+La)od?&2}AI=a6%1w?L5$ z3*VB60&&D`Z=@bTVr=zR5M8iB>6*U^gN^2ahJhlug%t@fp|o$;&q5;zlWoz$1@nt%E?oW@=9%;M1wW`U znHwR~SOqUbWG6x;Ll@V!JNdB~nt#yDTi>jV!DMcblO!jn21TBi=Hj(S?2cRkgNlzO z&L500&FxjMF~EU^uwZHZt-@Zt)WS@>E3aD1l$Wfm+^xm8S>S0H;7J%c`urE;)#>@d zG&FSjqmMXR_+#oGfdYDaP_R(5Mj^uim_NEwE>JH3^WGn$mV-h%{!7yr{|u2 zj7GKuO2Mn<^1C7L=4m zSw8kPy%+gHBmQ9b$y~gvJ((1Y-&-J+t$C|w-U7?Lx<-cLyeeM2`ub{ZW8L1r*)uQ@ zeoKgd`~Hzvl9sKn=)%0L$aU<)h|3HcoJ|-7idWF2e^fL(ZlSG${CgF=67AgnF3$vg z$m= z(8DM;C!9P+IFy85g4`Y+uOMRy>W1Ja_KiNMq;B~A46D8ec#b-b`7|&?U`s;j$v=as zj~#T%>~$6r2pB&O^HfZe%ida6>1bz8gDk0*ACxZDUG5Z_dC>{Nmipq=Gns>#M_o&R zp>-ry3VVqxIF^pDkprbb0GxOW3F3PELi*6OG47R zU&FA{#K@@@d-NFN)F4|yI|n(~VU(grZkOmckLxJtD4Vx2y9{bmReq#q^6myLzNTsu zV+%L*GgMsXN3|5vMl5w2zqB%csg4Iy0s#N0Us#?ncF?)xS8reNZ~lii zd>wsu=rlb^VKYIV}eIhWb+r5!-CrpM-~ zbHrznQBz+jTpfLlrPK44)Ii0hn5vZ=%k6pNs*uG1?W2nmk1~D1XhGj&y%%2w+-K?z z^LCzYN>#-4CCUBkbJVy-(fv8XLn>#Zn2xJb5qQEM$t8M||nf2a|ko%Y*As zr66sHwu5!(-!OOTVIxjC;MzOvHqr26KOAOwGo7-+NW*(fC-{xVf<^3zY|wG-(Cgj7 zs(;QNe0V?5-qtpqilDadl9Xs8Y0|#u@(prlZoIz$dePL;BBoky8-!4#s7}7}H51?k zJJmNcMw<#2t5$TVB#)_96veASwQ(;Uutk{1k$=5@?)KWEkcjahB+_=nP|ZNLop-Ht z-?sOA5&THF5!xJSVF*orfF(4=#O&5#_v(XYu^}>XOmWP+V2u|H_OG`ne259GK^6wj z<_1hMM8D~DJjuZz6o`}G)1=N2_H;KE@reex_v#3Rz0P)%OE;(!WLo3!YOe=YkPLap zhqC{1*MPs>9^IaCWza0YyGMzMlVqxs_?hH%Wc%bN+qw0fTxy$*YBZ7Ga!b7cySgG zNa`S-xnUCY_;iS2I@HUSw3Ys1n;|qS7YE|=uWvz-eFAhsppme+5@7!>?&Z8wUKII= zY5iJy`8pMNc#=$OA7crp^cJGC;BQR4iUVo4xZWix!qoDsMNi+j5=;wSV2}* zq<-B6h@f6jLmkwYv!M9qlaQq0Z9Tu~r?~}9%u1B(?KlE0PE$*}W}F&5OuNBZz4OR zSFSe9c5z;-!c03yKXAkbl#aZ|RZ$c^{de&KrWgT*8U^8&duvOb z^6Q_v1GX?>s|%$L-jho)M~@%3U+qoxp2(M)EI?uS*(BYXYR78XI#%P(L|$*`h??4J zT3+y<#=9cslA%y>oJ%?_JUko}2TIPnNaeM^lgRbLpGus{GWBSic9C@0ZaKxuvY?ZB zE5Mw5JGU3er5V6rA_rJDk@%*pP|AsiiHQkcde0QIky%KL;gXNVC*#dZw~MP+70QSi zs4d4nccWAc74xFCaJPEF9-Rz8_8B{Cb7}t2%BOlAd*Vhq9_xpQwcUO?g)euqafkQk zS1k9jp`BK<1dJ=W%}2~VoHPkj+zQh;`R{)+6#|0g-%0`F~K0TmP3*;(EjZFuoHVGbM;GYW2H6OW&(z{Y_oB+n+Z zbIBmHIT5m(quw4H3&U{}jZu7R1=!ggGkASTz$S%|?ow)&KEqKCaWnY7&R^CJQi3EPsIPYQ6{M{Bm6?apcJ_hOcAM}D>lE(f^yGHLRt@brZHY$hY;~u( zASMc@xOO)PIyw3+0D3OdRxEd0LHO{vLz%&ME(~J$Eq8_!z3T;~CZWS{TIBLsy9lO~ zc6-Rm$pmi?%ucptJBg#p^43QY36XfQ?2gG!2Ch~I+7{XvV{NRsCC1ejrnK>Oc%{Qs z^s@9)U>GmZ!8RPRDz(yMqf=u$1nhA|hek%EFb2)+<@C?X1dVxO_SVUhXv8j_UBPFK zC;58Rvq>s>6s5G|he%@#S<7B&w%oM^cI4)90T&HCI!z&1GFpDg5wlhyJhdFgiCgPg z<0mNv`ic*sMltW@FH1k)3|Xq#9>5_4$RPw!xFu|+(6gQoQ(Y+wnMJe!KgE6e7NT98#qS_M`H#46hS@mutC#0Tu%$iKnh@fcK^n+6*yE%*iY$|7N(hmgW?IkqWZhX6CK@$#*UVEjozZ z^V>6g2wo8AmfrDPCw2p(U+NajdtUXr#a&e0*SequVVY~fpjWTyrE`Ee8W%a+DJ~yD zHEm>a-np+;#Fu}R(iO5VFON_(|Q zjKgk)SA&Plh*sOqJV^Wkf$- zoa;dOSuW~8*h(F0?{tqOYSQTM)3Rsx4Fzxy~!D_ zy5YW|ozlBq>f}LogU6AEc%1@M-{7?u{68&gR!faCxgH&`Y8%z~{l;$}Is%l+HzcnA zI<&eSe2o_1({aLcO$jFxm5CCrf29y+RW=mdM}B|LYJSJ9NiI1W&Q-yn6HL3fwM|*~ zm)HG8SJ&eHlx0xlv0TYoTk-Y4js$}vr9oBncvbZBk{;v4eO0omk*yK7{&xZUBZ>&b zS;_p7pK?FX6-DLX|5!MWFQ2cIJ8^=eGqXD9)z2;og>{c_v)!~XIbLQu<7-yd8u?)N zU>d)CSl)TO|4eh(<=_tSnO&c~K_t?#?8C%bPPauFGwku3qjfKm^Pn{&RL{^Y5Xb&lczM_Ded5W9N@bfscv%=1&-CdN}XZJ_3! z)Qa}>*ZYlJwG%j{$G=>=fojAi3=Fyrl(}#zQXBa=9~KG`y(*rg;-aybWvSs1@}mFr zRQ3rJ3!|Fw?~V(;a(($mVZE(=FK60}rP<9l*Qb=PDytaNpGK^l+8`^2Hd^|#uaZ;3 zeha=C-3Yzp3~Y19=^XbTJ^DAl+^=YdG1aze%7>HG24LfxR!0dQMZ zljoc0mQejcnSV6i@w4hrFyGojDAnP+9pDNKSAV%_Y!MC=_^8?OW2;i|&z(h`d&kWV zgr28FkJVr?(GPYXaAFb6)2mZq5o2Bz%WE1sXD?rPuj)4MJipYeq*&LtK4j%mw(cJ{QIpSRD+{Lnf1hAZ^E##Lso0eN}7FRuQct#K6 z2i=U&L4t}ql(5T^n~JtAi@q6eU;cv@NU?ldm{SiFA7yoIDU{`xL5(izZE0p>Tsa2b2hvfumB_D(^L~B(+eU@()sr`q44;%CXveitvLo3` zoWNG?jwIgMX12`e9X_nProa_ef1d{>ROdJHNA=qEh#dOd#DcoMV?$Qt_nSG4iLH<7 z;fxa*eo7=?3-DY%7O4RFg9mBCo~kER#3jVPrYr4ovo3MZqIHjMSH$-t3-B~L$t+D?;80GAbPwlT=5_UN_GD8{pYaM% zf-as8E`MwIRV=aigUM2Fw#IWm(ZZpL9h1J%(I`QJdxI)DWO#ll`G{-jaaS|+&sAj$ zq>UP&y!>ec0jjNtqCe%K;v<~dyCTd94Mo$ZDqO-FXAHhcUb)Bg=u7a7|G(WT>pmC{ zJ+E+M>_9GalEA|pPp~rm9IxIxPi3^_tVJ%uE04i-b9-5o0@(F0*=6^8qrHo^;y%AE zcuQIzKuRwhU%+D+auf6KSe7yx!m0kBOP6v3^zb45mUSc3 zV*UOR?gxoA!`N7Ojgje3+_BQu^5Q>`WkqzY9>OM{LN7R9`(^mh7j3DV+-_AM^9@zG zgZT$T0x{qC87}1bvG%cNZK|q%lxTb#-QOY!O}vlCx6-pZ;3i~7m0N0$r+G63v+2j9 z0fIyV>}(0-xQVbo8AOz0o`c2$PtEN)AX$7+S&*S)5^Xi0g4NJHW3gA2YRG#>)jwPJ z&*2kHel$s@7ztwJ{Gs6aMtw>#J4%i(Bt^EltVI%P27Hb5jV{GU?hkD?<)ma}wJ*oE z92j;CMfr;+a6($<%d}oeu`nqn2WF*9rm3^eS_!JWei2kVagtHlVT+`o^^EG5_79Z3 zCpR%eFsXLUk-f&hYNRgB^%M35tYWxnCa2<+XotsOhsxZ7hzMSYUY9tcNtDoMt%^lA z)_l1=emgPd+->Sr!ieh(`~1~}a?%zP;+p7%kcR+v*il%ZQs~YeW#QepR6dh3@&%>{ zMBXAGBSumo?jP+7%)E#IDf&?=vb;0%zVdTzKfR^^=K{6Y+J2TV-9pAqcG{jlRz_-7 zq7F!3%OgFJ)q2JePA%a7F@q?=JhWTp+p8Za1C|NXTh@~QuBkyP^gaUEEc(AawAv^g zFwFQV$TujWqV2H~CL9dbsl?7Crr zY0+RnQ6>{MqV0)$u$>fAyvvv9rU+4Cjl_C^oN-70QG8!N6q ze;mNfn}D;B6WYdCB!+Mp7h0Z0Lt`sqf-s{;%Sy|bS3siI{XV$D54e!rAUKV#dJ;7H| zd%aMCUz&I0%iGMh@80*{HokjjKOQ1h@p_`zb;5!;l>@!G@Cf|j{t?vSE1tWQtqPQ= z?#0ybNqlJQA$bf>CY)~8_C`7K_AWS$1(Zg*-=~0!#7pOJPj zg1hTNNcfP>g@riyU}DOo<+6Htq3HVW z!&GDy+%nwUg0Hm}sP8y$>tiq3fIpGkeV#ZNq?_Q7w5tGS1xx~AOn zK}E)o+(Bcc?yIQ_>-p82fj$Az$d47=aDH7ckn)%x-~aHz|8nN4_?3#bYWT_3u_)fA z=xmAk-k83jMR#*%uLz2tM1OVIfAU!s+W$ZFDAa@S*x>(wqBb^U{!AHFoM6_Wa?Ylb z^Vf2iOn$?UH(wZd9+l|WM$2boMqsts7Kw(89T+pvaw8q%VT4l(gi=vB>aTC!OGbI1 z(Ooe5^DV>p#JI@u3e9Z9%d44Ar}-9U+PF@-C^;k~q98QEWIV|L+?EK>BZgNTFU($* zB6H^CtGTKWmq0!zCM97ocN%XwuD$!RoyT;V{qi(UOhQ$jD2sQkg*QkF#XXR^bh$@D zQ=fAB>aWmfwLwvv2fSN}`vBgg$IMnTiUv~soYkzl1LtS*&s9I74$dOe&uqx!?EDpY#{_YGMbIsU#ok;i2|%|dV86`x7IsYq{w zg)Cr_NsALN3?!{5gerumYek;HLfh4~ov!Hz1A2o$RZhn&J^er0X0wRATrW%g{F_`L znUh8MgY%ulu+bU|KG;88xO^yJb#KP=F9*}JRI~(zVP{V?blua{xznMYYgEhHGk!cn zYOjeY=NW7tuY!#V{pHC!flosz`fR=bO(+(e|Bv+T4;ujg{8qTlb}V6XuFU?^75Vb= z@A>Ul1gXRkBi9!z@eeGATRbjNz2H-$macUB(_$*K#31zqG^u}jL3<%r-u5et_uiM@ z1pkTcS}H8RMuTMR1_DG)0*hK^J7|*AUL%yFUG#IvvMyiuG7X{jFb=(*+8h+N%tu9k|D#UsvuUt!(Qo8F*sXY#9#3ZEDn_K=; z5|DW<{xB3ib>c%g5>IaJ5=;vF{YO5+{oW@m;j2jlGJf7EQzzI%xh;1u(|U&C1OWn< zV|#WAI+rB+6X>CKVPG8e^6r^j<*dKgOn_49Hf!1WQI2scC1UsmHq3BBGUCw<*gN`i zZ_>1v4uVb5f?T$0C>b9mLu=?R`D3VhM$M=Q&)q8$|GDd>YEgI~EP~`jnNiGln9|ImlCV~6Uy%x5Ut7|m=p~0~{%|p7t6t?lF z*Do#pY2_43aBpl{%iw&8Sgm%%z0xoAqiH)1o4KQMfEP&oaDw*U93EAb?WZC^=nB zK0BB0IV*ZGW*W}8mA%Fe3KTWNC0ECI7uOL%-IQul!pX{ z*MD;^O;bMgz&GN(=Z{AbJb0*%D0jCU99dy!zqU#rdWBTJf{CsFfav|!ljq$36-~nr ztMpbS51rPRa=2}`67&v(fb&$nTjhaUZAtDbiRQ`z9P}a~~FB5*iaJ#{O z*5V>cju8$vZ;nit6pl>{W%@)M(>S+FiNzZ)rGj)Gi){P6g{hgIMId*=g}0K@pWAoD zK!aF1&>g3t@&#F=1OXw{SmuJ@D1ndHtJ_c4Q7?dvDGCr=wn0nvl2-b1#^^Kw(=88 zFaa7e{PRV~?N%m+eVhFf_l0B8lQE8)+%}t-xewN9ydC;#7cv3hQ6pnc=t!sP$Dx%Bja zDVy&$>n|7ZS+Dpy(mi@`hxc_aih&z!LLC?w?&ay_8PS9{{>>Z;9qrDrZkf%g64D$= z$*QfV#L|!w#1qT94d8@V3Db}}87#+wp$!BkkCm--i^kE=ZqRbGw3}7}W_weO>v1K5 zj|U(VF95ZGqae4aKvq%T$>5g=wz(Np6_nSKJ zjoFU=a8Qdlx2Z-DP0kz@KQs^pxs!zmU0i3{kcsob~v)$wgSq;u?N?(;+{eAQ}UAdm%t(Qc9>DB@Mp|3{%+K~W9E6M2A zbVTpxujHGfKWZlyaFNi&oAWl+fWV^eG~c3nSBkg2g8VPQpBg25ENO>pBdEpA|1dKn z3+2cjmz)1VCa`RraTG{t1Mdkb$$i>$F{RpCXn6%u1nW|=Id9=ILGkg{ToOtzC_(#_ zX9B;R%ls%yt_!>u{UjZKm_eE5!{{lQRI3y{yD*M?%*Qsa|5i_^h_Mi30VgDxu1?!b zfFt>erIEggfQlk4RAsH5ECG25$^QwBreGj(&zsS)AMR#b#orCkOkaIY5%vN#IrW4f2m-voX1K)#j1*Nl9`WMsS)c(~xHx3p zwNP1~vZ-jj-r}n_=7yLC$jW7?^^eKk>tFVN`OUQI3d9D^>q@!`P^lXkqB{umF^^y8 z)cl|%3Av5zjf^T-KDE8d?rWW^A6Nq7?Y@AQ@g#W_M zxp*gaJ2&gG)Gpk#hk;pSl<)Dr{-X0A>O~6oo zqZKktf=J%FrS|p17S-`coG;tc``@J<3^Z`n;|q#WsSc=F?=+q0jt1%}=ec+wUu zR9nB>(fju5@B#aBA!)k(mB-KPk_(=V;RI@~wBrhqqflB*ftchF0Nu1FhMa(Xiedhl zTFSas<8rQ?89D+=j?uUQU)M;Ur!aZ%(-SW+N3>1hPx%!Ga-;YE*Q!wKZO`SH(|L@s z+i=vA?7s8&bkV=lhI7;VP11(mrvpZfk;iaLE>vgrlgVFu-se5Bu z1OihOO_S%_^sct<;ThP#WqoS%lUm7kd(7sZ# z9Hz+>pi$s2_4~nvY7jAmrtji-Y1V>C?&mn`*Z~9HaMnZqx$}Qcof~-wC9sw>>BkN> zdA8!WXByvhYN^eK`G0Czq>{S9(qvccJX7EOd{DIXc18d1*CVpeu|627ZUkz^&!n9U zPwi6`ID#Lm8QfB&6uBAQKbS47OcVIt_KR~Z?a0jWVlqo`aoGgWVE8j`Wkl8Gn?3_; zA7+oQw6z76f$rc=>_YMaIvaQZf)Fw7(CRuz(rwp8QAWnQIC8X+w3zQKO=Ue4MZdCs zweCIH_LgYSagVdDp2{sKlpdaAXUxVCZ_Pd2xGMGicYggJ&#N*-B!vD!YH&d0dpr7< z?SuqmNO_y&T8^yqmp=QVs}|3$AD2#SjaE7{i@NUELRkEDiqvQ7-J^r?j0^Q$VXnuf zv5x+NfiC8Qw7ilI1Lwptsuiq|C{bR6`r+J=j6g^hKs5n574wA0aFsyxG^UCOV@dtr zpiQpMn@8!*Dw#TQ!RCu@#ot`L8&sGvzOG@WQGF8_N15l&&+9mqv3-yr%sv{+iS%5* zFQY^j7U!|=9!f1nu8|x|O8UP3!SKWR3CmwC`+l7&IwO5Wza}{!eop)h>|K%7VLgxV z!Ee5J$2QH}!LOnx?{|tbckMt;mk~Z40 zHY}%m+#g8IhH6h2QYHi7wkN2dE?Oza@)otdUl1b-?F2y0wx!ni_QNLj!K z(tpz`!FwC+Tz`Ih5(QY~Fwe_18YC}g#~wfVp=tth#QNjcpBmXHKB_Jdc%zJ#)AUH! zwnjyO5^oV+jhjErU~9^>Zn6>e>7!j)4jdHte?J~_g9k8MeyvRSkv-<}Of8b2n0Z~B zD$6IbM;2&{-=}8;4NOgCWdo>?y+tesrY(Gj)3)rK4=}E!z`|e1ml(K3km}Qi8eDR9$AJCDJND7olA-4*#ic2B!&fg>V z&FLx%#g*9`1^|Hd<}k4JO!((2U#3zOtD9F95 zC@7Z*GA5N&lWWOJ0KpQ(y8psy!=pY(xb)yNEiKnuTj?ye-jY%Oqz=r4=GMLcmO=ZU zJi2kDF5e+m7eIDQo)5GF_tB7%uXgbdWsTlv_ac#J|8ZUI)rB1C z78{WufUflGbn*7*XEPw|(HW?tBgiBBDK~4Cn+_IZev3aAJpwKTs1zy1+$e$ln{}ub z8II~OlzXNmT}<~AagT51;(w5s`+NEHPOOh?s{3JZfYg63%()TrAe$TkntJT>`6f6K zb-BDt95=PHe$=q}Pj6ht&}1;8?(aKWgR%?VU9*bkrUf$GuqNZ{OS8U4V@`pWuR@h( zE^yKb_2ix^`AMx`{48KEYHmCs)xT)nddHJM7d)TVv*WU8Ct&S#t><>!+iNv{`sAlp zN?rVaMU~_@8e$ytALrAAb3ZgqAJGMa&IafT@QCJ+{mwC-5Y~P#t+^9xmg@V6@lSCt z(h9%ym^y1pu&bLKwWLm2Jj7BYg}QLYU^Y;SUR>RbnA-VV4tBsbQj^u{(4eAXLCd|$(u>Z*7 zH2JuaJZ9RcEuK&MA1}a|?@KP`V0vKFce`J*^}m`~Ez_1#M`P-ZM?+vD=BjeEHQz{X zM&#Q@x2ZcfOey;GHgu7232|9fi0Ojqlkk|$w)kOxwgOxkP z3eBtl#-;jJcu=)iakbX8d=LZom6H1qIhs0hF zK{KjX75Y^EI8;6XDNY3B1#i8|11BbA|#7zBd#f7RzcbkK_%dC`AcK08n>z={!Q1bl2r;yu_XJl z90^3|ddT7$Kvl_XtMXFKQ@VBF-5?pyTQtxGgnO0-gER+6TvDDYE7Qt^-tm6yKC7z~ z&B=g?vTm|&#dLyWRvwtM9oZL|xF7ODoB;4;7?4H6+L@}Xx5)&}SoCk{ zU(^V|&b9wOEZ6~2PzpKW410(c%Rz|;w5UxbU}Yk2ac=;#vbxH&mhb4+{UDDu`6b=%fWrDX?nirb@1AWFufK0x)o2oDvr`nJ{ zx1P&DK`3pb!Ap$g*ZXUe${|U_mL&EC+vTO*F7~ePiyS|hOmB@$f6a8SZ!`_;0msyh zU2lw`>RK%?gi;9bL@NH70@+6-zxiv61prJeoa{2TlF@DF9#LqQMt!$$tJq}5D@l`aG8O^x*5sW85X|j{ybw0TS8{x1^1b1`RkJ@R zFa`Wb3`joqQ)iV{wIOO%qzw6!p*p(d5OHLZZ?ZQxBPQ`micBVqW@yXW^%2BeEtN3* znKOdQvFYp9=nlhnvY7Bg!nI4zYr%OMGOy-OQkzqiBL>c|t?z(-Mo3P*UP=SW?0wm+ zX2TJ}GM;goA?MZ8Q5$!I z0s~zFfG&F(NQ*i8%Nm3K+qBnlxkIXbnIH#3#DNUm1cXX+vvdRlfO$Q%8Dv>xr7gf#iK9$ z5L|c`WM}U0$a-S>c!A!&n9_bkp2is65h=<`WxEp}(!s|>C&)<-GI3=@TIPkQa`xyfkcKFN*1az+lnxU`F}b~E zX8M$3y-4|0|I$a*kGmUU*W|$WBJMM>@t{)g`i z$7H`-15lfEX)6$wl$KIyt}JaCtusv>5rpmpm|km#pMmFfH>Vn}nbb+^H;GJ&MJbcn zvP}V5eo-^1SMcR$AlP6!AuJRTOF)4lWUsDT+rr+RY%N97a0Uvma-rbE>Byt;q3^8ex%>ov3{a*^J;J9y5&I32~X{< z;@Y01FCbm41MIz)x?lC)A{g$&xPut;u@e^Fbd&Yh355)-F+@$W_$M0LDlfc6CQkH0 z1`ypbB}7B69IHHC(V3A{le`7`7TV>7X5ZQS^r5xL&ndwZJJq;X$XC84_oVEoDhV{~ zH~Xy3-qhh5_a;e8Kk;jG-QWa1NEQ||m^G~6mq;t9?*hWe{0R+Ybcbsa$R53ARA2YL zc6WUcl!4%8?ykraQqcI;r{&ju8C4w_%en*O&k*{c zUg=oPvDOPs|1zDTmdHUv(3vNRNvS|s{HPWkmGs%IEXQUFj)-Ep$0}{$p4n!Tpy{7i z`elA8^;hU&RFklFeO;vB&dN@}&kq8Ip$csQp4XTdy!iE7Hm;i|n+c^Xf?7Z${0mUU z8H4FfzMAZI&&PljE9Bv{i$NdCyZ=c%fvd|)4wXU$BKJgIV$EKX)3{p^lv_@EGiy6h z8Zd_~p=KjqX`u|#QBX$^*=JE=`v>+~zyFy{qXBaUGP6-R)c)$rXK!Dt5L@e>AVW(5 z3NS492l?&t_^!9l{%Y{eyqfs{cgi>Z6f6J=CoT&XAtg6VV++KSBLa#ILw`S(GIYur znY4H)Xy#C4o}|bO!vR>a_R5_-m|i|uzSyFrLbRe~lWxcfm7urp0jl(q8n+jN*nK|kVrVfH_IqC*(*ha7aPm3~gWR216kV(But@oAL^dax z6rrKH`pb}NZ~NVLcR0~yD zweN&?yfYpUkm^AO$}>j6_A)Nfmj>&CWHBkJ``o9v9R0V!rKNYY|A2*+eg8EV&m%&y zi|*^D!$$+@%B)qjqa0CkU~C{vBnTn(Jfv^TY}{*@xRx+~UeI<9k)sXY7}Rp)$%EQz zjJtD*7Mf5~WlP_EXJK~ys&HguZ`J@`cW^XHyKc4elSE$qu1eG1V2D!2q2@i-PF9iO z(s+*c&W@cEQp8cw@ED8%3geq)^L9aX$zo}CE=$23S9`jV0w_m} zbA!JwON=~{*DQ%{nMR4{iHF?2nT#s-%GA`1c zT(kyhg%p(@uqO-Ky*4nkft|5H`(p3lq-j!US)xG2(M2m=nU+O>;2-_p?!7UTdihFm z@Dogsdi;8nF!I!~R0u!>@6$ICR~){_e*TWo=DUrXMM!OC+(RyWFL1t)@#*0>SWK^3 zUwjWYoQt?VVy@ZsCFKU|!70YMj+-ScHP+v3dqd1RbGugz1e0155Ls}Z(FB8(D4s`I zqf8R^!nZBO6=zIVX%@Ie>WUizo@OsTJ)b=|5o`Nh^aA4YvhMn39zs@-$w#*Gc;*2) zTai-88@LJaXZ9SLJ%uav*&sm?1u)LG(ELJ@jQ2KOFOuXb=?7VrQ>9 zz^Np*ut75k?|uIJ&VQ4K-oU(xS&Dge*r5;TstF(5AWi=GzM6D(+a6pHzQqZ2c|R9> zu6Vc|Zp-Eter<@qX+Vqh(sC|qU`-0pd-r;(KF6NxudC zjCEY`c&z(JrbIJM+sa(72nhS@^^tJUmr8e<0+yLMWnKD4|+;j@>ib438oqRpcL zSGas&ZkJ~s&_G#-E%qvKnJKfasynQ^eKIy^tu%*@m+AY<1v><#1NQ;Ql%`R$4--p1 zAWw-(MfB||1HG+Xv+J2@D?kYb4G7x|3f)WqUE3FJ&;rn!2uMJX!CbvWKWy?G;{BWyZ{{%A5=iTbX|RGPa~$S1e3Wd;%j5shU7o^2OP7wEu-Wq1V=5% z=!3re>Z)Z#dt*p_xPHk_oF`BR=g<#G#2*^UVBx6x?DA8K^ArP}mY$We zhq1}vFa9#|&))th*XVm&qRsv?ZwM0;$E!yP8>|owbPdX;gCm<_ml?E;qh?YdV0=yt zg&~u$s)iyC!3x5w2%`w05ZM{p`=bWn_NWZJ4g~=nqdwsd6j{7?aMI~8!!RWwlE-oF z{*zoR_%m1U17FadGSXOR#dWgts`@vgj05Q?MAh2HCWh!Q}Ps3SxI*68%^KmKZCImKZ9j1Ofh zGEn%;bMsctv(163wWp8-YNDM_yOlhBD;>KK5E}?+xGjl7VL^+S_gHmnxLgxw@U?RVxUYPsrIJ;ix!uvB*2=aDsxU7^RJc#YU@u(t@Tw9d5mKG=(I7!Uj4Q zB(KVr(1n0-2HOkjR8!aGsR#IL4E}p$bX8Ao`gnoUiv4Pl?VCoGN#8IEmIZRaF{MvI z0aa1$6)4Zt_?7eLRnlXBOr(dS*xgRLE5IW|BUmUQH?suc5aDJNbvZGBXpHZ44c_e+ z99p>1c{l3Z7D&O{v$BJ|9{n#=@^*CpYcFd@^?ia$z7#Wh-Q=7@XH}9fO4{ zFe8xPl;S&{u}+PXHi#qs?-D1JbH3Du>17t|sp; zRD?pZ{@djK@WYIzVdP59@ZxrEz-s#KeRT06mi(>HL{p_^QcoJ^tkSsa2!tN)NLZ`z zeC^Ed4d}bPz`k9OO5##W&9lc77!=aF+uy`vG)&yw+=8)PZ1B6Qgr+)`+b4MCPbN&}VY;VL1)Y(!rm z_Xp_@dYjE9i1JaQ2}iq*torhF=Uv5_TPonS68(Ox&9Ri5nl@%#KaB)I(_b%}x&eX0 zsKrR2_yWo1yHfhs8Y~DK-24=DKW~6M3sI5bzA!OVq;ga#{&{>kv^;kSv81n?{J>X; zwU1Q{i8|?bkF=5VU2tJR=BA75ASAqv09}kZOa@m|2i!JY?I9P-`wMF}y|w4;K+ter zx$U1m*_^Do#mN=axW+KRb8`HKOSB*@2LzI0y&?2?uCePla^Cwlv9~plt{($&{1gF0 z?CycIozY1DO{WYQhEdAs_0muZ(f@`ZD$JmjlX(6h>yV7*u^;~AF>_x5@n<8{lDWwE zB4@O$o&WG_CE?(s-IHvCGHy@%h>D6d4A?46giOfu9LWoUWz5qTM^##O3dGw#ckn*Q zesH`O-MJ*E@C0a$_z|#$O#V;UH9A%?TCNiNCKtXV}*)006bbk#9WYR>Mm}AS%EfKG(}nXF?|NZ z?j~EW?($?pjmo89+~=s^iS0_zv}%g%Jq6M5i@>DK~g@DXuGOSsS)&oe-(Pf5zGt(YVNAB z_iqb{1jZoXgr@aj$gee35}=mpgo*Dk@X^M|tJsvI>A07pVesK+6n-gu`q43<05!%a zCYL>5>Nv{GY@(P7z;Xbywi`;ip1+5=r4pu3;CyKdf)|Xqc@k$4krT28*X{hs`PFfY z&5zfDJCdM{g7bJUx=pg+1z>Z+P7=Kqtut2rW0kX-0KN6}>gh|wG z_qZ<7Ryb$im&G?rM7x5G(lQ<7agTEyu@-LpC-Y#J{-#|;PBCI30qrp36E$Ma^>0fX zUc8sY1+nsg^p=D9wBMY8|3Xv_O)Jz*A|4)gj+Z(;llraYxD!=1)MUhc=15^kAI2)Tn9g*3dD5C>uD3;Vei1gHsG~qgGi|ZdXz@CC~FaM(PmE`(Jcb# zvDne>$_MtFNdKCF_5Aj80f$vG2GS$0+!$*+lN0t6 zzGRP=&Q{ewt{GF#A37Zoq5<8(Q@Jf9G`u-q%=HHL_Un4xC<-sdoF*;&=^(%YW}4N6 z;hofxlP~?bQByq5?lCelb7uvvG8q8~KRn0=8~(D#UygnXSiS)#RIT$7N2lF79s?Tz z4|Q0s;ym{mOFP<#%>!G~*? z9Nih#MdjKzIQjty6qM*MN)kZae_}PNn7EyV-}vF)g6j|5mx-8!Eq*PhO+{CMz|$6W z96>2W6hwT5oZQO;N;GtAh+VCuMgmT?c-ebAwWJ_(3o81E%jh$wdB3*3P19b9g6-Gw zeUFLIkRQZ6I8B|o!av>`hfba$IE81JAN-hU;6!`$`Vpd#rR8A2<*oeIw$kXgcYU}a zwot!~Dk5|}i2}^Z?p}3j+V0`&%Ux2j4^rdRz3I?PqC5CADz`BA*a>HFWSD`AnZ`QE zTEs*3LyNw+;tRZQB>8Rox$$xe2|SfL;v)ZKK@`dO^v;Dgje?aM{9x6ei<*-= z+`o3PGgFUC>rC{6k&)DDdXAk6zC6(E2;vAaz<3jw*{m~^fQN(l_it?)VYb~;Ruon0+9 zm&lf9_rrxU$L&2@Du?RHkgWzBk_UNqc};-!(03o zmZRfA?k$e2J<^Jfeq*-Zm`3`!^KA#qsj01*_Be331K?|56LYFW-zqkdKV9G>k(#t%SBW0g+%5ME$FTuz)q9#yODvv-QKhJMl$|XaClhj zQ?WDABs>hD^N_1+cT63j#03j46dKy~TPM>V(!Dq6s~lhvebf9-gy?Jx=yp5}l`lZl zvv(@G>c3T%MkWcOi8qKxb{K-lSMERoib%1?5jwQmpxF=Ei;Cp{Qv3C$k{YZoE#@cF z!&~m}UnVwfKL+^p;8u$D6}1?j=-(NosKn%r1Ix!T@_v1R%80<`DCR^;^zqglXfJSa zQx>w%o^u>;O$HRqn2S5V_vr3r z>^M}HxV~XhCFMl)AYj*dvBhPG_5u3?M%le3SsnTfD~F;9@ThNQGXK-oNHhHx)yK!T z%6!32(0YFK$>mzzRj94s$~mt&k{5J5IkMGBLg7~Q_wqS(m7IPnQf5KuIl7X?ZGh+q zZV*;7@9*Z*Trop681TL{hyDQrZizKdV0ehXd_mPVm&e1Qhk85ZA}peBHcL}p z*Nta-+`-Z4v~@JXZ81Me&7n;A;Xs`C`ZDzd>f~wd2-Av*65D~Zzq`8w_$OZrJiWw# zunYj3S-<|Ki|CKn<5v5lPr7Q1WFmS@L0^O31_(RU1Dili6Gh)b_&d6FJ~2R+Tklcw zI5H64Jw^2u4Tl*sbJc5!?oPQblAz#!$M%asYtov?S1wGHe>kH)^L({ew%nEaRDx2^XT_92gTxMCuzfD^X z|M;O{wFsidYi9hws^O{0Z1E zM@SK+E`$-+K~7;_$`iL8YT;|&mD!aY*Mb@l@j=}8*bN!cwcugck>cE>9{%W(q0Gwd zFi$4DHy<4@YD^CBkKhi~8jn}2885?zGKs3H8f~=BSXnL*>JSM||5n%i!z}h5*S@nb z*|r}IhY66i7QH+8R0*mro|kv6BNn}D5}?vHy{(*y+;(qypAgE8{e*$`{$9G~jgS#i}H%evbqVlFhGwh2#9WP|CD z;B7thA1G_S3Vq)B!l1L_4I+YkaT}A8}N6N`40k^RU4sD}u})gSwhV$Gr-T;)Nw73EvUX&fv~sSfYP;{}2!m zbJI>Bn$aHOfejCrK5oDd;7TefktCS3DQNi8-bWGTq1}7mfsnIW%=TWjnek?R$w1Fr ztHn;Uegr4ao(c>XY0l*ACOkFQs=E#FxBf%kt-{V9M&blk$A_H#y-6zNOIIGs7mD>?vkz|dbhYaz)X+T*; zqu)H(6XbD2nr;~@Iqv!Bc^H$>4|G9tz|%rXwZnQ_sr?_keLMU1)bP;RS|krEl}Xt7 zyeX(9W;4Z9)#cmNqt0a-#g`#PJZ8<@IClgMA732H5KDstMt=@NXZE_(ab`|M%kiph z#xYoi{AilOL{3furQeljXcXOX^CxqwqV+ga;-;kC zX+ADHwc%f;W@x?>8+M1XlC?W8HT0s|F2^PaML$|eIXI&hb90-V;7d?2OzPAN zb)-73eiCa0)&}R3GRH^Or_J#vruWGoK|zni(aliwSl*kUNVwaGBP{A zE@5Xz{PYFq@>X44_wkA2t7IJ`{=_`i{H$N))He8b30Pud}-9yrD=f z`sDl!2Y3M4+br{QOX3o$!nUNn2IJq}OGzm;_H2*ezI(e+#8JsH<$R7w^{vuMUrJMw zu+7J+&sBPt8y9ArL(3>ETBe%1mtC2k5Q%z-x@{y_5)rKSQgGG2KN{~lS+BHOMu}N^ z-W-HP{?D-glDN#c^&wxbXhiL$TEw(GN+shHrwgY!egzC&_L{$Lm^gThMI5~*iC@2( zC@@8L*GCtvuWipQrbi#HySBN9g8rB!g%t}SqHj3~(X=5slT0)&*zQ37>-VGpN+`E?SIu?Bq< zOooZD?H*rSZ}C>G?$yFeF`0rFZ_M*|hHNiqc>vid6j`{<$TDRHFxJK{Ge`F+t zP5I`3NbTDmaCr2N3=gWnvXZj<&~1yny0vy!h%d3#r8AX{*_e4g`U?)$eWL%mzCKXI zAa1nFLHz=L-=D$XJrR!jysuL{2gb)`;3mB-J^S1h^zXVn6$H7=qI!d0_3g8Y@#PH` z{hX2!D{z>9UyYR}p?R729L*0sJ;{rkAVyyLcM>sJfXA$6T;xNZJ`qY(oLiER$9jtH zaZB)#5Frt@#NZ;!+wRmS;EnO7^Zxm>N;BrH9V^?X_=%oy&6?gaM}n-2iQo#fPwl8u zKY0E?;E@o|J>I0OI~`qG!U65^ay9NXvQ$|8G!nOV|02C8eACf#$ljA8@YF*Q8D35n zCL_@#2|k*;Mv7RlvOycb*O1%ZoNvB0ijr)EgsDQs9JB*ag;RrE^qaxYsfZKMCS zFh4V?@g+(tjRwOLBJFAIZ@&y4l#+HWPY3MKiB)((M=@LjOFKteP7VfA-D5Fut2~iY zq@}x;0$yAn3E#qMT@twXcTSk%bl&W*s=PoH0+pTRHy1rKU%>u%EU>AsZLXr7i>g7( zB_a+PtrOE5&|~6ZH;l|+SnBLpb4|X?hkfwtV&C(9dA(^cHa12`OmaWtSBz?&A@oLG z9+j>1Z3Mv9V5~8x7-w&inK_;i0e^eBgI!nQWDc}o&EJ}SK|#%!m57tmI#yx&q#3bm zSGy?@5*nD*G6qF=`peTSJx&{!u%F03nTIR8d!ot-3z~R|R83P^SXpzZsna*Dj}|h! zTvBX%R<0F_B#N~S`RjTRWgjw^jw!UB;V4pxn3rlv*`f7bUfux+D0}-4EOctvd+qP6 zmU_&~??=fI+~n+}ygRF_jDI!J4`REThPv(5SN3+~M=}={=hTQ-NZ*(*q}k7Uw>|u> zw}4JbCHrgpPsdnL8O89QKYtpRU(S5VZ?bVx21sM9lyaAo$6>II3lZHLyPu);VKzR` z+8KI-d&}!rL(^MH0cV#PdQoP$$C1NXS*==WP*F2g?tdi82>8)`>p?(%!^f*|(-H40MD zL0r$uw&i5Y2G=j1ud{@&W7pO8aC|kXqvVKx_eRU% zx(mELawMabg9E*tDy9XsMZ5wNxZl>c*UNw6LZeYvnu>hW*o?pyw z&WB3m{rs{M#@_j(>_`9Ppjl~ht*?l-==7*o_Ep8d`0C$hD}#f*1HH4)Mw$Y)DZ$_6 zL(HoLt8)+kSOIXPop#Td1@ETiV(Nu(&U=$=9wMWic)G$L3YgPA{rnR zNeva7@K|qEn~1E1*}&Ez|K6VKBUh}zPG#2$jkJxV~ySZR^qIB$y?oWL+_CXU7<7wIc%JS>+k30i3n`3$C5Vefa)-^es$W z{D!F}clD5GmNI|^z|jc^yS=xA8FuLW*huvIDckDlcilMEY}n-jpNob|AGXsGH!LxnLr@-**j6&GSyizB#c{Kgdr{iu9vV8R{PRwee!U-)$Dwvl8u;1-j4`c zMJ0F+sY_Jz7`YiE>m%z`wnnC>0pgwVP#wGir+JNg?cV;L{`uyb`%cMP=gCCdO`VV(VSv(>-`NMoCx7?5%W@PNt*vbAbViFg!OrSgJSWa&;Trs+y{d%di-J%I#-rBlMr2P0fQ*_Yq@(b3^{C4?X`YkhZXHr{?9yVHmNLbcH zId@@G1Eu!31E%M!L&XTi4OZman9*2fv#EbRGK=O_a3*Lt&_CuHle)}@D1sHqlzif{dZ z4^pO!-H-bf%1)J$T?Yw^T2%^w-agl9t}HF=JXMw>=-`W%lT&0gQ)Qvnre2_qt4Y!9 z-f+x<|3%~To7}>0%c^hrSCo=-zWub*tr_ppmW2KSLLNeeL5zw|n+Mu#M-NKA zWES}PyL8TElDp!1AbYUH{Pupd1_*m;mNbB+hpTnEA*K2KO?J&+k&6i(urBm1xv+Y% zeXMK=knj6g-M)i^epnmpD>2*IK#%`$^Zu>WM(Az{_-j2pI2zjjj`HBj{nG8U^&!n- zxa&@Rca!-n#^+&`&iOYaw}^?FXRc11!>RrkRbL%eW!mj+f`WodONo?7cY~sWbV>`- z&89n)5Rnp)ZYk-KZjh3e?$}CqH{aSb=e+Ov{unRjni1H0Kli%V`qdIJh zyhsm}kL?hLXKY$XQUP@jyjRoRsJ{zyaPpzv#qZ;vQvP#!Q8T&|rxT10a`c@!Jl024 zfSoJ9fPFJf(BlM!AkvC!K3}iKR=x9s=(buUE8(3VrDZ5#uMC->xTXjRy=ATMO-TY4 zI+w8StHBMkS1z{Z$tVEixcN0p@n|T?LtImL@=Ke(8Vx&Rz|8rYBd*3#0zM&0dx_^I4yaBc zS72unk%as?hvRLT4->u@(obd)sM&xonR zKgpiE`-gvN>o0D1jxZ2AmgcR()xEUuvSW+E^W*SdHAube4N8X&1O-dI@!g?>!?7w} zIT&KlDgp}Sll)ZS=Ell`?)WRd% z=mUYisAHbzm~YThh3DSJ{?VBO&HDqGO@i0>n`p^BBuwP9hROs5`F<-5pj|N0QGrBD zwP*t$o^nxmA_`L;(uBu&yJBsx6(p7(#rDQ?hijiNN5NcdeKKmI<$7Vy)?;)B_v#Ay zfMDV2BcRX(&i=e$&@Lfy^-Br33N<^gH0Vc($=bO&^Qj|wq3YoG+={^BdA8sU;m--J z!6598%vb;Q%bp&Vr^**M05I74^JG zB3nfW#;ksw7%E`QG;h^q$PR&F@q*To@bakyIlVV5J6mqMxY+3%EqqCcL77A3pGNq| zU>Ni>2B;R>uqtNC)*2o@gJ*r$ji&daw?sev?T_&u=DrXyZo#IL(~+4kuPy*vXiXJ< zH>Zy7a~C#=)FsE{klb;|JJ_`+woBv|mxgUA&#tA5_=m-vp5xH;?4Kmi-fG$h5D`o; z?Gl~E>a&`u{e^DbHVcfg{P~;s;$eW?8g2h-`J?qKy|mJPmNic~pgf7Y?xGIRsJrU? z9AUOgNy5k5)#SDtgU$ix$KTE#f8Gf|MQob#b8aglE0NO^F|}4)5Q)L%Nlz3cDs;A} ztaa3kNy(S7)@9CRG0E{~M%#T851&DUW!{!JxI_Qjllm{&7^}9p=hx0ui!5&@PABS( z`vG2fbXsUL^_k)-N4_PK)3FP3`LDQ@>b)v@jFgaGEi- zgOXN0`>T(yUy*T(2%r<XS zq%;IB3N;N)ML^?N4DW_vPlA8-V&AuYt94;v0DrQ7nFf>c`~ph7ohQk}y4zGX1kOmW z?Z%$7kjo+^svMVCaY?E$8jPIA?s74qt8?z`g z(Mq$jvUM&uu7g`~t6dS>c4DgBf5z79(Il>A{?KDDtAzg5yGwS%^q=V?+$mTwOy)h6 zpzk8_JKmldnDTCXC+KX&<0$AvK+I*?Fw|{RC^DMHVp+fF0GL$P54PuNDkG9GI~Fs{ z^j+n^jAH%+!jG0tiMF1WQf3mG#ZFVv4xX%fmqZbc4{OxL%mio%XWCqN|NJ>@|p<@GwN5lGlc$=S0qfE>0vXIkh(2tL7TW zuRq0;aDI0;aK=k-Nyjh$yS~_sx*=!ptrk?LhEwk1_x|l8@T>UhY|@St^$z`bA{v*X zf|H(I{CPl3R>NI@iz1TNbBl^XIiHe0h}3b?6`fH!&+VueoAKjLT6%`OmS)e_G`bgM zDI>%@?q`AB{!anPMp>_0M`qgQS&i2?p=r4{#;}{IEo^3Moy9mUG~K1f;rLFZO9hVr z*Y;R}tY5p5EMK2320%#S*S_aZK|B&>L^ZR=egEb^71xpjsQ2KqzX-59zYHI&=BpknKE5tA zwsSkZ#UQ$SL4LhD6(+`{^|`a<-P|e*YHWe+D-=2uspGB29mP@*4nuBv@#P$P7mG|K zI<;b5))3D{>l`HYzt`>m{CUNF_tqMRijl{sKq+@=!8q-)u3*L_)jK#J>g0AYjhI#A zSBJQ1hT8#RIC{5>pZzP(@3rR}*fHW8g~{vuhssgNiZ%zK&6&~?4%heCMWcg*TcLOH zZ<{Wf3B5p238AnUrg&-O94)ynlsZvP3ay!(#l50|n$z*>cwt+z+Mw+6$o7T0n+k}! zGyKfpc-;YPZdOEx8wZkMmZp;}ij4{EPsf-rqbg6pM-&cE+dN(XqmUNhCw`ZczX*~% zp1-wEXvv<|PTw2~w-)k-!o${UWYybz$fDO1xa!a|@NyNCq2OuNQ+_j_>UZmFgAe~FHN2H3KTi_0fa2ThC+!>tLUBMN3{9=IMZ;ia=Ox%0=*-51xTs&~pm<|Q(xB(X@R}wfA?aNvHYja@mkoX~^5R>== zo$w~mL5OBOJmq_wuFx^~mFgSFn78*So^aeE*G51~eft(h19t-fwk&o4MR#O-u@v9| z8eE?3jm%gIT%V7Fp%8xadJHl~9@obaLyozI>nEJbBhR#Y^Zi!KGpqedXcfl}rn!R2 z1%$Hq8!Z?u)~2^_X3_Q**-T-OUG^Z4j?}bV9U1q~HIjJT!r&VGGZ?Cv%9jKUz0gci z96N0fo9l%+nfu9o_|0g|?! zJJpppG1`LV>RkD3LKx)`b!2Pr$%Nd zHx&}Q4=B@D`>p4?ZUSiM3=!C>jw4k%&0p)V94#jsrm!c2yi=PS%%B@=f~NbEY%E*a zbwWJij)KcGv)C146H8N6lk{=iC7a&tlYa;Wm(Z}`c)>sP0dmgboNsk#Up?UoCybsn z@OD9DM?eU_F=@s6hCK`nV{2|zRn^6ILk|Eu7SH!J%gjpg@8pg0_fHB`7d%>&P;^i2 zUhVE|3veZ?LMIoVxPJd(&_`al2RZENHtP=ZhaAcPkwe{x;U(cMj}m$gWZ?Mf?UUx@ zF$q~9%xKveJ6D7N{=y+7;mJVa{7ZM_CzeHW9e8@Y4;Pccn&5I4-AT5J26XUZd!!yN z6c0PWVQ(|BZ^gM`o@eXi&^B{@T8T=9(~ascC41|nq~CN`w9=lOmUteo+n()tdNd81 zC|Vrsx>UK2p7iF`9GtCkUFY(negT5WiRlSZKzU$=POr6%!^6{qkNso)9c70?h7;%$ zfaMmCie@PlA4Rs9pUh?Gv`PN^EB^bZk0%N{n3~cnid}vQ1kT^t#20JF=9`75C~ywA zh!?>H`DKaR+HT>!<(uE^zg{Ki<4!_C1N{|0Lbs2&!oGR@_HEk!)%}{&lWKGKMapq8 zd4X(|kAnG+4BsL_T&W=|kN$IctB#K2yJ*SEk<)e|qbW*=ezoNCx(eSqOQG zFQ2@L7!tThXj3b5p^MGI&9$oq#0)49Ezdc72a5><`!1)nlj*iR!nyqbFgEfM?a5t$Mm@omu1vbU``eF4nK#s(L(AgLH+WugkNp=sPrsz0W~@O{AT zs6s+QBA5MDZ>1m-e)<`?_AHc7y;$Kf_^@Cu2W%9XMK0I^W2@zN4_j#PZaVuax3O^? zJOqFK_TB->q4P~N^sTiV&!*?A@54)Q)bsEoT=pb?8I(wQSPC% z_QVlE%xmt z;uBwMicA(>zmDOno4gKDEFT4?2m4<4{}svTP5h`}!r!QrA~(Cd-m&gD)ARd{f2ApO}FsPoI#nS^b`xlGbt{>#UB%R{xGE2L)5jI%dcsvrC48i^AdMbpsih z;K`~v<+8e3yjqVv9C@JopK^tbVH(|Gkbn0i!a?G4h zhsv!TtabO-b1p5ivxB2xRUQzY@CdQNh>TMfgeFLd_i*40_VIqqV!VYgljPIabPjWk=+pIZYf*>FK2^jeo7-E~tJ-qCH@+nS z*D&+h_A3Z?31ID}U!v_~gNOEP|IjmCB@>Cl%bxSwO?SNB--AU!{N&riod}(W&5y6E zMba@o{VQ94g7a(Uj24CY@KR_;8i4WB`2xEax+dWay|T1Ns`$C%3>(QU3eZMC~-byD7wUw$f_7}$kD@>Nnbip#d6s$FkL5O}@KgTkw0_-1%a` z7L>gb6_8#qp<<4iY%Ljh{YNe!q!+W`Ig1m}isVjlrpf-^<<<@u)Y)?P@G z;aGP1DF3JyxfyDL+dMdB`{t*wu=)950 zOipG+vd=+Tqkk9hb-#)wa%K=1zFwm~9mb-nzOTcm0Mh&v_mKPBK_pPB>}D4XIZ2IS zUJqcdWN_fR>vDaW3yqAXt&PQC`$bIYqywoDo|W2RhjbaYSV zc1f#EOe|xz+&9NBc(XEl#&XM8@BzRYXzSU6)2Mbg>1s4k(leK1>@V9yWnbSU{r@V# zg0j4NV`A@tg7(O}(jd6kUeQdAUk^o$F@p|pL9w_7#Oyr??yI=x430GTQb#>L)NspLjPn|&i zxtfBqW^2n{E^B=&kYUj3Fei{EG?yjtdJZT1;zBPpIoK3gE{y8Uaq#sO-pY5=dwN7P zb-v=Z(s~&gOSL-}rXo7`f?z`NyRu(#Nvb5cb{mzp%#WIOKZ*>_T zoNl|4?=BC^(ozBR0n=5@$)XCt1zi=>IY8Vxn2)ipxu%vW;zZpG@LSwkrGr_{5|>*v zH|f!2GA>5PLWp~K|4SoRbZ{%vRLB~!#VoC?l#R7^jG6qTxT|7gZQt10=K9j&%~QN! zneh)2o7}|cdDpe-rYR+2lz`CVGi}3d|E)UmOtE&qCq3@8UD4!?DIO60!-qqCuMx7$ zstDas5sEpujekc-s#Ta%4B-tyPN}@IlBl!EO?YMC)(ELm1Qlr00VDtG*iau}eyhBz zFhIs1${MOtOIa|o3QXfS7$g9%2ToeRDO2MJE;E;RBxvtbG;6 z2;>a@?hf=%KX*0bxA`Ps6sVW%+q=|I@3t49W=i%@GOJ*D1ig`F;a znpe_~`PnDtksuy=*Yk~+hmQ~IM@`J{Fd*|TcuJykf0L>5w&z+mc z&Nq?ctt%7CtegfbH%R-3QkCdp_}o{Xd3ngnVuRA4gzvsao-z(8!K+_uxRbR4g(W2~ zu1OiP=XEn`O4Yw?t-Ryn%INpe(8$o%>GD2^deGk>@IJDEZZch)YF|HlhzszmYzF+{Sh+2^B53&c4@s9Lbs_O+Ug3mV>vY15 zi*~9_%739+_!z6-x_|y5?tdP5_`)21H<%rEPB0N)T-Ztx7hmzn63LEEjx|eJX?;bW zgF>sOphB1#kd6}OlKq4{W$(vVfG&-$eTNSz_r@F#UG`lhb^>^9=NZ7kiQjgGzVqo= zEz%b1+${ML-0GGHQq8_hPdDd{Z_X9Yd)<^E*;{~#1xWHAd?8(cXuvEAl#s|Xx%Vye z{4nVPgwlU5mPZy65Qhou6RnU529`SGgLfP;5D)cOkTa-8GAb|FqP(;;G)dzbLbhPP zNec%?o6z_$pDDcEzk#BZKs+L63DB39i_V1TaWPOsH79}w^4_2ia%1HgN#e zIl%p|oaD-T$Of9jimJQr>*MMZD3K{zZR5P{I_X*r%xeIew{%b8MW|0XN5)pqtxe$8>@E}Ca*hB8G0oGS5v1lGx7s1+0RauE`tB6 z_84+<@>3f{qe~K6u|O!cn(Y*=g_!+07fy0A|E1djnnM(TNG?o1_!O_UwiZzOKW8z! zFIt;V42<+kT-yk4+(4gQS#aUR@)RlMDE<9g-~H{^1X1tIs6|eJ$EzUI1t!O*gtm8_ zq~6}%)&N+rZ%AD$8wZg{1HN(o_MVELW8qm#u6gLP<)3O6@r$zn-uT9-HoY+Dx&XuUf0A2V_W zg-oB1t{iFAr+v+$>XmA3?!<{u@Ru2pgnJ_^Q^5j_0FNND5J<0|Ep79}EUdf6c`$(K zODG6nVMStsALG&FrEyftJ+VGAg zxhQEgZt+B`xzOaJOplBn#z$yV`gj7@obV)Wf>cJjdg--AX6y4+mTy}G;$ej)BgslZ zLKSIw>D>#+iJ6(6tkg`=AB<{|Svh6XV@qpGB#yf266uer1bk)--$_YIgL+I6+DMj=Fi(Mr0&V9 z;@>5G`%A_Mi_G+_mKCZfv%HcLoYa2f{cRAG$%8DfcvbUbMk`HD0K%S+^zGZJ_MM2U z9HMdIvYJzGxA(}Vo&llxbI9VWzohmY`I4B3%{Aw!9R{Xnf{rUBV9JsD9Luo<0D~e_ zTW9~j{ycECQ}w=XYLvHo_75Id0b(_W*nu{g5HsRXm=a!I3DSN~u9!)Ub(UyhVmj9} z#t~panm}5KOq>`iw7%Y$(g^KgDwtZRZiB=KfnW}@4}zX0gy3>7tmWb>47eT5w3VAF zuGW>hB4xmk*M>}X0z7*4dshD6Zi1OVHo<=-N+a_=O}x(_e#e3w-zgZv~|Z9y8AX zjsqY9uPfN`aBMd%k?;qNR8aO*?gRKnXv z!==L}e6mT->ikPT{%r!#ZqeO_SGpr6 z1*g&cV(GG>E_yvKec=5PlmHyqn1^ zr=e)JeLP(^4UA)Ru4ef)M17s_-E+MO!1}=fg~A?C0Qts7%5$D#wTzVft8Wp^D(o55 z43Be}1x`P&e$Pt}0%}uU!{B&w&=k)Wzh{nER_G;|T>yxA9p(f48T=p4k_+2#p}?)K zzkMurNN%vr7!b|rIsB&Dqn-&A5b->AO)K-MFi~%ZYhQj%{?hP#cQp`=a@}HGU#tBO zcKs10L^I(h&GZjH2S@`Zp%-L1dHLy&Mr?|%gZc()j>qt1-9r%Hu5mbcZnWl(8k zrz29Fl`NvxpB1 z7l)u(Z^rq~<@-jn*2lDB!Ha`_#@`Wzm6p9wb>#N8RwI)VdJhiECp0e!q6phkL59CK zHUL^Iu{`DL?sJ^oce_ahtpYtS&MkiB<|eGHE$Lq_p2l#o$AH#lzdO!~ki32nIl z(OO$u%Pk{j))v+VL3G;>LsftkT&S-Z=gpk4WupH1&t=J!|W4&VaT@8`q14NvXO7ulPE#W#p)n6 zvi8!(+P2Z`>s4B9>P|M8Gyv5McV+bhU53?4I~^>1hD`0~ZgRak=YF+5=3`j)G!E7*{3-^M1| zU$v$LbQZ)`Oc_RMxfQDmbFIyBWQOY>Ibnjub`i#vxCtg2aC{eT7X@kls(1-dS<5*G zH<#LEU0rz;q^aec*Xa4%HftU5Aq`9nL^vIe9w(&uOd5|)SpJ9m`$t}SnE}z!N7Ixu zB|<_}-jj#qUByJ<{T$ya?}P3Olt_dGBioFzMdjtRCZZ2yFON*1Zpj*73M`^sOKQFgP21zYmG1}Ff0D_yjaMTis3WpQ%!_hb7~;Lx07I=V5u#nexDqq zJ9oxJ%_QleMjI`!q>oO7xIg#zi$a$b+GRxLa{wHMP8y(|N5jJr4LYo!+znBS9;Yff zy8)<5)3wcx@*0ePWaxzIVRP_Pt%|Seqd;$eM=0bE1n`rOzAp*bo!wUQy_T0637RV? zXmxd7QpzIv6Y3L7D{I$gw(+j+VQ%S7O|7arF7lmSb7OrN2r`JjB_R&;#Ipe*1-;1! z!0U<&#+Vk3-fO67**yk|#`{a*y)T43!eFNb%_A_MFu=2Wo}X*eF+Cwuur9C5NJOH` zLUN&B!0J%oNM#ucW-uCnD48EePC)f}Y-tNW;^Q)HS1lix34(7Cpu#`{++b9ad&|nI zw#FOw?^bW>GfrCTR$IKp<-=9oPoKgj28L+_h{9uViFy=;CsCn^aQsZHFHN$Rl_FIh z+$UxIQ`FBABL%lBAb>8Bzt}npqC`?2!AEv^-gYTwWxOR#piK22G)*eEOlLUS7w+q& zHRF=*h&`7V+B{o4m`413eDi--ISUOlk{aJV0!6H3OcX;fdiLPc$UODdL(hvljXK_^ zEl(pb>aPfIR4h8gCnOwoB9ckA9(b3R#g?o5CaZ-TG2B59aFuhOf(y4Xx;zR>9uwST5 z`dl;njkmCO7JJaS?(xz2)Sqwe>KhqiSEA{wE!gxJpr}Og{&1^9tNQFkPIc{({R!E~ z)23={m@gz116uU#`uMCiUmR}zbWB1i4cnbcs%YxLk9;3Bt~QZvfZDzzW~za=Plm+-s0;uQex)97;7L!V>@%>M~+22xn8)cO)}P~nHEP~J=7Aos)$p(0iJqRiFTX~ekPVOh3c{#vmnOt&aOYCf zGX{XRNK^AIj6zKvW48e?0Suu_nZ^{;U0)5jC8-1t``4=SDnDEoQZsfoM)K>YEzz<_AEyIBq*ODjJ|M~ ztZl5D##)H-hOfHjT?ZEja-+Ev0P7yJ&Em`5bHY23z0@#t_;CyVta)WyU>X6Tc>irm z=lE#7pCcOHN3a&LLbIH!R{!y*i9a&iEvHPeVD27TLAOUwR}0S9IscIrgWk;sWZlp6$OQ6~A`KvE0N%!L zsj1!@J(e(4LcwbSyHQIy!@~zcyF+?FW1t?>oU0syQ-Vv3H(WwKg7^kyGE&ad@K0FK z){6i?7LBtg+tk~sCtokCg28r?MXLY=XJLh)5rGkxP+cfgLMq?Cwu%#3F`S+>IhNWj z8qD>aX;-2EHi1tQZmzh3{-9QUHk7yLvpxON(Ee|59XL92gCY}JaQvy_FHqsk6APeH z!H@E|TBr0z6-#twWv$hm3p)mBY_?VnpEE=aiO~A0ClJ$#KLG#1>Snd~q+}%w`_P*N zriSD6Fv*Pa?@b|!-lUnKP+bs;pcJT@ziRZ0f)vuJz&3`sY%4$Y3jXisr$gABS0w)= zB31oR<*iybv&Ru8Q+y>$K2jZYGWBNz%-a_d4OWgu+)!ba1KysZpV$J701gE;QZsnL zYEGSxcZzpW$`8LE%R~D%bmx!ZMZLHtV961-w_h;+)gT7CI8Y4?o5rDJyXUhVI+&!{ zL#Dvl^?+!$A<1)>0JsmoA8_l$3%e&4e=A;PdcOeBC9s<&@0e}x+ZY3-e&wgZ*j1H0Ke@{4!s%5Yy)jlt7cdK#7La9@Z?Vm$2{U6 zK_={(_k}qV0gNF@$R)Um2UY4m%0dcN%E1)svX!-cN@doVZXB?rz`TC)4-!pJj1EZy z1qUsC+sR|vhOwcTzO9_NcDhN;J(ufMiDWQrn!wzhRph_WYN)QE4q8>+j$hQN(;1-r zJ2~Ci3P&vf?M8`@EHpqDt)enALiQ}xe()QS-rp?ngMf#r?G)khY@=oJwh-IZ6B?I=sJ z&-my3|F(r{FjtjNm0HRjNr5vsD6V=U*;6q~NtrflL}_?yK@tILxjO)Z1s!PN!oTq}4l80Z zVO9!0zyb^h&fWU_`ojNe0f-&=0YEV##xvV@5k$juzbhUlmz^7O)dFM%aB5MdO}I`>Pw|{e{KhsmDECc zZv$XU!e;Vvuod013cc>*R7&UhYFQQw!<~u)YIu_ag8+y@=_hh5H4L_` z5!wY5^%VaWAD@1pj?#JRM@@?$>fwQZ9N*3K-v^r9=;*Ha3~w8NiAhVfFuuA6s|G-) z4eFyGTjn|(VEHKi&7^+!E!DnasT;TR+lFnR7}x-g z1!UdBx{Xy8L?1xC2-XVp3r;Q~ed;TV65#z!`${F!gbkPyYUrFAAcb)s*g@#<>sPKm zgTVnM-FFwHm5u2FWeBk5Wc>$#?y6a@V(b(%Yt(2}S66rNBsGFtzjbdaSve<$7eNDF z`a&L+Nf|xvHTP@JLGl)QEBQY7%+nR_<39Kl=NC70B^K&9Q@XbPMVK>B7A?(ker;*O z55VpwfLA~wH`yspaq@*)SQ5VEm_0nbGQilyPcKw(fC`-E_QC)9RxK8O{+>8S>xdPA#ZRbn`eO`~Grk0eL4Nh2?SrGT?Ty;-4$f8q(S6#-F^1ZAo z_?wq5n9M=v`p59Eg0`OXl}AvJQDiYAK2$|iN08412-JX~EG_6xe)d|!*<@}A`BnUj zA_Ooc_|z1X9u4`3l5l%Vi% z%Bzi(p5$$vGJ1ZQ<(Wfmk^vH9XTLx*S4XVvNp9E`*Rgh7@+mj~MK>kOW&r(*Lc%j- zVz%@K?N>)jEeODdXs5QU2aQ=V43*G;wi>2GrpAA-zvqMj(5?uaBQtN4ERH5X?+60A zerwidrfW+hzXu^#F1_U%4=B9l!!Lzh9ZAVY>=!RD?fs0>h)mM%m*|Hw7BY^75&d~j zY7-s*BSlwYVqJv5^K@!z=E9-8a7N^JhIrgZB;ou|8S}O-ws3~R=$Q1m)6*A4+`$XM z4Qug>$j}VzBupd2d|KOH!NqfNfWOQ&cWKS%xS(}$L+iVulvL0J(Q*UQZecE_qE8iV zGlE>n#D0H$S)5^1WZNl|v<~K1I5YN|)XSEa(iH_NasB%G{T2bY;-9I? ziQ^W@%+R6)k6O7dAag;B)M-! zw?{{^Zd~&!S?pdQC^#d50k&VihHx@(U>WjHC)v-$htcKG z$k8(U9sP7ly2??kl2Mrg7FuP9W1y*#VQwI4SaU8Z@|}38BlrAXodO`)k2GHa(ttz$ z!cLQ8XfY?$z)_P3%C1gE`VUa*aM5@PWL|l}l)z;_`!iV^U zQ`%)=a{c9DJ@TS$v;OfSOa|*w+2!AdOo8{o^;Yaj6p)1d@UR&vTnM3BaW|ml8{sob zR|A2KejHaz{B$D;=hIS0<6N`7>I9hr)Mo4O4j<)A>ZdeeS!0SFN#P-!gTD|LtL1BB zb#04vF0~$-L*7h&oIm+iPL951YN@txdu#>KmmJAG6vsH9?@mx>ui6|atmnamq!51q&!PfL{d>p&4KzhqSpMCr(_LUp?A+>OY5N?)Dj=}7%M)O!@$!>2 zud&#_XRhKx#3DtiGiHUoU&}hlirD$4GYPNNOKWgaKZo`Sa@XJWmI;FF5-{d5_L3Mu zN3NJLqMx)aIJg1|9~ASH^LOYpiHaF0JK30`0_IY5T&i8O^70Kf58NSb)wZb(v_)Of zoC&0Hj!%@+3CYSJ+yl1DlZs)JA|o!HZ^DxgduK4Cm>!vl;)8EW^+m3#7lZMSSb7&y zSi}Mnp>s?4-esVp5w7_E$V;E?TUfUZ!8xEaVJww5W zq?jvhEyk-b&`Zf%_Y+DZ;|Nl?m(F`mv$DVITD`mXw*BXIm>~6jg5Z$wvWU zAN@poCn_E>S+8X`62J@iNz>Z2(Qn}{=nf#rycG}4+sRc!kV|P55S$h!G=89{s;+|Y zc%$>`)vSW7q^<35dZ{sjp7{Hvb%zlKG1G}!aI4S>D*hS{(vZ&?jJ_uIhhQO8C0b0> z0j+4e2z~++vc>SI0d{hax><#!9YJ1=U^Tu| z03zjX>HeRtcG2`WRY(HWrExbMr%oA$_E7ugyoT(({KuetzKdyK*eDfDK&~R?p39J< zKgGW)o)jz7a3j0W4YFuBhB=E(&S~u&9q26y%AKsDF7e+VncahZJMgIkdqpzs8C~HsDt?*lz?V=lW!d5&PJJv`rk*OAIyCD*M~-ya zb`o$aTW0^&8$oZ0Lk7;&_^dR;jIfBfS0_I>%yqZ`MhP?h(^{08cRgsh_msc&ia(Y3 zlASu0xI)b~NdR@b7j?S`WgQ|B9UVy65~i&D-Iya7j38BpWMLX0M}a)4d_8u7%F z3rD9cnod)$p|zCQU0H;lo(?SYLHRwFu+U0_GL;hQ2KSuOn}0#%xL1~1OmKXn>;rw$ zeCP+`I@mS3Mw={fBu=>r_i!1te^nh0_Ybe}JN*d|Aj2jLL5~_}sUtldE!QKzvh8pE z2TbO!hFCFbL}`X;pF28K`h5MjywAO%dH+4wCk5gQm?7a&w)xX=AJlHN?ruTKFW^T) zNf@g%Sd7-m(JDBc_nrhRlJ&+O@4K|euVu8bY!?W^LL;6_xL`C6&zdA#f zZoQQsUUyJ*C<~Nc&ie z6dF8&G_bPzw9&&9OMK}lPzvto=2PqemPQega2?DNorDHX%FbflXGb?PLlT}BkZZpS z99T}v&z32wEw6?*jC12+YE56Qb=_uQXzUfdCtxgq@2sGxCf$+sKtd)-F70ZH1VEw3J^%)YYtdo-6VZ z=7)u6EeFp6h$O%G<_G8BPB>LxT3tq}ZnL&HXjBBj7RCSu^|D)L-}kc_@du4LRV^rt zOV#6LW=vA_if*8XMu$0BHzK1z8^O{iY}G>JbsPqPjlAtro{0M10}yQVO^pRHPI4_2 zX)+N&tB{f|evIbu7~oNM!`?@HOyfqr8DK+|==}=roD7~&x&oGeHZA@p|M!#<_rH)8 zY6qjYRc*EP7=-;7sxsizU5dxWwq`FT{wbZg2u;jS6i=wm&q#!Y4MJMgp?~Pa;cPCJ zRa;WJ8Xi>8He;qjwj&JhyvlBYpKb9txH%ziK=iFWDP&&gp|u1?K|uwnJA^sOsk{+; zWzv!sJTNuZozGZ3T5Z8xbemOQw%71^|FzG`49!k>FHGyd@0k-Nc^uKn)@}{Hx!d|b zqr<Sg*+q*VKOG zL=;p`=3ng_nFu-OQ_inbMngx-Vkr2t)={FAP=M+O0m}jYZfSEp$*wV%Ib7)hJLZn+ zal5ws3e_40Jp2&Ooz!&m zCT`x7DpT9b|6Z&bl2)Ja%s#!bz8g6(M{=7eG1lRQ56zRj^U7IS=JVyx2{}2bIGF?X zFVOn}IfWMOc$UpKsW>V3A0C?fFm&Fzi>0G-s}XW+pg=F};b3nMt^Dw4Rsa_yAz77m zQ~2zOc0E@0)TS$BAAH=PiQ!(esb1R0$-vRYY&4hG;?I7WuB>;PjXu}}8~d*Iox_;# z5Pg5`nl#R|EYPloj>wC!2R<4;AWd;s=L6efYQB2qFL4kZpmL7;R_z%yip-mYn`+6J ziIqKnu5c?4j38-;SVo&_sCQx4NL=5x?<#$}DFrLFJnQ>~KmFl9xJsehSv}+_X0Sc@ zqb8)a%gR?9yfkY3_}gRn3(feJ%@6dSBl=R6wNJc3A{Wjn7;ra7SfR#**M5z|Y+9>% zr2hQQ@E3pI5U7`1+p!@;JM2V{D0mw;e-h=%xHb#wv64{NV#);e%3DZN=V?L&Qs~tF#oX ziCf>TAYScOsjqIS?>m+ycdQe5Y(v9hn?MojMpnIMPQ+inru$d#@R*b%%peBhrBUl8 z$_sGt@Nimkc}rlzyR(N~XxqsI1C5(uhDZ+pP)=)&)7 z{L_^5%80r|q58SHD9M~5xr*fXo0r{(MTy0hk(M9t0Iw&{T8eKkP~Hd?nD zbLe8Q=&%s=E!(LWxQRn9J-MDxD?vv)r2Gm@)8o7r#1Cl;aCfq4q@o%4Nn1yJU3A*k z6H`-v4D%>47m{(acvKq2$lbb%=JkGXl=clwA#JgKmg=(*%4M}_Z;40NhBHlb5Xtym zj}rOb-z^hqaB_MAjREcI%v)JWTY4bW;Z4;nY9P`M6uq9KrRv*;G?;Q-C0sisfAXgn zuI^5DpT(y=r_~Qo4?>aeli*%n5-cbw(7dG%A4-2^=*)F> z6ch~)J!=vWX%=a_vl3}Q6g81YI1D3{sJG6JJF(uKicD70o2qVeT^w0jm`Ezs@SH4$8EN-R|xAaR?)(G zsqqhWGMAm_aFWlazy8{3_eqG|%{;3^hnub8L0Y_x3+3v@N~`5z!WqW9 z+iW4u-hUjb>FDWFQylERcib!<2^d*TuoG^3k=oBa1ytrLvH9>s}Bqr=o!OsjyIyg>P zwBtJ-==5yDCzF_KY6|U3VBT*=Zp*!q@0JvHb?!1le1qs!%+CuYpyEsl@GrNV%;ssN z;bf1PtkEnoR{R&$pn1na$U;c#h#=L)oU0^A(u3LI;%GwHdf^YziOcC$1#Nk9Bl1Ab zy)Jg<>=1#XArKdT5x3%B;=M0;SkNX3OPT7=%C(DaSD8NgeJohfKVoJlyu(oE&T8$y zU32^^4s}t0Lxd#U?H%1e+7b}Sn4-cg0_ra@*^1d(-;OyKuVk-|7P?$7+Gd4T>F0Ui z|8{t&bF1CV*}HGY)a#Cf%OhdzKCqnWB=3jN0Np3x$_?s<-MZ`5#Bv|Lq0(8pR3N&qpVO4W>Y$5 zy2CLcXy3=eja19NeY&U79^Doo8;Hc-terpf7)Ndw!oYR5Gvg^PI}(I<&&EIQ-5E8* zq^ObW6mrt;z?F}0G*BpxaMeY7*(+lWf zZ^CMp+c{Vjb1D}w3xBLx1T&C*+V=3UIxlo!``@-m~bOPvWEwiog#Jx7qeUfX@ybEhy9*@3w4 z1LsMyU^2ee6>g!~miiN>6rCu?kQC({@JGK$SAgvz{MoVq{lHpM7BfpBs7WtN(f@Pb zz!!5LA7`cAR&j}>y1iV*R6>~410%*JlCX?jV9j+J`+Z}wSQt~n;@rMVR!%q%>lO>pQU%dezxD)DG3 z(a(|01_w22YvaWM-mcgRYFgqJZ%Ey1-NX3`+(&y{3+j9K&X!SQOvK5Kh|Ui_Mxue= zCT1q7rIi*+r_V00peiorXZ(9fgq~`P{}qp0x_^j#1}g->DUj65q8iot>bMdEM){(T z>TQ}YzAI$)Ew??*UVol5AY&@( z;eYMy#`D`^l}`k@Hgt!2i}BcOp*9-nOZR@WdVV#;W1ZEjuFyd;j0$w+>Oq^%@8Q^4 z@oc-$5XlINK6w3lX*5NAi{$Q1lPvc3W=3bpGN1u>8m0qGKh?gnXT0R;qUq#L9|X=za@=@5~U2I(A- z7Ii=xgaK*koV&+!zVH73z4M%79FH*Z#@=hKy%zb4;Xr!dn}6wEdAK)NwM#Oycf|As z9d`BC+85j~IisM$lvC2<0n4|fAhry&?i=U&$(`MuXtCSHB~&^y!SaQ(X9zmufi_`N zh9T9v91JmIu9g~pO_X7i+#ef(4%G=brF?Y#`-Jo0n)zT~tML%PkKzX6nD+fgQ)2=Y zj2Q^z?@E=y47HCG~&jKB;yhf5Q$gH&MJ8| zP)(ruRTEIoJ)hOyrxhWl_?O@7Gc43VorJlL{>*#~OG=kaGqxq-iWhmJ^|&1E^| zgDvaV!@mo{PoY&@oXFt|cQitDoN(v$v5RoM)00rjOo#7GqyLNz)`RT=yCVA>*K zn!m*U+WtgW>L=m(slmj)B9(OX*)~*I+h7K813K=UYHMpdla;RGUwwTHA_7O{1OJ=S zMndKXBYjJ-JVRxd6;JujK%Po(TBuDF*6%IMM)FoEF0CPaoN%l1Qg(gCyC&L&2L#G- z7y1(yr+4;t%8#PI+Z|f<1|N|VQD8_OAc$aU&w1VL&awOYc3p9gxwIpf%F;n1Bu!nT zpvW9`s7|WeZW6_y;fyG}*eLoAi$+GoXX5ViKDfBO4#}J0n0PjET=3lnm|?~HF}Eb{ zoO`Qe+KbTVB>Qw!e$N%_w{vhas1`PV%8C9*3vjc$ZIWU!& zc@4RNF?%tW9}TT%#}r>OK}kET_z zcY=0{1?1^%#trTP|F_BreisYE2s1qqh-(En>tNzN+{|X32t_5;HYa?<6W_~~NvGA4 z$7z<3A_xRbc}UnH+ZTT(ZVG(3*byXlSvB=YlmIBQ-@JW8Ohuj}q$d2gD6$)0aaA1w ztO`=>vVqhE4cbOL7UkRAHfr-$iWwh!NLkAUtTwEjdv;gdAu+OP%D>Yc@S#vD^13Nx z>6CE`sj=onK9paxQfOOVM^cgE7$V6F*Xu<-FKw)3I9+;$T*+#7#Hx{QRR|h>epBc^ zP7eTgD$g+-v+eD>0ui^;{Q|+$5wL2+%5L*>J165fErgou@6qL636J4FFrDx-Ckb&d z%sNqfp8W+|o!Om>`?T2&xxPENJe2~_6RcGMEfn@Zf9XDbFRCXSB;Y%4`?n?T*XRY+ zt(80-ZL^cO^&UK#;A{uSE71=hKTuXkt3b&CC%5|VDt`?d@4Kw)&3MHF0Z#E_B;mvA zS1gv3HP5yXXukNV$c~m4Ebg-5lg$l(&+ph)Mb$*{@0o`Ue4jXD*fFPdl{@gEEbTH0 zMmp?wM?ce79#4`I$J(1UX@2n#To07v7Pf!iYuc;X-b#rEJ^cBco?v)Sn@cms*Y(^L zv7&S1b#*J_Fki|Tc`gI-dt=RVJG37-reE>MUoIk`g2=9_`@Z2Z1MVYoPfSzinD10{ z_St((cFp>Ariwqx_DlsfgkV)9b9At<)0i*rb)u@Ld9CTvNTM7+}8HLBN+|` zkQ5-eq^PFKXsyJkysoR_=)pOeHVJQ|%h7=~9LPP(^Z7TH=qR>5p(us)I-+ZIOteV{ z5GoBB1~?EAw?{>El?Z?UKtj6v2AJWjj3Z)AFveS?=Y`^;X-c9K3kFQZXQ%h!w}83; zs9mZE->A zK9l77xTG3JI4hK8LE8kB{EXx2IrjPaJgd>P5o zK?dXl=AVH~Ov(80Dx^5_Bi))XKoPTw7ekOzo+(UAPB`zcA&b-nU!R?w1EgmwX_i+c zEyj8zpS!}CrC_;0R#t(C)UMS|;F{It-1p_7GRG0uv7w7x3r=_aI|PD3ovgJ+J`s`+ zPkvX0g~;AtoO)Yg01jYbk40*k`J~8aiyb#00fmL;9;m+_k5qWB`wVhJg$SsUj8BBH z*SCGwkpGKvl=?ig|8F5S+m-j@gJQqRO%@J$ZZO==Pc+c?;0fpv!+;~sgq%wy1Lhm zx0Hx2#eC@!*BxAP$PogM99S&$qatOyo=^b*H_uwF&?yTaM#>O(X*D1NDd(e-`d>t> zIVAe@@uv%3sghsYPg%Cw$jmDX3C?}#XE_$O#`j%t&P~Lx`W?FkoVf}=#xSJE{9jWJ zO3i|YaA)VPYI0o4W5x(Q8exbk^d2Y+jn>>c)BJeMk!Tb7y+EHB*rHl1VNSd?Ge)8gf~q)a%48uiT}gP za-8JHwRXhNi)$3=ZtQwm+dJ%va?8XsN3k$LZeFRSo2PQXG?qb{foG#(#%oT9tOGJ_ z3EvygSif|$%z)>Bh9#anR@WgkEcI1fC;?v&ZSk6jm}-K7#Pt0eb^szEd{uLfJ`6h{ zu~{)Lg+o?v0^Z+`@2H$85JNYj4iB5p0*`(kIcLoGqV(Oy85;~aq3BQg#_g+7@G$r3 z`&TKiFt!xs$bbuZSgu%LGta~$qUo#t4=y&5ZT`6FAhvh8ZOwh&rOsJ6tu~bdYG#IN zB4`8wACr0W0=l)Y`FXq`V1r%*0KAlsl-kW@DQoXMh6;$JO>4$sTDfa&FLNZiNpQ%9 ze`)7);c2`h=#czYN6WRgq~1?uTABb) zK;^;AC?Y*E{x6O651%A>6ZrNI><`Pj6Gvuq3$iZPPx?<{TMZ`*bug>-+IoE4?UcHsJaaaWuW)ubDYh^9z9$3{0OR!153dO zFsH24!{Fxl&az{XyqB>Q4!D}(jImB}Hyt-Gx{LZ^852x+!G$Ge(+8Ui?qDd2 zUol(ncyr&t%piTRWfl?eo|tU~{QdWV^>5oZ$N<%zBsk0WYgcz&8cU6Un$v|#@pyBS z3kI>FZ`rFS*a!sSnc>;t2Iy|20fzy47@%+yJiMFTlq%*Ag9kq)ux{-u#52D)SCJ~# z08B)_3Z5rKtoB5W@eVC_2_g>LcKwe!Zcw|24cY_E1GXPgE2o&-RzBQOMXENGL+oCE z_m`QCv(x;OBgugNbQ+)KjkV^xC`gq21sLvPS*iG4X?%xs@ysS%l)C=YF3>@G?jlQh z#A1}#%(yQk=~kQGfOwn1Btgt5kQIUX_igpB!#`RRVrzzwuFo`iAmRG)?#bPRi8Avi zZ&M&|Eh#>|4cbko<0Oi?$Vmsw%z&AX>SO@b-1K0Z-F(qbJ>fcpQvI5Qc%aCf$6N;T z9(Ui*?UqAoU|s;*3NVulK|PzQl-xUt!67hjvqZUH=kwVIsK%w{Pxl_>e1E_dAW)bd z5I>Fnvkz)f)dPONWVO;9Cn<(PL(eNY!yY%w@L&L~@O$}Yb;!={-gY7So1C6lm_)?(&!(K10Avr`>T)i| zLg^p{rVY^MgD>33j|q88gJ`bUDNo&QvCoe!?MOOaPmyNO2o zhnu4-TXUbrUo8x`zU3m3wn=Ki?$KBE(5aaYOB&gBJ#3x!&**p~+q@rPSn4scpFs|s zEdVuIT91_jjOegWgZKpbhDRo4d~7cQlHTlnijGftM&L9))e=7Za=8Ni*)M(p;6#};WMDApET7G{Owjat+#svES9s_BjnR*QWvc_Y@AwMOJ0iO~YC z_uIG1^i>sLWG9>XlJq{_)zi;!jERyA_%po>kARr5kaEuJHlJs{m5q#stx{m044QzM zde@OWAhaBZsc`oKj$!pokB_pSo!4U!qy}1#b0~h25K**X&aV9CL%tXOA9>sFhgif~ zo@{@BPZ4?x=vQDdL6;G9HYEZ6!dBKcc~G4+w2Ap|)=b~}w%3gO(}`lId;=oMsFC92OzB!iu*=5CBVG<-mb-B(jJ?fh~*DX8x? zWrkDzad!zt$am5mgF@l@Zv*?G99-3U`0zWP83n`Z_ zU4Zpxv zTSr^V-FJm!WnLH6@;BPHVd(C4VteI;wM4A7z(vr6VAPB9S9bjrZM9ex2x)J!di+e9 zmn4VI2I2?}JKY!(Ew+D+T@j@PbV>q)6Wew`Td?JO~R{$_`?X)DO_lhv4XXN0Da#T71Y%i$+` zcGJrCZ#Uk>ak>hn0^VRW+{CJQKLYjAk~Tl5DhL|zYZ8_qJ{|Pi7yg6#1q}b1==1mW ziSb%D0XR|NUgiZYaAyr?R#C3+1ZsVQu4S!>?>VkKZjnRtvHQdLh(!M_9@Ru*>70=j zOn+wCe9UH@Twu1bDIC-aMvzk>I(vV>n@;1gVF+@?=hYhdGQM}hR9$d>^A;Zz_bii+ z0hM~}Ape0KMZx>rLT}(9U{xI+RexEHHuj>L`b0tWCK>cp+7s`AZ|F|y<}S3NjKT%Q z%VoZcchxq@-t6w~4UgxXP>J(1**}qL0+#Th&O`OpUp#h)o_K&`R?V>>B4b}C#S1NV zxL}lmHdnGt1OVS>eX9~Wm3fO%#PnNZ9-N7XUqpqyH*YVXe5ZB%KhJ7Gjy(vXVrmip zYgpewsr3gv&-*;BhEmUm&bJ>Y2pehl0JRE&W!j3-k(ho_&t*$aIPu`1zI^FwEyaUu z!KKH($Bc`d>9O*nbp@%AL`W|vnRw74)I9G@P#-jzv9>2#!^AKC>>FbssQQ{+Y2zh~ zxMnzgaWkO>hSZ+s@h7&eKa4?TNJ?%^BkuM_>y{d?&j4Fi~ST56ut(@cMKfWIU3-rkWw&>`k&@!#r^Lvfp?ziyd}H*q{xfVo#=eMOO(V zo;1GLHw=gc0wyq!4M(pJ&ZHz2CNq21bt5T%ld0fi&tl*_ft>eUVtO8zFa7q;=bJgb-(7&?6-g=>#9=LA5g3+S!HVp&NQ^l&!M^uF?FUXHz# z|Nf)JH|I2hTd?MU^XS=m%=@75?J<2X4>(+IimJT>RRcW#&b{92*QjJPmEujYFuE`J zHeap{N=TIO=K;wJcpWRvS`&f-`#g3xYpR!`E@nq?2@v07x`fn#;!<7g7zdux=$|cZ z(6Lzb#hniTWi@z5W9B2DvPRrbiFa$ylenTI|AGAts_B>-t~IaI1xeTSzB=a+2VV_O zb0JK10-mujzV+$yJ_Fbr-pcTapGdBQe0EBS)6J2~bo0GH^JdR4$g3* zw>W#czue3D9+$8z%GEi)g+YbDmolOt8}4SpV(;+zPQ>lKip||fQ2&+@j$OV);fHN~ zx!egESlD+u|MCJmAq12kz@cY0%9}#V^Qw)Ai~-4qMk7gc~tfL_Y`WPXtkZ7 z`K|l7o*oF7I;=F)sU?NDx`FU=Zdz@a69859=vtz`yiIgUL!g*IK&71X)uo`CA91h@ z`d0h6>GnLp72xN$kE0F(?@A9wALv<7DsMcIhNRL#pHiR+B*u4c^b{z9R#zhCzuW5X zF9Zet5h{Z|K!W@`NMb&j?Baf{_?ny|{%A|6QQQw!5gv5-SQ70<*38@M^nApmJo;;CQc?aq=Y!~wVc@~G5EvTlz($bDKwlu z)f=-1v-1(-SMjgLP^vfngf89d0`H;Sh^2%i3V4^P)A@Q?glu&1>H$S5sE?0+dqe25 zI@sBkv&DX_rn*KhV`y^5er690=R6H_i;W z=lt%h$$!FOzth=4F+Myhn%Uk)WZ~v@2^pt>#7BR6!ZPBMQiT?uP}hvBx|&6r(gE3X zZt*#HkR6LOaDKbtp5i!3 z*q%#07cunN^k6AiGQjP|;#0k5fq~3UTPDG?(u9xl)k6{)nRX`EJpqTO`aJ}O#JQ{92Rn4N*dgE1^5}J1Hc3TQNADtn~_f81~(U1_e!%* zn#?f)gO6W;B+TBwGD?b_nvuMZsV{5a+<#9LyaIeMRA4aT!p(1a7Y;EYpPG%Q$NA$0 zg$0(Nr9b{e1E@y(Z0!NIMxJKY$jZ?2Gl*b>s{O_YPc15|9yp%Y_gv}kzP0&ESPb%S zz~6Fn^AGG|L6E*5%K@V9YCXmD_`ac`B{{`vE-H+avq^~rMp>YdCiyCQbnz32!EX!Q zFZrol zd2V2&`Zp*kyOy11+-f0R^8yOipJuDYNf3#(Fz~> zQ$s>j`jaEtXDOc1CXUEGlDn?;HUIV=uzm4p-$TjA=mBqu$xoQXDBROs|VvjDMF1d_F=K>fM1fVZ`Q4x2badWDH`h&eIOWWOrI* zA2<;KEjTlqPs`}4$8vY~S98q{u6UhSo%w>r)@1VuLA}IyXY#zIkddKcfRQ9f?lG$d zDD^sCS8n~j`**FMphj$}KCs--yav41Y@qa%{~ArFJ)om^bRT%i-75SSvl2{)>z0fy zZ4ibafKsRWzV1?YVAZd1m5^}zwE2t{4t5IFuE^$VPak(JxNwh3cAupkd+n=ld>x+` zZRxPP46=~n4H^1?=8!PlFhI4^KP26T!w;j6XvHw+@~^kvXE4*5<1@AtJ;^dbmtbS~ zL$QY641m9IPGN)K9P~Z~rPed;`^bvfpxBepjL#DeV(G)zHuDEl)Eo>6-~`w` zyOZPT)8>ZFAXHECt03X-D81q)PxU4DdbnJ2!CV~fTtbNrd=vAn4j3Bv(Dgw~Oaqo1 zCMyFYN&p>+d?PN*hS+D=Vf0C`28d?6|3U5k`LRtEpc)s8MJ^C=!ENOAupYUxu=#@N zWF!`yOn0_(5tv2!OyNz_bJY3mir~ZH*zgDU;{I)&-TwZ;rvq)h?>K>Q?GO#{4Fg*o zZE-a7WyQ;I0zjVB%I;TCaR49Td6?Kd?_1!K={^ue4MjBn??KfU=lo^Flm-*F5Fk%} z#9741!LU9Th*gS=Y+ug=2lswD``I8t$p=r0&d<2jIn}=fpC!g zQj);nNHD&Z?z?vxlqegbCm-q)J&Xck122Tg$}3Fl3_b66F?!{i@}WVpwEoJ6~wBzCQ24#!FZiZt-?*JV8}6K7`YaCDm6W$wio7r0e< z9NHnR_#yj`nCOET5<_ns{JwD~g}N7SI^abHIwNC5cltxEJ#3&1`=clfoi5H4nb#W$ zv{VNrNtArCRKn9$cTu1NG4$YRJwX&spZ5e;-9&OE?8ZaaKR^O6cD#C>v%YJfMh?8I z1)%YkP1{#t#)G{5k#R23ljSkZv&}-Cf}#&QUIbti1+$wyNeQ$0x8moHq@FK^T70~F z_oZ^UT%Pu~4O3iuptNsKbc@_W+#NE&n5H&kPRrvJgLc`Xv zph6=7;J90!U5_RtIJjdF8D6Qi2SUxw>P(F7b#NJgSz_kkbQ923r}cZ=iPJ}yIepPy zu(+S_$iuW1=HuC~7H>Fur&Cb!p$*yln)xNnN@!06An_k>t^*_hSRw6owc#{nVd!Ll z;>)RqGtGDN2PF)M9G*@4#=?hqRldDbI3GZ*UH;vMy>XxZ;5KnxGQWBv2V&%2lAdKyMMErQl~E+US9O0cxw0djn1kjGzzP4K{l|t9o8< zvxNSaP;&PY)aRWCDWKuQc{zp-bixKZJP7iJN+14^i%{Z_9@Z~R&S>fwDCpaY!f+3H zq3gTzQNarGL^o&@VQh4cWbRTqu0h!g)1cuNkF&XPUleDP-x%WSvYXgJ2R7#4*FTMl zwYA((ni1dN8C zF~2RAy|!Ij0E;Drt(rS)|Dy#+hbr98q*~+p8Y90UbVrR4WE6mC$kLQ6gKk$8R zb?_jvB4C!_j!A8x20B}Tef`bf9#q@W3tfIhOh(%UnpZ?)>=a@-Ctt|tHyZ@(VCl*P zzJ!!#Vg3-%enBhPb>d$~gjET63K?6nI?QFheTph5EzWPMe$kZv>fu+A%McI0GFMT) z#m@^O{@n;)4+GCwxEA0#furAq)^K<8DIGaIi$v%>$w;l!SB*T_W?~2L#2p2IC!e#> z{)@TIl6lNBj6t8i)wMCbSr@(fcI;>ik{btjJ%+`gnBz{7r#)Zy!~wjYgk0yYg08vK zWS>UleB3+9v|p9xozqD+(LXn^hf88OFj7if#APnYCp z8snMF3c#4cfgRlv-3$=7m>CCBw%hb<6Sz@gmx>CCo{r~_pQ!28w=6py6*bx5)s!7n z19kp*t|A`5n+2%-PugS8UIiXrP#GNTBsIo(r|o{(ao>NSt*N%vBqMPPX%r{Hp9UQS ztA8nIpkam|dPkZ!8Las=K5j&s;AB9BbYSLX4f>Gufz{jpzho@t7qvg3f9)UxiO5vv znZdCiu10UU3iQQizeNN3+p4GCAKAIMS{&)t+TEe` zi7AhF0pqZH=5C~5*8OSRy2$odg?hDy{v**v%=C~@S6%qAQbbgYc@Iq_My{Oiv*Yv2 z<@0ETA0bF(*`8ECw-z25qT2BkGt_&hH+nKmc2~(SmLEGfG{j}xpvENf(=&B=Fvr82MjoX-qu04{W(tL1>q@f&N&!1)lmT8K;7xr5HaqN@5Zs*9>ej65H0e% zrl{+Sf7{w);Z=gE05g8*te*}9N}H0HLR827D7J2E$F-S`SM#JBlD^Jr?>X@jI6#N` z`(soEY7td_(zH+%;Mh3WX#y$SX0Pa@6z|S0S&C$Rq4c;L zH(?+TP)kx?Jc~@tW24?Y%|eW115Qs%%%MNX+RK9=pqbzB-`r=@70KvZEr6Z_ApO$( z#QCt;3ZHS>=52%GEbVo_149rJOvN!o2)6AOjx57i2!UwLx+!9jMxtiRyAcCg0yBXD z;cJ6+-Mncy@*^So1Ow%QX~Ep?b#Q+KFvSS7 zwXrLKorNhLLkHPhMR^qcLH5YX{q%U*KW}nZeq2NJvmu(Az}}9voV9-l2K=H{|3c6;D%4i%=}-CuPwgb z%>d1>@^fJ<{aSpG4jS8X0bK%$J*Xu+m6V%e1ciihM;-WXa8s}dzRCf93)tD)82JOd zuA2@Pf6Zozniw&|5t|36Af2DyFt7X=VFR5v%mCVcczaBz|B}DsV{J4H`R{B;ST@mK z`TSSu43m6ikpX0O_$<7v*s;iF}l-hduojQj`8E+R&jeKh3Y-NrP^srNQwLETnYT~`|U0}Vjt4YZ6*d=l9O+6`! zBk}hyf9KTNo9npaqe-SJyfiqMzWK;pf3{}h;U8dZ{>eS>ZRPnFk{d|Y(iy*WuSHSb zdV%9rv5d0$NO7Z?mUV1hzuw>jH0)wcu|WWGbOY_QtjMVRBxR! z@JYiB9X+z^7Eg$rFsozwNQI@BqBo>F%{8~Cpj52EWI3fmE$MO|R<#?&T>>oB%D{ZF z&QzDar;qOZpn(|q>wL>PU&7v9@OqR*C@|e-wi=SNhlml+(@vy9RVl1?U~ujgH~nc` zo?i)y_HgKcP#6sB?snWTA@*xwF&f$d69(OK@17*~DWnyj?q`cTF*G8f5CRkCCoy%x zUH<;5Nn0-8K*cp;0|a#58V?^tLA!E0y1ibfGOOvFrFt4o&X;h=EtLn5;gegxW@U}X z3?1hwHaLsuJVUc^UP~x72!(I`JS;z~T|%#s3}dJ_%o#JYO*4QTO)V9TZ5H)v$>7yX zBXcF|mp{q{sr#?W45i}0(&mj!vor~3%6#?Irn#z~5rn&azIJ|R+X5bdz-LEc&Dq(N zrYd`8d{kw#d%N)53wmvH#iG+k-8PCp6qnHKY~(?G=ZVJri;qOz=X~7YLQKYC#8rar ziD*9-q!%KpnegLPqmc~h+KyH`UO+7{Npc40f~G{I&wz*g&T@g;aqadt$Sk;hTJ1>F zSUVtYzyWZb$YYdC%r}H9c`ER%6jXyeX*B-AAaUD1WN?iXQ|c4kd?7yLGrFY6s9icI_P_b zG@Lc=_us#bFg!WBF8xgcBSB*Q!pX{^X8)`dkIn*MJkvlgn96# z0|_&1$(Xaw|NS&tM><0!diPqUW30qH7?!D?C{I4zdEgm*R%+AzwTF1avU;ir&%FR zBock!%6a+DRuZ>m=*Xb+W9J*y0hKi1Y;2~DO-vju9@!ZE}udvT)}^O~5Y zh-TJSqQ}=KgTv4ZFpvDRuzF)=$XIN9V#IT%=OPj9bpj0+-|1}nOThVe!tn5OaJQ@Z z!R7o#wy4RAU&ZjfR5POR;a%BI0<_6WaXnLbtvB$&Mrtr*YDW>`m|?uWk9!Q63`l9ZQPzJj=BGJtA5;G~Ixoci`7CCJ*5Gq=m_ zl`I*x69BPAt3yS7&V;QJshmn`>Wgu@TE7mddvG=s3YN41kasQD^1bNl$ku zXM6cc?jw9*7;nn;EiFirqZyWvm&tlS47SWhqM~{ zdVh<}biUsySS>SIh!lP8`8uQ1OJ`DN$j+0oDu49( z=P+t(zwgjCKPVA8GnA>3As8b4Vl~xHY_Bwyk;P&2IC=8)kQnD2{1wmj=734wIkAgP>_%X7MKYYC}2+6`9%4guE3|;vkUfJkP;k& z?bCG6LuWr6p?qJlC1SU?%&wou^U$+yN5;ln1xMRK7qV^izG)*rS7w0DU6=g_F;dv@ zQXp2ec2FlPc~(WvHO|x7QY$+v7q8>YIC?gtnyXX}(^zQw0FaXzk6(;w&VY%M^Kfxu z4j$dG+c42VFer|uHazStJg^(L1PgALqf_$_W`FQDWdG51;?zkL0*k`L zW3=<-ye&j9q!Ex2Q6j#$SYUeX(6nOV>FGUF)!Y;V2~LS*{Dt3bnDcMVmxQf%{91l2 z!@&mIBMH3az&Ua&422E{$!MbO&em>ML*BOY@o5bM8_(6Po~%vr9`C%=~ zQS$1g^LraV!#KzTV9@Q+eHF!y_dbpor+E-(fDtbp#5m$Fjl;UoLZ~yj+PrT+wgm+6 z$3!_U*km7|rl)EtzVWo(idT*g{`Rs!*R_|DO;;dbR2`e{vIZ$kO9TPx50>LNFwTN~ z1oEHRoc(IXJM5QVw7xJiyav}+G!!ehIc`fYepBQ?i+`$GIR8g&G1~blEEYE=Sk8gW z#!3UE!SbP|;|cboji;`uE?P^aVZ+$mTioD7E&^OkB*;HZ!d}6V^%&2h)m&h97v5V4 z0M2YI_8YB&YYB5yuKUgq1P6!JHW`#63Or-clxPwq`jo*w(G)XWwB%48nn{4Y(^JDMs~lI2snn#sV(n7yK=sYh}?@I*sv!85Cft+#uy7lA%QI&Ln9jnA&2u$OyZ zvg;es{Tvs4V5o;#cW^lgPpbQxQiI%&y81D@_Uc&KMQ|Od6VkZ#_5QD8epkkZ#qG8%k^VZ)Kjoy?c}N%dq&QN5j_UPD{U$gs@4kzP{63 zcaKy+?(QZZy*MxSilb3%&q#&wCf7Y8oLuwV7UVS21WQ>_>at60THES{ic|?deijDY zY41g6m1GH+?q6$1jeF$e)qP)>2j9|`Z^>G(cs+1g)HBXkB=#)WPcn$xo_9zQytZ++ zEgp7G4no5Y%zFi3T(#Yb$ME~lfA7uk@ChkWxsDFR%*z0u^UKN0qh-%760C_vRpZ&q zI21m-^C~4hVhqpH1iB`e81h)jz5>+M!b$=YGaJrEaFiKG;oIIBW`vwh~1?=wET0? z4*bl*m@dXJvhnn>t83CiO&KR}2Y>pq_mzthlFw=MJ&Yk;Jt%!pnS*AdoL`ySZ13v9 zg(E+v^@}if+W8v2hcicFTDIF*e4CwvsxrXyp^yb648)Hs|y4Ko@UV9GjvGrjVy<-E%>tB_CNU8 zbeo94_2q$K=d?_vH@gLUYDi$DUGS2CBL z5JUG210tcTkp5(Iw@|}S+xhxOCiruzXxM|7@mkjiCt}_$P6xv-p82+=h3P4Of3eA5 zc|r}wR`-=urT+w*Gw0o8us6NimeZziy`<>jU2eFD17(8BJdlRIJ5$=JBK}|UHvT~2 zGUAXD3S%**z;jXC6*-o6gif11OAq{jnhYOUA)x*&kWJn{PTCA$MO%nexYJ$$U?hFWL}z zqcW>bM&|d-b3rwdEmLVe8crocc^6eu`dCy_d59~>2ihQQbOyD7`s0t7!<-C#h89JW zm1NiQRod3oJ?4kbWH_eP?e}=h;H3Q3m+ZWidK9~wNGA7|6H`Vek2V5S?0f*x%2=Nz&xI3QN2lIh`mNiYc=3 zJFR%;dNw3p)YCQfT4#Lf{#2+HITjB7(#+hB zp$mz!V2WBC4zUKD0lJO}7bJU9vzx@nFNF`DJct4SET^cJ8q#4-B}sm}>EVQ3V7kpm z0YE!kL-+1`!U=f(DCj{b*hfMy$YxZ3wZ~c!Rwxd^YLlmyJ0vkOD+UI{Ndle94gCsy zTc*1LB((Ce8Z_JYYhA=~>(hgVe^+8q0BAHA(JB%E2E;(%QC36lRdT)YOvqY^e-#*2 z`Mf z1oqhm78Y`M z{HOc!uZ8rG?=~T&&734!*SMc{nVYNjN;_F`K@|~d5UG^cmKx$3m+~s2%{2s+A3`zy z&ZH0~XysrR3K>YIN{l?^U%9PqSRl@rh=STvE=k2s`l|m~eTHzO;W?>u*o(qpDT6pz zkg-$4rbjGC4F65H%;CcONj`iylZAYas6jj%38)QeI_oN7F2spQsXXChF;}|>W^Z7- z0)4)dYMbj5Zj-M#4dsVV^VhtCXhqT-bj=K(h>^air)|Q6nQ258yqLM}%aEm>nO#XO z@Qw!F(EI6eQAGvGNn(CHkT-9#dGgx>zfmeyl(rTdF9CvzTgC6`HM(~XeqvUHyS{BQ zb|N~@HZl8S!<7)rNT-uc?uTz4Wq;28j25@tP*7zTs5DtAEovX|U7+OM$%fqhJmV0~whU86n2v6_5YLcLW zb#>9Ng%MW3@B?U{AY%a>v0`o(lI3jqG66prm*_**c|i_}JqTjI0QR6q$6>)?^jtz% zFf0-t6DwsUdkNHuGYe}M(xgz;Ee245kx~Zp*A%r@zI&fl-cTMuzyjL6Z5P5$r2@ zrhPx|Ciltrr3zk4*5WO57||n5W$#FSo&PG)$N8+Ngwn%;1(HfGF>lNzH{j+L2yqmU-qXwxRLmlQmw(!nAzHQV4s9I`sUfYSg0FH zL?Q5O)Ds~Owo#ZjZq*%mEy*AYY6V_PT&PelM*r9vVS(!Z1p5T~;Lz)s(L}P+C&8(8 z!f~YQiX?!#F;a43K6KNZl4m!tQ&V2<)|A3* zVcZ&zc{jOjAePj@pMdNkw9L9y0Q19-Z^D%$Wb91Ye%R~b)lU*UPGGo)g>#Qowr1Ut zx$DRqI&1KqW)k9~s37b6>`Blhwtv1!WO`uW1x#=#+s8Av_KxFOaPmS&W3q7c_ERr8 z=6ju<%1W&U(Cswek|(1XIchfKbt(_dKY@#U@3lx!`qs{@X zWqQr5^yeArl?xE-i1}1^`rP2&a-Z=np?|Vl_ee2*W*!yvZn0l>AoYgMfUE!G*xN6e zQq&tvC9hnD%?gJq0v7J>;2onB9acrczDD;*@qn;5KyCR z6~e6rOC>x!hN9dNV5dKWi#hd$KC5_tE0}cYg6Ck%-{9Lolt>zuylt^Jh`Q%_Pzbjtq}irSWu}vV&W?9Ds*EIR zJc?01@Mp}5z#}4=FCmCENKK$5b{t~jVIhHIE@+ZN=Ip0egRkMzS6e3TpNFkwDMk%s z8`GsD%g^8`-h+Asir$I<+G{)|wHNokR#B{3dKXDb8Z~DeE!I0Wxs(j|Bj(pu@FDe) z*il_}eZ;aRyST=>V_Z3#M4n%AaJ>zS5|-!Z_3-#`7$^WtsL^vaz_0}zXeI<8Evq|O z9DqI-L716H^C!M8M~oX}qIvxkeOT+;y@odU#S>aV-BmIl4w;0Ia?_+-0Gfk(llKv?hN*FQX`&*Z) z6zL^K$%*={cQz$0D~p2#8PPB_YES;N#Iq5O%YihBFkw7P!q4|K6Q06wZ5CH4R)S&g zb9bWvqsp2pV2%)LRnUJ9N3Z=6PTTsW=a)#w>3RcM!HT+E>dcMo_DqWWOtxKRc6fM$ z5cMX!q$wfYyTRa~p4f3C2fyocEn-r4Sda?6rVP&yc{5Mx<&BE``GO5UsxjX8_X-}y zc&os7!c+@Qr>Blg9Cu34cNAg(xggBB_(lo&=LE#57JbPtATo{=VCkHaxBqUT|MgQ> z?1$0&Q0EHUv5M(mitH3w+AC+vX?jnq8r@GYZaL6Di%s;Oy=T=|TWy*Gf8B~&v*1y8 zNaS6m>PfX7RsvE55Anx1 zVNF4?j)~cW!O_vYvj2SX|M{$zjnY0bb+VNPf}wcGiucAY$TsfRO`G4{&utQxzEe2FH_9 z9FZF{HDyRj{>FGCM*Q~O+wx%OZ09GytbSh_%m!Fg`(^KCO3GTVX7sR+KjjnPo7;KG zD^!Ul`V!geoV9x%3$zuKCK|J5{u1&ZPo+h#Y)3k+_X$5t(9Ya^LJXwO@%a4fxf$}A zFc7*jzh`OVY$43;hSHkf*2t`!ju${VkFG5zsh(2PO?Q=|XC4>xShk8j~p2NVwZ94qBdyi2O+>)4jK zd-?nE5M{);KYHhhRUKC*JBtLa|BEw+(#Hu;-Q3-?vMbL+A~u1$-Q-K~x*T0|?=HXk zIN|s>UCf&#kp3Rg1vnMibgQv?e{g^!0yUI$^4ASv4p~F%5rEp7TV-;StW+;*>40L%?PsC?;3vDp;o;#Iymej&n`wk)_PqkYA|SYhTlUr%&T>5v%NF+0gGQ zhhn5nRW>xH)edJA@1{qiLXSeFj`zcoMqgC_5NO!_qCa}Lr3UOf9jjoGtd0DaaA_+x zGS-1qq3G$eIS-+uPK}-2UE#^*ehV1|pNrK^Xzqcu9z>U@plQ#?&Nv3q35!}g0V*&( z-#nvww*@F_c2(YY>*>;r*hKWk_05uSEvn zKqPbFUBA2BIGo>3v(i+)zKt{v`Ua1KQof{|D_z+Ufh}8Gr3SLx!J3WM>9ZX{n1Q2& z!-I)3zpm^C%qx?rWe7z(iXXF^474g`dlK;lcNi%VlFPa#~kc;J2L`K zE#c&4V&k!^%_B6f7>f&Fp$j-XjU3(WDKW{o_BcTUO0UGT1CogC$jx8Pk}rXKf^nAL zKSMPE_duVPqB~*ccX(ukch1?{+k5jcM*Ks;$4lWPNPqF-0hF;9?tS6 zvVP%gqf19)L-kMJTdv=Qp()^zy%l?o@Bcj%{`>o!TQL<+n#L&40owh#e;X|qa8 z9q;KwQtm+7K-i=O-4GcG*?BaUWX(4u5*daKKnl6!$xT>+6Oq1kPIEj$^ke?N&P22$i_tbbnG z|Nmq5N*XVuj-29z4S7bY6q56sa zECB0AL+=HEJ^0u2t_SVyZf#3x+AxZ#%HNO2fz=D1i`Jfaq8W@ZsPG^Rz&$1TwIfw@? zOH7{~W=gRV56^~2y$lQsI~RC6yRZTeI@a5q>|uEx##XA({kV~scJ-EHYH<50W&J+MJh4ga^&Ov z2@oum41RnN!Y@r@Kv&Ge0Z`fhtoQ%<22v_OpSU%d;g7UF2Fg&U3{$HyQW8M*0andf zx0{zDkPyqM;Z{uq?pE06t-dJt!dcI2%$7484)E0nz8pf%<>mF(5_Na*z&86dhKn16 zMUJfuLnLLUr!Fcz$pD|$Nj`rM0trBRI}|qTJJZvAhQ>!$oT5N@Vdv$8XG&t08d}2l zO>m~&LsAy#wh`}LT=9fb{g}=|GA02ATsXI}?t%UO7v(QoQO}p zR!OG<=KXw|{0O~$E*cKP#q3+QPtT&3E;RFh*n1DCrqZo{IEaFyC^{CTJA#6Ofb@

iN{$LM&+wAspZ%$oFH zLx*yS_Z;*jYb(gB43K@52Vlc1{w)Ylq0caS2tz`}l|zYLzs7$apbR|w0urneoOX%W z%svqWso%1UJRiNgCU?mf29Mt6fY&}3Dv)Ak zmU(a|Br3ZP@GF1_6xsH>`d(AxDtkFucAZ&C+kG#9ZeRYfBak^Vg#T+Biz&NTh6gJ! z=PMwH-p6&?9O8Pdo%f51Bm;=4`?PY#;|w9@?{T3oBB-IJgAw-kIe*1^4}hecnfwj< z>Kua%Y2*oJ^+5H%joDR(J=yMb-<$U@{&t9kUHVBi1a=XF^(#t;7wHjfc&S%!;}7Yk z7g7s%o@4>v%k2R3RYo!ozw{o!F-baS6P_vSL<=@_7Ta+H>4jK;FVJt9oz)Ho0N8Ca z5Xh^#{U_$iPy%WTFqNd`l?T4wc0^IlA!9wW^(1VySBVYaV3$h#%d)VS&PoJ5!t`M`eIdgk;kdtTpl-_KVF&KEGS z{3fQZvFcUWlXzWICUsalRaK%r#_%*nD;=^<{gl`R{7SP~@zaSdL2rtH4KW2J6+HMP zv|GV>!B~n1@V?EPPe;Mqc}wy}U$VQj=P_>JXvs}^I?tQ|GROf~!UncgN$yYJ64Iz4 zG0)SyC$OOTVMbL$&r@(04=xZfu;sGlXNoBw`1R9muU@-4BJ$<}aKG#f4&MBl_{-xa z4D}?$Z!Xu&sB0y0d%)0(7 z0BtXCqzn^ezs`Fjd4J2@s>K`}kIl%6I!T)jh_pONu+xt-^ajnVaq8&aM>daO!=eCiHtc_#9sa_bO<}HL zAE`7`@_Pdud8hatJbzap|Ko41x|{6DBS)$rt>qm$2X_hgH<_sy&bvv!D@D>Q9b@A@ z`c)fu1Bi~D18eqOT=RP^An(gMXvRI8()wIKA|J>Fj=&F;zO^sJcuL$f7r@o3X1hDe z$W!N4!Yv9UP5s+$-j;Q=KF<#}hEZ0Pwd9dsWavtj`V=x&Gf}!=kV)@~ziQV?i?$z; zw-NydiQo>udMhazK!p^fhjLngy$q?yyif9Og;Ew ze;;054Ns?7wSgBzf9j`P5%9Y!YW&s!xBqDaoWFYhlJbrh*(Yuk8f06iXT7P03~tX9 z%y$SC7*#~>v*KWQyLIcf!ypu4YrC|A>1~gjUumTfL6^Z&bS9({gq9!yCt#M{lf{b$ zSM-dZ^!N8u5NyDE3m-jrJrlLNM%+7c&)$7MC0!cv;XHBREYI11YimiXumR?c{4S27 z9(zehiuG#1KH>6En@(GWZLX|Cc6uhDw*dYUzHcM21{?vS|NdRq2O9U>gvQ6m&uBU@ z!>{QO0n40s`5)6l_7^oOLfW1@mpru#{0Ot+-e(gALd3o|z$+)VCI%YI<1dertGCp6 z-@kf)CB{@fo(NVnka;c`yPs_8z<9nz-847??60dGHzxB3-#^%?(WCH`O9TnDhAjTa zG{v0WB5PlKT@ZFsxcBkqIJ(g>KQVJ@%jzlz<7Y5)Az*W{YP${0R@eryN&rCBlf;3T z%S|1zHDbYva#OcsZ5-|I7G^Ws)-5^5CP3PYPX^BWfTs00m_Y5cc^L$&gR3k{y>>(H zuGv?9(|OEP+BAok1b0;%errlP-1e)1Wui-~NLXB)Qp zu2c9{$Rxjqrt1SNG%W=-Os~%IALR`6{F`q4F`%Vdj;W~9r%MV zTOO%DeZXr>Xz*LjKz+-9r{p%`bE~-F5U;1zhpksn1A{4U)gVfItwC;SK-ShI(F_5TC)`aCs$y$vNT?h#d*etpka+cHiY4r366(pqQ zdUp=h=RNI@Ma1iI)El9x~r zhxFsTuUfRo8!*r~iLq5|#c$)c8WJ|P@8h$gxdyK;{jGK)o6e^HA)3E`tPKRWFFIbS z3{`f;Ku0znTb$A{UIdFDAR)D5DrE9{*?264=#c5~CNqt7`E8LRJ8WqGrN4hV$2e1$ zM_hhCTVOI(;_A4v5&c(Wg3w6K%=*$Xx@^Dg*Tc^NCT(ZKw$(2iA`yn?Z*-^GHlCn@9f~J zXyWv=V~iYXH?+p?9tpNxXWyTT*Fqn7&GYyOG+*q)NpQY_Vogj*VU)!-70SMQcU4zw z+NCdoKWxX=t*hr?(+zOoo1HwkGb%c2Z{Vbu|LD))XBUlr_B`wkL@llh9)v_G;IcUY z)xACO!3O5GE<{P)WtAZ(=Y;J9>t>uCy4CHTIik@b%4gzpr?J9U;@Z?fDc!$v?`@8N z4*FOVW^i|TH4fOEHqw$W=q6eh7nO=@K&%ypFk@MkR%${!lafCJ;w;a5$R~Oawh9j* z0!`&74;XMO+g1{a-n9>U0R7;NVESEf zBtDa<&3if@vSW&XB_u@&r+_r#;-m}h8W!4dbe_O89f&{rcXQ>p6F{c`q35;cQ`!nkH+jH-FobcfZax9~ALpgRljDl?8B zmow!)_q8c}yN<;koLco_QS@4)r+F!-5_L}8Zds3jQ zJB>mHl0w>BoJFo^`hya6S#y2716?hQL2{wVMAOe-POY;s!e-J>YhPYWA;M~lk5(8z z$>F7~2++XcqwQzbz9}qLWj+5P53_{-iHc;Shmh;9Y`px#EoQ~l{t#YVG$5r!JqZv%mecO*i$@S6*>;uVt-b303qZu? zfi1Simm$F+aG)_Yz?|fK*bB1-vZj80Kqc>cyz?@mYy8T+d2!a~5(+!Hs_&eUL40oR zy$gvTPxSkaoCM0(?n(aRCn_(aeSZSeSA!%5OlshJJk7Ys2aFw{T(bgn3?Kqyf2HO! z^+Qdx607+AYRmk=+B=CrGAnISyz%_T_U$fCzj?pKDg6Ib`yIKLLRYshWDcAI0aKI2 zp}=RedoJ#%Bf8U8DlDA@UX@nRMEwLEG0#}pTUCa#K%nk3BoH14!XgRl^M_j54rH_| z?vJUf0T=UO()Fw=k?_kPK4u3(nWc9r)A<9BC8IZI3ywO%4VioMZN^Bxd8jGdZ(V>k zr35;%PrLbw1L+weF}J_Y{!~V)UgJN`fBRi9#OgC3P4$ik$Oz|D30t*jnFFOt8%zLn z9}1dAzUP5R^q6Mc#!s|SoKa#SL6h-ctPy8`KP&}4q#U~*tIkqaVQ@llOshafH=qOp zW79Dc^J6qsbN{tdXA=Vj(gB$o$nxMijBXq)zyJ%4>8Ja_7^$sioROTD!hY6H)~tq@ z8Td8(PwPbtRqTZ!KQ(ZQL(a>-b8U8Rsd80)5-d2`Z>PXwDS0wB*qJcYneWPMj|#$H zw!19Q@4dlUwy-L9Z(1*e_t%i^=^te?e72qOFAkQ1NPPgLM*)9WVwE4HbDgDBNtaP) zbE>7h*mBIee8C_8ZX;^PUH_a~5g<5o9DR@kPEzO2?BZe;Nay{b63eJkfPppXg8JC~ zmy%q>M}8Bg?x)MzxBWx>-}mNh)j6Q7C|PqCDhGaBu?AZ5Q+-ar0&mGpfiLsDf_iG& z$K9`&+}H+0`z=6hxkVkwISesP`kO;4EKNs_^A3M%Jag9N{jWtV=jq1v_t6%uyuT$G z_%?y@Wd%!jXzPI-GvCQBryG@d+Z=bDEZLy+IIrFF1jbfR(rH%c z)Z;CDLq%&&_qrP7U(x()u;&42dqSF3)vvr`M|<1;%rJwpjS+Tg?ht#RB+#u z&$FMeZkuBIXf64*yk`32-2>CuIABUa?5CoBmq0w}P&9(`hcIR{{ilv?_Ap?9LrRt9 zsou$g;FJ_TsKYO+%`e-JJU(*w?3TISY`S<-Hw3`tm3h_tJImwjLNyNZonzXnSNx_fXmww&dF|H}1rpqY)Ex;yzzTOC@i99cJ>)Bk$t?5czT2N7%H9!v*<$7Scj| z8^2z!{*4RJ(0&p!v~Ah{cu@E%Fs?x%TWRbDcTd01$Vu4dar@XLTg*e8bpacspuO4l zFbo*CKY{o3x?uZ_nVzdBxd8K{*L)z@`?U~P_ON#-<%;38NiF`M{&tr8`dHra-~9s~z~1MeA~(EgA?H6b~VK{C2A=@jpS`#HZ}d95Df540@v33crn zd8S9!8MX7ZTV8&bGv(voIhVAyI=AQWX@4fxijVB&v@ew>zY>m@zZ<{2|AU<#(|6m; zSm}N?Uccv;S-GQdA}~a-L2vz_km%Ntw9ov78|dkL{lite>6ql==LAHgw=-`v+yl%t znxo)wb0Yo@r@pq>+y(~C9Ls6EyC?A`&kb?*SjU+W;sZJEGWsWaLLQx5p?h|PU6QEX z%k*03#B=#ySRqj_zmTA^i6a+8-L7g~zZ>#O3lh=3DqXv2rU9`DAYFnR>jCf%XgcwP z%hm)Lz}>LG2eKH{D){Xvf%P4b{e%DHi3^Yl2G%Tj=^XkRy6+kIAod+oKMheB z-j5#_GOt{`j_#3#eV06>{klKsMA%*B-w$iuRA5|!89{UEAKi8oguk-7`$@<|3=zWk zD<}5s=eS?-`82EvJ`C)y5I$arA2MA^Y98584Y?oFC1xfrTfQudJqUY+3k5>9cAB06 zRthaf36yEjcy~fx{S01s)7uK*3;#hpf-Kx0c+Ue1g9~_HB5Z+s&Qpq?Wtr+4I#Dqa zJBWuB>Kw|K33R3QW2nQ>Z{oHdUAqU;W#kxG_5~{ZO>KO6rn5uBXUzwJKzTGsWYK5> zj$3J7UYw6s{3R{ZD%;jvAFpMdb^99J)(xBAAlYfbjyS#N zJpcKQd-T@Bizo0DB~rH!ES&je_s{)j+%mn_yl&K`9p&j)1j3gT6P)|J`xsTLcm-De)+dut7ob_lp%JqvO+d% zU)QgVJGBk+w|$8{g7@$p08f0G}t^0NK)_Z`P+?^FQ7gl$buexe)Lfbi$|9NH7?bRgiK( z?#_A}x8uPU;8(#5$e9ex^yjY*-_>~ci<=>C$7*xB0^_*q^pT%-Tzs`f63GqsZ|rBh z%4WjP29w}Rhw*CZHUxYXQen;X3U<9?{XSRr?y8GFl~A0x8}YfV$1gCQw(x=dPG5gO ze}m(ysCA4|;}7Kc?Ppb;)9IoW-F-}mZ+Qh`n?|;BqwVB_w)UL^={Tp)oeQ6%)~NQm z9%$ArA7;5<*+}Hu@wc*uqiyE*YVEYc7Eg;Zzi4=M((sAu zMw*ur_c@O~*C!PnrV#^)+GBpq_bjm3yrm@-ywqFQsHFt)$tr!P?AJXwZ>5V>h+T+T zLtBTeq#m5itF8>@6n#?sxwkE8N%ROfdI+DsO=o6DNFCgHj1Dn!Hv&5-u2>Hxn}zkZ%BXr6T6nN0gRdvv;? zX|P=O+S#Y#%3{rLWjNZ2?vAW0YA@Ed7O-=VM82Pvc4oWX42VNc*&KkgV%RXR;r z_3<`8)xf^<``wn7FCEm)w#O>UyP8&t-!u31k{vqNq8ifZDEGv!!ziB~;U}=+Ck*fN zx*Ygas{@k76y=Sj5A<&Q)>MSygiO9WP??gl6_c*hh>9T3+jD(x$XMRG>A8|Awmw7Y zy-*doGuZCh-r>Paz7yGn;jX9=mq8swg+}|d{c!jEgG|QUDU58?Pu!N|vri@XT32G7G~o;GuRi>rOSb-k_6r?@ZzG1el`mHFriXSPDi5I@HZQ+$23*7$Wq z#qOfK+!ySV9ifXYZzO0JD6=IdnO7R)VZ_1Ed88a|E+dVypImzIptuDF$gwR5k37fh z&?gHs_Skat3=bq=-L?<(RLSL_1Z-h0w23WMWz}=O^3Oc=r)>Nk-4)@V%sb!)#^^?uLu58D_64VWS2Y=h5rR zY*K3`CK!D4YTM%4#``+B_zhvk%4WkhykjhsmnSfnBSvO4iqn_ww+4QDDvBqc`L&7N zG1aVWqx$C6rT3}utBKY6kG-95$xkX1RxeHy)~unS5Qr&WG4oO}DRhRvEMxT$ zaoclUO76UktImTu_sprSUksHKvwNj6jgINgClF?Q-BU#_Bh};r*-6z9V{;_5wDq8l z$e|yFexsW?R?(^u$=MUZ3zL5I>wCF4{+$S4ORS$%O7{A#$)$uyWO36eDDgmxb?ZYH z`{m*GeHkpeO0`R2m*d_r!KF}qtE}>OORFpWHKH&pURuwp1%4W>&0;Jv!EICq=jWE_ zitI(?>~ruikU?$)W)y?PQd%>n$RV;kS0`NWY%b0>7T4*6fVx`q*!=xX#tekby zR4s(5O=bRDW%jL`UhkYm;fEekjLQn)=UPa|R3%G^F(vc1x{?QXrof{@OtPb;gt0FQ z+g(C#Wym}i?pp|apUQw=`Ftx*e!jPTW=D8?*2u=Blh@(hh}qPoH7pD;40Cu>GR7}z zX;9v0wjnZF-bl8)!h8OkYb0A&7_W_c`~2!hmKLJ>j?E{)ugzWGO=S|g3^Bx|E z5ECmZbxy0K%>c92IF25aXxk-;SQ9yxQ9|APs_)OtALtN@`sw*`q-#nJIlMyNNK0Hhw?@o#1A)rMOvJrL&()9eI&Wq%&CQ#DikB8<>|$tDitY3 zo?p$l6c#_VPc}T68SPy+q)kz|3#BdfwDNo`6i^L$KI+9KGn=I5w=ygMzgfxHJ@Zbf zW{ivAvoYO<8J!5+x%r!qwv)t$J8D?`Ut=lK7?AjUba&uZ|H@>%cX_eTZRR6WR zZFAz^yiK(2FJDj0?4Zr9M)4fYl3;k^!%|!6y@sTj~^lMxTA7~RqvG`I_h}HClrAP3LW=MhDq&IP$lwaVH z2?Nkfu4k&iWwc%snyhbe4GTKfdul*04?Xe5u6Id@+!LwY9x#% zJBZWo*T>6HWR>y7ZdFIWi{u1)%>x<|oyQ&~&$oUV6FVgz%d3CAG(<|6B4U?0vROX+ z%)%Jj=bhY&!_GIoR!3z%e%2YhC35}8jIV>~hMt|pHGTRP$E#YJ|C|NCt}Y}TtDfC) zGG;fT)A1cvqaR06_FJcT=67Met2)Z-&XG4lsiKXmR|egno>OkFp#clM^ynA433a@6 z>k}hozRPlvrW=Ph{IZ>85oV?1ZqR1UN+X#KN~<5uR(tn(uf{onVBD`E)d_llLaw69 zl8D~YVLHu3htZ`c!lR$U^jgLp!-E(R_F^DUSPp8ShfVHZ+6__}gIOi9rZGnZNqtUh zGYFcI3rh)7#3(!PgSs1I>2FFO!$pJzCN<9?{2i<<%#UZsQB9hJ%VPa-5paz~^`C;3jVkbU^$Qp<$7b#t^e(!W>=?? zddR1zO`~0F)We}Xr8`o1R3{33G&m`?S(!#nb}S}pAu$-V+&Et_7sXaN;Mfknt%#y0 zAmsdfC=X&xy^xcgk4kD>Cua4uv&lyFhhMVxN8{uv+?Q(x&b2ku1e1Sk%2m5z7;<9` zHGab9mZGJ+`mW>RI-1Z*Gp5KyT2ISadm4C;)eFE3c>+Q&lvkT4-+TIwQqz~NeGgjT zCeBiSqx>m4!sL>jD;6FfU*@M_N>=sr805~&%YDYn zN}28coZH8oq?>u5sRpmkq&YIlN0A4)>BK3AoAoeJpkhI1o_xuK=oC8Gku}=3Kq$5F z9sM>n=ezL%V|CLj%}n%=(RW0Lw|IRjDka!Y0f!_M7X@Y73yv1<6dtx>@p~57Sa^3F z;U5CZYJR?JW*CNfw@-Jcwd)Z)H+4wfZ6FnJ@B5#i`fq#Gpb&PjcO5gQm(OJ9*X}Ud zT!Hz`XX9zG4O-}^v0?eM2)rNPjPaTc;8)(dxJUFbtkAa`0&aA1*|z!6AVaywS4}ZV z@yOxP?TwM?N_7oZRH25+g$Mh#Ad)f)!xy?sSmvvf(DepL@5vQc=JneeoZgZ!Un|{w zVPK*0syqW-s4Ti&J4eQKb|?HVx>83 z$h?tMY0dw}M3y!=a$I*wDbG)BpkFGZ_zCDSr@VFp4Gg#lIQvL%ERcP zRNbk8EfswLs%~XsL!RtRS!f@L*hKu0Ncq^qlm)9`YH6<+@<`aFBxfzS^|Wqc5%^?$S+*p{Sys<6 zrr}E$qwB5ZTgQ|=k{lPg5Yx6;6DfhrB%_yTz8MqbNtpP&Qus-aKw@J@5LKV6m->3) zox|qP93P@e5>N}-3!@`?0JAOpmRu-vi`0mXmYVs2JK4AFtwjU7pHVsK287yYNcnID z2$F2m^%^4sU(ZP_h$FsHE_{dPx_+|CQ~YVw?<0)>+0g1jRpRi&v!E3FJzX6K-9QKf zKAE3Kq9eNIHe?DW@2CY68$QgsO|&JO{=B8a_!VI$rLU`YzWQuduXM!aCsug{9w#>z zLuo)Th^QEu&9P<`cVs7sHkO;OWx}mys8yeyPS*wZQ-w>)7X3noSL>732V$vGjG+F? zQ+Xn}`l9Cs$6M1J6X^X*FW?CH>QP(T9GM7QC@Jtd1`U)+Qi&e&Hk(QGo?gLtJ)YkU zapOFv`74x<`=|~kUSmolfepebZC$mN#j&1QWg!ihXGR7-ZFGrwOPin5X0-`Q(aj7C z+J)$4!1@V;si)s}9n^7Q8G2(0W@Vob4zOr{AM>c^4D}vIOnJw~uRTW4GN7k?7b=~s zNGp^ww`CY^TJ%%4?=yNalI6_sAizyaU$1U2aYXNh)eRG9x%>h@1zKN!d>{sL zsi+@wD6UCc1sBe@TM4R}dRO(%hfO=fnH{ET%uBhjt~r$VNUzfH2niRVYo!1s)Qb>? zMjBBInyUXWt9DGqN=zb8F0t_ulazAHn_Wp0$3zgF%c~oHJ{yjba+%SZ&zw%)@fiT-;b&mQ_6Jr^z+G0IT`V1VUY6KzT^tkn6V41^qx~%@NO;HC@Y^<k zG;S%UO4sV-j0x%_3yX%;&0Bh4bX(cFNz$rEo{cijG|+65kb~Cr+30Viq6tO~KFFhB zY6G?d^Q%xW8=umqPgw0YD2%HTBaNgc;>SMp9`A5Ai=Op7;@SJ)ftbD#cD0VM9Y%YQ zZl(@Rm&YLof?Wu40SoWPadlmjPAMHT$sIG6fcjuiCl_7UOq6`#Q~2DrFx)mN>XQ5& zP>!XP0BZlFC!qG0@uFhIGSbyTTLr#lg*Z^pHc(_}mlJzgpa<#n(%W6ENBh?YL2%`F z5I!lzP^b%)feYg)7>(>$a$7^BDK-YgpLVt+;AVJfrM4R7GWc`GV_suSAW!!@Y@?e` zbk|go6WQ4XOt?lG#9ru3uo9J#5B15sdQ1ct_GoZi9)<6q!B;=8cK~nSKNoY`By#E`&RV+#{Gt@Ey1cHbDFQ10%!om&%!uKJY5^ zTZqZha@S=<$emVF#St!N6Vy{Wy58Xz$2{XHy)@*=rv)Vk+N{l3zuzSK*3;!SMw=4|DM+dsg`dP zL+YMKa`bGNX>j^U!oWS6`daJu8Z)R0i$G@e(Ls9C`g?1bdeDK5%y!i5XJ<8n{C&_aqGh>S| z^+28gARAwl?91Pc@O%z8cwTM&5WEkPNYRwtRL9ZeAl}<1#Vyy;3u&?-Vw$WRC%@`z zqPS){Up*4xidwn{ZY7i#HY*HlZZ{q9bJ?^;u{u7r^sqP8Xi|927o-?rkQAvucJK`&JY#)OMk2TG=FvfS5s$fyaAsT*M*%X{kq3h;?O!RRum*v9_RZ21R(%Hz_) z3?>SdA5K)U+G&8S6BSEDg7I*1x(g+$%~N_wWh;zJ!}WD~)xt|BnWvC~H6BVY8>IU2 zr?YdWH=E8P}YjS&HG-_bVgJkZIR!Q#+zSEL#-F#LXq7ehMNsJnvd_bzbdk zpJnol0AnUHSAkqzhpQDMp~S{mngAd~LQ#En4Wyd8jWkPpE|t12oNG`Z@gOQB^;Gi} zUsyU=Pb$?sJeF_u_4MvWj)ptqNKpwM(ym@g(P!AZDK%NT~ZRs<;Gy2 zk%5mA$fArwy`pKcrO)ROoic!Vz^(?jQE^rm5hELNPpk&28V7P0s_92hFW$R1m}7mu zJxvf%5&QLCAVRLDysoNC%3C_R&jT=-B1gQwdV2nI4bO!t_VV7(%!nStk*a7iJD884 zqK^!LGGdRqm^slw^E6E{EgA3xSWY(phG?EHa4|vzMyUMp+Ds+=CKMal_`5Y3sB*q3aiu3@7z5w+>)DR#}D|d53 zZ|C$~^HV&n&3!3fGRAUa6w&x$wqW5MooMf&I)JJ$GZ^#RO&aT~Z?-GCG-mZoCeLtb zgD5F^v(v1b4v{~d{J(O`rDnRKP_)OeX;MqidN10~#|!VEw&~FRZ{L^S$qlA0F^OeRJjE|GKLFXLQK_x>B}XX8cpn(mpWMSb}m7WL6|$oOIZ` zB;-BW_wAD06zG4twt~g-_RuBDOffYAzcJ<+%<-u>m>SUY(mLnbU)(jG>F@7=ps(LV zr+>b!zv!lW>Q5&{;=j9f{&bRw|3BiEZ@~8=S4W83P|C}>o#gfuPb39XqNmvU^<=BZ zlBs!%rs0%^X0-mQ@MWsSI8JaQ!aZe;Aiol}xU8_uj!Q9Ry;Mxb2BT(R&A(PAWmR2o zBO>YdpUIU{{V2;*)Ku!-qAIeLa9}G^Pn2-IR%@f(RB`0u#oxF91Ziq-zlRT%MHwBV z>o5{-4*Q|HB#2}BG-e3+KR*2r_x`;||LUY%A=MG77#l!Eamj9Uu=cz)k!>3DkXxNu zmUPl)b7Z6!$5d)f`ix2mMFh>25EpUezppktW7><__$V5UdNmTP&~1xepTo5L&!SO( zQXC;$NyIW!-37)lJv|yTIAde=bdO2=b-N)c>L&+2IV@V~M@#;zV*ld`62DPYNXdGA zU9Q}6@6d|v*=VCb?Q^A<0}fgD46f&Quxp?bpZg%it8i{y6YHUJ zsa(eS?z4F7-GJ=cUeX>C)Kc*#j2uuu;sbbH%UtFChS0K~|rbg+#h z&3gy*THh_;mNDM#La53ce``v!(*+UzCphU8!#6Sa%Auu3*Dm)#+*&6o%XsB(61 zputoJ0o=6DfVz6~>o* zlFWa5A_9SfJu1V4DiGfTwzxBiUuJ(XgNDj#vzNTA^80o1BCdb<^yBq`T3kukze%$Can1LCI{e#cf&M=q2QJak zJSl%V{(H+mHpl<(+A^)^8(olOVqzjJAwKT&^qj7?cBp>7ZsH}w7;;zpNhp#_5O1)Y4kKeWr1uL0& zJq(M%;s0sD8%RnY=gGADjZlAk_s{PPm%0~>DScT`=+^l=$AbUk*MIK}i$8n(@10?9 zuI#Y>2rC3&GfG@9ez3xZUZ|zv{k!*}wk} zFK8cZLc6(%_x1G!5LY;2(p*Z6gnYLD&whv(%`h&N9E(Fv|1yJf8L5X)9vpmW_H7qv z(ThJHr%(NxMC+UGSARF$5GPdB)zu9Q40d}{p_OmPJp2bWLIjmf{Ke1X5B*;lgoPKF zonJfjzCDFmZB2oK9labroR8UW8Xx-;)r*n$wLllT;4u=rhisUiMaIN3ho`u!0n`clU*FUIrP z9&^znUT$4>bi#+TcC!jJ}(y@?yovcA0gD}ozf)i!vy89 zZ>KbxrdTzEzp6nXj+F?M`=Gw?sIld3dTl1p*?Dt z@{u)VESund;R_)DioH^zq9N_=?dSS&Sy@7^qn}u79-WB({`4vy^YIDSy;~8^Ub}*y z9#D9HU$Qg*C=Y!A<$^Gx;?B!<-%GX=$t()h_a%D!ViSUvX-|cK=}&gIGG4?gU@rvI zK?54!yLTa2Bw<&9gE}cF<$D<{Ig!e220_9;PgTbvaO)OL3Z&k1{b+y{`lTL3X$_sO zACjCg)4TyS-n)E-R@l(qpIE44;^IQbX)UmM1Zg2LSYb`{+P?1(b6SfPwwklxKIy|G zok6>YvyNTNDT#N0X<{?zdI&WMYZd%^_*bw|F-+^bhA&}om{dx}^T-oMtlW*qwN zVdVGw@z5(AkEn)n1ReS=R4WXdrAdo|Mjq~)ZSY}1iK_O6=STfT=o(+T`UZhQ)q?^NqjHBdc$`+oHob2s6YHWVo?)QI>wQp||2ZdI$ z+-oNXsQo#Q;m;TTl|J65tXpDcKzWFZB{i_m(XQl%c#wbQY3 z^Akat<5Sj`H9b8;UZ2oB}N)H z_JVyxq7BMj1-6Qfiwg>JD=f5d^O9B-wJMl~ta@`dD04G$tE0=c>QfnYh-g zh#oveR#M!+G>dvt_Qp*GJ`4!R8!Sn2B?LP*x@u-$M)veCF1w|art{EmY_PJi`J#v~ zQg_){zYHH2vwWRv;}KUZ&#yQ+uH)yC}pEb?wTW6PDYsi zprEr&O0LUOWiuqD5DTj~{aSLcr9-L5h>DaErV+1Dj3@HNco_H*Ns2My+ja)c-hv{} zve9I$2;I}bz>yIaOe2p{Np=n^TeA9PSOztZVp^}gVdyTzu@BlDxQh^caFhm*99D;US>FV&v2iBsTXF>$f@!t)#E7P%A1w9A6j zp6R_3)HB~==%}h^X!kJSp_X6hinlP)8QH^v=U#OpHKf% z0`;hsiH8ks*d%-0Bz)BtifGC8)QqEiOe-Mvk{=QwW$CRheXALg?1$oaO4;MKb`4_b zTv^F~|oM{-5qDNdi^D9qP6)PM6w5$#cyd$k`Z5Cf% zT?i7segYPElu1Nu@(b{|t#1 zRv8&u`rD!vh{ofS+-jNxGi8+;t!vl9#&>Cp+Km>7N|8H!dYHadFe@!xZjOIVoS6$@VN(k}J%Su`EMe^vP(G`V`6xt^xY^ z<*a7+jrkLSffkdafhF*@O9KfPnHep4P~qpz>QubWckM_one3NqZd+>SdOD|TLwBZcNgmag`>59xJcx>RPY0xK)4`n7ATo3cPNFW4zRadw0=d!?_dGg_s%6sWw-RMiQ(ZmZx-Wg91V>(2H$~4yW=K!=>ev{M@wX5< zBDe`?Jx5B@l0J3Ve0*j`z=wCU<@{A`NO8s`bEQ(ZdV(B4$gRKn7+U0?taD=I5g&I0 z3vQK+ax@h|-v}3$y9S<94mxP9Qc0(#lcr=XF6?9z)8aB$k!xZ3s;%GbN8#5FV_9WX zbMA&I?}ab`iX0CfK2(|i&>}o$SWMxyYsbBGkTY!QQF@Cng!^#zp~k;WH$2qy80JgI zG(EM^&bJ)TQBY(j8X!aMaue7$S_1+BAN{z8H=?%f7C$kDX%5pPqdkj4lZ;FXK zkP?%F*%DP;q_n07v$3xSQ;4_!G~N(|UzY*2wzXnwJk7OawuOz0tzNwYeX0XJf=en* zANn=MwGWyqiMooJPoWMvxi;6OBpLj)OXUs>N(V!eIyAWfjZr}_n}K-Jg5A}*Xo`V$ z`2UoJB4zb16L28`PNfo6_twf&z2x1C3rEVG2rpsIt+(@0`9Yy@EW2(&IGa^hU+T)S zQ7LyG3U*new`=a|i^$a)j}L}McsVBJh<_4o(nuei)i6UHm0NJ~8;X_jpxAeei3!2% zWC$=3xU>qKsy3m_Ob0|ROZ_oXH8 zv9=lf-3K+CpI38pbLDeK$MlzG&#f8ff6Z8!Z?el$7+}?qAAix>x*~A?3oM9;LtCO8 zS0={L7G!92V+~Y~e#!7-Pm2+677MbCKyz)U-4Wh|z#j^RVxUNx}FWAyH-- z^-C?Ut*#+S$WPEq@@|A!+Al}yJvX|&a4^-uph@_Wo7=lOaE_cVjJCkpDbBOwo(ogR zDa>RJq@3)04Ru7!+j|mrRP&xf5*iXpDF+ps6s2=UwG@g=Fl$^Y&68u%o?A^pCxhVQv>-{8A`M1>fL%fmApP!8g;Oq1w zjN@8_--_(ikp88`s_~M#197ruVsg4Et1)_EDymJNG>gw0tSe+?>!jXhTw9&5AeDz| zxP6b%W1Y$#W@eQuro?!f*!nUqCI=bkiFcsZ znZ-IgJWo#=dX`6F2<}#^@7^t!U~`j>vS3Ll8~vOC|DX;LbLAlYykY;bZT^YInuODi zDw&x$EZl1*?Yr%(lBSt|$w(V=*TOlyWyt zxU-9j%Ns0SM5vuU;7R&+8J5&)`pt)e-p#XUn13MbdxL{-3&P(tGb>9m0y`9Cv>ar& z+j^t6);SHQvyi+^JS9_F4%Sa8v1jj5F_|*#skB;GHq$}1 z(72u_sGTBNHuwaxNlbi-kr7CVDLTT3aq(%w&W7~*EvSZY0i4mY@vW9dDv)L_Sk z9%0;xrKPwWu#Q25dYhY@dyghXEEED-DKTn_OcB)EjDS6T($`Zw6331Un_e!GGM2*T zp6Z}b0$j%#6<>Nk3DPu1ZLqfkq)UPZ3_N*K-=&(hWMCqFp*GfY7=+b}AO++yati*s z%&1F^zD?q@n9i*!AL$U^tD~o9^(D_jzrbGqOQM?fPEGk!9eyhn4augy6Ls=S`IzZq zf6Ml%aHFLJ0%ZA*lQH4hQgQ#3i1dr-8T~N4FR{qJyJ#&Z*2+HP1x8`nUXfJTvk-w^ z?2&Oi({MwMB^3Xy^Lo2N@o_Y%%`ktWj-|Q#ODMRi7q5esJD_*ErLQk`ZMt;k8HrHp zNtL<*+K)qnhlj`8cd5#IrG{i-ZS9P)OG`@=MGxkZSGZBK9jI@nR`oAZ8C(5LjEzJ0 zg$p*QD9)wW*gDqf;btQX3a8h$1h}NWz{q=f`jJL<$&3^na2l7OL#*kZ2y6b*6$-g) z-&?+vbg%{Ja&4+b2ZV(|k8BG8H>2d)*4}1G>Fr9*RPdF0TU6-D*=l9By7*9B3Z_D1 zr4rq3Gcyt7vIj@2;!c)Nswh<784Re4-PfSAuBW~zYHI3r?je##IX!Q!W%YRzMj{R< z(4rtBs!YpE7E66f^$n99*ztGzCr^@VO*fQE2Kw|;{3h-gCtlMt#h8Z+HoBu{9|@ZH z*_O{+QLWzfEStK}@R|451jhL$<_(XD6g7o!9a|6;2?Aja-UDNk3ticL;XqPf4Vz5S zNNva%Z_(1zi=LS1px9p*iV%~P4KKfwu(-@zZgc!101zT`R@144=mkC;i6Y(@Ws=)5 zecQQ+Xy3?98PUt9JWxL#wjYm}ZPziP#ZE=aP1dv^#9+*6)0Zy?JvB}oZ>pVj2FvW- ziI!x1ZEI>vvpmv+E!xsb*WA_C&W_A`j2_#Kc7_6{KvpVM|9(=wLzuZ??i- zi|SAMk&5P)Y|&9-R{eLy#l>HzO$<2IDzLc~6_1p2_lFY=rHG!5z1}{_v^Oc(Hw6yw9`hnyn<_*W?ss6}G+&XE^xTpUepQZxkiJ+mMkL~Wk)c5dk6iT71 z6sgzUW6;vo9|dB)wY9x`*-t_ep7>yBSzqR52?N7wz9*?^st+rsKwZ&~6Sc67vMj#U z)Ju*ObMk{}{j`UM6|1Vwb>-VxbuUkASXm)yw!<~!Sg_%4L1hgLMw2b3%K*l6KNo!X zkRhKO7LjMwkr!!s^6$lLa0gqt6R9M<4b$;_s#F z;mR8=ULc%rNk~eLvy6nSYLfNxR{N379er4Wd%h*U$%)c~j=0?K z>5#@e-wi4^O%3eZZF;7$*zDs{`y@*52!Bs%zw86C`uH?yX6-xTt(g zJ0}(4=Mfw~ZvQZ4BrOkwvP+mYdM|ZWPq|k)6>*%fSPm!rvSuzOXo z>bF_N#iR=>tmP)=aIRhTLFQ3XYw|oA6T1rLKHB${oytUd zreNKj06K@1>27K1Y>iL(KZv+B_lPg4 zXt|aW_9ih?g1Hx26Me9@wL`N`W(=n=_J4@_>aeKdZfhG+1O=o+C8fKDP*9K-knZm8 z7U`DmmM-b;&Y@wbp}T8b%;0OXl-Aw^vZK1_MJx4Ohqt|xW#H=61bJx-4P`yU4r+Nsfj<{jn9fMCNB~%b# zT+RS40el<7p@OuHGI-&{8Ho(@zY=kH1oZcK*B_6NK7i{lG1RpESzli_!4gsHds98D)W zvnmKZa4L-FNJ4+<;_4rxirenY4foG6x*87iBQil4C!UbF{{dJMMMIHPH#hlIrQ2^D zy!R5-=FSuijt9N1pFBaTH9j#PS$B%NQTvr@?N5~S&=sy`>|4XMH04l9lymVtu!d^h zc5}$txT69&?{TcJxZVz!6WRP|fUxTa5Q||U#6p6i#^UePgOWjugd_?odT3Au6q>Er zOlz3tDw89bR(3coPoqCwqIgH^Hk!ZSEIp^mk)J@Aq1Y zxsW$%cVVWoik@28F943gj@nt38#Xqbw|xyy`v(ZiO*bzp_ZOnt+PtRWrLo+n}oF4^C&Xl{09Q|1037!YUVjXPYp zoa8<4EHGQVlm*alyO-I;z1*qKjhugJ3xkyO;Txtoe$$tEAVCMUoJlU8b0oEUI5cu( zm8#OR%p?hT8OsGA_A3;trrldZ`Rb4nX@-isUJHBZ4~R&p7&IbVZi8DOOXo>77Q=_` zH;yO$8==Vg#rcV3={rl$NLohM9;Z;QjLE5~jV49SB!q^$`DoUIPhI%LB*P#dFr1Dp zMyktk>WeYB93pK%PUkeT&U*g?fDOZKD@uj#YWL0M#-ZUFDQW$Mbj-l|-pr*92OmOh zgOSH0ZJOp42KvUfeE3F}y|liW-q}5;sr#bCXczZ-vaOt)Y7(dl+nSpjYY*~^JlB>R z8$H}F%;sV*_!?Mwx#B;*dIgegrc+$T;T1WlzNNt=Ei2CO%d8 z`gnD#YL;X@VwH-?y6qQf_e=b97qRkkF896XU-9vOV6TnKYKB)=V8y6}N^8Ate*fO* z>(@7Y?$59`hC4>PPU1hR;=g)Eb8`9(;*p$rW@pkbW&~EPX4T<~M*elQ>l}cS9=5&{ z{6H3n#~HR|%zE_l)om#r66`274-H+Tq>8|IjJ)m_+o7c~GSAB`&PRRVN=2o{$9Mb^ zwN1e8xsY&}-r2h3V_Q;c~7y4V9dL`s$obKbqXM* znY>!{-QQ5xy?PJ4VRF58c z*Eww4C@h@X`h5-glwVnFJ7LBVkz$=#%I}_biRGVPTG*z?q5_+e&~>K|J+H!E)3-h` zqO&+4@xchz+m@uQLwfc6t!d#cto&xc3OBz@p1k)6t)9XOp39psF&I|voi~j=t0gU` zi^6t{E(l^N+yomi*8Sc(u56OB<-O$q>ry*DWX8J==`<@hT>E<}nzEr0^G7nTQev{Z z5Uo;4A1eO*t&vN1aQT_TJCf0Sv|eEhI84kytZeTkC)iQ=-xUw$gF)A zxAIF(iYQQjdMcDo)1I*ge)@Wm--840U7@E;Nzd+8Dscb%-=Qch7G_C;&Iw17CngR1 zRIlmh+HyJILTW6u`p>xMuPQWOvuYO%A==G2${aThF81ej%H|i_BJx=`WYyvakoPVL z-be>9qWVhc3g+@ByVLfj*W>Q!#&j^jQ$2jusA_>|wk7J7X32Y6KHrOKg+hClgpi3J zir?kbAHueY1R6;dIUPTUgc)=dxGGyC-T4ILhfPfKHaluGtF~t!qW2uESnZXZ-t7+~ zNwhbTO|Q|?BlHvcPh^`fFNvi{_&&+&!<-QtMYaN^zZ&VnNEgGyOG&dbdu(rm=IT7I zXxIqm+ASWas3u)`jPpcF5mzYj8}HEh`Cx3?oqSw$*>?u_^_blq*A*8~XBA|I{P9^O zf;6Uix?jG1FXT7t<^fK^<(-Mi?CkWskUlKobJO#r*jR@#mJ>$pq5nBAwXMVo3!HBb z8D$jIL)(n%_>}&gxc~d$-%54$wcI}=+WKy|pFSyZvBuGN3=No$Y}%fl=v+rq=%w_Sl#ztYR~`3eT?ow4X>_ZDa<9#qIHkrYiCJ#HQN) zgt-qz*Ep}B-K`DpW0Qioy(se}Mb{rz8!}Wp1 z*}?*wf_wxqX?Rl))e?3-`972CTW^#1syQZW3w z6l&#naKHdAn)V&f7GY8A<}UP><{kLdte}e6BW`8m=osviU`+v`Pg+Om5OP;Yu12XY zyyWoeXkW4TnGH4e3vhuCVoBd8Y*}Nu7AcIuMx|DsKE%RR`D=u~N-0rCO`BrGMu0=F z9yY3_wU$@*B|NG1cg7noE}qQMnG6|IGY!cvvHJBqkjpi1&a3NF3}aKi3YB2`GZ$it zuU7yrqY|-u{=L4kEv}9s+?A}U7|n$b)mp@9w`#LbOuB5_nFH@E8VC1iv_&(k@lSvz z|1FiEV261@INiWjAoOR5zX-?`H zkxr2)B-ySSoW^WF>VJRl_IGy1RoKzW7@Ii%xuUm%=>`C*+1Wd4 zGZDzpV9(4!me#g>nwpx1+}+FRbeXnBchB)co`ndLMY`o2%6UBkJIqyzg|nu#z?v-K ztj`(WZ^&=eZ}nLcz%VjSaFSq;&F~3@0hyEw#rY-Wztp&QLlIJ}O!1s9B36UxfMXE%Ofudg3Z_vA29l2Xjp*2vPh z8)mMB^?QgC>k0-dx*D*>HZ-_+o?7;WB`KedI=#9wIx!OS!2KfpiQgQD@QxUzDCYXr zm6^+?xzxdYV4soma??wMaY^y|NRDMb1H($Q;eBU7AYad=@&eo}r~Z!*yXO@a>^QfG z;llAf$S{nI-%~M1WQL#xv9K!NZ808Uy!z_=ru(@DgF#ea;+Sl+(>ldi7;#tn__?WB zLUTh>|F1_Jam^xvd;Av8H*pfmwa-Ox<2^oBtCBr*8f=T$A*hQ@ab?k)aZ#sVgXdj_UvAHKeg?HGID)--=P7v*TZyMlkK zLOnRz7Dk98kyD%%jCPzsk`ST)ow{{c%DZVpf^To1{qbERBd&66gdTp=Vt|5Ss}`FM zuPdL}(+dK22Z6Tp-g1c=1!=jHCrGuHIeJ*&%o0J3E?q0ET_#h&yI6T%h#;d1(W7O2CQWb&71q4ywdVrOH>4S zXD-d~OA3>iME|r)ft$zPjH2Ed8l`053SM-(l2+(2eBk$0`1oev-r#Ho6=zpQSwl=i z%5U;EhUV?K;c!o1-&N)Wq1MIC*TK{lqwhVx^%MN+RrobeIZUS;y+_kYN310%u>gEF zo^8T`w&tnR_o`-jYmF_h9v|&l7|DD`%l(z&i4Dv3Ew7`aW0~bN&dsA=msh$iZ3}|b z19ihs5-a~Ycc}1t+)EPdxRi|PzN~;r2&unf6u5^F5bh}1L!s6PJT;W+&!vU~-HhB0 zMw+EwTrYfo$&4~T@%1^Mxi6GnT&)OS%xNgJZ9V4_FC9M_nw{k0;o+*!PM0KHA)9shpq9Wy2t0o6-pK?3B9zv8I9$QA_>B}B5Ox) ze4hBG%jOT$<>`#bDQJj+8R4S&fomI^-<6WX+q7CIyJED3$naQA&?<7E?_DPFemxfY zUO4H!(*ib8VaH%cM#V`QI6T9L8DnFwkI#>x{cK4S(9T#60UC5(o>8W7h|ZVJZ$!jI zNPCSY4QT_ZXXmf%`T8eJ*gO1%U*iyjiDhTz*F~&a_sqCzaeg%JV`H1b!p0knm(fjN zcSmQ<8Qz=Tqy~pvEybk2KW~@f$#h>=OkSUym4j^qwXQPA%1cO6LlU^7d}iD$IoXxx zO8}-_pS*a~(T;RSkLvq2mCwb|e2eomR~q378d!a9U9Azv%Z8rUtmxiE73Xr1Qa_w9hRywwy%i0e+ zzxc8;yKrUsp|Fzo;zB~tFE~Xbsi}+KH~f9{bGBQ7r=e0|cXVRx^IjiY`HNvo#$;~h z7oMbZJ@JkXrXT4FDG>g+R*kT47;zxEZ*63*rPouD~9Jr{1n#NfRWq(Zc}j|X81ahA(UV0C6=&H~%U z9QUWM0N~y!2L4Z~2rg%{sqdOS8|S8dePV=mmxFZxYG>+=iAQy*(x;R(*o!iY;@JU^OJEqqHy3AR-=RY#MChpQV0Z9_E+ ze0iIQtqeZ4$!P>W3Apd`U}t`9?aj<3jW}i>D%DKwK~yj%zpa9y2&AFfW1#aRYXZNk z*AVS9bu<-oPEGd58yS7%BgWh2^?u$H;mzv=72>=8 zLoS+GkP}4QP*d)DFJ<%6ruYIHKV(o)K$~AsvSuet1G%v1=o<|z8)c@`sEuCdoc|SP z2!AOKp5t&h*SCAvXi&!6D)3uT(<$YtZE2ZO3Fp?8%c`PV)p0P%KhVCp2)thnQ$75W z@#DLkYDbCt9IV0+oHnKtdiz1pEaj{DA-P>)M$k|P}1P_<_(!lp*i8*{NiXg z3Tey0osg?*bRAd2QD>I)$r2yN`0P|W65Nk~G-u^s^#tC(jamQkBevpxL}OzxzNeHF z`D(RC3M{g_K+b~m7i7Zf%J}<3Lka`L$tkFT0iX0&4?m~^vg4N@dPiGdpUDGBaUnkY zw|pJz*E$>tebSN?`&%+WEOaOcm;(_OmE(cd#SkYJpZ%;n1U=vAp z;w(v`0?xmpJN%U#?x8MCxRK;Csc}7yhfSmw@Bj;T(#}I7z9UezGfGfHC z<`dS|rfBEyPkw%v%rq>~cg^{C8giRcnX|f}i z9x&!~g)AK;Z7{o@P|vHC9E`N}&r&)NiLE5`|e!hgi)=h%M{`< zvjrsKNUg{2=CJbEJWurSdzmtz#}NhK#Sl)s^i2@wDFr(oqmkodCOpYC97TY=eZxYc zwLB0O<3X#~p`^}Yaf8&~`W+ zz9BV8JGt28>U=uHSsV-ZyuU3{soE)5%WczIs+!hVy*ysU8SwV@c3zKBJVGE^$dQ`+ z_4V~m2X&jheSKxlX(A-fKI;QE$B<|`Xp+kgL+x2=mgnw1mFmoNrZkT<{eah+KR6x%N7l z$0I-p>8LY;!T)Vd&7dM-&1aR3lXGPgULhk|I$dVi;|Q2S^@-C*Y@FyUykkmUsu-mAzM6} z62e(tay=Lo?x&n~4wagft2pbMCt$xFlf?>lV{y@rVX#mcgn8?gf&ihad^C>j$~QVW z!{J@~PoFg$cnF)Xa741+p`rXgg*O{5RBgqv7 znsU0238&o!g__n8DQ$TrG*qng34wA3D^3Xt(kw}m|H}7sQ-OM;WmZG{P+`oKu#@Gv z(vy>xDYTs{NzsrBOefVJrO$k1<+REBN7T0s4g#w z+rh*Cce7##8DA$w^3uYjWt6Ixvsw(f=#L*U;}a7zS*6mEI>?( zB|pRL2}`L8NqK=lq|u2bH*3eL!4E-P~FTG)%V3in_>NJcCuK=s#gQ4-tw!q&@#(ClqT8 z+amB#s6D^*Z)0ylpH}*C`{hD2-Z>A86j6^Jn@%;;a+SbrQqng*yMW&K;~9urGQ1;B zwQc){o+yFco!l~Wja<3`>}ZA4F_a`O?6}YBa!Erk zD-$UKPj-a+3kgO0sn-B_IPV=~`}kjGH1^Ma9yACX=cZ`9>q8KQi05VxRbTJ|>{a;u zv}@g-NOEu^4neo>_cY4-Z%xzwZqdcCMKhcfDzXbEcgy?0%#donVGQ*CtE{3!3>$hT z45fDm8i3Qe*V~sVryHM`ZM!D!xiZ<+9e6}%o4?(9m`A6t&bW%^? zetgnx#AeOLAFQZoNy!R7fs03y%c&?!3&|xJO=CUz{C9LRCH4Jtg>R8xtgV@m7yE=~ zWzk~m7!QfT4zU<9UzK8mv#n9ytF@@XBu)%cQd01sJmQh8vNCL!4|qC;)2UWx#Ujb2 z()i7*_#+}BjH;#-wMq8wK$sc*=H|wmE0t32oYvaONdXb=>yn zPr89*DJd;z*dZRX$==uP-!b|L)v*SP9{bLq91n(xZ5nG?QA^(~KaE{l-}GzB^di>2&XzDwXY!bazX1 zkHYYatCKowvjAU+)H6)3zTQh5q8yW5+1}Y}?V7a7Oa5HCnN&p@kq~QE#SHJ6PBX=U z_lD9QXc={W`$5AF7|MS_#J>+0RE|l$ePbvZt6vT{+xsp8;qZi?Demy7K@?y{V0Qb0 zogw6RcT@uiHcIZ}CxN+P$}kbh&fd{^8{{pWQY9LvV>l~ZGRq6Rkb~u1=bbuSPF2qM zxY?+ofx34yMX8}O#cEMmzEBd}TE2FwPdh|4>z~$uREshf*Wn%DF=;@YYf5se+)v72CLk4TM!mgx5M($Hm=m*7P84gT3O$_XRxxcMlDhffk# zMQ#N{9v?plT`BO<|=9oUJM3SpO zc=C%1DVSNKqH?8ec{syo{a6qCg?>H4XE(!>>j}}8Ju_#d*BDad7&6?^`Uyf*0Uw7Q+=4hals?SeUf z_f7(d^^@fmb!{2`G`Hh7+*hzBsan(*Xa?8O_?u0TzH$i*4|K-n3*KYVTqbevB&Io# z%x{<{jS!;UY#isJqN3l5QiTDk_tWd zIqR{gb>l^8oG4U`aHRH2iiU`+$s_%@*%MRKHc(=Ot|e#folKJk^uQf5bF?rGsJr1H zd`N15c7);rmkjujZ4Sassg$%YrBdI%#_@|P-2Ez06P!eNCnUJ+b8Zfz+i&uG&3(Rd zx-0=hv?lKgRx&CHy*6{&z(n4sREoH0JjBlV`Fodouded*o1SIsQBjUejr`O!${Vt; zjl>%^j*>BD^9d#BYL6+zgP3wyXe`D{Qud$Nz)rhV#==QO<|d8Lm7Z=JgUHLG*ZD1Y z9SEz|Cr_vx_jwN+H_ILE$UhkHP8X?(o>k%txRLm7Z-Zn%wNkf-?x1jgM+eTi8((hp zCL@BZq!)n&uAIGV%WpidP1)UqvlUheG24suIhowIS=_w6WvF9}Ado&$F+IQ*e;gR% zKE&r&Gmq;aEXx-?Y#ZH z5z{Kbj9WRKI{qQ%`&B(~0?8_A2;NWgkXRbAqGV8-IsE$TUr|LXV(N;UaJ7yn8SClE z4Vh)NS9{XDXyJWFOoX#GaDj0jAeId{V=y50I}OrNLOFJnFoZ~|ca8D>+ApM$Wzahw zLq%2@irjVhS17iA5vk0Zj!d5<)Pi z+0@zFHzO5NhEH@OH4hDjuHD%Fu3IcNm@xOaMhW_9IQ{ zdQ~{b=|UA93Zsn5WOB)=VjIa_?|?2Dojm69?5jX*co(=7%UdoZq`KRU}a zkr5G}fvLacu@Ma|t*Dq-{g*0CP)L;M)xq$|yHwE=tnBSKoBdOlj^Vu1x)i9Eqyj+3 z*Somcy}T_YS?Y|q#R-2eifKP7&y;VdY_HEP4D~l-$~jfpfzkpuxUP;C)%7X4aTwuuI|#bE zV-BM0=jlv%XLb&DXZ4|RHKkp<_xZi$gMF(g+$nDTCb}Ac4dJ~)wy@}TtFdiLpdBBd z8v+hIB!wx#5$+UmlRQY$^Lx+$_;osyuY&Oim~Fm?hNo)q&!4ug-rO9+FP?uytv|Qu z4oT`1xVxkCM;unVgp@~qNK7K2e>MWp)nUUJopi?B{#~}`Cno6Kxjha;-5aE>!p`Z% zF@?^W(S}-uPYWNCRyJzIg$Qhs2Tzu?aNyDHktVSdD)P?4>i8gxiH8kVes^n)fY|g^ zPM0q#XSls%0O@aASyP4Sc=n)pNRlrsOEy2A$(g6c{o*U7M1q%?y$6<0n&)d>9ZF`0 z8@;=$71oLANu?rbaa*$bm(SjO3Bl`H{`28Mi11_6Mc0%XjoaW!)9c$b&GqUAlC9HO z2+!^sN3~6jJ3YV0yv6V*?Ca+ehBzI2@LqH8e3HFNy z2V*fXRJ;4XdPWBDzebXqYi4zI-A1!&8m%cZwzMNqFknAQ-Kkdb z8dj!G@962M0~j@(6Y_O?&RqED_xH*5^~( f`!gRXXv3_NzY|YbWQ!>N9i3@hk>( z0;(QZrC~RBn?9_EmOv6{Ja0?7bV8P|&1P?&mtOey z>8K)y>8H*ApbEM{1-?7uKGB2JySqH|=bF2W*(#w?F}|h4 zJ4WVRW^9>6OONEk7#ND2_N;N>?id!EA@>iDP+w4*tdJ|*Z-;|Oa9E}_qZ}$P&o()w zQT33`5C-94G4I1XaNXc;eEb;NfU9wLhZaoIaS=ZK_sSblz9`0Ez ze3B6F+a)+EP`XMOypzIBm=b5)$$eXz8oc%d2VVZVh#S@0_@u_OpR&-1Y|jb_e$lXk z`{g%z!r8oRs>?Q(%?nosm3iQwk{cfZtk6A&`aW$hw{73crP427od>4+5GmU zl_Rke(nwAmV(E6m*ArQeoaf+SiOySYd(0mPzREqpKR1VW`t&ZW$`m&yCWiNPkhMsy zPWI0ds$ytjBEiYY2_Co2pCM>5r~m{6(uSpWjSm31(e6nFG$LV>N9rkj5Ek&=#|@x{ zUBn2)^EB?bQq2eA7L74LX{94UNzbnX%f(&o8tv`vmG*A{Z0mlzJksb;1h`YNp+2yH z!P^ZSA!2d9F`_5stkkq0iR>OoDjK59ae8(*!rd(F2{|1x4z6KYMN0OhVH{6{0FPej zUR1rY3~TYS0NxF=#lO{>NVVR@MK5F4O=j)>K~e<|e(|}5p^RIIv1P^lgTwvRp4ArT zFyo4i^XHUatk6kdS{s=i8*H#_{H87mT}qG4bYmHh!e?R2?71>u-`N;8cjRvcA8m?E z;>oZb14ekr$|8!VsLhQv&n&%kYe{Kg&Rl;G=d-4;EELD_c_quT`Jn}GB#Fzf-4acU zV{Qd%>@T4t1B|I|ev0?ku0bcZE@t3q(0nZafS|B4W8crFjg9WL+FdE&TQQPlcBfAn zjhHliWay=!SSAMns{25o@OgjvKNT0`0(fXHFQ?c0PUI0xAQ%OwF(*6w84PN)0xwJ+ z+>-lePGJRBaW$fi57(PcH2N(rrsLi_6iMH{U{4#1nO@CSGmw7u7!cbILzX6x zFWHK@@V~LeYI1k)zyNxTL?1W9V`BPJDB?tuMn9OR&;~ zugp5v$$5xi?DUj5$yKvh+LFnJiQns2xy^xEb+o@l(^Kcx-QC-7k@{jZw8|CRkg5F{ zQ6X{hmhjQ3(Y9_rSMCItM9|L}<7(OH^+^++rpMw(E{YwicmzYhD~#bDG}lVYH*`&H zy~okmEH6%!Sxj*RTJ)ce>+1X+S4;kR#84a?>WVnNHcT6zF{yKxw`8+<$L|#rT&lY^ zV#S;Q7VLh0O-D`7m{HmI2f9y0M1&9Fd4>^AihB%{(ORFVJ%L(8=5%3GR?44=Cl<{f zUDBAJvi1)QrWf~d^=}lFo$+%qrCC1c=#>T|CMPD3Mh!{PcmRFBzi;l==)nH@;b)0N zCjC#V9QH26)*R2!oL!ulnGME%BD4P^Qh3AkG7FoUrp-aU{9WzW)gzji7lw$%$3}Jw z>LBty8+(*dR$(|iG*EYEh_SS{VPQ>p*!n4U-j*?&8#bZ%LutO-jAd+c6v1a;nnPLWd|I{KF&&iXf05j&yUu{=kwf06MixmF+M^QU zck#8{KW4VwP%aMc9uD)RwG=5XTb=W4F+ZLi6eg?d69q!^67lO1vfh97dM(ccd-d3e znR^?9E9BMP;(?;XJ8>0@kCvXz-pQH4Sp2pnogso6ZjO%`1>7;n{GV!$-pa(-OWpvQ z-x~-uilAJvbZ<`i{lL;dV$1z58%Adhluv1nZjI=pgo?9gWDQ~DSnRw$R0ryQg{Mo8 zg3Tsp7rQ`xQnCB(MNaT*@CHC*UA%UKkXQ`QyN~lm_j~av`mR@D$08P5cDJ;8I=7ac zoNLjycmbT=2Y-heo5v63Xmsc?{7{$w4DxHicE7vVA=-f|#oBk7`sAI&+4*IuBkzTd zG0;9xJ3ZG}Y_6@XXC`;69onq=k~?l=vT4wKedeB6iN_#yYpf!a<%vXV%TO69D#5XCe&p7J#N>< z4`IZ!173V{OqzcQ#0(8ljPnBr8yxTtfR`g|tU>#Y3|*5m1xeErjE~V`-JKFTPHWUs zRH_>9NA*}n7Y&@-B>}gOO66K&dCXmdG;VgH>Pg;Qgq}+4Jzp6W^l=R?qB= z=Ger77dyKqdHF@@)j61)oESjsnBIH=IY`wA2?|?2x>Ls)XsG8I*=BHm5W1L7#lDTb zp8XYP06L%6rPi|Ipr%fb(a9~I@tuj%0JCP{{9VjC#l)lnVn#T3JTlQP2ykcf^L!|0 zj(;kGX*o+9lNv~!(9{n!(*D;AfB_{<)5cZgG=Qr5v>AQMmJM!^0V2&7sM{J;MLe{* zQTr0GV}8iWDoDB_6E1+u?(6q-MvrEVFX05;c@aW^MEp+-sr*0ROGN4?gbiD~2e{-xsm&Q=^ z!+5EEFc>J2%0<8^T@x$|Y+R$eq0!Ou5wQ`{Dh3bV%Guglwb%&9Jd3)bd zwA=$yW_-JkmU>f9kiZ!KX>@FgbNI=Lo$ePsywyyygx9Q*Bd!0B(S$ zjEst?sErGjbjGDCGL})K})Oj4g3JE3&Z)yV0zA#|K0_8|#PL6P4&(6*PIXj{pW?CGu+hfBrId`g2txfVieq)DFe(tO76io;?`z)0(L*>{ z&9o7`Mljy@Bx>tJgOwKHyWgFxz2=%uT6bv*Dkxx7jTtr{I>EE)Cb z@P0Fv2u~j0VUB^QX+AgiI)_POr|YY86%@w%2L^QN!cu~~?Ah6T{iXv0dDOi0LDb7y- zN`A2i&cPx-UHz(*uyoR*;an zt97kp(P+RTBy=-sG+|`#BDER)*ZJ@x=uDyW2>ZvHg@ziA7U5Bbhi;u;;(CMf=DQo^ zt|QSQtD8aJe4tv37Uuhhi$)VB_4W3D4C4OkYZ5)xXIDrY2uc*bYjQwgr4nX#`_`IW z8$x>N>dK^j#zeG_(%f76|LYN4FLKef&COFVdi!|M_3#Zo{zuA7RK>=N3kRtbo-TL9 zf}z}lP`UviNJquTJIN@tz5xfcgh4Dj=xCYjIU!CUooGA-Yov0!9tWVqFB~q6_O2~{ zkIul(3CguO66h!`Av`?FDQs%$%H1!aH$m!i&H9rI0#rJk)M^X7R(5uRx`vL6uF7k@ zMvGTYk0a>-IrRObJhePcv>1h0_9I`NrIf#su)z#>2<&K`puaz;t=%3d8de=AklMox z&1FL&Eoo1+z0Q+1;Pnit)GK_?YjKeJFQ{F(7o8OWtNkg4-qgHlpn$bc_0%bq0^u3Y zNd>F1MPAR)P!{}f2^~F5B9SNRy>`d--cLt+donh&yvYXyTv${8?7|;E_GdK?o~*qS zp)$-FOb*uk1O|M7;K6FZxYWA153KYT)QtVmobpC&&6b}JZ~0bsaa$#87A3WgUI!yJ z@s5Dm-^$+BaCY#jw>$)9CLB3vcxTFy1@5)8Ij8^!64MoB7pnw#?qo>|T{O{%&5Oyq zNVAG3ZGQg-)g9-e4emp)_Z$nF5QwY6kb3W^x9d!et!?-N3a-=B!grj(5n0`{XRK#} zWFn#3$LBRm2g{RL5*a^q;3Ajd@16^Kg0~&j`@mTmZOO5}-cXE^UufA5OuZ}#%t_g& ziy}Y3VUEG9Qt=fo?j9IOdCNJtS>~^RNw5;$l+wKcR|H5ux&4u@U?~gAQDJ+Cny50% zlpui@nv=SzkpKO5bADJVEQFNvm+PTYce}G3>Jg&Z?C~M!_F?A>H%#-4TN~$As5w17 zUJKNmZ@YC|_*o%$@8N(%I%DyZ|GxZls%XlZFa*L_#^jE*T_wp_yrz=yXBlYVcq&PW zL;$|1vp$^GXJ#eW#lr(Juj(MetvGdZmuLxV+g!e=CDtE1Q+@{yp26}Lp|J+zS-ivS;u6* z90jyzs~Bm%viVDM^kp-~6CwC~$(sD-L4%qbS+;kauRt6Ohg^dr%^RG92vczHBG<^hz+KNGW!ja~cD(5*yE1?X`g zFgi4%Vj~X;SdI`l69F2oumewS9Wf))bueI47Ds%!{lAfPoTp!zh z0)wiq)LJfu7lFn7PYxBC=q3W60L?`zm1nr>v{ZMo=*k=Y1K0-%Qmy5!r2yIdPBV`2 z;(g)5Nfz@HJ2mxAn!bPeg|u1vY6tcUzi{Gd!=)ELvw~UqKTq{A_eJ`Qrnh8v7o01C zb}k{2@Pju5w(qmWz8)Pd;#RV5+2b63Rdj_of9z90ybLDzy5@uG9$qM$j|aH7rGRh1 zn(V^+xW0s;-_|xTfe*!%*6U|n63`jBSiSwk23}@RM~eXwsTGu*W;<{>YV1p2AFKU+ z$KN(v>0PfWquM@M$bOg|mLgL3NXBCn#jMKC{;{@6rfl-~2Q?nQ1yK}^hUxe%_`h0L zS5{6m-zqONK7pPJ;9X8MG5z`=6nr`Pf}G~3S)(pnY5s+*=!&0y`;zw&i4SfJ z9Ac=xLLJSMr%MtDgbOtOVkQGFak;D;=%J=)WS#!8-9fW431ibeeeT!^JxOk%YEP_t zZanDP6yK2glGR);_*`E(C$BNq(lIT)P0Z?dQZN<*tRRmPXz#*wFVP+agBUTWDd+5%II6ZlCeTq>q^0yw1oOWoasSkUifa0LMz=a=Hn--M#EVxIO<}KRG^UZt| zi8L3Own8a^$r+DeM`Yqit8645pua!>a0u51tgNoW01+h3caF`#plFbp^)9)v zygahMU+mYfU)N@>z;khOVgrORM!EfeIU;SSUxGhf<1oWU2p+j4m*p zNVHI{r!{T9@Ldr=!uXFt6O`SoMFGt~egXBar9ET)bv;49cu-L+0&xdCO@QZ%=h0=L z3h!1yt2Q<}`@$iPgHAA3A6S^3@3JTNAVz$x z!vo`?5DV2o;u?+V!z26$xVTipBCk3YHM}(}c}wY7bL0KGar}H#G~R<&Jz1$uS@Vo6 zE?&9-P>|e~gjb#3{D4X;#!cz;KL4rUYQSV#QBpdwT0xbn%xlrK52-4A7>Z7IO+?dllcKQvbT7q-1-lfMuGM!G+{Kr zHzCr zCCA&zgi#H!th2K>UM`#5T{Xa2C|OSY+Zt|1vnR^|cbZgf?_j5Q;e$F8{EqiuTj*Al zSoQ}UCGtXr`8Ima;?9nR$G~)`{Pp9Uqj}fB0J8B2J4H%FSb<=FpKlHwl=?uJT3du1 zP_^FQw8m82Zlj~4OC&QTs9M$!0e`iSE+5dg;x>LEm%FnEoKMQ&D~Hn+uSSQ5!#-l= zTEkN?j>2m+GD;Rt)?zHo{~t|P9Tvs^bp-29PO>5!0+4gm$}?(Xhx0RhRSL%JKJ zyStmEL%QD&-{14{C(rKA&g{&MbIv`N^YHMnw_-7?d03n6=kFwOLHU+05AeTwn_XKX zubsE|?hhla5Nc~%d)?b#iTB$Ha!&$oP(os5hwWpiEF@Yqi(^ltB5BKVF+!%gfD(ly zKp!1m!tsNF*N47#OOigz_@S7gTo|9cOqPtYto!meDh zF6HDNz|Fd?&6ZR)_h#11n3(u@sqY^JT~|`dXA%_n7bkVD zg@l>RPdvQ5&LobH_h2reSlv^}zC^fyO5Ke9sNOBPrm1!> zk%IP@EO~T`ND@X$d&Y>KeZ3wZmdNjHw4Q0tQcF#qD0irAi5fHZl8RYejQu}n7BG8w z{rKr?A$gkW6~A9ce}LzV)$#!QfT7)u!GwGPe4$P6^E zq##!L@sW9YmRbQ#H^52`+%4TcDCwx~hAL&+Ftv-eAmOyGL=9}|8XE@#f91==1u-ox ztzPD!prE-50|c|_qKxcp5gZCAYJ_OVv$y>A2&8nD2pwHG{OeDmFGfaG?Ck7=iL7u- zIB!e&AyK!tt_@VU-{?@_U-^!_si5C*@>Bxw$ z|D!9}nqBkIJF0Cg^kcwZ(H-cn9W|}iH-ttf@l^Iv&}Ne@Xx_f4 zJ9S`p6a4B3a8M}v7=N8m-EW!crP=b=A8K~c^1@8U)6KI z7e$ZW&M(_$lt6o<_o|zu4VEY>c{oz3NfX)h;`-&gccGHO6)BBhU{vBy?;d8~+?rTe zuFkGKOn!@#S62L@vYTO7vtn&xW}$>p&+|pTkDh*P<1Tm9xLSWpy2Cj9D@OX;>xZgZ zjtyIGX0eFlM;1Jp&23EGcV{YS;L?SFeA4W}@p!)I>Up)v!_)Aqa58WpQUWX%Onkz|Fl+t^Wvr0jJ z3gw%3_r|GhleCIE1JJ?y*Ui2i+LdNJe_+J|-!9Gb5;0Bv4zKu7FAP5-MT!CW;oQda zwAVwu*;eq!RqVv>`V?pF$-t9G+MlG{5LtZhqh*-qFS~nm2n|}fqcx$?gZl!=OEou6 zxrZBv$Lqm;`zu;%-ZyQBr~mSx(-SLJ8(N+x7xGh{lu`p4wJ;rNUxy_H@zD@N3gRWcC@!|YyiCJ}!Hb$1*j{pTwp*zA`B0&# zSnqmD8neDJ_GhY&mk22?p%S{o{3b9Eqn6ux{Z(m8Ysh&vxo#$9+C2dh>%Y8cNUDSE z)dz5Hux)t(zc#-;?k*o&FHLc#&p@PmM|bU|2Zo@ZRv!5o6A)rgpIVM7B)ZPq(T*PG zYj+R#b+om;TYUH%+#109)opIRXXj)GoOc8kE~%tkBu2ql86>^CI&}Fi526z@k!+4V-|nP|gFoq24$;`rcv z>TmB(13e*8X*C=K9UUSNDjR6I^?T$oRpp?bV)uJ_=5q4voFE5*v=DBN_^QRGbXc|r z4UKaL`=9Z5Ua)MMQ9(7gJO_ULEfURR;-<2evsWE7(uS;wuU`u;1l`^3nx`$gmOY=* zs|;|72@1{iN22@FJtY6JUN6sBDqujk(W_OvirVR0F1o&do30(7<>9d!C5A7%vwn@~ zz<{0lV(8~mUrW~a^SNIZy9ZSwEzT2Lia6?C)X~3mHj?=*x$ueJaUlC{L}eJyHzkG&|ng z=-}%~RE`OA2|q&*58rh5JRcf4tvAEc9fjP>PV9dJwASfptyRbFKNKATNHZ0<0uTrP z7ZIzh11a2g&L?efel>8gFz)=smbJ-gX+*TNwAYZQ8{!(vB|Lk4;8#>yng<%Z_I9DF z%=uXt;jA(1{o-299TRn1dj|)b^BsB1hTUA|*T(9M*kC5${QNv6Z|*IK{i_O~kw`)M z1oZsB2nD2fqy4HlI6scAh-t}Tdo(XOH>1nh1Gq}PcH5Do>4p=eqmJd_{R;)f@3KQw zKE`9rHsq8!KjV@|z86|k1VNMK2{TiIlgk)F?9#Qddj z?>C(OQs!jK-8^iZx$YkArRV+$_j41-ZKw!ePGfvy@nBz7%fgGnd%EL3v9Q$l*~@xt zEPHd(20|CWyF zmK~d$>e-NM6*6>X(Q5MeKI&;16(qp>@Q$g(Eox0?Yma2ZV5JTHey^4Z4ckm-c+l65 zba`e*pcj&+HZoA!Qu+gR8-ue%bNJ=T!@IZBCx7;7GUN*-#AV1Cr&%PuyP;=xC*s_6 z2_wRJ_zx{RJ9}HRAxZLHGy7E-n1(VT`P~6cUfu1)KV6P;oUT`To+L-O{C^B*R}}>c z@Sow;&lP~6hI7VqB=D`(>&KFM=ak z?4Gb&do57nz{QSC)6{f%@_0O--)D28zMF0r1kLf@-i`H>XO|;5>Yl;C zHMDfOE8OHp1V`?60eI^io{lc9jBIN(-jo`pWfW+*5H-K#aJ?0qr{akSggb6duv&D) zf8oHu;C?ms^p0nQqSEN}?@VjtQFte&pQfCk(sr=t;nGskN<-W*k-^=!!ZGWM%z(wG z942NfSLW?t_-`@(Bgc8wm9Y`T$LqCTHwYIDSpn0BvQ)`V$7>B+11fBG-Q1W65CM-{ zZtQwgaD{wUCqP9E1x?D22J1v!^IuAZgqRyfA8~$^AH0ias3%Am2pHJvT(j48dcSdX zBOxm(SV;J3jn_6DmMr6->8#Db4m&0`4tqS0LIRC>jG3G&CsV2L^iJ&V6Q3s2FbOQ1 zhez||)C>xIxa@D3-vVy@0Tb_fs*=xn_!5}Z+A4uCM(i!icIBiRoUNpU-TTUBJMRc3mqmcQB$cTXIO%kM3sCl4)acaVg}v zyxJ^}&Q=dirkb?0^ckV78J&FMSt!xyENUM`KTzuXe=Wdqp`tN6Ui=V~;Zy}T??bIjs7KO#!<<`HQXGO5CGWAF zpwkLT+=Yc;mp<^xggtV41Xmv|YYNEvp!opPifgFD1^g z<>Hi2s3t5GwmNTd`%d-d1%OM&fT6VBZPoiKRm3@yx7p;~(J?5(?$Q|1lw@5lYtVFQ z|J-Ry(Am7=gCH2n>J5EI2xfa76(JB?mCa?TQT_X64tte-_~Wuo-QodLacF)XzVaVJB1GXOvf9@=P{QIB!CSBe`#IkerDr3Ua?m z)%19~f9=ba!NbGzJS|qJT#5zD<{K86l0q;(K7PHE6(xGV1fspXeB0_OiM$^D@AIoG zMOD>5)I`=dTL}~6<3bugNmQS&uzLgG<35vkdpHMAZ@;-bp6k0T#-|bR#L1iR8QJgJ zzW_D5Tn!;p40-gY;@~^46Lin3*%4!tKgf95SfM_Tns0913jS~n2dORV9d0~zXgIC( zd@r;jWn2!Gbg)E+^WbN?bF?sm1|`?;)-Ywx1VNoAOdQDb0yTLpEb|RUIX=4wzjJ^h z)GmCHQWPG&1c@f-@-QjJ0}*j<=fJoFdd8}Gh*2ync8=4|9}B+E_}DU9Rc6?P4)l7= zPqBh2YT=8T6uMx!XMP9ZXEsbcr0k0+Y%a;$6N(q&_OBV-A)u|8E&uHfC%%ctrvObCUcg$j?4(N<}7ZXv7mBKpfi|0sp)K~k#ES&g5 z*UrG`Q0XrVVnPx!hPQqnuDSP#aW-4M&R}z>OmioR_V$T4i=AriHuempnfejunNZ+Q z_L9|ZmWZD?K7f=lOUS6{qQ@E(^`6?GPH2{@3{4TB768Lv8u}9lu6)$4;583 zo<1c&m5;g6aixs~iLl@=dZ71S|JGS=pCsnx@8=uo4Kv>PqQgeS{mxONgp@SSDdVU~ zbv7mCzb$;L^p8}@rBC~E`BsDs-(W`txOzVkTjp_bVsEI_D3hS9Zgd0{n3y6lBfpG# zScP+coHzTpV^6lNGrHd=DYMqXLmhX6*ByAa~h@Xk)z>wWx!ViF>X*7AyNKGv6P4ZctQ;f0vTr{b>QeT(kt1OeNPB>&(D7id@1hR z4CNb9#P7`*k$S$^$$-rH`D69_xm0K2Cg=!U^GjiO_k7Iz(TX^Vv(S&Y9RuHA z1al{_HJSbH&-cCY5<(h@l_oFaT5^eX^~O@9wEUb*#_NWE>{3S2X}X9DQMpt3iCQsIf{uScmq#zG=!T5Q zprM~b5$^Udq03PY62&RAe0%M43!oSba3}b7ksO8S(p4==%Zm)H%Pj zv}e~IjT@$CwhK6gaSU0>hRp@rTn=V%4%(3IgZ(R{s$2$kzIaWvc5BeYUSZL_z<*LOv z5#4sS$hQxkz}nix!Xl@#5||JR-g^pQg#~PG>MJNJf>&cNU9@7HxHf}^WOQsy(8`MG zCFI+?I*c?XpVi7h`T^kiZfpG1bN_HSGkd?BT&LcKjgW^HC)xD}-|CCi$A9CMa@Oag z{3+-~i$IQInGpLYbmJaMCa}73n0zzs0q!Z(tgHx2blrBY(j>l5>05d zQ}s1!Tj^8kxk(_?4eXMj2#HW&1WLT>er_WBrEav(8qM9I4u5E6SFf~YyW&n+lt`~d zEPj?Bm1VTSq+Fgp#TQJ{+AEU0e|~;5PVOF>F1MACh!}|}%xy!%tH^Eb#>+&$fU@nU zfpsIFz2lX<%GBSl@Tb51#QIGv_p_~4r}H@$RzG@q7?Al1_qRs#6bFW%lUJ}PhJ3^w z&7UIiSs~Uo?v>RLrjubv4SkDh8I^!|e7rheh-J4wr0nm&*xm*Zn#;z8o1s{yiLnCp z)UQBy`cN%(bNY0LmxwE&sOzRmZ)ljXS!r?Sd2ktBa}%u>$5v|}V;n(P&Qv-wviy(} z8YhWX#Js7WlQ|~MxbshFSzzs}suJt^ieY6)Os0B~4YP4I9<0v`3qrnM%2TXdIQJS$ zOA{lPgmCp+-ErivQPWT1V9jr|iPiWQu_w2KBSLgtc*UXEiK|74Za&%ad}>;n41w(+!SIS)P@5y4)Ok`tG2IS z`U3-nj?hTL%CWL+HV*=T8A1)tf>Y98#5|Xej2eJhK`ZQuS1bc8aq^{F*P*^d2DPVYc?(IS0`WXQg>A)sMT#GzZnf- z(s9eH4LtE)-RJWoC4@cnNbM4q$w+z|Dha}PJzja`k2K2c5$pYO0=sN0&wb1*51Bf~>7#@~Ke zFPjmOJmf3}YR!*e-{*6N4Z%X9isI`$l^W-6d?Wnw;rhXo4Kn+iDi3W>RU9h)+ho?T z_LF4MxF)DVs4Flk|M607`mMg9>vliY!0AT`n|}P)t38T;hjb)^GWreI>$} zTuWJDHJ1y4CYHx-$7fm75z5|_zV`>e-eS|CKevS}Lf)bzIB=kGXlmj$97&S_RuLwr zVU@T91mDch?jC-{#hrrvFS?ZxCDxfU`}OOgbU!Ne-_E1>_~sd#n5<1HYvmM5spQzK zwFjs>EjjGu{WaOTaDC<~)M_nrs;bVbj6+7C9lGT|V`AFRcPA3L?I>r|?GVA5_HJ!> zEegAeYj=JS?R$WI1jfR~#zsBZHxm*Pf;&?OZVW(y@L#XKL?Yx7(a<0)E-tRBt=+gL z{|d~o^{Q3-1_sE0zCMxF$QD@U#6Tck@^4QuFfe|C@mzzg-o4IZ+|~2 zXl7i_w#!YYXp5##FxiE{<8g6u(FD}Q1YFju=e!M#jeu{+D}4gzrcz^$3)&n?X6B~N zr?1Yed4BsTmoGSic(n@$$-jSGzEDy3ykTO?9HCqss!x z5Ns-*Xf-q9*Q&3tAA|@%=t}B*i;9ZE_HNMN`K9cus!tLG1lrU}TSc6V?p!peUjyt) zy?T$**7oeidce(m?wVt(CrVoKRnL}vnZbsdGq2_2L%BfAPYj-5ZCWlzXMyAuw^*yi ztBx{1Lc4$OKx!IWn#!29Bqqu%I{!#fxgEf6cpA=q|Cp#ObP0B)Y_xhk>S?;&%$03Z z*HwiiX@AXGf$BXir`{iDJ&6K9@7f*PQr>_?d_2I07heVy{nk*ikX=N?y+K6oHEdg# zn3xgMaZ4+!aaVjI#f0;w){()O0jqdfce+wxw1G8O3b3UrUpxMEeTdF9fh%&7*_;(r zQ)BrBCOW=`5s9$)>mnm9ZH$`bk2Y+9)=(F_(6S$-W1ghJ2WL0CyM(K&Yqn`#E-y<< zRx-In&D1!-q5SZbnS}F_@13kkNbFXvxm&NNASUJtn%;0fTJ({S`K&o9zSJvHmYK5c zCB`TC^cmv-2;AK1tNizol*HF$uMEs=+N%qK}M zbzgJkcbdS|&!zwm1VF03O|ku}o#8wiM6ep4UnaB*HLZX{km zrCyMzpEJ3iXs&%ul`%0GQA%$AJRNy7K=V?*<=4!60p;ZSV?Fc&ymONC4YK&<&As`f z_6qe_YGu9d4R6b1T(U}yGY5@?5*r>nNW6jF95ogzBgc<*k^Or%p^9k%6T92ZGL2l$ z*}TS(gF%ePH7QP8sKrc7T(tYG>*RFp$x$~U@6`zU>uL>w(K}d zL_Udr9@Xy^m(m>`gk6I?Q(hw^pgAy0VEn*IrZ;%yO)gQSTJsBWL0~=04-WHgl7n+% zc3D*PpSVpsSkDsq?G3j3fTYY;d|}CZ8h*P_ zo73NM7t*3+h4G9!hi0XEiYUVcW|kTCS$e<9zSfB(58Zbpv2Q{?@Y}tUhzX8B@?d;! zi%C%KiTgrmIz{NHBBU`?;%#^c?HwHyzN>xF5N1eCJ+!}aR8-LrRFvA-Z8yZGi$;wW z5&hkU*$&gUZ})8Ar}ItNxx6{f`~B|9|wDT~HT6!(Z=_lTPN{e%_1 zVQ)r?K*q$MKJkzqUY|@!=6G?Im<_*oqinTTEFbl$`gmII2kj)3S9S>MSzS(tkZ-QF zPq=bePjBSjmA%Tc(uRkO2#`|t9a%HuWcI4Mn@|3}n(XrJ*~n`CRQFp>C?#N$1eP!3 z&j=^PV@Rn>JA2ALUtIsF=k}o+cw%9=xVYfF4+M0~ zvuZjt^c|7cy^X4>DBRVTFJE-n@Koxob$%dZ5E2p=X*9-HRIqeiT6yfNse4>Z#bTwK zn3;i&Ic+@vTQj}7+F{aW!JNaK1Al94%Y3%Xr}goobP)u|`&BsL?#VCkCS?Sw_i%gB zblQy{dI)y3Y}71R;wQY20mSzR@7eqPng6JYj1OD=4Z_00?-3E@m6d})LZrv4sI06% zZn>-5i2O9Xz>O1DRmBF#nxGCr6B8P6LdNOdBhZb}4B@;lPQ&D6WPVjuRWy*&mE*iW ze?qIxW-1)^mB11s>o*1cvr9`dPEMRuvMFGu2bGw31Z=JATyk0&+M6m22@Op;;ce1n zO6IUw9}^?aDJly0@$r$@Maa8sH5wsW1}hhr-Ops6ZQZtqk~!BN+WYLvoQ^d;ANI9a zfphO4J_d#e0Fi};hISK}fiyuWMS_x^lLM}gvk&L8vmZJ*E*1t)Y$c?$u8!*+0>aJX zRqKeuj7E?NS{xY&y4U5%$fuoFkFkjRGdf_TmPr;XRhH9mJy103LuvZ&!2|UYbq93z zID>Zx)BH)rVZ)-cyGzdbNYR%(Lm*1mADcbK zUnx&4Ci{$WmQ8bV@3avfJV$Q)k3Y2pZmoV1aa$u9zE&+p8p3#e;7{rrzPj<)vivpz z|EI7r&NzO!vBSobVJtZq(tbq)LXzD;8yP0hXw%hf>F4~j5@-ry=PV+S=M#z{VHxiX zU+1;095*&Lh3*=~J+NCV7>kBnUJ~8jig_Wt#E>1>e6(6>YnuC#<&KLNk(AxZRnqAA zHsH02z5Qi{-i%I{4Tyy~S(ORl=h^9J2*6ULpvZC+`gB1>`v=eWGMSk#>gl>qfW^qL z?U&Nx*TQdr`5+-}7k3)zeXSkT@#V|+mq>YG)OMY_t+h2yc1=x=lRv=@u;I8=Bed__i=#+1D;K1cWA{5*eU)uax%;>bAZ%FTdw6cn)bk0> z)xIxvw)pk8&%K@gadfv$xOg_jPF8~IJ!Y(}YWZf@L_$0!)axE9=0_wZWp-jWHZ{?+ zaBR^7J%c-}bni2Nyh;#Arxq~tKQtKKryd+3rJ{?gniTQ;M=VDhiXGqwW--{RD{TfC z0|S4(%x4{sbw2SawSyMVL~;>>YjofAejMl;z&jqdqE{4GW;smnrfV2EtQI=r5=N-_ zEp5LSUbqq}m#Y6Q=F}3DtiUYo5i@aNuxhG~$n>QjAE#vc%T$A4+A-aKSP(s&rpg|0Zj|DG;2btNTrfqc!0>jbtbouMBFs~Lr zYKHZwE)QPu)!S(iHuN8K|6`VOWO$e>Z^kl-ycVJ0y0~-a^4$y$=T0CYoyi-;k5Hskp!nZa7E)(BWj1uZjazx{&4-sqGpV3xD)G3s zTsHNhP;D4MLBtd>@>**0`s^CC$J_8^NhHj%jJ4PcWn{*v z(OepVAjZ%tvp}= zvZDAU9URz^1@1qBbdt~X;li+VW&ljnIZ=$6l|)An^4Z^)$Yown zo8Ft)a~jMpCMgrLzBgaIFl8+Y%1vMP9ccNY?-STb&2ImhLAFq_{h6(a&(Fs4}L z4NEnW(XqB#4^7*+YJK9XuCBiJ{noEUiUL16HPr>yk<>w(F*h2oVVDxHBC;l$EM&!fud^v0#Fh*ZoK*5Xr~v;d<&Hr z;c2pUS#axi8@8GnZQzOzmqI<>m;FsadOTlc5zuzOKu#=(51x^mxh9Py6dNQz|2;V`GJN z#SAp7z8W2Pewa$OQ$gEn{I%+1$#OSIkRcv&1J* z;>E6@x4Rc%cxGFWQXvU#t&b-}lA*SK?|k>{RZE&pQES?C9xsfdCKmB{Z9d^>mPP>t zk-9XW3$V1TaWB&ivq1jM=OQBItIH2v-{g+{6E>w4(DvARcgXqob1? zG%$D(qC;|?_Q)#@cO1HzLwS&e?=$qIt($&6Ct7X?8#`Wd^5I^&FgNtjYroAyWPUdv}7 zrY$eFZW99-FM(>_dC5}J$z#yhtkhsTCWow(k(PdC$M4h=Tf=#%x=8VM|P&5d=?SH}n1qzA} z@&2iX6-J0}QO- zyDR{z$rcLcBn4ZLq`F%LrZ+0r=Pw4dyAeIu%htlJsN9{w!Ct>?gQhDLFyei(pogvC z-MbzZDI7~5lRmgJDpLHYUAb`)`J^g}bAg@~`wW))lD#s1v>Q7YV+hBOsiOv)WOE zIE!1k&8m0l4iZF2C1qs(M5=2ixBC}w&Rt(tLK*ch7X#e)8wCt|IsTsN z1yR%SNq?DR=9yQ`FQCn8E#=&C9*S_7ocb1%mdN_SZ8+&?XTsy!!lL_r!bdTc9}XME z2P+S#pZFKqBwxtJ8!WG!ya`Lt=v&`Xjx?c@i0Q1rR6e_^)y!%5=dTNCNwZvbyZauR z3lxi=od50AcWZx){`r%jYU$7xgwK%3n{z7pohsM$bvb@G$!9$u9C>g>r@F1LDUkMg zGzzTmWUQ_rL~m37L&%F&{@U)_H9!MSG<%bcbVxogYG(hU%R7`?dA1TfQ`2D?J{hWvD>dH688_RAKgA z@jzdUmo(qk&y`4hXw5~#C*M)#t1ao@C$4FEySp0vdnfQAE}{w#(aHVD)UxHs_SQnUs@iE{Zc0ke?h_y1%R%}? zQOOtw=>8~(J89JXdOOdNxH_PjmH2Z1yaqyIC#OzfO3rkhneO)GcQv3u< zH$u10fW1HEp_Nq!6dy7&yJl~*`jtC%$s_cThUV>JnJexXmfdK*I0y@QTeoHj{}!mU zavO3@zCn`a4I>E(bstkg?!o|x-_@+u=OpKTk$<;A{MP& zL^NclQMyS< z+0X@t(5+qEm>j=7GCMlj?;wUx$A#k6%a>ApVmBDqmG}>8f|munB=`~WHMf0?5ozo4bWWV$;Na_Too?V+_0cvB4i3$x`v0Il<7h)Z(SXMg<> zKuuudhDdoSV}Dv!mva*q9_7a>j9Y(=@G-K3iK5OLTsCe>EE$xi$c5eZtuj5fWM#u>9SB~9Oz~7xkMzwKRX_wEo0t|A)mf`EcW4|Em9mf-b&Q=0XJ>clTmjMkw2$UD zDVfdt2rN|VPpMU$lZqA2Z|7~UH`EP1?uX@CbN^1NQ7K{ri=2i`k?-vE<~VsA$QC}gwQiwXuO1p1@#;a^cR zFc^rp?V47rCW8F&Yw;L(ZA?sb5IDw4vrhJAZp1U!Iot?j>>sV3 zd{F|A*L7V+!`b!qi4ObIYlsTbtCb*Y zkoSyW>O%|!{Dyn3Da`&%D!{(XonhG=KM3geZ7J(~l(t|;1Y-=Y&Cbn4U}H575m6_{ zsF<8GRfn)2d0uX%b>F*lBX}+l?CF`y={0QFrIgiyraPxFD+0N;sYhyrPeNwPScm-u zpqGcHpNQ%T~wx-8Y-w?YpwsUpY5zeQ2- zP#&z*WwZ%OA|LlyDwp`8U>bbjY=wV#eJ}fbw|>o^qRkwP7D*)QlNM<8Os%UkKIhqb z*W@ScnC5cIP~&0mZ4hRp_#4tU7~>8~S_z3r;?pI_FPxsM&!P*N(8qtpKI}H9#%bT5^n-Vkl{!$8SxHEm`Sq`Q0g^%&I z&*9)X`M4rlp1v&hzsI);^_B0+x}z*u@!z|~pl$foxOuev)TIZOU$K4OeflKqH}^`p z_6_Qj>{aWwGz%AR-g7RE)$=dEjV@`@E>hSWCYQ0O6LFMM2)58KXQtBm{vLkX^C~ zVdHyvkxEJm3yacb%Gm5OGLyAcx(G2T4jJV%MNVmbUQai3Gz-d3UILa;a1l1{?Hxd& z3Jjy5g9((+<#C%4Vyft;5Ar*YKAxICt_Xm#nY=?X%w5r2>2B9+MeAKAqiGyogsPpy z;P+?J77ACaDc82>LT;e1pp{F&1U+-*DltnYS`hpX=^nX!=@8tO)d$YZ42%pZOu=5Al#w<7Px zd*rcnQltAGg#6k>EQ0;*HuE<+M-ZTq09YY0J~lKK4}*oG+9nH%yWuXw?Q7Y~6HV zb?#tPC?``5IB>_L)8SK<3KPG&KP%qgg}_FxHZ=I$pCu)Xn$Byl`-QnmfUN~LH#hb- z<4xW6HS3dZd@JiYg9MwaB`2+&UH|N?EI&}t+5!{>)+fYWpGL28&UvpO??<&MW`dUI z&1{E&q0T-OxYE!t}4@>5`IuR*V7GxVGE9b=xN zjXX0Bkox}H5OdZ!BsMm&{JUGWaq4W{NHBZo2t=1*@tT63I&URtOayzzOslhVjViV; znI=|dI#}iIksa@L$bqNr>D@iLLS79exy?h;LP%r?-$UBi?ghQ7<7;9|m#9Ovr^u(G z&oEIvLnVZ2Dh=zjN-Nhl%!S53GWF{2`=<_Wjp>Sq5JNr8Up_RQAH?pQyB>m zN>6t3^LElP@pZH;5l6VG0X1!v?|S;7MX?OQY;EiS78CJABu*9L$+VeQMs+=^nJx>B z3GNT{nBb5Ist?Ryk)pR!UZe3#*zdHMCb)Ar;i!L&Gva|z6zOdI+^zf(8#p zYn1;U_b`WKlgCN2Q>^pJ$K#BH=B|gM>+-V3(9v>5T2>P#H@KV*>lLi0Y2g63yESqe z2}cI|hs37)%(nu0&rbvMhHZU8k;&#wt7Uq#-u-&XZb1mpbmp{_%aku2BY#z#k(SY5 z70pT$F{6JAxo_29lg85E@QYbjU*E_anQJU>H1Y8as9lKtH)K?;egjJ?Y)CYFx1C#`m+(KYa z6d$-t}3Ys1mv+X|rr$4z)mX zFZ@Q`YvbEDZ@_5%+j-583}N}bC;rx;W(G44f(vG*$u{*%=na-Y89SJ)VL#@n_Z;bR zLV@v(Zd#M~K>#I7J6p*sLy4G=JLiJ2VGlkpI*{(DpF(*>Uo3YnE5>@&0{475FM(sv zDnHIzi>#KMxX3i3H;0$Ze)3FjQWSr@(4!ALL~?IK>k&$yGX97gfzA6v={Y|7j<9H! zCPV=a#i8`kU~!$bD`0|#mO8gio9D&TG#fPSGMQr2~(f+NJ@-H z;*k9B@U@%x=b%Lb59FMr7OF(7@r*guC0J2X^(X9d%tg?wU+@ zwdUJHd1?A(OwNl{WBR(?F$UzhM#0r#Qo;GZaVkPX3%JU(%9{)R_?E;QX7-@)0cfQ= z^qpgKBTVUnjHxA2{56*@8d`8Hqx`n zuWLq2PE=6mhaS(N$~Q78nf-kXYwAK@Q4Of z!KyXxgQT@Dh!Vok8$k!d<=Ui3Ze)a>{EKb-$MbpCaWUiB4>Asa;O7Y~X&F0a@otEh zMT(n>PEHKB^d;>OcBDG?utv#qJHL(V*jf{Jc}dmLC_3*=8|U9;-md>%;Y9~jfW~$G z63qlpYcYIbBdb>6`i_GFaIX{f5q|hgHu0loqh8LN$(z}7GJ+gob0VE8<>GHrId-aF z>{KG^)zWz>s+;dr;}w;+g(cZv&EpW*{$v}@)a3W_e)IvCc&l2`&Oh4US?Wj4e*t?K znN&&**lm!YfM0tR2s^k4*x1-kPCAe)O(rOtmRJAyNk~Wlq6DFtzi!RBAVaAT=w;S` zOBzy6IQ`(d^0&J|AeL0KEPD8P6E`(9qDedyD+S<;BJ6q@tYs*RP+) zU)xhsl9PY^`t>hZh9yP_Jz8o6WNub^x_0Nv!omV@CZ5SJ2X@}9hJz#@KKNuX|9KtQ z0aQNDx4U^JN?%=rVtuMWRZOp5_tWjDKFDAo9xRXr)OuBQ^)t%BVLtm=O)Yty9n|;c z^Hm}1>$;CCPj>(cr1LtcQ)kq4-KB^m{DfuO0)UxC*Ztx}&$~nEWDdk)rGdYHUo6yG zK@*MA)>hm6gxt5^1BmCi^@-o>>2Ag5wMsbH?L@EH^xMjP1D{^2g&=Dzcw%=Gy!1bt z|EI?XuGC@qg}FmV&#T52W}{&O!{HPftXaT7de3ZRw3dpUUSWQ=`iO##ALaTUl^QZK z&=OS(e1fs7z(n`w_^pYN3UUcxA2qL*JlpC9N1d(l3&Pbu)^P{0m~fi)bf7U{6ML;; z9S5jQ)h-cFNqbj5(Ft)(J1`tJ&}%q!=p?V?>K}O-FTdbWQ(N&E*ZR?J7?!2-Oz23s zc>1H)nu3z117T_NJ-ub^KPyu$E6!H011K1JzFAjpmC&`NRXv;~WdMT&V9rep`EVNO!FB#-~qLb6!!3T z?tXlw{+RT*0)QUAWTt)2c~2pGi!6LqkZ-DeR>P;CO_gc~6MGPj5sF|pTWOX309smr zi09Q>dBey0BsCZ|G~8nJ{O0z+%RNIOw=Q|c@MR?6_xXy(@pmsSrx#&YV_?IA$weGmMxiV-?r^i37@qAK5A*Q|58qDrI0xJz% zEis3qeIv)xm>Sn(4W)9hvh`CJhqkK>EV++AN*#faknMOK73YV56!KR>$l8C(Biaxb z>iTUHC{&uSG_R%^ul&mW=2+EYD6j{4TOqB^#VVDCm*ay;wf9=MiE9}sucG2sUZ|Ix zS@@CV6y^n0;9XhWR)YOxf{v|B8`fNW?>f|Z7$4}1IPmeE4!50vilQJ!p@?Gv)0U}3 z8uZQBnf4_Urp)+=@a?niwLn4s{1^T{Vc?5}2ja2&HOB?wMWzR^=c3GdDrQ&up95(&;sU4Pi%>^MN%WesXOf8>{ z!m6AFKI<#6OeBO1n6x1C zOx|{6VRd{xjce}mynXLxkjvr#)Vb`Eu8~3F; z47^5q#d1`mndl;VlH`9aa*8||T*+9qRC!~>GNOnFA>f^H;3ANFoWtJgmpJ;`2p}aT zrwM+qXzmDUm?xy9Gs3A=Qk>_v6k}`S%}2h|J^ZymQSvqJTjWQWj#+yXG&~rX>P&g) z?A|#D6eGbTCXT14P}M=E@tEMK)dxF{-1APS}%fI^^HUIi6H z>QWsX9=dIYv+W%oezH=diZ9h|{Z)|e9s*jRi}KFZ)eHbek;$eGD9Q2kg3NNdUzzO= zyRsnb5lBYoO|%Gr)g~i#2~ldmtp;I6d`Q(fkMtyD@K;u?-r#El>mv z7VE9%gk!dwo0~NntnnOnMhsS3+^mo35<@{%UGciniIf<#v$NxVII5x1tc4y;M2UQQ zd<+T=H5!Z|<$k)HYPtHW<>(Y%igGz4DJ1mvwUjzJZNwjXfnnf4ROJfj2pBjxuG1TM zL`1WVp5WI!f|koaQrR~^MimEevZUKv*RbkX8PT1Xsg(u6ljr5I4EFerOr!>9@Hor4 zU_3b6>>kg)P-1DnpO{<;S+*8?5@8G>goXDp*yt(d}_>#e)bU|flZc}=C zkiF(dxaP=NHD3y{E~n(xx+hzJSgJm``@_w|>?~LD;q?Wg;h*VwRE(X_=N(d!g4b%qfHnDnD*><+$}A%(kXS{ z!wK5Lm5kf9LB!nZ%jL&6zMZ-GkzzS^7&S+yhb9;IF6{2+d| zTNPoj2#c9zdeiaui)vm0-0dNOcL2ml)PEEi^WWKjvt~Y{36XqJo;&Z}2YzwsWb~x* zRs%a^(7E4Hy;a45ho@CmomJAA68AR1B0ra6ch3&G|A{Ukb|MZYq-ocDLoX>$9)2M3 zVkKUOUtA-sr(cOp;eAS#$?%S-=nVz4z3euxE|ODY!O-Cj(yH;=dZ+WDb~Z*z^6=nK zf9Q&JM!vB!(vgnIe1E^@D1k<5!gBj?XS=12JX&@_Y)_WsYxnhxf^NfOU$HD>-4N3> zo`(Kq{1U`Ls^>6%Uoux5OMR$*f{bMJLOP7E1(v2)K*hN)LGuRR{OsCvhC#kk~k~IoR2CqYUSv3gMzu%G*plI z?9H1rYG3yasK}E;eaHNhs8f-nuxb|rQH!Iy_U+wkd9bfIFLP*!|4jVp0+J`IcOBA@AL` zXm{RRh7TUi7pi!aa7`YIs?Xq%uSbRJUbmHZ{r$Zf8f2LH8 z-Uv3^H@7xj&PdD$D?4O=niB>%bY$i8=aoK=?PyI~|Y-)#4m{O0(A zo$h<5vFY8i%O~tZ4OF3>At8AR@$@JeamSBNCV60GDTrMr@a9*Rgay^H=UY|dQ=e=> z>_(2*+{L83dt1g=O`oC5bWD>4)W+q>*P2-f729% zGTrQ@?P68T?uG<2PV?%kRBE3bMR=IV^TRb$MPOTHh0HVQ{Ro40h_A1-K$^V%!Vlx= zZi=Cq>A!uQ2w8Y>TUAM@&^PTneT@o|*;wATTDyASfmK&V2(CYbsuDf~BOC7D6&V!`#0(}H z`hQmgi>DeQe1t85R;#`eG-eEVQ~mvcmgP-90h3kIH9!pw37W1~K4T=EN9*zz*n&|~ zQoa@nWmVO4&~!>gGJx0A}>_gLHRaihY8A|Mu+ctZ?SQVN;JnMMcFE6s4GBP+7$Yyy5@^iC3ec zjiy|t$^`1W_jOGM!%6p=jFn9Iw{EjKG3*@o<^%n|rJLqMhWc~PxYy;BI0XN!#2Az^JzzwM^= zbaH8Fr1`Y%&{@vI-F>drgYC8XuBy)I8?hAse=UHFsOiY0dF|~7s2o+xy?!02?DqD^ zaDtZL;q8sS#S>C0PW&?2z^VP*oYwfqO|iMw%}@0{zkTmH$KRJR!4&M`tbiGWuXS!J z@Q{L*4vB~EbPYfKu6J|Ev)^SPojH7M;-3RCzP~@~f~FR`(+2e6s#{uXbG>9bA0t)5 zKw*_n#I3q#(@Edr-1EK|Gg}jv;Dft|>rGi=a)T@BWEBj4KME>r%oJnOuCLhbDLFxY zU6Vw0#WL?ew9o^)-6fI&!x<=2{j=i);i4X38r`;WW<_LvzWY!}CkJcD5Jq{6BoFvQe7$6SDl#R5=jH3L5}<*a2Ze8qjigaPe*BAKlZ_hD7lj zP~$WzKiXtaH*10cx5h)496jXuw!is3$K4*q>Dr(7Bh{Uc!+}2SM`aL@UcrNtET$fw zzaA88Iq%BYKR}WDr%*IQqvQ7W4KcAi23+hs9DyF+;PvlcRL|G_s}p~g21c6TV4%5k zoUQ8+Ce-zY%dJx8tDHba>2d61__}mQfI4X=eGWG9zU9reTvEI1yR3DExl&c^150y@ zLZio?qQZ9DYdAlCTXXgcjEXJul{cIVy?Ds|kz3~@^!NqU*>U?d=t#(Tp{ga_`;Op% zJ(EHYbC>I!_1@L#0*DVCZlS6%Om2NDB)U{2Bo0mhI^8?QA!MV^h-^3_YR?RN^kcL* zz_w%O=F1WDZ%7O#;B)+NE${({Z&}Evy_w0U5zzK$V;{8Ue;e_^?aHi2q$D!gTnm4^ z4?RVq=ho@GmuVK44eDH6betT^QKXcRi4e(C{!Pk0%T{lDt_(8>NRD{rkY{U3iAiCR zh$r0*dr(jH&zfpgAV8y`S`xl_NBV8iElS9(fL%jg(nN(vb#ZOtt@Z@R8AUA(^dA+w zTu{rB#E{q>64F<^fB#-O4#V*O*{ACMKzEdEz>3q3j5c5bjKXvu+E&t6TftP59F)xc zgEJca5*152*P|N8n`=WnTw{>wUN9`;B1ix9R&~-jTvJAzzr*c9kpk5G`n`SZ-_p){^JY(!(iI5%UtbRUz~7deTlyp6uj!<7&0uIcy^ zV!$F(1-hVL7c}h01k}tyu3y8|BGmutT+xx8o3vA@Ms_En{aroWHV+R2dl&tB`!Qk% z!*naea{dmS3I5l57=?gPmV@eT+y_s#2UaUoT23c)OCLTlm*ecU2_i#9MVRpL-mOgs zy7XRNc5yke40rsZM)H>K&ZIkauVWX&Vudv}R+})k`Uo4qvN| zPU*uGK@lD2XzliJumnzE=Ql9;YjSx-wlhn6#p8rN#*iE$_)}jjyLT5FhEJ|C)nF<| zXk*yNkn^6+{$p00)0ZcoB1&3=z9CETfsw8w4>Gq!)-U{4UnxCUJ$qKdam)*g<1XEP z_Vh4%^-4;kF#LcG@cS`B!)P|PF*U3A>3FkpalZ9g$i4Y@lG78FqCMHhC#3I+d0o~u zlj=IX!8{(fTj8^Xz_#UypG1Rhs_Q{35Up^&qAV&Zx?>rkP+|03`*G#Lk$_yT87}F~ zCH&m&Z1ZB3Yf7W#3^(LcOW@UGMWNNkc*Qrv#TBEElBk7>6j1rbq|xb^lnW;&%u&gx zU|2!m{)rt1OR42IlYZY%(qoc6j3cgfeF?IxUJ2y$){F-O_!#8--7VU$$>8XpY58kT zZIMgRWja?k=GBhvo7h#=JKMSgZe6j?Zm#;q`+rnKV%fyL@z)I%Ewg_JchY2$V90V! zzH|&h-8ncoHhY;QRvj58ny}eR4ZP21jlfWqS67uXNcQmFY?!4Z=Aj&t0Yl~#4OzoW zp_4I#C$ogbpcLd4X&2hOR$)|We@U83URM5n`0f<-Bqpb%OPhX?twbS4A zc}>sE=-5Frx3;z(E!4p^>u8Hi>VL6av}!=cz}U4T0dA%wZZ}3ytd&(%ny@$?9v*;c zGAAo*9aPqu`LIcI;VgM1JiPFV3VIOqwtEG%%0z5ziDP494v@@gX=%9JuJX#?$(Whr z=H}EvnHSAbP{cH)ICAC!0Bb>uHa7#FKIjpb?1yQE(m~*GbXwXd0F1q^PnJVx4)8|~ zZP{;cHzVD)qQ!!OgE=b73JL;&_HJT%nU=FnwPbq%h#b#0dWB$5ghfQ`rexT_FaoKs zMa}k4Y%qz#z3Dt2<`yU~>X+x|8vx*rN6ql1M_o5Ee5E3}oR4fh#@PSsk11(sNjW+1 zLmYAk20WJ)id24x2xm?0?;W6?PJz6*#fqm`e-D=pt1|#;uG=?Z?w#foRr%)kaYhP$ zH2Z2}*=gO7@#)8RdZ^PsHAxMwdAShbKq0Qa0>IEAXc+i@<7>`cR~L3%Ffw3rNmH?S2kMIi}Q;J)t+WFs-@z85Z9-WSS#MPaX2cIBan}6)-qB$lj z*=TX-80Q-=&*>}pXD+iEN7n+9p>Us1R|8N|qPlK_l@yA4ZbH6deM)LDe`nKrEm!*X zq|d6v={pZsgQD#@CJ0o*lre=8=&){hZJ|{}C{;@w^>@NJLNPO#(DKfQUTWbH;5YAa zPS)t#Uu*_f4?#T|k_>TuIaWJ6zZNJ`-mqogJ1{TE{~coubB6W(as=0OT<^ecmG>{m z1Mm|f`Mfv|OQ@m}2TtOpL!X(5aLsmZfBz)AxO!=mB`pu^OirwW1IIDcJ$cCA&Nep4_d;VppWOF=)9w- zsu0@U&178)7`-($K98ulEzIY;KM&JV>>+_tVrGb^6|0?-n+0wMXC6(*`6d|4cJot< zQNZZOvPJJ3`LgLiQeDLZQFE7yfT$j_`3x6}RB}49B{Ki?akS1Bw zvvEvH8T5B~R!nFZi;JrJQfd(o!GiRWS!i$|0kZssJ}Ft1Jk7=p}dH%3(D(7#Ax=4+DQ<7 z|Gvj7jx#M{;9sFbuY4}xlTxX0n>Psb)E=K6gF}ON_ku~XkDJ^+aM;sCaxm?>gI^aylI`LueSXFM57WC|1w zptGcX@lGS*;_F7i4ARW$41m*mfuOx|s`VsYc^NLO)7&XNyMzC5H-4EUb0R@>%se`TH+FUw2|IyYGk}KH7tluA2 zi(&F1I*`&o{9lvUNP!BOJUCgZ!yB8C>aA0)=V698{(MLxNp;}1_LtQ7vq+9&aC__# z!UwVQ;oEJ!*(&Orxc)B5EZ;Voj1>R&ibnGg*%BgZmiP2Iy2{`4Z94h_wGbH{46&(q zc0StGebL9+npzf{i&dkTCnu##l}Jv9JJv-2Wl=l~W_#_-aPpcb5RTop6y)Ue=O|0= zLSA2MxmkAaa=-j7*mDF45wz}U@BL+gwj7kB^NBQSbq<>ixBjowRDGnv$k=<-Q~)|O z-|eU}1T~D&EM(qDItEM|p6ygZO^|pnAvgJ~&#c{&?GbJ8=5zl~3D}7(47Ibr`8>iu z?l;-O)Vu!_zh7BwiToQu_quQDC=LD!C0^?E-|aTs9iNd$PIAG*U!=_V_0MPT=Zg#v zRQcQA$S5dX)h5F~Nok0oA*TxD%({NS07g%uR?fkJ2^c*AsHvxLQ<# z{KaOah3K6a=w2x(g5JTQ>jPr&=7yYsA^P7xTS!Pq(1mswGp-+c?8i#0P|a|Vdx6$0 zmP}@2OYP%4LZE+43`%iH2@EVOVC;^$rn zP*HEM8`^okP2z(#C0iu)b-Dq8_idJW25_pr&K+>Pyt7CFW6588La-DNuI=rC)?X)N z9O=y;v)$;}SRrTU`YoQvd{EOyUt2++O^H}CPzQOoR+B*6a=tZa4AP96ZC4hvQn-@vWS2+Dbn^Zh@Csxb~yJ3RGI`GW3)gs(bCilA`Q} zf<>9{z2?kzyI!li>MPy|cysT3jsD@_lH=T?D+AGH%6Iie+m6#K>p9)aI%%Sl<#wb zDW`VNZ^yl*j2&CHn3noC`ioOe)*L%w5Kb0J?^Bw!=|EDUxfPZKWEj7_Cg+XUjo7WH z3>0^qB-(IAbF|-M&Nt0=gom|E^Y-#Fgx{Qz5#~n&N$mKyou8nf-bMB*2Zm1?oAw>O z1Jlmp?=|g6YKFJ7^O~X)c0?zyYs{TEQ)LSZ+04_dr?y)oFs0tq~R{VKMPAmI3X*8*m6=I%|~MPA%f`DGIFG6aZaJhCv@ zNvKV`M8EPImN zq?CpUON)*h`sWZ)5&uZq`@#l>sCSD2VC5{ku(7L5CQCjlEuK5yuUxI`zDd!RGfi3f zufq$X&!}1R1tu;kMi)N3A0c6V*t>x5gyM_Z?>`)Wl!SKh;LLAscF9%ZW)fTb<5{Q8 z?%w*>s3Z74#wNh%?3#?5JqbJ55Itaf8<(DrY;M&@Jfl+$t5TCgWv7Yod+#vZ6-9{% zTSJ|arjF1z*W1b+b6|+m(U8yzbT#yo%^~7u#~e-k7UM^Vv_bG)fE=HF&z7B{s5rl% z0e9BQ&|KZC-M#!%LFgct%6A=OK5}Ttbpe5;<^y+7b;K*%`7GjL;}wHJ z(1%S%nPSMxNjI3MzInPSB-KCR*QY2P@Dpk8P)bQF#oXho47?H}lW|J;>p6stuwfPq z4`zNyG1BuPD3F33#L57n61IzMfZ+!=@DCT)7k_Z*-MQ5)Er*jv?j%9_DU->Dvh(p% z*bcf2pSO+|Eq%*9H+m@j{#UHRcE#Hs0Jj1ST9V86@C;93IU7WPb$+l^ES=S(r!bqZR0Zz%{G8POj!8PoA9P*T#g zZpyB>Uu$R{Scp7)T=+Jv7N?{~q%(3=QaEjazUs`ixjS-3(PGb`K!GZ)+w!CS=X>7A zUB119%0Z6zAHEhBWqpNyb5xP7SfXLSXhrk;iuF=7RxBl*5V|L?z+pItZ)fJ^K}(bL zhnKc2xm9?0EsJ}E(*u=cd`PKAP1F7dK7gPOT#LNc^b)96*=`-novH_UUaf1~?tRoq z*RRKWJ9SrFcjg{+Q$SUCyfEJ*@(uw0^3ag2Pj68ZX3kG`T0NZ!Gp@(*Q}GR->#$iu zO^q3BBssC`8f}!akdmUXhf@dlTiUz}%rnk^iF#ijwZmNB5Rp*A)@hR7M;x3Sn{{6c zJ)UF`OiWXq1Wp|?(Om)cn|BE7L%+FGn{D{FtH!!!!@xwua^2EJ zX#4khOJ7d+&8cT~VmSZ^+?Ow4r#480MHIm^%)lBKwq+hdeNf4@bU_+2WZ`yl6$zNa zC=?uD!OuIR>$Wj^+PzYzR%RVk))`@u2nM3ER71%3f5szDi=r+x9z92B(pY@G0^zxbV|w(GUaGR7dq$_9bTV;aFD$oQ4_<8@+I4#;PJ5S|B)=n#Mc(xH6_YTg zLp)c~5H*d%{P-RSJO*RscqIh8;tMv|u>n^nuB<#=39rw~%L}qyYfOgG6~rDpIv~J! z{kh|71vLy@DPZvVnjk;(J}4xFlf0G@l`J+GSc{BH^PLqys#zEgn`~`aRaNw#KZ@1W zW6GO!uU#84G&>U$@Z&Kb+<}e^G&|Ju>A!wSD*hRWaR?VBx!4&s0&jH@TpmEq0jFz? zVQ+|oASw7IA_8Gyagnqi9uAI#j?P+rP#s{-ySx3l60x^iv2uI3f@hBw`QUh6f?v^= zfMv;`j8p*W-_;c@!66Rdz~fNZ(E(&l@qm^~iDV(KC{!xnmMJR&=2v<>+7O zy$x*-6yDDm71%-Ge(nXetW$AFQf_K>H@$LxTD`vyafOVRDByc$6w#^=FK^qD{jSSD zMCh&X#OJ;Ti#)V$Oq&WIpRaK!(P|#$*w4RToIZ=yC)~S&8(UmS@BuFFp;|*^)~|UDin^$NY=1 zx;X^pAcIkTMq|w1@ofMaDd87 zf>w!o88#nZm#YdKUb0U_Z=Pd-cMzDiW_^SurD>vxI=fd=*mdYasDK>!C9~ffnW!@d zJ#+tP-(WnNM0!xX-vd*GTa&Aq-SYtT38nsBtE`x^c#u|JGm?)_m%l$n1)7}XAU$nX zV{u3n#P~G_yMqtx@bJ8Yv2k`kM7;plI};P;`k=)htzn}0>K>m~n+)Llh&-7{&{`j; z+{+LM=qn~+CVu=t!p0<|%-LnF@lQ<;ce9q`N~?tFpGzmrlX>01WV*A&jhNRZ%(%@LgspA# zydgId3sp_m31E7UHEMcFIvL^N}jYm@l7sR}OjDI<*`>#XR1N08z#B7S# zX>)r1U%Chkl{F+}3}2=Ve5*7b&sv_u$ofdSbUq%+kmlJDa*B@rP49`>{*mzE{2Bu> zy0qn!Dn2@lRCb-ghV(~j#HH#VcTmd09<(^s&DDe5V&DyR|Tb#s&RVZnMafEUR;WI&EIg z@JUtRJw1+z4N3gIU|oXd)8F}nOc~x5KK*)zjg7%@x>1Y+BbMXqpKK8SB)}p358eJ6 z_0HVbSX98(l^q1etd}>R5Yy5g+sm1aybe~2?g)}CAYouY4(%0;4YHd6aMI+YZ})nm zA(VNlEIF8&mQ74*2J`Wwa7k-*cnyg_o+9W2fsVuWV4703RNa>l8l=2VQDm=`sg_8{ z$%(NH5$XWA#>9lw%WFe|4v=*~Fw@f7DzeBEosb|ZE)I=`hL(K{I=$Cu0CN7R<>g~L zIxzL&_T!W|JXi_Ab`}Jbd*y&psMy$r6^XKv=QlS5*MZrZWqqr9zYP_D{sX=@WSQ*vPUH&KjH?0iau~Y08y*ZfgCmc<)ErKa z1C<$apgr~uZYAKq-ooO(kJY<41$B4_{xmpbChUjAl&Qn43%=>lWz(KxwR$?(pU2DT ztwPw8E^_C(25FWN^RoI)Z($OX2tr>rAmL++8w-DlMkbn`VY?_heTh*D*x1~=`i&f1 z1?~5}ccVAv-IO}F4;9ErLq$_lRMwUdQNb&K9j2rz&n#_8cylgARDUqht`R&EJ@O4#dbbd4{tMDTT}8AoMTY3 zf09P+I&v13mKU_!=nu5%k#>yxWn?tuLX7G(l&^K#EEB?A4FYkZb}!(odVU7RdTitx zXUN^0kPuOYoUXK>`w)x^#M}0@PXS#>34zs`ah0QAMdF~aqT^+N>=Uln+ox}f?HM9d z1q>`C-J80gi>W%DQ?QrYk-82JhbHG7@gmqc+>~*)plZJ8vHBYCbmiIT^&$4p)Fiw? z0muN^$QgJ3vylbuj|zfsSi-{{rRLIzMS)U)_ZVGMuX5<3xEkP^DL**uy^jq z5Jy&-n*nA4`*2^+4yaFr)jz}S(ysRuZ8|#s4u#=NLqNbA%&Tqr0aD_Su~B}4$9~&U zVOD-PdNZ)7q$0j_0v8W!)UPS@OA4K~u{I!*L`6+0paT3V8`-Trj$V*(kq6CATZODN z=97ZdINc%n+`l>tEinlijr>Yq< zqs=J1Iwa)WZ3PC0ldbVP;iWz%%w(r!Cz|F9YvR}#QI>@Ny4(2Stq~~vhwH1y9iP?p zo_(sx+mtHef${}iBZ`Ox4UpXP zKTd*gJp7)@AZg%48bQZ~@2+366I`0}suD7PHlXluywkdZecMa#PO9Tlf0hB}RUm1o#O(VgD{F3PmG1-C_d zL!@JxinN9-pTR})eMQA8CC6-9)15L~ZB#72ocQ;5Al!NRB(uCId})(Z`L9Ii6{$#| z{jr}!_L%_$XBjp@jl%iu&jbYTo30PI)TR6dDpNDdVqcx!u4*(V9b(9E%mV2h5WX~i zy_1eWQGBO9;Ef^HE+$XBjMt56K7!t2>SLKa**$xsQ|Xh z%F1f)8>^!b6p=0!xHSQ@HAhNyg7xveprBwmW@l;@P+R%}ifd?SOl}1W6LW2AFcyT{ zWGPcoQc&pYckm>CrlJZ-N=i}~2Lh~#zaYkf0lLGS8OW@>1#b8NOxe-FFY)FVfH#48 zSf~R-a`S&ZoBQWt6!SXG?3^6Z*#=BdxH2;{OC&q?1kJ6kX#D=257XCJCVDuzGK__w zOH9tpi1@U7_@#k`cP?{kzv*u5L7{lM<#|Kz>sNHd#s@0Ewj}5^Bxs%f&ghA=o1UFP zLctd395k|iKbL-D9kFw4_P3?i_2Kq+c-~+0Bz7mVP9lSr_7fJ}`+fqfy+rh=f3H5^ zpDCy7KebMYcq!59++F>sas&Vim=QJNM>?yhoh0Xy*vzjXq(e(k;S+cU9!Ye3j#9fB z3wW$e@^iiRTOVAG+Q_TZma&s@-d?J6d;%DY@SmkegETr#KaV7sf@H)6b0#BPPXYOu z%3sFhaq1y^mKW6lon|}z13nm%y=zImmZexUnDiyO-{=10I`Sm=q-dHc zV!HFB3>@JEJa^P&vdICH#%R66#-z|%HgC;h#rj!ZD9JkU{v z2>4_~O%Xcf9%Eo)^DqAJJ&W(}`x)r1Y5+YZO{*!qi}xW>w&6y_ZyUH*=p}KARHbzF z^HW*AYKeN4F`3I!OX}VL5+o#zt4A`!UfnLKMSZQd>%S#=u*T_sr}ur+Hj?x!V1`v( z%JM^58QwGyT@bg&sXe#)#uiAD_M2NB=T;X-z$8mxe0FD`BTavCzqHPm851s3WpJ=S z55vk|z7XJEEqfw7AQ9j5R*^W2~a*xrrJ z?dumZn1VZbPrSe5dD(4Q0Wuk$vL0d$=FN&l`hC+h*3DnBH1>m`-rsL*TJ`-y6G%pg zUJF{+XFdoHg%K;5nc!PXznxxM5csMAuVmi`Zz*$)sLtWOQJb3-P44N{dIr!;+%tAeKLUK7r$|zEW5jlJ3>Q4 zt>59rJGOy$MY`g{3%{c}#h}sJqJ7hvu>>W3%LBdw5s@&I6dc@NkWu&N1HqqZGAo~? zgm{tf;)2n-1xA{x=;LkozxM3vI|Wwe8iU^{6;!$|Fb}pKeV-XhXnjy`-(rmaZ z<=nfmO|_bkb@0G(4t!fDybcSP*&0Z@-okeN0FAWc{t@tis`QuxIJnf6SMt zGC9pNX2--#Y7;SBGa3aFLaRf*)v&v1h59L0d6tJk$>IF5pqnAvuGXDrmhT_og>@tH zOO!mSnrl`>4cLt}WDs2ZT@trdW?o$N4qYnK+CRi^A~dphMtJQHL1H9BcxK^I;;!MB zl;jpy7-X1W&+`T9qxnkT2=p6BCLv@b^s!&3JUzf{a0UY%b(8bj;HfcyJ}Iw;EK`x{ z?k;0>Yiqwa)IOcihrru&)#;V+Qf1=yhz5~m%?-49+4a%mh!XwmE{<2#PU8&qCmgIB zZp#kckmbpIc5@EDae+q;h%DAGx(n1BzE&8w|BbL-d2xIR>bqv#Z?c>JNS5E9bNR0g z@%(1}IR)e5E=vg%pi|nRVjRj6d|mu8jh4~(s`SZ{ruMbQ1ZAQ?MQLW&Hnk0&eLgs$967dhxjV&n}bSgM1xxd`YbA9^*KXlon4 z!A`cJF|D~v{r3aaz4djX1n{$gkrX|>z0q-T!j6v203I~2TPRtO&s8lEbMyjAHhOw` z310+o$E~ep8neU+q3WKASsd8-; zFQ`2@7_xsP+(A$KXe{Yw}BjcotK7HCF2Q}aqJV8tEm{rPG8>@UE84ySb z?-W>U;t3SY;xGG~`PY16|I+nAJ&D^3VYa*}KF&_VyzaL_unV|HefXocc;I2eG5>N& z6|hx0ZeyQEZreSImz!sVoEt`L<6+iAjLzS^dmm6~zV8HlMn4t~V0o8sd7E84C9Pyg zSxe9k`T3li)4%~a&NAp%3BpwY6G}o%_48}ZZVe~{KYog2&+G$V4#OPaMF&HXqxrCoE6{-0d<6XPITozit^TVRdl*mgqQF9s3GgeGCwujZB1Vsi2y^c*k^Kf#P>W3 z;L{C%r;15=H6b58de#)Qz;4iho7ZsytC6oS5skV%Jlm=h-}AFCMVq=7X!-;c^CNeVlTFxjMfO!rsWdk>{*WSd;5?>Sbq+GT15gO}Gc3mOOtrh3KsR!7O+w%^bD z(@^ctv@4{_KOrsgR2SKr7}p&A$Dqo83klFb^_&8y!G7SuX9pDz+Te;3kED>YGAWLURu_48*5< zHg3CBuhJwQ90K|a4lQ5sjeyO73-5B*i^rWw@c}&zvSgX!`r&c7Ad+p!k%%T2=j~zp zD)Gd|aNNkpFn1oXsDUiHW#F}IP7%7h<9-;jJmPVl-fZHAS%!dy#9uunjaQjwVMWG6 z2lk+k3&)r6A2Vky#+QxKRENW0HYGxW$!>Zi+T2BeX4IwrTb7OfcY+l=uCI1>d$ljf zTlkflgY=Z!z->4}iOffxs$s)KZX6q(w0GwIAE0V%TrLeD7BbEVQn*AV1r-6Y-2j-+ z)TpqA*@jq&j1lk|o>H#Jrl`vD;r4eyEH54BH#K1IPD%0c9jwbFqXlf%V|y%ngF z96j%)ZaTZKO9~^KNB>{vGB|Yh#^yEmI<~9 zuo=G@;Jo>gGy?Vm`j;q_`st-U(67!iSX>%=Ml)KlY!U?tn~bAe&m6iKE{vAT1)(hA^6QiY@Pv4o zkb1CL>yQ7ux^#3l#KG*au3<2LP<+f3^%pG=0%=d_Nj*z5mIKM{5dMQd);cRVag{At zkb{Gl?qUnn?Z&-umwpHE;QXCZ*73K>e37Qf8GT>Qbml=7{=QB$bS#sckR~;GR;PbU zB=ig(mKWoJ^~j+_Ssl_JIs`Qmz7FcWj{M1w|DKj!@U&cV!UL%d5)M{RPP<$1TQRvQ za+E*^qQW@In9Z)+$<|r>Wxv5aClZfc(|qBJn$wEyg7G$ilVB7OR};J>VKDJkz1bce zakUAh#?JK)JrPa-2^#Twa~E-0TEz-3P&RH2^V36Y42x90-R_kG0}fBWFEuJ82i6() z)4x&frve`(omt`oJv>U$)`XOV@wF3{i?efiY1tgrn0Rb|$)XNAMRvK6{J#j_{GFK( zE^Y``Oy7c|#Vqu(t}Ioiv^`g8oJt1xNhHHO9z9qNN(VP(|F;&5uwmVmDDU75(>5%1 ztv+8j4D*DS#8fQy*%bDKS1ewA{NUfo@9xftBTNPVcY0c0T|G#(BriK#@bVT+LU%OE ztvua-&XD2%joID_j7y+FHsUQ0<4R;y7+JRsdVdUbm7f6JvE)4N|N1uKEC*G{$=#l8 zTwicZSuzA}fY1BKc?_|IB^5AGKta8aPU4OTgNYRvmv$TFe zYNP?C0=KmfFCtDNSa*gX&`!gBB(Y)JEvb;|44CnXx;go?rFooumI9{TNy6bc$)f*VU z?~-|4<9tzh)M_=ZcTUW#(+=l*L_?CHg~b$k%_VSPxVF441+qfb=ln1d4%&;6njOfS zYUF4JZV%-2Pp=#O?oL*AaVHrP)e@Ono-ZAVZZ-DMmp6EdnC1EP(e?G~K0ZJj1bDTr z=1A`Dv)JBjulo~&zC}%-{Us}6g4gP`OFsGg`zLZKSD$W#4a4rUGgiHcrg1{gvs0-T5@;x> zTNEs0!Ifw~1CBp$pJ;Q*db5#>6dj0cs)n1q%b9^y8D6H$(fKN<9w*%0bbNq}l?-S*J-!Rtcn8+((RA&NfBB% zdG#Si$kiY9S~y_j^4gaM@jOJL*e`oB<5Rt!^ff;Fm<}R$ct!;qNa2FIy+gu64vTES zva$I@DZwB;hUKEN;QXpKXglc<6SjYPbc<6}n>rDYU$JgbTK&^?s_UM-+Io3jIF^iw z*IVbXyJ3Ytx^n0VB!n49^HOeBw8N%jpJ{m!T2=c&qg$0t%zCsWA)?co8wsM@ssJ(z zg41(jIe`)P>mUlK)cZ%st8&V0%;&cR>D!yj8BCUjtVGYx>^c*F=Lb)O+&pU?$XSC0 zk-7!S6p=BoeA{~kfi6d@d3c-nbeaaJr3N+miXFrVwV=!@&v) zG#M>NfvUZLy7AY(eFAk;Z1rDTW>h9O8rG!&_NEVXa!Sxfh(Htg6;{ zMNLPYU)mCPbjD%Cd_j#NfD{xFw%*U@>ev4GYU~{}={V5%KJkexC-|KM#@U`wgzw_J9&r`l0_*EbCX`><_S}oP>PMxu&+jD)rwOj6t$5VVX zA_$Hj+lLy9l{joQ%ei=$zGdF$B-)9Mi2mjtmDhoqVLMrCn zRMRwnqc`*MA$*+RF~j+Q-hFuIdMn1bdN*daGg574*0y@%lHzQ&P)PD{eb6zM7L)sZ zO<#ocGX)Eq92l?`5)^vlMiFX&Y0s_`A|*AIi(@@KXIajpG`v5a(^f7k>*FKROHRQ{ ztCK7u5+Wg+@PymtnjIVCxJy62%FYYXac6)DlKJEXoI}!|2OF+NhEER&PD>~H#&3#$ z_g!EX)dvNCoY@mtzAUo|Q?BqEQSC=`GpwP3KS(ag7;Tr8tPm9JeEZ^^#v6J zuY4XP(OK2>QL#@vJT!#{8(F5*j4#kmUyv(pHK*X{yLNF;)d9d4Wf=7CaN z-b#SRz4>SYD?v_nb)S)XtpO%1!sQgng((FktKj(a)9OOqOHjWE#NyC)rDabCvDeKG zF0FjE%qY!Hj`6iTHAmM}|2pa~9%P=4?)t zb-KqtnK#RHzs8!I-%kD1{RI^TJHPEMozm)?L?=!7&dw}s4$~OXh~%w{XPGHQZO?2l zT+C~Q4MLj+khm%u8rx2SCOTfNLvpIcAS<%BGGsH)#Qkihm(#QMPIzXDs>EV?ZjbkZ zagtdk?{gm|IhP6!+v&eX;^BN+uB*N-#Q21m%(cw#3FCQH3jLC@t~$Qm)zWu&&Bnjl zJa_?1luJ#HFvazvTQrklZC&4*T`l?I`ot5CKdI;c;Bqr>H;&84E+250?Cx>Km{wY7 zQRihWT!`{1+v{4ef$20OG<@+`)x@Ok;1Gw~+UDCgWevB0gTsSudv}6db?<7$ipL9` z<9Uw(%ar0`Y!+tMw*2|z!Gqamey#fB>&DYbqdfPGSsx`q&p1(pW9q~bx3%@NKlV?j zO7sUW&(}YP0WE;(W zyN-0BbvM)bl%sIw{(L-nwA%57XSPIc{IQ-+7Y3s7`{SebFI$Z&-n(x(x<02oGio!wwKW=)|PR+OO@px0X_CufndQ&7NvYc9L!u@~iirQPoz zK>Brcoy5ZY+vPpSyW0n9PVSaoUi3%HXz-p)e!;^)%Y#Wo^C^DDjkC?f(^6sbCh1rShr zR}e*z-n)qO-a9xVFrt8hbRmEcdQe*EsE7!NK|*f#p;z1xen#dCODw{_Wr1&)%qa+rRZ2{W?xo>&&@nChf8=?}sPy&y=HgP!?SC2aQGO zLfk3tPd7OP`cR zYt?0U2_aV>4u#P0nSQ<9q6FN|$8kB6qFWJu1c(ip}7}6ueFG)yUcegf0$=_5i zcgH;wVViboInZ4p;B|>xN*~?Mo>N3md` zxks+<4Ty|ONGQn9U%hpfO-EN336e)7M?q9rI4>jPT4ZFJUf^XFEoy3N88w8=$sGHV zS~@Bz-=!>A_DooPQU4jBA>>Wz;=O+(#?*gaSiF|bM@2Y$R^ZE5eqiHD{Eme2e=iTMv|KUob9FPxhn@mFuy z9zN(7I7V}cy`ZE-2?ANnXb(S%qQzQ<_}P7niT125tL>M`KCenZ2OFk$bX*;FX}{M$ zpN4=`7{a!bgh;>T`Rx3>Z&EEi7)rjBBNfV+x-#`Z z|EadI#rcV&32k|w%Fb?XiW||Vzvk7#=N*W6{Ai_LcxH|?efMrMa?dvT-ujYpL4NfO z=bc%3`URYdeE5pp+IB%+-mRNa@dSKBJrcM5ZSq94j*iK^aMyXEvx-{9OsHwIAfxi~ zG#V<0{t6@S;oXO*YPT$;+No?lgO(Cr<@2rFD`Pk9R})T-biMPI(e(9gerKJfg&ALR z!*+A=7}Tt>+N{l?>NBnpYteE$JABeEm{snXf$g2#+}uv(V!j?((_M7$IOMIe(|2`7 z>{q6y3TPIh%RYaO&~pmwSvipyUqoK1Wp`eALtVWP!E^k?(Ye^2yPTZe9UbWw=T$u_ zqh}Uf+LOetDVS834TzjMcSH1cYNiRzntjt2J)5ZwSowICgy^0$c;21!1p~DR+vHYN zl`l`zq{}>e`E<~nwNnM}9)!fDq@gVBd;Ny$cNfiddb=)Mc;o&tt8GkOS1Dp-$HgEe zx*x7s63b{7U@f&O-6B9|dG7q|$ZTk&JM1;qD)}x4D56+e;h*449 zcV-I9$a;T#IJkGctJH{JdcD(ZWi!f=U2m^Uk+NjloRgjfFtHy17@|jSLZOW=74o1|Kxcgjf<`d}VeAl8u9s z#nxz5m_-dXO%5`$PNWbwC}Wk1Y93(B3x=9%ZxuaL-I8QS@W+y}2R>tfwvw<@r+GeTfbvYg8Q*63&7Iuk@lLgaOf#-E&#OtEEVmMu)!HfA6b_?_{M zIw{&jbLuyx`!=c!185wX!sH%o3kT`*uX2Cv>iV*GWuS(qY>`~feaV@eWE$X6i8LND zI~KO4iALe)B{-NNpUbMK&|NjvtzHhbXpYK5p-}B_Ih@c& z+1c6og)34A-j?w0qoFbLpccB+-PPBZ8xs?=?3XI3EGPG%6N@c3Zw#{CZeWQF;vmWBps8HU{YdQ(;F<+ubRx9tsAHfg`E*|nh|<9++~rEcP& z!B3yQ%9qe)K?)1o9~U2A!Sg2j`@Ch&X9G;Eg8^+qhk^C=VhhL zwD--;=t}aV^JyI~#N7kjc-u-Qh`86Xi`0~%ks~(e#dYreN4$p*oq(;4O;@J3Y|r!5 zb;5!(MiY_dOG~$+XYta=f1Rx-<*`f`&deyYj&!IXBkpA4Dt+V@futPXh&m!qnwbo) zf9btb*Et?UIW~1U7}cb#Ycnu$gfPIRLQS2s$HLx@P-c&zu3IfWwEB+cv9@2P?;q{9 zO?&+>2uin1$I6@VgbG=hD1E~zm4C*5F}oy9KdVuhZkCKYx( zJpiA;*mL*b%Jk4k-xq~aj=4@VZy`H+$AdXaegh}@@;I_t21@NmR|+x z7F&NSE?{vStiLJi`}LIcX3F%v(%ZMfl9Q9Q^~{O~1|*8f=s0sGCDjOQSqo>G)vzv` zjAmz{ijGd?iS5AGBSzIqI8e4>W<5sc8E>!LvzcDl>$fwBUd4#wdS4&K#O{!dRaKY- zuZQ{?Ol$K{jc%9pI}mV1iVc;)^$Q!{EDWp01iQDyuHtZ$oio4hUEMJ8=H}yWwZ^T7 zt4(AegwhuTC z)xR7*BlMPKQAB>ljxzKf}q7NVtF93-rex&d#BPo@1XjiAe?lolB!jc3>B4>qSEb7Su-DH zC-gL~D$Lx==2~vcc+Sky5sO+UZwV7N_5v%Wu&);5!q!{m$x;hSjzddHG8;32ffcqP zYaD$!TU&nFIhMHs#fp2k=rx^s@?I+8}y~J3noqEth?gPBSzG5_3 zfl0TjT0W}jyg+@M%Ke8f=pxbrJ-6SxIu6uo%!;D0)+r)3>m#YwdzxzQd20CCh532S z76C{M4G(Q|@-Cb8wY%<IIMERt?){Wg{J{7 zKv+m5AI5#;!nvvx)kw_cdyx6m0C_Q`{dv2@>fB6#^y6O_z4MHml4(n{Jm%x%%k6hH z0!xBYWcpv37s5a7BV&ch>Y8sayZZygkN)Pi$@=+;{D$*b22ic>R7Djo}8 z|F*Hy1%OAK&CTaPxp*(Rp;d@Z;o&~p_xGKcyGje9QZMvwwbzsrO4+tNu&RzYcg1eA zhi@k=xp*XnRmnBp8kp_d3Ss+N%+ZMDcX~)$7?!u)&mHOpROdPR(Y44Jt)eX3pJhUd zwjsVf3QyIF!K+_`RPkBobK#!+Caf=ry-{PRF=4Ax*t2cSEL(==n@uUkRLpl5JdU^T z-kKi>o~S%~QIJbj-1Y#$;AuwMW|C6W$(9*%I{I+a2SZ*Qvskl@|-m)swa`$Z*W{S!^kM(FGe00hkkxgfIcyE3EUI8^!V_Nc15`qOPPnURU?Pj$!`}XB2C-A{sXP3VvMbGXj#@gD_vZ?9v*n<%LTlmXWS&{lyAr*z2H~s5& zy1TmvYJE$&xw(%s_0>O8hf1EEZ%^p*LOAVir*9%M?~gxz-AChZ%9evdsX`J0bzk2q zSP4C-O}XVYA})Ajj}3u9@PLJ%A~YRPlH!y7)vnFsY;2bxTun#MU-fZOZZ7ceefvN23<);s>|e(; zy{DMwP288bOese123_-TS5_T~VK!WrTpF~2mG$1xC#`enSn8B-Z%YNo1I%nE_Xs)m zSKKU+9iH?2>Ju0kLYS?iVH)IBc76R!!mqGQrC=!_4r|_<_-ya$PPqNbC!FbeAM*8= zBKdG-kKJIh(pGWBx}yPdYvmKRj3C&$zO|@=DYF}kJPht)`lVWM>gR9@d(Jb_aG;Cz+IuB(LMC$#OA|k(ZkAJ|2*=jK! zVIIi176@>Fhsywm@Gn(;o?yF*Yl{OLKk4+#QpRgz7YHX3)_-F6IO?P+q&dz)rx3jT%O6EC+xrFVS7;9u?V@H|E z?q-e{RcTJL`9z;!KbqM_L*K<#YMrNSXIHaMLp7>to)rIJPl%y&pDIK6OL>dT+%&)P74k7H0P zMpZY+s|1yLWxJLggMZbApXp;lxew!5utF2WY3&lT=P~1%LQXdPh&9{^kv2{B`-ti+ zu3Tlq2UaT8bMPAhI)nI(r|DGmGtXX9OS=UOG}tCDxr@h*x^bU5C&x+Sxoo>@<|(sq zkm~Ae0J@O-zWbCHC^?<(sqSu=T14TFwcp5|GOHi57;W1;&dfZ(yU~pI;68u;=GfSn( z@$lg)-iayLPSgE!&whXDDD(Vr&-53J=tH zJE!b1JDXQqYYa(UI$L7X&bOu(0I<~2(?jQ_RaZ-}Nq8uF%9vg&yL9*NU66@Wo+L=X zHo#iDkJ2fX0&XQ*!gf9l?hDO0O!kX8?M;$z+J)Bu@Zt~-JG@{Pn#j1_pZdg<^X9%g z;_Nq_pdh~XG#Ono>G%tD`kh7rA;0Uv^mBR>7}T8-x*+XQ z*e@43FKNV24o-Wf2JRZ)KKeCN*iBz{AL;br?9=>OXWpjCC$DpeWyXxBg}hap{c`Ch zdPGd6<YquwT*_B% z-9be~yb=?$QkUY~c??&lCu$r!;Qd`tcTOb0!f5sT$Q}{r?jGt+*~h?Wrr~H++5`fz(#e|u7u;+@0-iF zq$RjyOwsL$8O_9MEsJovXKi?GZT&y!!FKWDw*hM#^;b@bA_{2Ci;S{| z4|Qin=;e{-K7k2SuLgy)^!UjWpNTVLM$>jz(N0VD;}b49jTsty_s={Vbo#=H9?swX zR46k4ozgj9dwy|hlM_Q-> zwWN6`efkl-_26rZ@SY;0X-|I#heEJ@J`ZN4MMXqL>X@3Q!5o=zC4GowDZkr93t8U% zezWlOEQh0WAjLv13_-}=v)j1_QE01#?gO?-B1WaNPUyk7n3zn^ZfxRiSt{h~H*RnX z3zr)tw&S~lBz}WQ0Ke_(%vP3m!j;*^ng)|SJ&7ZA?Z<9H4%^JogD3rl*;!a-UpG8` zdf3w1dN$R2E zYs#0P+|tq#5gn}utpGM2UpZ9?rgm?$}e0}Ib4cp%d^#^ z>m71di9;mZIc9e2kLaGbW8rq~IYu6fi?6K)m6h98I}09nH>myYR+F_kSCxdttDBq7 z9=qdSqIH~Qmh`CqWFy)AwM!Li^1GXP zOb9Ea{TxvgzEE>*eYdx`%#oDFurUc%+-pcfGmi};thHhaarxPYy14A%+n*>mXfkt0ffJkkxifh^I|ii=Il zFMEGIo9paH3|@*zR%|9GyAR&#S!8pr5&; zC}scpTt@9DI+&A)1Np9f4t}FZ7SHiATXvF~nuB;plK>r~7e4d(hL4H3+e2LxE`43E zsK*)+95l7uqNt>8?p3jWZ3Z>syVxJLMO1sXXb>MrdA5H9G#r?B zt4$~_J~7={B`Na=nKZjR>|f?ID^i53&|#6;HLh9XP@g|}P|D+$vX0-~Ouxy=yqiu@ zZh}{yA7<>H8@5%J3kg~67)tSMTfwuVX*Q~!$Hn-4snhydAiN2d*<*A6;71`9U^v<^ zZTVh6s}9upqbZOWmMIix9HXbF_uC~y_A(ad7Z^9srHNax48k46L+50I8DDtg@*6TZ zf<;&TwtHi1*+4MF6si!w4h0da%opxKyZ0xC^|HZ7y~vH=6U0tvD0hALfQDy>WO z5j{pfra0H<0YJAJq;FIQ0tK>F^GDpLYZUnWi;vGi+@>Ar{rl@&=gw`FMcs!H#dKM& zVYA&?d%bD2y%hMHnY^4dO;^{~`{|jx?skB;r3aFutiVcjJE*U~J_rQeGtQf!V{`f| zVClpv9(yyGJZmPszXKAj$Wh*QHv|XMGYOU_dGyEKi#dkeDBfqjWTU;Ou$ zfaxmmM;W80V&$-!M^|Rnr?@#UU27nZw+)m}r#DZFI!fnM_9;^IG@QptV2L%wnKv00TN|odl*>lV%>A-&& zgl8!QnS3Tf{_lLcMMYZYy_6+ZYrZ&@w(K;0J#Ni`6;ffmSI(~4cNBcYCs+%C)*+-O z)`bn?%q!`vols=r~ccD@JuYCm|a}F+TwGeV6VxC zysM|$N1ly$ z21RBKBNiOs0^;W5)0LBZr*QeeI{`X{>(?&;J_e&&2bNYno`-bt+TgD4=uk)W0QtVP zPAvX?FO40f24o*(6?3_INM<9QJz((~J-d|t&dyFd^t40Q+Wg>Z!!D`853>E%vGeos zsm7jHi^n!9g^Q%9Koxa8kU9(;GIOrGP)mjYe#LGmVdHLTY3aM2bb*HlOAK05wt@T2V&^-kLN zJsn||O0U60v7Kbg<1@H;DaZ~0-H9;s zk7E$i&{0o_iOZZmv8}voy7kCwK_b2JrjoX<=|zZVcshEusp$syC1KM9dP3eQt(!a- zFZWpYUXNjAs0pK+L*)c72a;EtLjA_#6b9BdopiJ-=1=JwD%I{^n?*W*PRgPVJct>r zqob;Pr5cU|>(&3@3%6L6@GPWI|-?`+dav#!Vd^|{B+tNP5p^d^oFusIk7$pdGlrA;7lS*J6U zxUG21nR)C za~C8fjZ|~AK>^0CW&;y8KR0*Lj{#<4&zs-(*7d}^cu{y!3u&{tG-)~r z#hTP(grFc2J+qJ^REz-c8LaI$YzV45!Kc1Yr*QS1xJ$**zH9b%`7a49*sD9?mxLXP6m%YhmY6!+454^V5^F zlap39Gfz#KJJ5=^9$hY}|83QxP<>cn{p2V47t6q#W>Z$c!zIVraauR9Z zJ)mM(?iDM8t5-{@nCr@DZ1&igan;dD-SWrcaM_8uE=?bv(t+)~fd2r)vuR898hVCyPh_*~(?kiS|4su4V%}GvVM`pjdx@)z&J8?ih zE}j{eF;25@bN2I~mF>{XHPgYBg)ikz1L2J8wwJp&WkgDv$daTTTl`je_D<>QLQ+mM zd(|*$O&67Y*0V*a`z5YM!Qv);XJ>SeP4__iK$U$11FKYi@_FEreo(i*^0jodu?A`V($t*G01@Aj8oek;uO4fV*VNX(bvY^F_ihz2f z8ZVBH*D=TJAc*GP^~5Wk$GW)$6*Qh@%=?WTJv{1-oazCv_x>Rh{DwD@bwH1!NIxgi z$jgrOIdGuf0>Qw>r=c*J_bT<{AL?B16)p;L-s(@fNReMa0&4l_yQBo8RIut8Cvm@- z8qwV9>Pze=4uU&toryQ zzb;G)-nJNU)`6J`IcmX=dj&E&nY>D7VPPq^YG-ao$eMvYNz1?xZ)uQUT-*yFwagmB zUOfFJjuL8~&RT5LZg45Ca>s6ng_2{yj8W!|`78glJZ)Fk7PBY<}xyHgf{U^hQdt_;xl*`B@UucdCA<#=DrCCq+%Mxtz~ejlHP=wFE34HaiEw@d?(|8o2au1K~Z{YGyuw? z?%g_JcNHgxSa(cD0`OglyvPasmTcVf7afn5Xmzx;A#L%kpdi)I@clg8+_%j$+Cxr5 z5J;f^q(8lzZS)_Dhbf~efqnr>E){q{Ob9o3zP5o*(bkq$#yhVX73v)ruPb^!`mf|3 z?6pJ>4v^O~+pX{s-x5;Y+dK0P$KkbZqWv2*8cDWYi$QD>NK1ocA(XzRYxr8iM67>4 zM2Wdq1)1q+Kh4g3Q0Pu*1laR1+-7?+M94#tpG_Q4VM#1VOHFzCmoHSq-@SXl$ao}w z@zvpe2eTjs>trMlc5$((eyW}?+Dmvr5L^&BCkNiwjV#R7V7<}-Ba2c4f@zn6!af5y zi!T0=8=I^Ks)8qhrFZU*A9G_GyZB8m`9o7&&)yMoVP^59H&Z z1xgTr5PG9OFwQoaFo-U8s{M8YOslPYV+6n}SpTJ@sMxVXCK)*lmfP!rVn{{1M1##Z z3?;lJphzqvbe4`jBNPLkl3R{_uQG#1W}<;09a5Y9IdxJjp>Cd zgVGi5HNzBO%D^tw_tljXtN|vbK9v7d0Vx{Ntrgd;B}4FGZH4{(G+=y)y1sDsEM=7H z0Y(ic;B!lZ8G;r=IA%J|V0h|#ySi)uTMtw?8l{lH0hui7oOUCU+0%4S%=gyJ0jah4 z>P~3*K5suZ@S7bOI}`EiA$*r%hu6tAvioIt#@|6A-d3(X-AuaIenWcWW3gp25|Ce9 z;^CLT(ZFE2zisZ)9E(uQWkV*LDS=H}+pMGROWn2cEzp#V0(n(b3XKfmfA(7<+m zugpvO4b8p{38|>O5Ys^8f`G`&ln5AtY8}JA)2Y>j`~r!ga7zPbQKuIXj~`<$zP|`( zg*wBCEdc))&DH;mvCuOMEO)BLp92U(%EAGgppq(~(`(|L3f?^aR>bA+O>zJ8y(DDC zd|v_j#3ig?u)m3{&vtaR?Z_cv&$U2qcJ~lJx_ZM{>ye_D&8lSsdX6S_J&t|IrC1$y z^vxR&xdo+fdC|;r{MS~cqB1-GO9xfzXULR2(3TK#FV?ey{$XQc>MUsNw?HqL*BMdKv%1!2=Z+$RcSglmr(MK^ zY&uR$?`(`G3z$Eco7eY+rNZi^)|j55DtF1_a>qD0iYY)bS<4+)g}mnU#Zy1K=T91gTgVpRX!Yh)3PwDU8zuwfr|5+;B`f8>R_*!T z6f<-g8E+$^=$v1-PY`Rls$p8&GnMzjlc~E>6$K;TZdg!mbIievG`0 z#9!L*{_=DT3-yb*xa_9KC%t`k)6-9L=!Pg$;xyFMdMU#ZNEvPC266@HM5AGhkP7g| zCKM>j*RZ{`%zhSDQwqCHC@S2zK>*vWm#4V4bBc_-os=2Q5u^`!V@h~uYkTnaKv`9< z4cNy$f1YKwvy|WmV0&O?GoV{X&mXKctJ znOvA~Xj|O#=f}x2H)T5$FIxm0J!;7m2EDyH^D_FoVG7`X&_fzHgD`p)B$P9~DIZ3s zVQD!j-uw07<;$0oWrOQM!+xi~Kx@zM1dMH%r`ZF>q0Kun6M#r@eG0c7yP0QZ|8 zUu;Y)Z*PfEl3wZ~damf_eTZVuf>gEO?1T6y@Q8g6ZP`Q+3EjQ%Hf=p70KeF^EttYO zwT?ZHKgtg@-Soxp0Y^Xe*5YerzWbDBfy|P`xhpPd0MkMAgP`vKpqZGCva-&tG6T~99(Jf^M`8LXL>#JiNr__Z3{-m-fRzP?5*K># zQcC+dkWrxzAA*IfC_3WvlCNg&3WQ^o`)2-b&IGD!)lf}IDPqoxmv?C9Alm_mFG&(PPH|RV8)&@_mGLJa{@v6?sbd zDyD8Fo>BFh1p*O6z<2c#?5TRM71#asJ{r10>4_F&5ZJ_Z`s!|+6fiqJ!gy11OOlNbDD-85_Dg~Se!bZoDwVx*mNs0snIn;JHdK` zB*^6RJSC;Tw9YT@)hl%vhh_sfyoVi3FhIq+;Zd@V141e(B1I~m`}gm+oBEWLEMyx4 zRff|Ud30^slZBxdHE-X30yMXdv2h{f3kMOQj<-uAH z5KSPy-&@BTtM)!aARr8EC8v58RkPA^9x+34ZPy2q&Zg4#of5cne`^Kre(p3u$c*d{ ztO(!=fcFaxyFn(wgl=4CZXCevadQ|f~x98R1eF~ zG!11SROTMOy4U#Zp1pf1vkLqSnWf6#AnXC4X1-ZN5TUo&%sRpS8VE-4I`T+LLU@1G zJ@}NJm34uOOJTe@n)sXcD%R5-DV;<9vdOYG6d!_^TQn{Sry52k!)nBUZ z_Vr=&d}x0^#DkoyRHT>2Wo^hGFfcDCV(Z;& zvIrR@r;btTVarlp#L5AxtJaSqpL7Vvp7ZiL_2UHJUtQip@ht1ill$}@!59m9MNar$v?c)iaLz4_y_%K-=uWbQ*NlKoG@=?ude zII#8Pn-p74N$q*T_PuP7YiDoqC1^bxJ41W$pzhs(ClHiyoR#%L+ZD-8-XG`r@w!1v zE%f6(Rl;ek&~Y%E(%(S-{j=U);{W{SA8#ZMv}ygRb^rPee15|fLB;&v?EiCD<@KL> z;BQAIT-amtx1B$Bj2=~>`m3z_@f}bYnLmw|f9?MB-F@ZbKRw@Hzjb-^+kIGI^KUjj z4ZykoZu`%}{f|G5;iM{(9^v zEU0Y*uUU#j4PC(7xw*ecf;YSNL?_*GzD5CthO=6q5-)4BAmxVR&Rw{qIFl*9fBk`f zsoO=5`2#*wa;k5%T+Zw43H@~*$?;3g>j6!N&ge&Mjaev0V<8p)>D^$I{qDQQ=rfN4 zhW?=l?$ho4mz?@>V+yDKr6+#uyI}Y9Cng}@e`Ro7q<>HiK~1&tX5r;|{ITx`&M^IS zy2D=U3I6~1f{N-w{ty)aQSSjD6QALV3k%EI+uJQ_|BQtGaTRzy{SuA?mIaua-l*-M zD@m;5!vFkQ+taW1xePVcOjWkvOHfRd&EsknHOR+m%dPcmHGxF)Ejp_7^QN~Cb)Ki9 zy80eVS|1!5)u!yEF_r39Ms)C)8IySfp^=M6r&Ge5LkTt2=z9aBQ|R(I zS2zl`nKHG8`ebM+*d*r1zq!@ybSM0I|5PJle|FLv9-}Rnx;nwF3vika_nZ^k*5;Js zVcWssmuKf=F%2q|4jv*+_hQe#VX^*Gs{2?zZc7;Wx|F^0FEDVt!eimCk%afc4mE4&CwoBw4~fBt93{k;2r z=6`PSdwcwU)rwuG1Ap%1@9)){At(M|f4}$=Y0RaM&$!a%$;oc3zmG*9dtjTp`ONAA zCf4ok?H&F7Nlg6tj(mSVBh{(@;{#7vkz=u<2!*OBDk|b^Ff%rm`e6UJCvj_~?|5?K zrPpL@+)vZ-$6>DS_`i*Os*eKynGW9@=>L*X*^l45`_, `Online Strategy <#Online Strategy>`_, `Online Tool <#Online Tool>`_, `Updater <#Updater>`_. diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index 6947d6678c..443cd61ad8 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -11,13 +11,27 @@ This module also provides a method to simulate `Online Strategy <#Online Strategy>`_ in history. Which means you can verify your strategy or find a better one. -There are total 3 situations for using the different trainer: +There are 4 total situations for using different trainers in different situations: -1: Online: Only use Trainer. -2: Simulate with temporal dependence: Only use Trainer. -3: Simulate without temporal dependence: Use Trainer or DelayTrainer. +========================= =================================================================================== +Situations Description +========================= =================================================================================== +Online + Trainer When you REAL want to do a routine, the Trainer will help you train the models. + +Online + DelayTrainer In normal online routine, whether Trainer or DelayTrainer will REAL train models + in this routine. So it is not necessary to use DelayTrainer when do a REAL routine. + +Simulation + Trainer When your models have some temporal dependence on the previous models, then you + need to consider using Trainer. This means it will REAL train your models in + every routine and prepare signals for every routine. + +Simulation + DelayTrainer When your models don't have any temporal dependence, you can use DelayTrainer + for the ability to multitasking. It means all tasks in all routines + can be REAL trained at the end of simulating. The signals will be prepared well at + different time segments (based on whether or not any new model is online). +========================= =================================================================================== """ import logging @@ -207,7 +221,7 @@ def get_signals(self) -> Union[pd.Series, pd.DataFrame]: """ return self.signals - SIM_LOG_LEVEL = logging.INFO + 1 + SIM_LOG_LEVEL = logging.INFO + 1 # when simulating, reduce information SIM_LOG_NAME = "SIMULATE_INFO" def simulate(