From b7e1c4bffb88cf67d3cc21efca556e32870620ad Mon Sep 17 00:00:00 2001 From: Eduardo Salinas Date: Mon, 20 Mar 2023 16:01:51 +0000 Subject: [PATCH] Squashed commit of the following: commit adcaff296d6039532d11e801c7fc8d508d7dfff6 Author: Marek Wydmuch Date: Mon Mar 20 15:30:30 2023 +0100 feat: add a training loss calculation to the predict method of PLT reduction (#4534) * add a training loss calculation to the predict method of PLT reduction * update PLT demo * update the tests for PLT reduction * disable the calculation of additional evaluation measures in PLT reduction when true labels are not available * apply black formating to plt_demo.py * remove unnecessary reset of weighted_holdout_examples variable in PLT reduction * revert the change of the path to the exe in plt_demo.py * apply black formating again to plt_demo.py --------- Co-authored-by: Jack Gerrits commit f7a197ecf7adec58b791e8e5792d2e1a34c4f474 Author: Griffin Bassman Date: Fri Mar 17 11:01:17 2023 -0400 refactor: separate cb_to_cs_adf_mtr and cb_to_cs_adf_dr (#4532) * refactor: separate cb_to_cs_adf_mtr and cb_to_cs_adf_dr * clang * unused * remove mtr commit e5597ae1b29a72f98d9953fa38c97130f5a8b84f Author: swaptr <83858160+swaptr@users.noreply.github.com> Date: Fri Mar 17 02:21:50 2023 +0530 fix: fix multiline typo (#4533) commit 301800aea8ab2cd2c6c1261e98152645b169873a Author: Eduardo Salinas Date: Wed Mar 15 12:25:55 2023 -0400 test: [automl] improve runtest and test changes (#4531) commit 258731cd116be6fa42d6ff6d2e59d06b9b790dc0 Author: Griffin Bassman Date: Tue Mar 14 13:28:03 2023 -0400 chore: Update Version to 9.5.0 (#4529) commit 49131be7f8e8852b9842136bcb1962c5e2fbe1b5 Author: Eduardo Salinas Date: Tue Mar 14 11:03:24 2023 -0400 fix: [automl] avoid ccb pulling in generate_interactions (#4524) * fix: [automl] avoid ccb pulling in generate_interactions * same features in one line minimal repro * add assert of reserve size * update test file * remove include and add comment * temp print * sorting interactions matters * update temp print * fix by accounting for slot ns * remove prints * change comment and remove commented code * add sort to test * update runtests * Squashed commit of the following: commit 322a2b152e4d5886cabb9c5b33ac1a86b8527831 Author: Eduardo Salinas Date: Mon Mar 13 21:51:49 2023 +0000 possibly overwrite vw brought in by vw-executor commit 0a6baa0f61a071d3b326eb134ac26ef1a01a9ac5 Author: Eduardo Salinas Date: Mon Mar 13 21:25:46 2023 +0000 add check for metrics commit 469cebe6df1b650b7577459d990f40db72149cb7 Author: Eduardo Salinas Date: Mon Mar 13 21:22:38 2023 +0000 update test commit 7c0b212d9716bf9065851768281fd57658d8caa4 Author: Eduardo Salinas Date: Mon Mar 13 21:11:45 2023 +0000 format and add handler none commit 533e0676cc6905f95d977837e4d91e9cddaca818 Author: Eduardo Salinas Date: Mon Mar 13 20:56:07 2023 +0000 test: [automl] add ccb test that checks for ft names * update python test * Update automl_oracle.cc commit 37f4b19a7d30c75078de963b8e08f052a0579166 Author: Griffin Bassman Date: Fri Mar 10 17:38:02 2023 -0500 refactor: remove resize in gd setup (#4526) * refactor: remove resize in gd setup * rm resize commit 009831bf283d29dd7e57d9a9e1028a0d388299b8 Author: Griffin Bassman Date: Fri Mar 10 16:57:53 2023 -0500 fix: multi-model state for cb_adf (#4513) * switch to vector * fix aml and ep_dec * clang * reorder * clang * reorder commit a31ef1443fa0d56600e3a7aba913b77142ab74e2 Author: Griffin Bassman Date: Fri Mar 10 14:52:50 2023 -0500 refactor: rename wpp, ppw, ws, params_per_problem, problem_multiplier, num_learners, increment -> feature_width (#4521) * refactor: rename wpp, ppw, ws, params_per_problem, problem_multiplier, num_learners, increment -> interleaves * clang * clang * settings * make bottom interleaves the same * remove bottom_interleaves * fix test * feature width * clang commit 8390f48c5ce7b7dbd0c52b28ee586d072138143d Author: Griffin Bassman Date: Fri Mar 10 12:25:12 2023 -0500 refactor: dedup dict const (#4525) * refactor: dedup dict const * clang commit 2238d7053f1f86cb8db6224d65e6e34cff10381e Author: Jack Gerrits Date: Thu Mar 9 13:51:35 2023 -0500 refactor: add api to set data object associated with learner (#4523) * refactor: add api to set data object associated with learner * add shared ptr func commit b62254096f81d6a2234b0575ef863830362344af Author: Griffin Bassman Date: Tue Mar 7 12:16:39 2023 -0500 fix: cbzo ppw fix (#4519) commit f83cb7f369b916bf4484ab30715f8e72c6d85115 Author: Jack Gerrits Date: Tue Mar 7 11:21:51 2023 -0500 refactor: automatically set label parser after stack created (#4471) * refactor: automatically set label parser after stack created * a couple of fixes * Put in hack to keep search working * formatting commit 64e5920e8a0e43c42462caa7a7d25614274d41e7 Author: olgavrou Date: Fri Mar 3 16:20:05 2023 -0500 feat: [LAS] with CCB (#4520) commit 69bf346b8d6a26be2f79a426a67648d04111a821 Author: Jack Gerrits Date: Fri Mar 3 15:17:29 2023 -0500 refactor: make flat_example an implementation detail of ksvm (#4505) * refactor!: make flat_example an implementation detail of ksvm * Update memory_tree.cc * Absorb flat_example into svm_example * revert "Absorb flat_example into svm_example" This reverts commit b063feb0a61bc97bae28373b67001fa484cba65d. commit f08f1ec5213c7e4ede216b19a3b306f612f3f56a Author: Jack Gerrits Date: Fri Mar 3 14:04:48 2023 -0500 test: fix pytype issue in test runner and utl (#4517) * test: fix pytype issue in test runner * fix version_number.py type checker issues commit a8b1d9138cdee1b1d086c6777a17318f989af538 Author: Eduardo Salinas Date: Fri Mar 3 12:59:26 2023 -0500 fix: [epsdecay] return champ prediction always (#4518) commit b2276c1337ad78cc31ae1c8d147402e271e1eb24 Author: olgavrou Date: Thu Mar 2 20:18:23 2023 -0500 chore: [LAS] don't force mtr with LAS (#4516) commit c0ba1804be1b94a12db1b7ef5d891af24ff37748 Author: olgavrou Date: Tue Feb 28 11:27:36 2023 -0500 feat: [LAS] add example ft hash and cache and re-use rows of matrix if actions do not change (#4509) commit e1a9363dd24dc67dad0164c43786f9bf69272cf4 Author: Eduardo Salinas Date: Mon Feb 27 16:35:09 2023 -0500 feat: [gd] persist ppw extra state (#4023) * feat: [gd] persist ppm state * introduce resize_ppw_state * wip: move logic down to gd, respect incoming ft_offset * replace assert with status quo behaviour * implement writing/reading to modelfile * remove from predict * update test 351 and 411 * update sensitivity and update * remove debug prints * update all tests * apply fix of other pr * use .at() for bounds checking * add max_ft_offset and add asserts * comment extra assert that is failing * remove files * fix automl tests * more tests * tests * tests * clang * fix for predict_only_model automl * comment * fix ppm printing * temporarily remove tests 50 and 68 * address comments * expand width for search * fix tests * merge * revert cb_adf * merge * fix learner * clang * search fix * clang * fix unit tests * bump 9.7.1 for version CIs * revert to 9.7.0 * stop search from learning out of bounds * expand search num_learners * fix search cs test * comment * revert ext_libs * clang * comment out saveresume tests * pylint * comment * fix with search * fix search * clang * unused * unused * commnets * fix scope_exit * fix cs test * revert automl test update * remove resize * clang --------- Co-authored-by: Griffin Bassman --- .github/workflows/python_wheels.yml | 18 +- cs/unittest/TestSearch.cs | 2 +- demo/plt/README.md | 11 +- demo/plt/plt_demo.py | 24 +- .../learner/VWScalarsLearnerTest.java | 6 +- python/pylibvw.cc | 5 +- python/tests/test_ccb.py | 135 ++ python/vowpalwabbit/pyvw.py | 2 +- python/vowpalwabbit/sklearn.py | 12 +- test/core.vwtest.json | 16 +- test/pred-sets/ref/active_cover.predict | 30 +- test/pred-sets/ref/aml_spinoff.inv | 2 +- test/pred-sets/ref/aml_spinoff_bag.inv | 662 +++--- .../pred-sets/ref/cbe_adf_cover_dr256.predict | 4 +- ...ccb_implicit_and_explicit_interactions.inv | 2 +- ..._implicit_explicit_ignore_interactions.inv | 2 +- .../ref/ccb_lots_of_interactions.inv | 2 +- test/pred-sets/ref/ccb_quad.inv | 2 +- test/pred-sets/ref/ccb_quad_save_resume.inv | 2 +- test/pred-sets/ref/ep_dec_spinoff.inv | 2 +- test/pred-sets/ref/inv_hash.cmp | 60 +- test/pred-sets/ref/lrqfa_nn_preds.predict | 8 +- .../ref/lrqfa_nn_preds_rev_order.predict | 8 +- test/pred-sets/ref/oaa_probabilities.predict | 398 ++-- test/pred-sets/ref/oaa_scores.predict | 398 ++-- test/pred-sets/ref/readable_model_privacy.txt | 2 +- .../ref/readable_model_privacy_no_tags.txt | 2 +- test/pred-sets/ref/sender.predict | 624 ++--- test/pred-sets/ref/slates_w_interactions.inv | 2 +- test/pred-sets/ref/t288.readable | 2 +- test/run_tests.py | 8 +- test/save_resume_test.py | 6 +- test/test-sets/ref/active_cover.stderr | 12 +- test/test-sets/ref/aml_ccb_metrics.json | 2 +- test/test-sets/ref/backwards.stdout | 32 +- test/test-sets/ref/metrics_qc.json | 2 +- test/test-sets/ref/sender.stdout | 12 +- test/train-sets/ccb_automl.dsjson | 9 + test/train-sets/ref/active_cover.stderr | 16 +- test/train-sets/ref/automl_ccb.stderr | 24 + test/train-sets/ref/automl_readable.txt | 27 +- test/train-sets/ref/automl_readable_cubic.txt | 27 +- test/train-sets/ref/automl_readable_qc.txt | 27 +- test/train-sets/ref/bs.prreg.predict | 2000 ++++++++--------- test/train-sets/ref/bs.prvote.predict | 396 ++-- test/train-sets/ref/bs.reg.predict | 1984 ++++++++-------- test/train-sets/ref/bs.reg.stderr | 16 +- test/train-sets/ref/bs.vote.predict | 388 ++-- test/train-sets/ref/cbadf_automl_readable.txt | 27 +- .../ref/cbadf_automl_readable_predictonly.txt | 2 +- .../ref/cbe_cover_e_dec_legacy.stderr | 4 +- .../ref/cbe_cover_e_fixed_legacy.stderr | 4 +- .../ref/cbzo_constant_invert_hash.txt | 2 +- .../ref/cbzo_constant_readable_model.txt | 2 +- .../ref/cbzo_linear_invert_hash.txt | 2 +- .../ref/cbzo_linear_readable_model.txt | 2 +- .../train-sets/ref/ccb_simple_invert_hash.txt | 2 +- .../ref/ccb_simple_pdrop_invert_hash.txt | 2 +- test/train-sets/ref/coin.readable | 2 +- .../ref/coin_model_overflow.invert.txt | 2 +- test/train-sets/ref/dupeindex_self_cubic.txt | 2 +- .../ref/dupeindex_self_quadratic.txt | 2 +- .../train-sets/ref/dupeindex_self_quartic.txt | 2 +- test/train-sets/ref/ftrl.readable | 2 +- test/train-sets/ref/help.stdout | 5 + test/train-sets/ref/houses.stdout | 34 +- .../ref/ignore_feature.interactions | 2 +- .../ignore_feature_default_ns.interactions | 2 +- .../ref/ignore_multiple_features.interactions | 2 +- .../ref/inv_hash_load_model.invert.txt | 2 +- .../ref/inv_hash_load_model.readable.txt | 2 +- test/train-sets/ref/ksvm_train.rbf.predict | 246 +- test/train-sets/ref/l1_l2_default_model.txt | 2 +- test/train-sets/ref/l1_l2_override_model.txt | 2 +- .../ref/l1_override_l2_leave_model.txt | 2 +- test/train-sets/ref/l1l2_migrate.txt | 2 +- test/train-sets/ref/l1l2_migrate_override.txt | 2 +- test/train-sets/ref/las_egreedy_ips.stderr | 22 + test/train-sets/ref/lrqfa_nn_pred.stderr | 8 +- test/train-sets/ref/lrqfa_nn_pred_rev.stderr | 8 +- test/train-sets/ref/lrqfa_nn_train.stderr | 6 +- .../marginal_invert_hash_readable_model.txt | 2 +- test/train-sets/ref/nn-1-noquiet.stderr | 14 +- test/train-sets/ref/oaa_probabilities.stderr | 6 +- test/train-sets/ref/pistol.readable | 2 +- test/train-sets/ref/plt_predict.stderr | 10 +- .../ref/plt_predict_probabilities.stderr | 10 +- test/train-sets/ref/plt_sgd_predict.stderr | 10 +- .../ref/plt_sgd_top1_predict.stderr | 10 +- test/train-sets/ref/plt_top1_predict.stderr | 10 +- .../ref/plt_top1_predict_probabilities.stderr | 10 +- .../ref/rcv1_multiclass_legacy.stderr | 2 +- test/train-sets/ref/rcv1_raw_cb_dm.stderr | 14 +- test/train-sets/ref/rcv1_raw_cb_ips.stderr | 6 +- test/train-sets/ref/rcv1_raw_new_cb_dr.stderr | 12 +- .../train-sets/ref/w_out_slot_ns.interactions | 2 +- test/train-sets/ref/wiki1K.stderr | 4 +- test/train-sets/same_model_test.1.dat | 4 +- utl/flatbuffer/vw_to_flat.cc | 2 +- utl/version_number.py | 4 + vcpkg.json | 2 +- version.txt | 2 +- vowpalwabbit/core/CMakeLists.txt | 1 + .../core/include/vw/core/automl_impl.h | 17 +- .../core/include/vw/core/cached_learner.h | 4 +- vowpalwabbit/core/include/vw/core/example.h | 41 +- .../core/include/vw/core/example_predict.h | 6 + .../core/include/vw/core/feature_group.h | 4 + .../core/include/vw/core/gen_cs_example.h | 44 +- .../core/include/vw/core/global_data.h | 2 +- vowpalwabbit/core/include/vw/core/learner.h | 54 +- .../core/include/vw/core/multi_model_utils.h | 66 +- .../core/include/vw/core/parse_example_json.h | 12 +- .../vw/core/parse_slates_example_json.h | 20 +- .../core/include/vw/core/reduction_stack.h | 5 +- .../include/vw/core/reductions/cb/cb_adf.h | 36 +- .../include/vw/core/reductions/cb/cbify.h | 4 +- .../vw/core/reductions/epsilon_decay.h | 13 +- .../core/include/vw/core/reductions/gd.h | 49 +- .../vw/core/reductions/search/search.h | 4 +- .../core/include/vw/core/setup_base.h | 4 +- vowpalwabbit/core/src/example.cc | 82 +- vowpalwabbit/core/src/example_predict.cc | 19 + vowpalwabbit/core/src/feature_group.cc | 25 + vowpalwabbit/core/src/gen_cs_example.cc | 7 +- vowpalwabbit/core/src/learner.cc | 46 +- vowpalwabbit/core/src/parse_args.cc | 13 +- vowpalwabbit/core/src/parse_example_json.cc | 8 +- .../core/src/parse_slates_example_json.cc | 14 +- vowpalwabbit/core/src/reduction_stack.cc | 8 +- .../core/src/reductions/active_cover.cc | 4 +- .../core/src/reductions/audit_regressor.cc | 10 +- vowpalwabbit/core/src/reductions/automl.cc | 61 +- vowpalwabbit/core/src/reductions/baseline.cc | 2 +- vowpalwabbit/core/src/reductions/bfgs.cc | 1 - vowpalwabbit/core/src/reductions/boosting.cc | 8 +- vowpalwabbit/core/src/reductions/bs.cc | 6 +- vowpalwabbit/core/src/reductions/cats_pdf.cc | 3 - vowpalwabbit/core/src/reductions/cats_tree.cc | 7 +- vowpalwabbit/core/src/reductions/cb/cb_adf.cc | 93 +- .../core/src/reductions/cb/cb_algs.cc | 12 +- .../core/src/reductions/cb/cb_explore.cc | 14 +- .../src/reductions/cb/cb_explore_adf_bag.cc | 8 +- .../src/reductions/cb/cb_explore_adf_cover.cc | 27 +- .../src/reductions/cb/cb_explore_adf_first.cc | 7 +- .../reductions/cb/cb_explore_adf_greedy.cc | 7 +- .../cb/cb_explore_adf_large_action_space.cc | 40 +- .../src/reductions/cb/cb_explore_adf_regcb.cc | 7 +- .../src/reductions/cb/cb_explore_adf_rnd.cc | 23 +- .../reductions/cb/cb_explore_adf_softmax.cc | 7 +- .../reductions/cb/cb_explore_adf_squarecb.cc | 7 +- .../cb/cb_explore_adf_synthcover.cc | 7 +- .../core/src/reductions/cb/cb_to_cb_adf.cc | 2 +- vowpalwabbit/core/src/reductions/cb/cbify.cc | 16 +- .../details/large_action/one_pass_svd_impl.cc | 84 +- .../details/large_action/two_pass_svd_impl.cc | 3 +- .../cb/details/large_action_space.h | 8 +- .../core/src/reductions/cb/warm_cb.cc | 8 +- vowpalwabbit/core/src/reductions/cbzo.cc | 29 +- .../conditional_contextual_bandit.cc | 3 +- .../core/src/reductions/count_label.cc | 2 +- vowpalwabbit/core/src/reductions/cs_active.cc | 12 +- vowpalwabbit/core/src/reductions/csoaa.cc | 9 +- vowpalwabbit/core/src/reductions/csoaa_ldf.cc | 3 - .../reductions/details/automl/automl_impl.cc | 23 +- .../details/automl/automl_iomodel.cc | 6 - .../details/automl/automl_oracle.cc | 29 +- .../reductions/details/automl/automl_util.cc | 4 +- vowpalwabbit/core/src/reductions/ect.cc | 9 +- .../core/src/reductions/eigen_memory_tree.cc | 18 +- .../core/src/reductions/epsilon_decay.cc | 53 +- .../core/src/reductions/explore_eval.cc | 1 - vowpalwabbit/core/src/reductions/freegrad.cc | 39 +- vowpalwabbit/core/src/reductions/ftrl.cc | 20 +- vowpalwabbit/core/src/reductions/gd.cc | 121 +- vowpalwabbit/core/src/reductions/gd_mf.cc | 1 - .../src/reductions/generate_interactions.cc | 2 + .../core/src/reductions/interaction_ground.cc | 6 +- .../core/src/reductions/kernel_svm.cc | 133 +- vowpalwabbit/core/src/reductions/lda_core.cc | 3 - vowpalwabbit/core/src/reductions/log_multi.cc | 11 +- vowpalwabbit/core/src/reductions/lrq.cc | 6 +- vowpalwabbit/core/src/reductions/lrqfa.cc | 8 +- .../core/src/reductions/memory_tree.cc | 52 +- vowpalwabbit/core/src/reductions/mf.cc | 10 +- .../core/src/reductions/multilabel_oaa.cc | 8 +- vowpalwabbit/core/src/reductions/mwt.cc | 2 - vowpalwabbit/core/src/reductions/nn.cc | 13 +- vowpalwabbit/core/src/reductions/oaa.cc | 6 +- .../core/src/reductions/offset_tree.cc | 6 +- .../core/src/reductions/oja_newton.cc | 9 +- vowpalwabbit/core/src/reductions/plt.cc | 97 +- .../core/src/reductions/recall_tree.cc | 11 +- .../core/src/reductions/search/search.cc | 29 +- .../reductions/search/search_dep_parser.cc | 6 +- .../search/search_entityrelationtask.cc | 4 +- .../src/reductions/search/search_graph.cc | 10 +- .../search/search_multiclasstask.cc | 5 +- .../reductions/search/search_sequencetask.cc | 2 +- .../src/reductions/shared_feature_merger.cc | 2 +- vowpalwabbit/core/src/reductions/slates.cc | 2 - .../core/src/reductions/stagewise_poly.cc | 2 +- vowpalwabbit/core/src/reductions/svrg.cc | 9 +- vowpalwabbit/core/src/vw.cc | 4 +- vowpalwabbit/core/tests/automl_test.cc | 9 +- .../core/tests/automl_weights_test.cc | 10 +- .../core/tests/cb_las_one_pass_svd_test.cc | 177 +- .../core/tests/cb_las_spanner_test.cc | 15 +- vowpalwabbit/core/tests/epsilon_decay_test.cc | 8 +- .../core/tests/example_ft_hash_test.cc | 180 ++ vowpalwabbit/core/tests/flat_example_test.cc | 22 +- vowpalwabbit/core/tests/merge_test.cc | 38 +- .../vw/json_parser/parse_example_json.h | 12 +- .../json_parser/parse_example_slates_json.h | 20 +- .../json_parser/src/parse_example_json.cc | 18 +- .../src/parse_example_slates_json.cc | 27 +- 216 files changed, 5508 insertions(+), 4939 deletions(-) create mode 100644 test/train-sets/ccb_automl.dsjson create mode 100644 test/train-sets/ref/automl_ccb.stderr create mode 100644 test/train-sets/ref/las_egreedy_ips.stderr create mode 100644 vowpalwabbit/core/tests/example_ft_hash_test.cc diff --git a/.github/workflows/python_wheels.yml b/.github/workflows/python_wheels.yml index 5b2102d948a..4bfae45fee2 100644 --- a/.github/workflows/python_wheels.yml +++ b/.github/workflows/python_wheels.yml @@ -70,8 +70,8 @@ jobs: shell: bash run: | pip install -r requirements.txt - pip install pytest twine - pip install built_wheel/*.whl + pip install pytest vw-executor twine + pip install --force-reinstall built_wheel/*.whl twine check built_wheel/*.whl python -m pytest ./python/tests/ python ./python/tests/run_doctest.py @@ -133,7 +133,7 @@ jobs: shell: bash run: | pip install -r requirements.txt - pip install pytest + pip install pytest vw-executor - name: Run unit tests shell: bash run: | @@ -212,8 +212,8 @@ jobs: source .env/bin/activate && \ pip install --upgrade pip && \ pip install -r requirements.txt && \ - pip install pytest twine && \ - pip install built_wheel/*.whl && \ + pip install pytest vw-executor twine && \ + pip install --force-reinstall built_wheel/*.whl && \ twine check built_wheel/*.whl && \ python --version && \ python -m pytest ./python/tests/ && \ @@ -272,8 +272,8 @@ jobs: shell: bash run: | pip install -r requirements.txt - pip install pytest twine - pip install built_wheel/*.whl + pip install pytest vw-executor twine + pip install --force-reinstall built_wheel/*.whl twine check built_wheel/*.whl python -m pytest ./python/tests/ python ./python/tests/run_doctest.py @@ -361,8 +361,8 @@ jobs: export wheel_file="${wheel_files[0]}" echo Installing ${wheel_file}... pip install -r requirements.txt - pip install pytest twine - pip install ${wheel_file} + pip install pytest vw-executor twine + pip install --force-reinstall ${wheel_file} twine check ${wheel_file} python -m pytest .\\python\\tests\\ python .\\python\\tests\\run_doctest.py diff --git a/cs/unittest/TestSearch.cs b/cs/unittest/TestSearch.cs index f5e06e03170..b8ce765a50e 100644 --- a/cs/unittest/TestSearch.cs +++ b/cs/unittest/TestSearch.cs @@ -146,7 +146,7 @@ void RunSearchPredict(Multiline multiline) rawvw.EndOfPass(); driver.Reset(); - } while (remainingPasses-- > 0); + } while (--remainingPasses > 0); driver.Reset(); diff --git a/demo/plt/README.md b/demo/plt/README.md index 060eeb13436..74b015841f4 100644 --- a/demo/plt/README.md +++ b/demo/plt/README.md @@ -1,5 +1,5 @@ Probabilistic Label Tree demo -------------------------------- +----------------------------- This demo presents PLT for applications of logarithmic time multilabel classification. It uses Mediamill dataset from the [LIBLINEAR datasets repository](https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multilabel.html) @@ -12,11 +12,13 @@ The datasets and paremeters can be easliy edited in the script. The script requi ## PLT options ``` --plt Probabilistic Label Tree with labels ---kary_tree use -ary tree. By default = 2 (binary tree) ---threshold predict labels with conditional marginal probability greater than threshold" +--kary_tree use -ary tree. By default = 2 (binary tree), + higher values usually give better results, but increase training time +--threshold predict labels with conditional marginal probability greater than threshold --top_k predict top- labels instead of labels above threshold ``` + ## Tips for using PLT PLT accelerates training and prediction for a large number of classes, if you have less than 10000 classes, you should probably use OAA. @@ -24,7 +26,8 @@ If you have a huge number of labels and features at the same time, you will need as many bits (`-b`) as can afford computationally for the best performance. You may also consider using `--sgd` instead of default adaptive, normalized, and invariant updates to gain more memory for feature weights what may lead to better performance. -You may also consider using `--holdout_off` if you have many rare labels in your data. +If you have many rare labels in your data, you should train with `--holdout_off`, that disables usage of holdout (validation) dataset for early stopping. + ## References diff --git a/demo/plt/plt_demo.py b/demo/plt/plt_demo.py index 6de9f0e1f01..b92cb49bdc2 100644 --- a/demo/plt/plt_demo.py +++ b/demo/plt/plt_demo.py @@ -6,7 +6,7 @@ # This is demo example that demonstrates usage of PLT reduction on few popular multilabel datasets. # Select dataset -dataset = "mediamill_exp1" # should be in ["mediamill_exp1", "eurlex", "rcv1x", "wiki10", "amazonCat"] +dataset = "eurlex" # should be in ["mediamill_exp1", "eurlex", "rcv1x", "wiki10", "amazonCat"] # Select reduction reduction = "plt" # should be in ["plt", "multilabel_oaa"] @@ -18,10 +18,16 @@ output_model = f"{dataset}_{reduction}_model" # Parameters -kary_tree = 16 -l = 0.5 -passes = 3 -other_training_params = "--holdout_off" +kary_tree = 16 # arity of the tree, +# higher values usually give better results, but increase training time + +passes = 5 # number of passes over the dataset, +# for some datasets you might want to change number of passes + +l = 0.5 # learning rate + +other_training_params = "--holdout_off" # because these datasets have many rare labels, +# disabling the holdout set improves the final performance # dict with params for different datasets (k and b) params_dict = { @@ -34,14 +40,20 @@ if dataset in params_dict: k, b = params_dict[dataset] else: - print(f"Dataset {dataset} is not supported for this demo.") + print(f"Dataset {dataset} is not supported by this demo.") # Download dataset (source: http://manikvarma.org/downloads/XC/XMLRepository.html) +# Datasets were transformed to VW's format, +# and features values were normalized (this helps with performance). if not os.path.exists(train_data): + print("Downloading train dataset:") os.system("wget http://www.cs.put.poznan.pl/mwydmuch/data/{}".format(train_data)) + if not os.path.exists(test_data): + print("Downloading test dataset:") os.system("wget http://www.cs.put.poznan.pl/mwydmuch/data/{}".format(test_data)) + print(f"\nTraining Vowpal Wabbit {reduction} on {dataset} dataset:\n") start = time.time() train_cmd = f"vw {train_data} -c --{reduction} {k} --loss_function logistic -l {l} --passes {passes} -b {b} -f {output_model} {other_training_params}" diff --git a/java/src/test/java/vowpalWabbit/learner/VWScalarsLearnerTest.java b/java/src/test/java/vowpalWabbit/learner/VWScalarsLearnerTest.java index b705ce733f9..9390006ce30 100644 --- a/java/src/test/java/vowpalWabbit/learner/VWScalarsLearnerTest.java +++ b/java/src/test/java/vowpalWabbit/learner/VWScalarsLearnerTest.java @@ -39,9 +39,9 @@ public void probs() throws IOException { float[][] expected = new float[][]{ new float[]{0.333333f, 0.333333f, 0.333333f}, new float[]{0.475999f, 0.262000f, 0.262000f}, - new float[]{0.373369f, 0.345915f, 0.280716f}, - new float[]{0.360023f, 0.415352f, 0.224625f}, - new float[]{0.340208f, 0.355738f, 0.304054f} + new float[]{0.374730f, 0.344860f, 0.280410f}, + new float[]{0.365531f, 0.411037f, 0.223432f}, + new float[]{0.341800f, 0.354783f, 0.303417f} }; assertEquals(expected.length, pred.length); for (int i=0; iindices) for (auto& idx : ae->feature_space[ns].indices) idx /= multiplier; @@ -1657,7 +1657,8 @@ BOOST_PYTHON_MODULE(pylibvw) py::class_("search") .def("set_options", &Search::search::set_options, "Set global search options (auto conditioning, etc.)") - //.def("set_num_learners", &Search::search::set_num_learners, "Set the total number of learners you want to + //.def("set_total_feature_width", &Search::search::set_total_feature_width, "Set the total number of learners you + // want to // train") .def("get_history_length", &Search::search::get_history_length, "Get the value specified by --search_history_length") diff --git a/python/tests/test_ccb.py b/python/tests/test_ccb.py index 5be3c4c7ce1..4960b24bf0b 100644 --- a/python/tests/test_ccb.py +++ b/python/tests/test_ccb.py @@ -122,3 +122,138 @@ def test_ccb_non_slot_none_outcome(): # CCB label is set to UNSET by default. assert label.type == vowpalwabbit.CCBLabelType.UNSET assert label.outcome is None + + +def test_ccb_and_automl(): + import random, json, os, shutil + import numpy as np + from vw_executor.vw import Vw + + people_ccb = ["Tom", "Anna"] + topics_ccb = ["sports", "politics", "music"] + + def my_ccb_simulation(n=10000, swap_after=5000, variance=0, bad_features=0, seed=0): + random.seed(seed) + np.random.seed(seed) + + envs = [[[0.8, 0.4], [0.2, 0.4]]] + offset = 0 + + animals = [ + "cat", + "dog", + "bird", + "fish", + "horse", + "cow", + "pig", + "sheep", + "goat", + "chicken", + ] + colors = [ + "red", + "green", + "blue", + "yellow", + "orange", + "purple", + "black", + "white", + "brown", + "gray", + ] + + for i in range(1, n): + person = random.randint(0, 1) + chosen = [int(i) for i in np.random.permutation(2)] + rewards = [envs[offset][person][chosen[0]], envs[offset][person][chosen[1]]] + + for i in range(len(rewards)): + rewards[i] += np.random.normal(0.5, variance) + + current = { + "c": { + "shared": {"name": people_ccb[person]}, + "_multi": [{"a": {"topic": topics_ccb[i]}} for i in range(2)], + "_slots": [{"_id": i} for i in range(2)], + }, + "_outcomes": [ + { + "_label_cost": -min(rewards[i], 1), + "_a": chosen[i:], + "_p": [1.0 / (2 - i)] * (2 - i), + } + for i in range(2) + ], + } + + current["c"]["shared"][random.choice(animals)] = random.random() + current["c"]["shared"][random.choice(animals)] = random.random() + + current["c"]["_multi"][random.choice(range(len(current["c"]["_multi"])))][ + random.choice(colors) + ] = random.random() + current["c"]["_multi"][random.choice(range(len(current["c"]["_multi"])))][ + random.choice(colors) + ] = random.random() + + if random.random() < 0.50: + current["c"]["_multi"].append( + {random.choice(colors): {random.choice(animals): 0.6666}} + ) + + current["c"]["_slots"][random.choice(range(len(current["c"]["_slots"])))][ + random.choice(colors) + ] = random.random() + current["c"]["_slots"][random.choice(range(len(current["c"]["_slots"])))][ + random.choice(colors) + ] = random.random() + + yield current + + def save_examples(examples, path): + with open(path, "w") as f: + for ex in examples: + f.write(f'{json.dumps(ex, separators=(",", ":"))}\n') + + input_file = "ccb.json" + cache_dir = ".cache" + save_examples( + my_ccb_simulation(n=10, variance=0.1, bad_features=1, seed=0), input_file + ) + + assert os.path.exists(input_file) + + vw = Vw(cache_dir, handler=None) + q = vw.train( + input_file, "-b 18 -q :: --ccb_explore_adf --dsjson", ["--invert_hash"] + ) + automl = vw.train( + input_file, + "-b 20 --ccb_explore_adf --log_output stderr --dsjson --automl 4 --oracle_type one_diff --verbose_metrics", + ["--invert_hash", "--extra_metrics"], + ) + + automl_metrics = json.load(open(automl.outputs["--extra_metrics"][0])) + + # we need champ switches to be zero for this to work + assert "total_champ_switches" in automl_metrics + assert automl_metrics["total_champ_switches"] == 0 + + q_weights = q[0].model9("--invert_hash").weights.sort_index() + automl_weights = automl[0].model9("--invert_hash").weights.sort_index() + automl_champ_weights = automl_weights[~automl_weights.index.str.contains("\[")] + + fts_names_q = set([n for n in q_weights.index]) + fts_names_automl = set([n for n in automl_weights.index if "[" not in n]) + + assert len(fts_names_q) == len(fts_names_automl) + assert fts_names_q == fts_names_automl + # since there is no champ switch automl_champ should be q:: and be equal to the non-automl q:: instance + assert q_weights.equals(automl_champ_weights) + + os.remove(input_file) + shutil.rmtree(cache_dir) + assert not os.path.exists(input_file) + assert not os.path.exists(cache_dir) diff --git a/python/vowpalwabbit/pyvw.py b/python/vowpalwabbit/pyvw.py index dcaa06c84a9..2506e45a743 100644 --- a/python/vowpalwabbit/pyvw.py +++ b/python/vowpalwabbit/pyvw.py @@ -640,7 +640,7 @@ def learn(self, ec: Union["Example", List["Example"], str, List[str]]) -> None: elif isinstance(ec, list): if not self._is_multiline(): - raise TypeError("Expecting a mutiline Learner.") + raise TypeError("Expecting a multiline learner.") if len(ec) == 0: raise ValueError("An empty list is invalid") if isinstance(ec[0], str): diff --git a/python/vowpalwabbit/sklearn.py b/python/vowpalwabbit/sklearn.py index 4472215a1c3..789904ffdeb 100644 --- a/python/vowpalwabbit/sklearn.py +++ b/python/vowpalwabbit/sklearn.py @@ -738,12 +738,12 @@ class would be predicted. >>> model = VWMultiClassifier(oaa=3, loss_function='logistic') >>> _ = model.fit(X, y) >>> model.predict_proba(X) - array([[0.38928846, 0.30534211, 0.30536944], - [0.40664235, 0.29666999, 0.29668769], - [0.52324486, 0.23841164, 0.23834346], - [0.5268591 , 0.23660533, 0.23653553], - [0.65397811, 0.17312808, 0.17289382], - [0.61190444, 0.19416356, 0.19393198]]) + array([[0.38924146, 0.30537927, 0.30537927], + [0.40661219, 0.29669389, 0.29669389], + [0.52335149, 0.23832427, 0.23832427], + [0.52696788, 0.23651604, 0.23651604], + [0.65430814, 0.17284594, 0.17284594], + [0.61224216, 0.19387889, 0.19387889]]) """ return VW.predict(self, X=X) diff --git a/test/core.vwtest.json b/test/core.vwtest.json index 3307c0c9e99..98749cc5c37 100644 --- a/test/core.vwtest.json +++ b/test/core.vwtest.json @@ -5082,12 +5082,13 @@ { "id": 394, "desc": "Test using automl with ccb", - "vw_command": "--ccb_explore_adf --example_queue_limit 7 -d train-sets/ccb_reuse_small.data --automl 3 --verbose_metrics --extra_metrics aml_ccb_metrics.json --oracle_type one_diff", + "vw_command": "--ccb_explore_adf --example_queue_limit 7 -d train-sets/ccb_automl.dsjson --automl 4 --dsjson --verbose_metrics --extra_metrics aml_ccb_metrics.json --oracle_type one_diff", "diff_files": { + "stderr": "train-sets/ref/automl_ccb.stderr", "aml_ccb_metrics.json": "test-sets/ref/aml_ccb_metrics.json" }, "input_files": [ - "train-sets/ccb_reuse_small.data" + "train-sets/ccb_automl.dsjson" ] }, { @@ -5884,5 +5885,16 @@ "input_files": [ "train-sets/automl_spin_off.txt" ] + }, + { + "id": 455, + "desc": "large action spaces with cb_explore_adf epsilon greedy and ips", + "vw_command": "--cb_explore_adf -d train-sets/las_100_actions.txt --noconstant --large_action_space --cb_type ips", + "diff_files": { + "stderr": "train-sets/ref/las_egreedy_ips.stderr" + }, + "input_files": [ + "train-sets/las_100_actions.txt" + ] } ] \ No newline at end of file diff --git a/test/pred-sets/ref/active_cover.predict b/test/pred-sets/ref/active_cover.predict index 26535713806..f55becacff8 100644 --- a/test/pred-sets/ref/active_cover.predict +++ b/test/pred-sets/ref/active_cover.predict @@ -10,16 +10,16 @@ 1 -1 -1 -1 -1 -1 -1 1 -1 -1 +-1 -1 1 +-1 1 +-1 1 1 -1 @@ -33,7 +33,7 @@ -1 -1 1 --1 +1 -1 1 1 @@ -71,8 +71,8 @@ -1 1 1 -1 -1 +-1 +-1 -1 1 -1 @@ -110,7 +110,7 @@ 1 -1 1 -1 +-1 -1 1 -1 @@ -192,7 +192,7 @@ -1 -1 -1 -1 +-1 1 1 1 @@ -250,7 +250,7 @@ 1 1 1 --1 +1 1 -1 1 @@ -262,7 +262,7 @@ -1 -1 1 --1 +1 1 1 -1 @@ -387,11 +387,11 @@ -1 1 1 +-1 1 1 1 -1 -1 -1 -1 -1 @@ -404,10 +404,10 @@ 1 -1 -1 -1 +-1 -1 1 -1 +-1 1 1 -1 @@ -419,8 +419,8 @@ -1 1 1 --1 1 +-1 1 1 1 @@ -469,7 +469,7 @@ -1 -1 1 -1 +-1 -1 1 -1 diff --git a/test/pred-sets/ref/aml_spinoff.inv b/test/pred-sets/ref/aml_spinoff.inv index d292f2989d9..a382fa99cfa 100644 --- a/test/pred-sets/ref/aml_spinoff.inv +++ b/test/pred-sets/ref/aml_spinoff.inv @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:-1 Max label:0 diff --git a/test/pred-sets/ref/aml_spinoff_bag.inv b/test/pred-sets/ref/aml_spinoff_bag.inv index 38cba4a8d95..7d8cd56b5d2 100644 --- a/test/pred-sets/ref/aml_spinoff_bag.inv +++ b/test/pred-sets/ref/aml_spinoff_bag.inv @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:-1 Max label:0 @@ -8,8 +8,8 @@ lda:0 0 skip: options: --bag 4 --cb_adf --cb_explore_adf --cb_type mtr --csoaa_ldf multiline --csoaa_rank --interactions AA --interactions AB --interactions AC --interactions AD --interactions AE --interactions AF --interactions BB --interactions BC --interactions BD --interactions BE --interactions BF --interactions CC --interactions CD --interactions CE --interactions CF --interactions DD --interactions DE --interactions DF --interactions EE --interactions EF --interactions FF --noconstant --random_seed 9179422484841000272 Checksum: 3172229801 -event_sum 22 -action_sum 66 +event_sum 6 +action_sum 18 :0 E^236*E^642:0:-0.015755 E^236*E^642[1]:1:-0.015755 @@ -42,7 +42,7 @@ B^441*B^724:156:-0.274478 B^441*B^724[1]:157:-0.274478 B^441*B^724[2]:158:-0.274478 B^441*B^724[3]:159:-0.245706 -E^1133*E^1705:176:-0.0357544 +E^1133*E^1705:176:-0.0357543 E^1133*E^1705[1]:177:-0.0357543 E^1133*E^1705[2]:178:-0.0357543 E^1133*E^1705[3]:179:-0.0320136 @@ -104,7 +104,7 @@ E^768*E^18535[3]:431:-0.032717 A^1530*A^2536:440:-0.0120241 A^1530*A^2536[2]:442:-0.0120241 A^1530*A^2536[3]:443:-0.014432 -E^73*E^378:452:-0.126915 +E^73*E^378:452:-0.126914 E^73*E^378[1]:453:-0.126914 E^73*E^378[2]:454:-0.126914 E^73*E^378[3]:455:-0.11363 @@ -155,7 +155,7 @@ E^18661*E^20228:572:-0.0028089 E^18661*E^20228[1]:573:-0.0028089 E^18661*E^20228[2]:574:-0.0028089 E^18661*E^20228[3]:575:-0.00251507 -A^468*A^709:596:-0.188027 +A^468*A^709:596:-0.188026 A^468*A^709[1]:597:-0.105936 A^468*A^709[2]:598:-0.188026 A^468*A^709[3]:599:-0.100092 @@ -368,7 +368,7 @@ D^33*D^41:1320:-0.119071 D^33*D^41[1]:1321:-0.119071 D^33*D^41[2]:1322:-0.119071 D^33*D^41[3]:1323:-0.106633 -D^6047*D^31213:1328:-0.0627285 +D^6047*D^31213:1328:-0.0627284 D^6047*D^31213[1]:1329:-0.0627284 D^6047*D^31213[2]:1330:-0.0627284 D^6047*D^31213[3]:1331:-0.0561658 @@ -476,7 +476,7 @@ D^42*D^36354:1632:-0.0285705 D^42*D^36354[1]:1633:-0.0285705 D^42*D^36354[2]:1634:-0.0285705 D^42*D^36354[3]:1635:-0.025582 -E^1865*E^1890:1636:-0.0313556 +E^1865*E^1890:1636:-0.0313555 E^1865*E^1890[1]:1637:-0.0313555 E^1865*E^1890[2]:1638:-0.0313555 E^1865*E^1890[3]:1639:-0.0280814 @@ -538,7 +538,7 @@ D^33*D^161[3]:1803:-0.04984 A^932*A^24036:1808:-0.0286776 A^932*A^24036[2]:1810:-0.0286776 A^932*A^24036[3]:1811:-0.0344332 -D^857*D^3899:1840:-0.0306832 +D^857*D^3899:1840:-0.0306831 D^857*D^3899[1]:1841:-0.0306831 D^857*D^3899[2]:1842:-0.0306831 D^857*D^3899[3]:1843:-0.02748 @@ -865,7 +865,7 @@ E^236*E^1133:3060:-0.0473717 E^236*E^1133[1]:3061:-0.0473717 E^236*E^1133[2]:3062:-0.0473717 E^236*E^1133[3]:3063:-0.0424165 -E^1865*E^2538:3076:-0.0294642 +E^1865*E^2538:3076:-0.0294641 E^1865*E^2538[1]:3077:-0.0294641 E^1865*E^2538[2]:3078:-0.0294641 E^1865*E^2538[3]:3079:-0.026386 @@ -920,7 +920,7 @@ E^1221*E^3577[3]:3219:-0.0214738 A^229*A^2848:3228:-0.0817616 A^229*A^2848[2]:3230:-0.0817616 A^229*A^2848[3]:3231:-0.0981666 -D^533*D^4003:3232:-0.0291562 +D^533*D^4003:3232:-0.0291561 D^533*D^4003[1]:3233:-0.0291561 D^533*D^4003[2]:3234:-0.0291561 D^533*D^4003[3]:3235:-0.0261087 @@ -955,7 +955,7 @@ B^606*B^2036:3352:-0.034171 B^606*B^2036[1]:3353:-0.034171 B^606*B^2036[2]:3354:-0.034171 B^606*B^2036[3]:3355:-0.0305949 -E^236*E^1479:3356:-0.0440054 +E^236*E^1479:3356:-0.0440053 E^236*E^1479[1]:3357:-0.0440053 E^236*E^1479[2]:3358:-0.0440053 E^236*E^1479[3]:3359:-0.0394141 @@ -975,7 +975,7 @@ C^545*C^3845:3408:-0.0153372 C^545*C^3845[1]:3409:-0.0153372 C^545*C^3845[2]:3410:-0.0153372 C^545*C^3845[3]:3411:-0.0137367 -E^564*E^2477:3412:-0.00690412 +E^564*E^2477:3412:-0.00690411 E^564*E^2477[1]:3413:-0.00690411 E^564*E^2477[2]:3414:-0.00690411 E^564*E^2477[3]:3415:-0.00618213 @@ -1009,7 +1009,7 @@ E^1062*E^6578:3512:-0.0337328 E^1062*E^6578[1]:3513:-0.0337328 E^1062*E^6578[2]:3514:-0.0337328 E^1062*E^6578[3]:3515:-0.0301999 -E^2115*E^3845:3528:-0.0309565 +E^2115*E^3845:3528:-0.0309564 E^2115*E^3845[1]:3529:-0.0309564 E^2115*E^3845[2]:3530:-0.0309564 E^2115*E^3845[3]:3531:-0.0277168 @@ -1129,7 +1129,7 @@ D^334*D^22746:3952:-0.028281 D^334*D^22746[1]:3953:-0.028281 D^334*D^22746[2]:3954:-0.028281 D^334*D^22746[3]:3955:-0.0253218 -D^33*D^690:3956:-0.0779985 +D^33*D^690:3956:-0.0779984 D^33*D^690[1]:3957:-0.0779984 D^33*D^690[2]:3958:-0.0779984 D^33*D^690[3]:3959:-0.0698322 @@ -1156,7 +1156,7 @@ E^2028*E^2166:4048:-0.0290303 E^2028*E^2166[1]:4049:-0.0290303 E^2028*E^2166[2]:4050:-0.0290303 E^2028*E^2166[3]:4051:-0.0259962 -C^545*C^3762:4068:-0.0129575 +C^545*C^3762:4068:-0.0129574 C^545*C^3762[1]:4069:-0.0129574 C^545*C^3762[2]:4070:-0.0129574 C^545*C^3762[3]:4071:-0.011603 @@ -1175,11 +1175,11 @@ E^236*E^1389:4084:-0.0484128 E^236*E^1389[1]:4085:-0.0484128 E^236*E^1389[2]:4086:-0.0484128 E^236*E^1389[3]:4087:-0.0433507 -E^3577*E^4599:4088:-0.0135782 +E^3577*E^4599:4088:-0.0135781 E^3577*E^4599[1]:4089:-0.0135781 E^3577*E^4599[2]:4090:-0.0135781 E^3577*E^4599[3]:4091:-0.0121612 -E^161*E^2166:4116:-0.0783629 +E^161*E^2166:4116:-0.0783628 E^161*E^2166[1]:4117:-0.0783628 E^161*E^2166[2]:4118:-0.0783628 E^161*E^2166[3]:4119:-0.0701739 @@ -1225,7 +1225,7 @@ D^3380*D^4194:4248:-0.0176538 D^3380*D^4194[1]:4249:-0.0176538 D^3380*D^4194[2]:4250:-0.0176538 D^3380*D^4194[3]:4251:-0.0158052 -D^196*D^973:4276:-0.0639216 +D^196*D^973:4276:-0.0639215 D^196*D^973[1]:4277:-0.0639215 D^196*D^973[2]:4278:-0.0639215 D^196*D^973[3]:4279:-0.057228 @@ -1312,7 +1312,7 @@ C^1188*C^2066:4480:-0.134393 C^1188*C^2066[1]:4481:-0.134393 C^1188*C^2066[2]:4482:-0.134393 C^1188*C^2066[3]:4483:-0.120323 -E^478*E^789:4492:-0.0676739 +E^478*E^789:4492:-0.0676738 E^478*E^789[1]:4493:-0.0676738 E^478*E^789[2]:4494:-0.0676738 E^478*E^789[3]:4495:-0.0605979 @@ -1436,7 +1436,7 @@ E^1059*E^3800:4868:-0.0526034 E^1059*E^3800[1]:4869:-0.0526034 E^1059*E^3800[2]:4870:-0.0526034 E^1059*E^3800[3]:4871:-0.0470995 -E^1130*E^1305:4876:-0.058911 +E^1130*E^1305:4876:-0.0589109 E^1130*E^1305[1]:4877:-0.0589109 E^1130*E^1305[2]:4878:-0.0589109 E^1130*E^1305[3]:4879:-0.0527499 @@ -1796,7 +1796,7 @@ E^642*E^1133:6012:-0.029963 E^642*E^1133[1]:6013:-0.029963 E^642*E^1133[2]:6014:-0.029963 E^642*E^1133[3]:6015:-0.0268299 -E^73*E^1036:6028:-0.0917762 +E^73*E^1036:6028:-0.0917761 E^73*E^1036[1]:6029:-0.0917761 E^73*E^1036[2]:6030:-0.0917761 E^73*E^1036[3]:6031:-0.0821953 @@ -1872,7 +1872,7 @@ D^55*D^9252:6324:-0.0200115 D^55*D^9252[1]:6325:-0.0200115 D^55*D^9252[2]:6326:-0.0200115 D^55*D^9252[3]:6327:-0.0179218 -E^1380*E^2362:6336:-0.0986824 +E^1380*E^2362:6336:-0.0986823 E^1380*E^2362[1]:6337:-0.0986823 E^1380*E^2362[2]:6338:-0.0986823 E^1380*E^2362[3]:6339:-0.088345 @@ -1891,7 +1891,7 @@ E^73*E^33705:6400:-0.0131691 E^73*E^33705[1]:6401:-0.0131691 E^73*E^33705[2]:6402:-0.0131691 E^73*E^33705[3]:6403:-0.0117927 -D^196*D^1465:6404:-0.0655309 +D^196*D^1465:6404:-0.0655308 D^196*D^1465[1]:6405:-0.0655308 D^196*D^1465[2]:6406:-0.0655308 D^196*D^1465[3]:6407:-0.0586837 @@ -1977,7 +1977,7 @@ E^973*E^2248[3]:6607:-0.0402995 A^468*A^1189:6612:-0.104018 A^468*A^1189[2]:6614:-0.104018 A^468*A^1189[3]:6615:-0.124904 -E^236*E^236:6632:-0.0451333 +E^236*E^236:6632:-0.0451332 E^236*E^236[1]:6633:-0.0451332 E^236*E^236[2]:6634:-0.0451332 E^236*E^236[3]:6635:-0.0404201 @@ -2138,7 +2138,7 @@ E^73*E^789:7088:-0.13481 E^73*E^789[1]:7089:-0.13481 E^73*E^789[2]:7090:-0.13481 E^73*E^789[3]:7091:-0.120699 -E^478*E^1417:7100:-0.0674917 +E^478*E^1417:7100:-0.0674916 E^478*E^1417[1]:7101:-0.0674916 E^478*E^1417[2]:7102:-0.0674916 E^478*E^1417[3]:7103:-0.0604441 @@ -2303,7 +2303,7 @@ B^1340*B^2376:7552:-0.0114072 B^1340*B^2376[1]:7553:-0.0114072 B^1340*B^2376[2]:7554:-0.0114072 B^1340*B^2376[3]:7555:-0.0102185 -E^3825*E^3845:7568:-0.0185583 +E^3825*E^3845:7568:-0.0185582 E^3825*E^3845[1]:7569:-0.0185582 E^3825*E^3845[2]:7570:-0.0185582 E^3825*E^3845[3]:7571:-0.0166207 @@ -2315,7 +2315,7 @@ E^478*E^1038:7592:-0.0707452 E^478*E^1038[1]:7593:-0.0707452 E^478*E^1038[2]:7594:-0.0707452 E^478*E^1038[3]:7595:-0.063351 -D^3291*D^3410:7596:-0.0353553 +D^3291*D^3410:7596:-0.0353552 D^3291*D^3410[1]:7597:-0.0353552 D^3291*D^3410[2]:7598:-0.0353552 D^3291*D^3410[3]:7599:-0.0316637 @@ -2363,7 +2363,7 @@ E^1305*E^3825:7776:-0.0242411 E^1305*E^3825[1]:7777:-0.0242411 E^1305*E^3825[2]:7778:-0.0242411 E^1305*E^3825[3]:7779:-0.0217105 -E^236*E^269:7796:-0.0917762 +E^236*E^269:7796:-0.0917761 E^236*E^269[1]:7797:-0.0917761 E^236*E^269[2]:7798:-0.0917761 E^236*E^269[3]:7799:-0.0821612 @@ -2842,7 +2842,7 @@ D^293*D^8277:9128:-0.062108 D^293*D^8277[1]:9129:-0.062108 D^293*D^8277[2]:9130:-0.062108 D^293*D^8277[3]:9131:-0.0556234 -E^169*E^4599:9144:-0.0451333 +E^169*E^4599:9144:-0.0451332 E^169*E^4599[1]:9145:-0.0451332 E^169*E^4599[2]:9146:-0.0451332 E^169*E^4599[3]:9147:-0.0404179 @@ -2865,7 +2865,7 @@ D^2234*D^3899:9220:-0.00640082 D^2234*D^3899[1]:9221:-0.00640082 D^2234*D^3899[2]:9222:-0.00640082 D^2234*D^3899[3]:9223:-0.00573274 -B^518*B^1321:9228:-0.0249396 +B^518*B^1321:9228:-0.0249395 B^518*B^1321[1]:9229:-0.0249395 B^518*B^1321[2]:9230:-0.0249395 B^518*B^1321[3]:9231:-0.0223311 @@ -2895,7 +2895,7 @@ B^1100*B^2360:9344:-0.00689861 B^1100*B^2360[1]:9345:-0.00689861 B^1100*B^2360[2]:9346:-0.00689861 B^1100*B^2360[3]:9347:-0.00617674 -E^73*E^2259:9384:-0.0753089 +E^73*E^2259:9384:-0.0753088 E^73*E^2259[1]:9385:-0.0753088 E^73*E^2259[2]:9386:-0.0753088 E^73*E^2259[3]:9387:-0.0674193 @@ -2981,7 +2981,7 @@ E^973*E^973:9632:-0.0509488 E^973*E^973[1]:9633:-0.0509488 E^973*E^973[2]:9634:-0.0509488 E^973*E^973[3]:9635:-0.0456382 -D^1434*D^9095:9652:-0.0113788 +D^1434*D^9095:9652:-0.0113787 D^1434*D^9095[1]:9653:-0.0113787 D^1434*D^9095[2]:9654:-0.0113787 D^1434*D^9095[3]:9655:-0.010189 @@ -3035,7 +3035,7 @@ E^3577*E^6051:9896:-0.00713287 E^3577*E^6051[1]:9897:-0.00713287 E^3577*E^6051[2]:9898:-0.00713287 E^3577*E^6051[3]:9899:-0.00638681 -D^276*D^1623:9932:-0.0216123 +D^276*D^1623:9932:-0.0216122 D^276*D^1623[1]:9933:-0.0216122 D^276*D^1623[2]:9934:-0.0216122 D^276*D^1623[3]:9935:-0.0193524 @@ -3164,12 +3164,12 @@ C^2162*C^3008[3]:10291:-0.0450986 E^973*E^1196:10300:-0.0517618 E^973*E^1196[1]:10301:-0.0517618 E^973*E^1196[2]:10302:-0.0517618 -E^973*E^1196[3]:10303:-0.0463538 +E^973*E^1196[3]:10303:-0.0463539 E^1130*E^2028:10304:-0.0331385 E^1130*E^2028[1]:10305:-0.0331385 E^1130*E^2028[2]:10306:-0.0331385 E^1130*E^2028[3]:10307:-0.0296725 -D^1422*D^36832:10344:-0.0795602 +D^1422*D^36832:10344:-0.0795601 D^1422*D^36832[1]:10345:-0.0795601 D^1422*D^36832[2]:10346:-0.0795601 D^1422*D^36832[3]:10347:-0.0712376 @@ -3195,7 +3195,7 @@ A^156*A^3370[3]:10395:-0.0807389 E^193*E^18665:10400:-0.0184845 E^193*E^18665[1]:10401:-0.0184845 E^193*E^18665[2]:10402:-0.0184845 -E^193*E^18665[3]:10403:-0.0165504 +E^193*E^18665[3]:10403:-0.0165505 E^193*E^18666:10404:-0.0480224 E^193*E^18666[1]:10405:-0.0480224 E^193*E^18666[2]:10406:-0.0480224 @@ -3235,7 +3235,7 @@ E^1389*E^4045:10528:-0.0232477 E^1389*E^4045[1]:10529:-0.0232477 E^1389*E^4045[2]:10530:-0.0232477 E^1389*E^4045[3]:10531:-0.0208151 -E^973*E^1263:10536:-0.0251218 +E^973*E^1263:10536:-0.0251217 E^973*E^1263[1]:10537:-0.0251217 E^973*E^1263[2]:10538:-0.0251217 E^973*E^1263[3]:10539:-0.0224916 @@ -3262,7 +3262,7 @@ C^55*C^631:10640:-0.195248 C^55*C^631[1]:10641:-0.195248 C^55*C^631[2]:10642:-0.195248 C^55*C^631[3]:10643:-0.174844 -E^478*E^2323:10644:-0.0527683 +E^478*E^2323:10644:-0.0527682 E^478*E^2323[1]:10645:-0.0527682 E^478*E^2323[2]:10646:-0.0527682 E^478*E^2323[3]:10647:-0.0472474 @@ -3333,7 +3333,7 @@ D^3298*D^3483:10852:-0.129673 D^3298*D^3483[1]:10853:-0.129673 D^3298*D^3483[2]:10854:-0.129673 D^3298*D^3483[3]:10855:-0.116113 -D^3298*D^3474:10880:-0.0622634 +D^3298*D^3474:10880:-0.0622633 D^3298*D^3474[1]:10881:-0.0622633 D^3298*D^3474[2]:10882:-0.0622633 D^3298*D^3474[3]:10883:-0.0557567 @@ -3420,7 +3420,7 @@ C^55*C^820:11108:-0.0398321 C^55*C^820[1]:11109:-0.0398321 C^55*C^820[2]:11110:-0.0398321 C^55*C^820[3]:11111:-0.0356661 -E^2433*E^3008:11132:-0.0525861 +E^2433*E^3008:11132:-0.052586 E^2433*E^3008[1]:11133:-0.052586 E^2433*E^3008[2]:11134:-0.052586 E^2433*E^3008[3]:11135:-0.0470965 @@ -3456,7 +3456,7 @@ E^1454*E^2323:11220:-0.0207217 E^1454*E^2323[1]:11221:-0.0207217 E^1454*E^2323[2]:11222:-0.0207217 E^1454*E^2323[3]:11223:-0.0185568 -E^478*E^2433:11228:-0.0610193 +E^478*E^2433:11228:-0.0610192 E^478*E^2433[1]:11229:-0.0610192 E^478*E^2433[2]:11230:-0.0610192 E^478*E^2433[3]:11231:-0.0546405 @@ -3706,7 +3706,7 @@ E^1298*E^1417:12076:-0.025048 E^1298*E^1417[1]:12077:-0.025048 E^1298*E^1417[2]:12078:-0.025048 E^1298*E^1417[3]:12079:-0.0224343 -E^169*E^3800:12092:-0.0700685 +E^169*E^3800:12092:-0.0700684 E^169*E^3800[1]:12093:-0.0700684 E^169*E^3800[2]:12094:-0.0700684 E^169*E^3800[3]:12095:-0.0627268 @@ -3741,7 +3741,7 @@ D^209*D^3474:12212:-0.0271433 D^209*D^3474[1]:12213:-0.0271433 D^209*D^3474[2]:12214:-0.0271433 D^209*D^3474[3]:12215:-0.0243053 -B^518*B^724:12216:-0.0355722 +B^518*B^724:12216:-0.0355721 B^518*B^724[1]:12217:-0.0355721 B^518*B^724[2]:12218:-0.0355721 B^518*B^724[3]:12219:-0.0318486 @@ -3829,7 +3829,7 @@ E^1455*E^3536:12628:-0.0383832 E^1455*E^3536[1]:12629:-0.0383832 E^1455*E^3536[2]:12630:-0.0383832 E^1455*E^3536[3]:12631:-0.0343717 -E^751*E^8467:12640:-0.0296464 +E^751*E^8467:12640:-0.0296463 E^751*E^8467[1]:12641:-0.0296463 E^751*E^8467[2]:12642:-0.0296463 E^751*E^8467[3]:12643:-0.0265458 @@ -3908,7 +3908,7 @@ E^161*E^236[3]:12847:-0.0745011 D^357*D^475:12864:-0.059282 D^357*D^475[1]:12865:-0.059282 D^357*D^475[2]:12866:-0.059282 -D^357*D^475[3]:12867:-0.0530882 +D^357*D^475[3]:12867:-0.0530883 A^465*A^518:12900:-0.0276558 A^465*A^518[2]:12902:-0.0276558 A^465*A^518[3]:12903:-0.0332002 @@ -4296,7 +4296,7 @@ E^182*E^8284:14224:-0.00704502 E^182*E^8284[1]:14225:-0.00704502 E^182*E^8284[2]:14226:-0.00704502 E^182*E^8284[3]:14227:-0.0063084 -B^514*B^759:14228:-0.058911 +B^514*B^759:14228:-0.0589109 B^514*B^759[1]:14229:-0.0589109 B^514*B^759[2]:14230:-0.0589109 B^514*B^759[3]:14231:-0.0527706 @@ -4491,7 +4491,7 @@ D^3384*D^9252:15040:-0.0308523 D^3384*D^9252[1]:15041:-0.0308523 D^3384*D^9252[2]:15042:-0.0308523 D^3384*D^9252[3]:15043:-0.0276272 -E^2538*E^8395:15044:-0.0252476 +E^2538*E^8395:15044:-0.0252475 E^2538*E^8395[1]:15045:-0.0252475 E^2538*E^8395[2]:15046:-0.0252475 E^2538*E^8395[3]:15047:-0.0226066 @@ -4550,7 +4550,7 @@ E^1214*E^1263:15268:-0.0271129 E^1214*E^1263[1]:15269:-0.0271129 E^1214*E^1263[2]:15270:-0.0271129 E^1214*E^1263[3]:15271:-0.0242816 -E^655*E^5313:15272:-0.033555 +E^655*E^5313:15272:-0.0335549 E^655*E^5313[1]:15273:-0.0335549 E^655*E^5313[2]:15274:-0.0335549 E^655*E^5313[3]:15275:-0.0300504 @@ -5310,7 +5310,7 @@ E^269*E^18666[3]:18999:-0.0541213 E^269*E^18668:19004:-0.0466516 E^269*E^18668[1]:19005:-0.0466516 E^269*E^18668[2]:19006:-0.0466516 -E^269*E^18668[3]:19007:-0.0417781 +E^269*E^18668[3]:19007:-0.0417782 E^269*E^18661:19008:-0.0120859 E^269*E^18661[1]:19009:-0.0120859 E^269*E^18661[2]:19010:-0.0120859 @@ -5435,7 +5435,7 @@ D^50*D^1881:19932:-0.0202891 D^50*D^1881[1]:19933:-0.0202891 D^50*D^1881[2]:19934:-0.0202891 D^50*D^1881[3]:19935:-0.018167 -A^646*A^2543:19972:-0.0932977 +A^646*A^2543:19972:-0.0932976 A^646*A^2543[1]:19973:-0.04463 A^646*A^2543[2]:19974:-0.0932976 A^646*A^2543[3]:19975:-0.0597125 @@ -5455,7 +5455,7 @@ D^55*D^4003:20152:-0.0155932 D^55*D^4003[1]:20153:-0.0155932 D^55*D^4003[2]:20154:-0.0155932 D^55*D^4003[3]:20155:-0.0139619 -E^800*E^3536:20168:-0.025612 +E^800*E^3536:20168:-0.0256119 E^800*E^3536[1]:20169:-0.0256119 E^800*E^3536[2]:20170:-0.0256119 E^800*E^3536[3]:20171:-0.0229321 @@ -5682,7 +5682,7 @@ B^678*B^2949:21116:-0.0584598 B^678*B^2949[1]:21117:-0.0584598 B^678*B^2949[2]:21118:-0.0584598 B^678*B^2949[3]:21119:-0.0523475 -C^230*C^3008:21152:-0.0684374 +C^230*C^3008:21152:-0.0684373 C^230*C^3008[1]:21153:-0.0684373 C^230*C^3008[2]:21154:-0.0684373 C^230*C^3008[3]:21155:-0.0612844 @@ -5801,7 +5801,7 @@ B^518*B^2330[3]:21587:-0.017714 B^300*B^12301:21588:-0.0708493 B^300*B^12301[1]:21589:-0.0708493 B^300*B^12301[2]:21590:-0.0708493 -B^300*B^12301[3]:21591:-0.0634575 +B^300*B^12301[3]:21591:-0.0634576 B^82*B^const:21600:-0.0223671 B^82*B^const[1]:21601:-0.0223671 B^82*B^const[2]:21602:-0.0223671 @@ -5846,7 +5846,7 @@ E^800*E^1865:21716:-0.022474 E^800*E^1865[1]:21717:-0.022474 E^800*E^1865[2]:21718:-0.022474 E^800*E^1865[3]:21719:-0.0201246 -E^1305*E^8272:21724:-0.0321191 +E^1305*E^8272:21724:-0.032119 E^1305*E^8272[1]:21725:-0.032119 E^1305*E^8272[2]:21726:-0.032119 E^1305*E^8272[3]:21727:-0.0287567 @@ -5877,7 +5877,7 @@ B^1100*B^1321:21828:-0.0335983 B^1100*B^1321[1]:21829:-0.0335983 B^1100*B^1321[2]:21830:-0.0335983 B^1100*B^1321[3]:21831:-0.0300808 -B^5593*B^9217:21832:-0.0646546 +B^5593*B^9217:21832:-0.0646545 B^5593*B^9217[1]:21833:-0.0646545 B^5593*B^9217[2]:21834:-0.0646545 B^5593*B^9217[3]:21835:-0.0579025 @@ -5987,7 +5987,7 @@ E^802*E^2115:22276:-0.0398148 E^802*E^2115[1]:22277:-0.0398148 E^802*E^2115[2]:22278:-0.0398148 E^802*E^2115[3]:22279:-0.0356535 -D^3384*D^3410:22280:-0.0640386 +D^3384*D^3410:22280:-0.0640385 D^3384*D^3410[1]:22281:-0.0640385 D^3384*D^3410[2]:22282:-0.0640385 D^3384*D^3410[3]:22283:-0.0573375 @@ -6053,7 +6053,7 @@ D^286*D^6047:22564:-0.0274166 D^286*D^6047[1]:22565:-0.0274166 D^286*D^6047[2]:22566:-0.0274166 D^286*D^6047[3]:22567:-0.0245479 -E^69*E^2477:22592:-0.0956631 +E^69*E^2477:22592:-0.095663 E^69*E^2477[1]:22593:-0.095663 E^69*E^2477[2]:22594:-0.095663 E^69*E^2477[3]:22595:-0.0856703 @@ -6195,7 +6195,7 @@ B^217*B^12341:22936:-0.0165215 B^217*B^12341[1]:22937:-0.0165215 B^217*B^12341[2]:22938:-0.0165215 B^217*B^12341[3]:22939:-0.0147918 -B^217*B^12340:22940:-0.0455324 +B^217*B^12340:22940:-0.0455323 B^217*B^12340[1]:22941:-0.0455323 B^217*B^12340[2]:22942:-0.0455323 B^217*B^12340[3]:22943:-0.0407677 @@ -6292,7 +6292,7 @@ E^802*E^1865:23404:-0.0349518 E^802*E^1865[1]:23405:-0.0349518 E^802*E^1865[2]:23406:-0.0349518 E^802*E^1865[3]:23407:-0.0312983 -D^269*D^1029:23432:-0.0815262 +D^269*D^1029:23432:-0.0815261 D^269*D^1029[1]:23433:-0.0815261 D^269*D^1029[2]:23434:-0.0815261 D^269*D^1029[3]:23435:-0.0730112 @@ -6323,7 +6323,7 @@ E^1305*E^8081[3]:23523:-0.0335336 A^2543*A^const:23560:-0.426997 A^2543*A^const[2]:23562:-0.426997 A^2543*A^const[3]:23563:-0.512584 -E^225*E^564:23564:-0.0368331 +E^225*E^564:23564:-0.036833 E^225*E^564[1]:23565:-0.036833 E^225*E^564[2]:23566:-0.036833 E^225*E^564[3]:23567:-0.0329787 @@ -6605,7 +6605,7 @@ E^1479*E^7351:24528:-0.0344703 E^1479*E^7351[1]:24529:-0.0344703 E^1479*E^7351[2]:24530:-0.0344703 E^1479*E^7351[3]:24531:-0.0308722 -D^24*D^1213:24548:-0.0275381 +D^24*D^1213:24548:-0.027538 D^24*D^1213[1]:24549:-0.027538 D^24*D^1213[2]:24550:-0.027538 D^24*D^1213[3]:24551:-0.0246596 @@ -6621,7 +6621,7 @@ E^3027*E^5158:24588:-0.0185886 E^3027*E^5158[1]:24589:-0.0185886 E^3027*E^5158[2]:24590:-0.0185886 E^3027*E^5158[3]:24591:-0.0166418 -E^802*E^1416:24592:-0.0590932 +E^802*E^1416:24592:-0.0590931 E^802*E^1416[1]:24593:-0.0590931 E^802*E^1416[2]:24594:-0.0590931 E^802*E^1416[3]:24595:-0.0529023 @@ -6644,7 +6644,7 @@ D^428*D^1308:24656:-0.0654267 D^428*D^1308[1]:24657:-0.0654267 D^428*D^1308[2]:24658:-0.0654267 D^428*D^1308[3]:24659:-0.0585771 -E^1038*E^2323:24660:-0.0457146 +E^1038*E^2323:24660:-0.0457145 E^1038*E^2323[1]:24661:-0.0457145 E^1038*E^2323[2]:24662:-0.0457145 E^1038*E^2323[3]:24663:-0.0409386 @@ -6939,7 +6939,7 @@ E^312*E^800:25640:-0.0495754 E^312*E^800[1]:25641:-0.0495754 E^312*E^800[2]:25642:-0.0495754 E^312*E^800[3]:25643:-0.0443907 -E^69*E^673:25648:-0.0342759 +E^69*E^673:25648:-0.0342758 E^69*E^673[1]:25649:-0.0342758 E^69*E^673[2]:25650:-0.0342758 E^69*E^673[3]:25651:-0.030689 @@ -7095,7 +7095,7 @@ D^52*D^5671:25996:-0.132172 D^52*D^5671[1]:25997:-0.132172 D^52*D^5671[2]:25998:-0.132172 D^52*D^5671[3]:25999:-0.118382 -E^312*E^836:26008:-0.0760724 +E^312*E^836:26008:-0.0760723 E^312*E^836[1]:26009:-0.0760723 E^312*E^836[2]:26010:-0.0760723 E^312*E^836[3]:26011:-0.0681279 @@ -7195,7 +7195,7 @@ E^69*E^516:26300:-0.0324023 E^69*E^516[1]:26301:-0.0324023 E^69*E^516[2]:26302:-0.0324023 E^69*E^516[3]:26303:-0.0290195 -E^510*E^18535:26308:-0.0458988 +E^510*E^18535:26308:-0.0458987 E^510*E^18535[1]:26309:-0.0458987 E^510*E^18535[2]:26310:-0.0458987 E^510*E^18535[3]:26311:-0.0410991 @@ -7219,7 +7219,7 @@ E^276*E^14387:26348:-0.0513367 E^276*E^14387[1]:26349:-0.0513367 E^276*E^14387[2]:26350:-0.0513367 E^276*E^14387[3]:26351:-0.045964 -E^1038*E^2248:26368:-0.0583297 +E^1038*E^2248:26368:-0.0583296 E^1038*E^2248[1]:26369:-0.0583296 E^1038*E^2248[2]:26370:-0.0583296 E^1038*E^2248[3]:26371:-0.0522277 @@ -7274,7 +7274,7 @@ E^1389*E^8256:26476:-0.0113419 E^1389*E^8256[1]:26477:-0.0113419 E^1389*E^8256[2]:26478:-0.0113419 E^1389*E^8256[3]:26479:-0.0101579 -E^225*E^2259:26504:-0.0537564 +E^225*E^2259:26504:-0.0537563 E^225*E^2259[1]:26505:-0.0537563 E^225*E^2259[2]:26506:-0.0537563 E^225*E^2259[3]:26507:-0.0481363 @@ -7378,7 +7378,7 @@ E^312*E^1038:26848:-0.0613778 E^312*E^1038[1]:26849:-0.0613778 E^312*E^1038[2]:26850:-0.0613778 E^312*E^1038[3]:26851:-0.054946 -C^147*C^1910:26860:-0.0315889 +C^147*C^1910:26860:-0.0315888 C^147*C^1910[1]:26861:-0.0315888 C^147*C^1910[2]:26862:-0.0315888 C^147*C^1910[3]:26863:-0.0282908 @@ -7586,7 +7586,7 @@ B^1574*B^5401:27340:-0.019414 B^1574*B^5401[1]:27341:-0.019414 B^1574*B^5401[2]:27342:-0.019414 B^1574*B^5401[3]:27343:-0.0173828 -E^1062*E^8284:27344:-0.0127806 +E^1062*E^8284:27344:-0.0127805 E^1062*E^8284[1]:27345:-0.0127805 E^1062*E^8284[2]:27346:-0.0127805 E^1062*E^8284[3]:27347:-0.0114443 @@ -7796,7 +7796,7 @@ B^934*B^1100:27928:-0.113582 B^934*B^1100[1]:27929:-0.113582 B^934*B^1100[2]:27930:-0.113582 B^934*B^1100[3]:27931:-0.101714 -E^1118*E^2477:27948:-0.0272648 +E^1118*E^2477:27948:-0.0272647 E^1118*E^2477[1]:27949:-0.0272647 E^1118*E^2477[2]:27950:-0.0272647 E^1118*E^2477[3]:27951:-0.024417 @@ -7867,7 +7867,7 @@ E^2538*E^5597:28188:-0.0166343 E^2538*E^5597[1]:28189:-0.0166343 E^2538*E^5597[2]:28190:-0.0166343 E^2538*E^5597[3]:28191:-0.0148958 -E^225*E^1705:28192:-0.0464781 +E^225*E^1705:28192:-0.046478 E^225*E^1705[1]:28193:-0.046478 E^225*E^1705[2]:28194:-0.046478 E^225*E^1705[3]:28195:-0.0416125 @@ -7967,7 +7967,7 @@ E^69*E^1118:28420:-0.152492 E^69*E^1118[1]:28421:-0.152492 E^69*E^1118[2]:28422:-0.152492 E^69*E^1118[3]:28423:-0.136587 -E^179*E^3845:28424:-0.0413158 +E^179*E^3845:28424:-0.0413157 E^179*E^3845[1]:28425:-0.0413157 E^179*E^3845[2]:28426:-0.0413157 E^179*E^3845[3]:28427:-0.0369917 @@ -8087,7 +8087,7 @@ E^1081*E^18661:28848:-0.0033132 E^1081*E^18661[1]:28849:-0.0033132 E^1081*E^18661[2]:28850:-0.0033132 E^1081*E^18661[3]:28851:-0.00296692 -E^1081*E^18665:28864:-0.00637371 +E^1081*E^18665:28864:-0.0063737 E^1081*E^18665[1]:28865:-0.0063737 E^1081*E^18665[2]:28866:-0.0063737 E^1081*E^18665[3]:28867:-0.00570769 @@ -8264,7 +8264,7 @@ A^156*A^7052:30224:-0.067507 A^156*A^7052[1]:30225:-0.0183066 A^156*A^7052[2]:30226:-0.067507 A^156*A^7052[3]:30227:-0.0620318 -E^1417*E^14387:30248:-0.0298286 +E^1417*E^14387:30248:-0.0298285 E^1417*E^14387[1]:30249:-0.0298285 E^1417*E^14387[2]:30250:-0.0298285 E^1417*E^14387[3]:30251:-0.0267108 @@ -8489,11 +8489,11 @@ B^441*B^2343:31520:-0.0154875 B^441*B^2343[1]:31521:-0.0154875 B^441*B^2343[2]:31522:-0.0154875 B^441*B^2343[3]:31523:-0.0138697 -D^757*D^31214:31572:-0.0252129 +D^757*D^31214:31572:-0.0252128 D^757*D^31214[1]:31573:-0.0252128 D^757*D^31214[2]:31574:-0.0252128 D^757*D^31214[3]:31575:-0.022577 -E^3027*E^3825:31576:-0.00957195 +E^3027*E^3825:31576:-0.00957194 E^3027*E^3825[1]:31577:-0.00957194 E^3027*E^3825[2]:31578:-0.00957194 E^3027*E^3825[3]:31579:-0.00857092 @@ -8525,7 +8525,7 @@ E^3845*E^18656:31756:-0.0216426 E^3845*E^18656[1]:31757:-0.0216426 E^3845*E^18656[2]:31758:-0.0216426 E^3845*E^18656[3]:31759:-0.0193786 -B^1091*B^2344:31764:-0.0604159 +B^1091*B^2344:31764:-0.0604158 B^1091*B^2344[1]:31765:-0.0604158 B^1091*B^2344[2]:31766:-0.0604158 B^1091*B^2344[3]:31767:-0.0540898 @@ -8577,7 +8577,7 @@ A^2524*A^2543:31884:-0.0403298 A^2524*A^2543[1]:31885:-0.0143156 A^2524*A^2543[2]:31886:-0.0403298 A^2524*A^2543[3]:31887:-0.03235 -D^52*D^4194:31896:-0.0499312 +D^52*D^4194:31896:-0.0499311 D^52*D^4194[1]:31897:-0.0499311 D^52*D^4194[2]:31898:-0.0499311 D^52*D^4194[3]:31899:-0.0447028 @@ -8661,7 +8661,7 @@ E^3845*E^18535[3]:32299:-0.0228511 A^236*A^7052:32336:-0.112851 A^236*A^7052[2]:32338:-0.112851 A^236*A^7052[3]:32339:-0.135516 -E^2538*E^4594:32360:-0.024085 +E^2538*E^4594:32360:-0.0240849 E^2538*E^4594[1]:32361:-0.0240849 E^2538*E^4594[2]:32362:-0.0240849 E^2538*E^4594[3]:32363:-0.021574 @@ -8728,7 +8728,7 @@ C^230*C^631:32644:-0.0141074 C^230*C^631[1]:32645:-0.0141074 C^230*C^631[2]:32646:-0.0141074 C^230*C^631[3]:32647:-0.0126342 -E^836*E^18668:32648:-0.0283016 +E^836*E^18668:32648:-0.0283015 E^836*E^18668[1]:32649:-0.0283015 E^836*E^18668[2]:32650:-0.0283015 E^836*E^18668[3]:32651:-0.0253401 @@ -8854,7 +8854,7 @@ E^8081*E^8287:32952:-0.00670557 E^8081*E^8287[1]:32953:-0.00670557 E^8081*E^8287[2]:32954:-0.00670557 E^8081*E^8287[3]:32955:-0.00600467 -E^276*E^8615:32956:-0.00433155 +E^276*E^8615:32956:-0.00433154 E^276*E^8615[1]:32957:-0.00433154 E^276*E^8615[2]:32958:-0.00433154 E^276*E^8615[3]:32959:-0.00387929 @@ -8866,7 +8866,7 @@ D^430*D^4194:32976:-0.168664 D^430*D^4194[1]:32977:-0.168664 D^430*D^4194[2]:32978:-0.168664 D^430*D^4194[3]:32979:-0.151055 -C^315*C^1529:32984:-0.0462612 +C^315*C^1529:32984:-0.0462611 C^315*C^1529[1]:32985:-0.0462611 C^315*C^1529[2]:32986:-0.0462611 C^315*C^1529[3]:32987:-0.0414187 @@ -8949,7 +8949,7 @@ E^52*E^131:33196:-0.0460009 E^52*E^131[1]:33197:-0.0460009 E^52*E^131[2]:33198:-0.0460009 E^52*E^131[3]:33199:-0.0411982 -E^464*E^2323:33212:-0.0890519 +E^464*E^2323:33212:-0.0890518 E^464*E^2323[1]:33213:-0.0890518 E^464*E^2323[2]:33214:-0.0890518 E^464*E^2323[3]:33215:-0.0797381 @@ -8975,7 +8975,7 @@ E^378*E^655:33300:-0.0563081 E^378*E^655[1]:33301:-0.0563081 E^378*E^655[2]:33302:-0.0563081 E^378*E^655[3]:33303:-0.0504081 -D^3474*D^18618:33312:-0.0947955 +D^3474*D^18618:33312:-0.0947954 D^3474*D^18618[1]:33313:-0.0947954 D^3474*D^18618[2]:33314:-0.0947954 D^3474*D^18618[3]:33315:-0.0848685 @@ -9225,7 +9225,7 @@ E^301*E^2477[3]:34211:-0.0129715 B^99*B^2036:34212:-0.0473977 B^99*B^2036[1]:34213:-0.0473977 B^99*B^2036[2]:34214:-0.0473977 -B^99*B^2036[3]:34215:-0.0424342 +B^99*B^2036[3]:34215:-0.0424343 D^18535:34228:-0.0548939 D^18535[1]:34229:-0.0548939 D^18535[2]:34230:-0.0548939 @@ -9454,7 +9454,7 @@ C^236*C^3006[3]:34931:-0.0983604 A^613*A^1160:34940:-0.0690838 A^613*A^1160[2]:34942:-0.0690838 A^613*A^1160[3]:34943:-0.0829555 -E^464*E^1865:34964:-0.0663638 +E^464*E^1865:34964:-0.0663637 E^464*E^1865[1]:34965:-0.0663637 E^464*E^1865[2]:34966:-0.0663637 E^464*E^1865[3]:34967:-0.0594278 @@ -9481,7 +9481,7 @@ E^193*E^8421:35024:-0.0463566 E^193*E^8421[1]:35025:-0.0463566 E^193*E^8421[2]:35026:-0.0463566 E^193*E^8421[3]:35027:-0.0415067 -C^558*C^631:35044:-0.0610193 +C^558*C^631:35044:-0.0610192 C^558*C^631[1]:35045:-0.0610192 C^558*C^631[2]:35046:-0.0610192 C^558*C^631[3]:35047:-0.0546405 @@ -9509,7 +9509,7 @@ E^378*E^1081:35148:-0.0263841 E^378*E^1081[1]:35149:-0.0263841 E^378*E^1081[2]:35150:-0.0263841 E^378*E^1081[3]:35151:-0.023634 -D^55*D^76:35156:-0.0520048 +D^55*D^76:35156:-0.0520047 D^55*D^76[1]:35157:-0.0520047 D^55*D^76[2]:35158:-0.0520047 D^55*D^76[3]:35159:-0.0465602 @@ -9525,7 +9525,7 @@ E^5313*E^7351:35224:-0.0221111 E^5313*E^7351[1]:35225:-0.0221111 E^5313*E^7351[2]:35226:-0.0221111 E^5313*E^7351[3]:35227:-0.019804 -E^378*E^1133:35228:-0.0497143 +E^378*E^1133:35228:-0.0497142 E^378*E^1133[1]:35229:-0.0497142 E^378*E^1133[2]:35230:-0.0497142 E^378*E^1133[3]:35231:-0.044515 @@ -9544,7 +9544,7 @@ E^378*E^1118[3]:35291:-0.0416302 A^613*A^1122:35300:-0.0727642 A^613*A^1122[2]:35302:-0.0727642 A^613*A^1122[3]:35303:-0.0873471 -B^180*B^1629:35316:-0.0284818 +B^180*B^1629:35316:-0.0284817 B^180*B^1629[1]:35317:-0.0284817 B^180*B^1629[2]:35318:-0.0284817 B^180*B^1629[3]:35319:-0.0255056 @@ -9595,7 +9595,7 @@ D^55*D^161[3]:35459:-0.0764918 D^1730*D^3188:35464:-0.0538528 D^1730*D^3188[1]:35465:-0.0538528 D^1730*D^3188[2]:35466:-0.0538528 -D^1730*D^3188[3]:35467:-0.04822 +D^1730*D^3188[3]:35467:-0.0482201 B^24*B^5401:35476:-0.0343098 B^24*B^5401[1]:35477:-0.0343098 B^24*B^5401[2]:35478:-0.0343098 @@ -9877,7 +9877,7 @@ E^1110*E^1455:36676:-0.0668236 E^1110*E^1455[1]:36677:-0.0668236 E^1110*E^1455[2]:36678:-0.0668236 E^1110*E^1455[3]:36679:-0.0598302 -E^1110*E^1454:36680:-0.0875596 +E^1110*E^1454:36680:-0.0875595 E^1110*E^1454[1]:36681:-0.0875595 E^1110*E^1454[2]:36682:-0.0875595 E^1110*E^1454[3]:36683:-0.0784022 @@ -9936,7 +9936,7 @@ E^101*E^4594:36852:-0.0337632 E^101*E^4594[1]:36853:-0.0337632 E^101*E^4594[2]:36854:-0.0337632 E^101*E^4594[3]:36855:-0.0302331 -E^702*E^1038:36904:-0.0898848 +E^702*E^1038:36904:-0.0898847 E^702*E^1038[1]:36905:-0.0898847 E^702*E^1038[2]:36906:-0.0898847 E^702*E^1038[3]:36907:-0.0804822 @@ -10007,7 +10007,7 @@ E^1416*E^8273:37140:-0.0156778 E^1416*E^8273[1]:37141:-0.0156778 E^1416*E^8273[2]:37142:-0.0156778 E^1416*E^8273[3]:37143:-0.0140407 -B^1321*B^2039:37152:-0.0270479 +B^1321*B^2039:37152:-0.0270478 B^1321*B^2039[1]:37153:-0.0270478 B^1321*B^2039[2]:37154:-0.0270478 B^1321*B^2039[3]:37155:-0.0242228 @@ -10015,7 +10015,7 @@ E^1416*E^8269:37156:-0.0305747 E^1416*E^8269[1]:37157:-0.0305747 E^1416*E^8269[2]:37158:-0.0305747 E^1416*E^8269[3]:37159:-0.0273794 -E^1416*E^8272:37160:-0.028974 +E^1416*E^8272:37160:-0.0289739 E^1416*E^8272[1]:37161:-0.0289739 E^1416*E^8272[2]:37162:-0.0289739 E^1416*E^8272[3]:37163:-0.0259422 @@ -10204,7 +10204,7 @@ E^702*E^1221:37708:-0.0793866 E^702*E^1221[1]:37709:-0.0793866 E^702*E^1221[2]:37710:-0.0793866 E^702*E^1221[3]:37711:-0.0710837 -B^1712*B^12301:37732:-0.00323783 +B^1712*B^12301:37732:-0.00323782 B^1712*B^12301[1]:37733:-0.00323782 B^1712*B^12301[2]:37734:-0.00323782 B^1712*B^12301[3]:37735:-0.00289952 @@ -10248,7 +10248,7 @@ B^1712*B^12339:37836:-0.0297765 B^1712*B^12339[1]:37837:-0.0297765 B^1712*B^12339[2]:37838:-0.0297765 B^1712*B^12339[3]:37839:-0.0266642 -E^52*E^1038:37840:-0.0875943 +E^52*E^1038:37840:-0.0875942 E^52*E^1038[1]:37841:-0.0875942 E^52*E^1038[2]:37842:-0.0875942 E^52*E^1038[3]:37843:-0.0784259 @@ -10436,7 +10436,7 @@ E^169*E^const:38416:-0.0106062 E^169*E^const[1]:38417:-0.0106062 E^169*E^const[2]:38418:-0.0106062 E^169*E^const[3]:38419:-0.00949573 -C^24*C^147:38428:-0.00903403 +C^24*C^147:38428:-0.00903402 C^24*C^147[1]:38429:-0.00903402 C^24*C^147[2]:38430:-0.00903402 C^24*C^147[3]:38431:-0.00808901 @@ -10742,7 +10742,7 @@ E^7351*E^8615:39632:-0.0236642 E^7351*E^8615[1]:39633:-0.0236642 E^7351*E^8615[2]:39634:-0.0236642 E^7351*E^8615[3]:39635:-0.0211935 -E^655*E^const:39640:-0.276908 +E^655*E^const:39640:-0.276907 E^655*E^const[1]:39641:-0.276907 E^655*E^const[2]:39642:-0.276907 E^655*E^const[3]:39643:-0.247966 @@ -10754,7 +10754,7 @@ B^14*B^254:39680:-0.0129817 B^14*B^254[1]:39681:-0.0129817 B^14*B^254[2]:39682:-0.0129817 B^14*B^254[3]:39683:-0.011624 -E^269*E^7351:39688:-0.0718558 +E^269*E^7351:39688:-0.0718557 E^269*E^7351[1]:39689:-0.0718557 E^269*E^7351[2]:39690:-0.0718557 E^269*E^7351[3]:39691:-0.064357 @@ -10813,7 +10813,7 @@ D^1029*D^1459:39840:-0.0413221 D^1029*D^1459[1]:39841:-0.0413221 D^1029*D^1459[2]:39842:-0.0413221 D^1029*D^1459[3]:39843:-0.0369952 -E^702*E^751:39844:-0.0822151 +E^702*E^751:39844:-0.082215 E^702*E^751[1]:39845:-0.082215 E^702*E^751[2]:39846:-0.082215 E^702*E^751[3]:39847:-0.0736134 @@ -10918,7 +10918,7 @@ B^19*B^2036[3]:40295:-0.0231562 E^7351*E^8256:40308:-0.00828462 E^7351*E^8256[1]:40309:-0.00828462 E^7351*E^8256[2]:40310:-0.00828462 -E^7351*E^8256[3]:40311:-0.00741811 +E^7351*E^8256[3]:40311:-0.00741812 D^433*D^4206:40324:-0.0183327 D^433*D^4206[1]:40325:-0.0183327 D^433*D^4206[2]:40326:-0.0183327 @@ -10959,7 +10959,7 @@ E^673*E^4599:40472:-0.0352902 E^673*E^4599[1]:40473:-0.0352902 E^673*E^4599[2]:40474:-0.0352902 E^673*E^4599[3]:40475:-0.0315934 -E^378*E^378:40520:-0.0497143 +E^378*E^378:40520:-0.0497142 E^378*E^378[1]:40521:-0.0497142 E^378*E^378[2]:40522:-0.0497142 E^378*E^378[3]:40523:-0.044515 @@ -11018,7 +11018,7 @@ E^702*E^1011[3]:40855:-0.0934148 E^52*E^768:40856:-0.100053 E^52*E^768[1]:40857:-0.100053 E^52*E^768[2]:40858:-0.100053 -E^52*E^768[3]:40859:-0.089595 +E^52*E^768[3]:40859:-0.0895951 E^73*E^25081:40896:-0.0771482 E^73*E^25081[1]:40897:-0.0771482 E^73*E^25081[2]:40898:-0.0771482 @@ -11174,7 +11174,7 @@ E^2028*E^24559:41468:-0.0145542 E^2028*E^24559[1]:41469:-0.0145542 E^2028*E^24559[2]:41470:-0.0145542 E^2028*E^24559[3]:41471:-0.0130307 -B^233*B^9217:41480:-0.0454977 +B^233*B^9217:41480:-0.0454976 B^233*B^9217[1]:41481:-0.0454976 B^233*B^9217[2]:41482:-0.0454976 B^233*B^9217[3]:41483:-0.0407263 @@ -11206,7 +11206,7 @@ E^464*E^464:41608:-0.232555 E^464*E^464[1]:41609:-0.232555 E^464*E^464[2]:41610:-0.232555 E^464*E^464[3]:41611:-0.208207 -B^348*B^2343:41612:-0.0830133 +B^348*B^2343:41612:-0.0830132 B^348*B^2343[1]:41613:-0.0830132 B^348*B^2343[2]:41614:-0.0830132 B^348*B^2343[3]:41615:-0.0743251 @@ -11372,7 +11372,7 @@ B^24*B^2495:42300:-0.0364763 B^24*B^2495[1]:42301:-0.0364763 B^24*B^2495[2]:42302:-0.0364763 B^24*B^2495[3]:42303:-0.0326648 -B^9*B^434:42324:-0.0462612 +B^9*B^434:42324:-0.0462611 B^9*B^434[1]:42325:-0.0462611 B^9*B^434[2]:42326:-0.0462611 B^9*B^434[3]:42327:-0.0414342 @@ -11468,7 +11468,7 @@ E^1036*E^1110:42704:-0.042114 E^1036*E^1110[1]:42705:-0.042114 E^1036*E^1110[2]:42706:-0.042114 E^1036*E^1110[3]:42707:-0.0377121 -E^1110*E^3027:42740:-0.0265924 +E^1110*E^3027:42740:-0.0265923 E^1110*E^3027[1]:42741:-0.0265923 E^1110*E^3027[2]:42742:-0.0265923 E^1110*E^3027[3]:42743:-0.0238123 @@ -11542,7 +11542,7 @@ B^90*B^655:42964:-0.139217 B^90*B^655[1]:42965:-0.139217 B^90*B^655[2]:42966:-0.139217 B^90*B^655[3]:42967:-0.12469 -D^678*D^3384:42984:-0.0260111 +D^678*D^3384:42984:-0.026011 D^678*D^3384[1]:42985:-0.026011 D^678*D^3384[2]:42986:-0.026011 D^678*D^3384[3]:42987:-0.0232916 @@ -11755,7 +11755,7 @@ B^24*B^2949:44068:-0.0143482 B^24*B^2949[1]:44069:-0.0143482 B^24*B^2949[2]:44070:-0.0143482 B^24*B^2949[3]:44071:-0.0128472 -C^820*C^5656:44088:-0.0779985 +C^820*C^5656:44088:-0.0779984 C^820*C^5656[1]:44089:-0.0779984 C^820*C^5656[2]:44090:-0.0779984 C^820*C^5656[3]:44091:-0.0698425 @@ -12055,12 +12055,12 @@ E^301*E^1110[3]:45575:-0.0259067 E^789*E^1417:45584:-0.0790713 E^789*E^1417[1]:45585:-0.0790713 E^789*E^1417[2]:45586:-0.0790713 -E^789*E^1417[3]:45587:-0.0707959 +E^789*E^1417[3]:45587:-0.070796 E^516*E^4045:45588:-0.0285488 E^516*E^4045[1]:45589:-0.0285488 E^516*E^4045[2]:45590:-0.0285488 E^516*E^4045[3]:45591:-0.0255591 -E^1380*E^24559:45596:-0.0283016 +E^1380*E^24559:45596:-0.0283015 E^1380*E^24559[1]:45597:-0.0283015 E^1380*E^24559[2]:45598:-0.0283015 E^1380*E^24559[3]:45599:-0.0253401 @@ -12068,7 +12068,7 @@ B^254*B^254:45632:-0.0122876 B^254*B^254[1]:45633:-0.0122876 B^254*B^254[2]:45634:-0.0122876 B^254*B^254[3]:45635:-0.0110034 -D^24*D^3483:45676:-0.0550935 +D^24*D^3483:45676:-0.0550934 D^24*D^3483[1]:45677:-0.0550934 D^24*D^3483[2]:45678:-0.0550934 D^24*D^3483[3]:45679:-0.0493429 @@ -12183,7 +12183,7 @@ E^2115*E^const:45992:-0.171892 E^2115*E^const[1]:45993:-0.171892 E^2115*E^const[2]:45994:-0.171892 E^2115*E^const[3]:45995:-0.153917 -E^298*E^2248:46000:-0.0440054 +E^298*E^2248:46000:-0.0440053 E^298*E^2248[1]:46001:-0.0440053 E^298*E^2248[2]:46002:-0.0440053 E^298*E^2248[3]:46003:-0.0394141 @@ -12231,7 +12231,7 @@ E^1196*E^1214:46128:-0.0558656 E^1196*E^1214[1]:46129:-0.0558656 E^1196*E^1214[2]:46130:-0.0558656 E^1196*E^1214[3]:46131:-0.0500293 -E^1110*E^1890:46136:-0.0594923 +E^1110*E^1890:46136:-0.0594922 E^1110*E^1890[1]:46137:-0.0594922 E^1110*E^1890[2]:46138:-0.0594922 E^1110*E^1890[3]:46139:-0.0532692 @@ -12303,7 +12303,7 @@ E^1196*E^1263:46332:-0.0356329 E^1196*E^1263[1]:46333:-0.0356329 E^1196*E^1263[2]:46334:-0.0356329 E^1196*E^1263[3]:46335:-0.0319004 -E^378*E^1865:46348:-0.0306832 +E^378*E^1865:46348:-0.0306831 E^378*E^1865[1]:46349:-0.0306831 E^378*E^1865[2]:46350:-0.0306831 E^378*E^1865[3]:46351:-0.02748 @@ -12552,7 +12552,7 @@ E^301*E^702:47204:-0.0402139 E^301*E^702[1]:47205:-0.0402139 E^301*E^702[2]:47206:-0.0402139 E^301*E^702[3]:47207:-0.0360101 -B^9*B^233:47208:-0.073793 +B^9*B^233:47208:-0.0737929 B^9*B^233[1]:47209:-0.0737929 B^9*B^233[2]:47210:-0.0737929 B^9*B^233[3]:47211:-0.0660909 @@ -12560,7 +12560,7 @@ E^2477*E^8256:47212:-0.00650927 E^2477*E^8256[1]:47213:-0.00650927 E^2477*E^8256[2]:47214:-0.00650927 E^2477*E^8256[3]:47215:-0.00583038 -E^702*E^2614:47240:-0.0807228 +E^702*E^2614:47240:-0.0807227 E^702*E^2614[1]:47241:-0.0807227 E^702*E^2614[2]:47242:-0.0807227 E^702*E^2614[3]:47243:-0.0722894 @@ -12622,7 +12622,7 @@ D^433*D^1881:47400:-0.0279762 D^433*D^1881[1]:47401:-0.0279762 D^433*D^1881[2]:47402:-0.0279762 D^433*D^1881[3]:47403:-0.0250502 -E^8081*E^10752:47420:-0.00950471 +E^8081*E^10752:47420:-0.0095047 E^8081*E^10752[1]:47421:-0.0095047 E^8081*E^10752[2]:47422:-0.0095047 E^8081*E^10752[3]:47423:-0.00851155 @@ -12669,7 +12669,7 @@ C^240*C^584:47560:-0.136371 C^240*C^584[1]:47561:-0.136371 C^240*C^584[2]:47562:-0.136371 C^240*C^584[3]:47563:-0.122095 -D^50*D^1110:47568:-0.0315725 +D^50*D^1110:47568:-0.0315724 D^50*D^1110[1]:47569:-0.0315724 D^50*D^1110[2]:47570:-0.0315724 D^50*D^1110[3]:47571:-0.0282663 @@ -12713,7 +12713,7 @@ E^789*E^33705:47760:-0.0211784 E^789*E^33705[1]:47761:-0.0211784 E^789*E^33705[2]:47762:-0.0211784 E^789*E^33705[3]:47763:-0.0189681 -B^130*B^const:47776:-0.0528058 +B^130*B^const:47776:-0.0528057 B^130*B^const[1]:47777:-0.0528057 B^130*B^const[2]:47778:-0.0528057 B^130*B^const[3]:47779:-0.0472807 @@ -12893,7 +12893,7 @@ E^464*E^564:48440:-0.113918 E^464*E^564[1]:48441:-0.113918 E^464*E^564[2]:48442:-0.113918 E^464*E^564[3]:48443:-0.10202 -E^378*E^2362:48456:-0.0779985 +E^378*E^2362:48456:-0.0779984 E^378*E^2362[1]:48457:-0.0779984 E^378*E^2362[2]:48458:-0.0779984 E^378*E^2362[3]:48459:-0.0698425 @@ -12901,11 +12901,11 @@ E^464*E^533:48548:-0.240468 E^464*E^533[1]:48549:-0.240468 E^464*E^533[2]:48550:-0.240468 E^464*E^533[3]:48551:-0.215296 -D^276*D^8318:48552:-0.00769899 +D^276*D^8318:48552:-0.00769898 D^276*D^8318[1]:48553:-0.00769898 D^276*D^8318[2]:48554:-0.00769898 D^276*D^8318[3]:48555:-0.0068922 -E^301*E^33705:48560:-0.00990815 +E^301*E^33705:48560:-0.00990814 E^301*E^33705[1]:48561:-0.00990814 E^301*E^33705[2]:48562:-0.00990814 E^301*E^33705[3]:48563:-0.00887234 @@ -12953,7 +12953,7 @@ E^301*E^836:48732:-0.0290217 E^301*E^836[1]:48733:-0.0290217 E^301*E^836[2]:48734:-0.0290217 E^301*E^836[3]:48735:-0.0259785 -B^90*B^234:48736:-0.033555 +B^90*B^234:48736:-0.0335549 B^90*B^234[1]:48737:-0.0335549 B^90*B^234[2]:48738:-0.0335549 B^90*B^234[3]:48739:-0.0300504 @@ -12964,7 +12964,7 @@ B^9*B^104[3]:48751:-0.0426236 A^932*A^1028:48784:-0.194399 A^932*A^1028[2]:48786:-0.194399 A^932*A^1028[3]:48787:-0.233406 -E^378*E^2477:48796:-0.0291562 +E^378*E^2477:48796:-0.0291561 E^378*E^2477[1]:48797:-0.0291561 E^378*E^2477[2]:48798:-0.0291561 E^378*E^2477[3]:48799:-0.0261138 @@ -13054,7 +13054,7 @@ D^430*D^8277[3]:49183:-0.028478 A^438*A^757:49212:-0.143766 A^438*A^757[2]:49214:-0.143766 A^438*A^757[3]:49215:-0.172616 -D^3615*D^5671:49224:-0.0611846 +D^3615*D^5671:49224:-0.0611845 D^3615*D^5671[1]:49225:-0.0611845 D^3615*D^5671[2]:49226:-0.0611845 D^3615*D^5671[3]:49227:-0.0547949 @@ -13284,7 +13284,7 @@ D^209*D^871[3]:50275:-0.0790043 E^45*E^1214:50276:-0.108296 E^45*E^1214[1]:50277:-0.108296 E^45*E^1214[2]:50278:-0.108296 -E^45*E^1214[3]:50279:-0.0969593 +E^45*E^1214[3]:50279:-0.0969594 E^42*E^50:50280:-0.076697 E^42*E^50[1]:50281:-0.076697 E^42*E^50[2]:50282:-0.076697 @@ -13308,7 +13308,7 @@ E^510*E^8421:50380:-0.0413678 E^510*E^8421[1]:50381:-0.0413678 E^510*E^8421[2]:50382:-0.0413678 E^510*E^8421[3]:50383:-0.0370383 -E^533*E^1865:50448:-0.0686196 +E^533*E^1865:50448:-0.0686195 E^533*E^1865[1]:50449:-0.0686195 E^533*E^1865[2]:50450:-0.0686195 E^533*E^1865[3]:50451:-0.0614501 @@ -13374,7 +13374,7 @@ B^14*B^4523:50676:-0.0130422 B^14*B^4523[1]:50677:-0.0130422 B^14*B^4523[2]:50678:-0.0130422 B^14*B^4523[3]:50679:-0.0116775 -B^104*B^254:50680:-0.0095417 +B^104*B^254:50680:-0.00954169 B^104*B^254[1]:50681:-0.00954169 B^104*B^254[2]:50682:-0.00954169 B^104*B^254[3]:50683:-0.00854497 @@ -13410,7 +13410,7 @@ D^1419*D^5845:50784:-0.0152418 D^1419*D^5845[1]:50785:-0.0152418 D^1419*D^5845[2]:50786:-0.0152418 D^1419*D^5845[3]:50787:-0.0136513 -E^42*E^179:50788:-0.052187 +E^42*E^179:50788:-0.0521869 E^42*E^179[1]:50789:-0.0521869 E^42*E^179[2]:50790:-0.0521869 E^42*E^179[3]:50791:-0.0467311 @@ -13600,7 +13600,7 @@ D^288*D^757:51364:-0.0370905 D^288*D^757[1]:51365:-0.0370905 D^288*D^757[2]:51366:-0.0370905 D^288*D^757[3]:51367:-0.0332067 -E^1890*E^8287:51380:-0.0105242 +E^1890*E^8287:51380:-0.0105241 E^1890*E^8287[1]:51381:-0.0105241 E^1890*E^8287[2]:51382:-0.0105241 E^1890*E^8287[3]:51383:-0.00942469 @@ -13615,7 +13615,7 @@ E^42:51396:-0.00301368 E^42[1]:51397:-0.00301368 E^42[2]:51398:-0.00301368 E^42[3]:51399:-0.00269844 -D^208*D^208:51408:-0.00575662 +D^208*D^208:51408:-0.00575661 D^208*D^208[1]:51409:-0.00575661 D^208*D^208[2]:51410:-0.00575661 D^208*D^208[3]:51411:-0.00515524 @@ -13654,7 +13654,7 @@ B^106*B^465:51484:-0.0444565 B^106*B^465[1]:51485:-0.0444565 B^106*B^465[2]:51486:-0.0444565 B^106*B^465[3]:51487:-0.0398047 -D^288*D^664:51504:-0.00970223 +D^288*D^664:51504:-0.00970222 D^288*D^664[1]:51505:-0.00970222 D^288*D^664[2]:51506:-0.00970222 D^288*D^664[3]:51507:-0.00868908 @@ -13718,7 +13718,7 @@ B^106*B^476:51688:-0.0159337 B^106*B^476[1]:51689:-0.0159337 B^106*B^476[2]:51690:-0.0159337 B^106*B^476[3]:51691:-0.0142659 -B^2039*B^2277:51696:-0.0391682 +B^2039*B^2277:51696:-0.0391681 B^2039*B^2277[1]:51697:-0.0391681 B^2039*B^2277[2]:51698:-0.0391681 B^2039*B^2277[3]:51699:-0.0350734 @@ -13751,7 +13751,7 @@ C^161*C^584:51804:-0.137551 C^161*C^584[1]:51805:-0.137551 C^161*C^584[2]:51806:-0.137551 C^161*C^584[3]:51807:-0.123158 -E^1890*E^8395:51812:-0.0268657 +E^1890*E^8395:51812:-0.0268656 E^1890*E^8395[1]:51813:-0.0268656 E^1890*E^8395[2]:51814:-0.0268656 E^1890*E^8395[3]:51815:-0.0240553 @@ -13762,7 +13762,7 @@ D^1266*D^1730:51840:-0.0134827 D^1266*D^1730[1]:51841:-0.0134827 D^1266*D^1730[2]:51842:-0.0134827 D^1266*D^1730[3]:51843:-0.0120735 -E^1890*E^8421:51868:-0.0307743 +E^1890*E^8421:51868:-0.0307742 E^1890*E^8421[1]:51869:-0.0307742 E^1890*E^8421[2]:51870:-0.0307742 E^1890*E^8421[3]:51871:-0.0275511 @@ -13782,7 +13782,7 @@ B^105*B^934:51940:-0.0315811 B^105*B^934[1]:51941:-0.0315811 B^105*B^934[2]:51942:-0.0315811 B^105*B^934[3]:51943:-0.0282841 -D^357*D^3013:51944:-0.0022943 +D^357*D^3013:51944:-0.00229429 D^357*D^3013[1]:51945:-0.00229429 D^357*D^3013[2]:51946:-0.00229429 D^357*D^3013[3]:51947:-0.00205406 @@ -13841,7 +13841,7 @@ D^1417*D^4345:52104:-0.0217597 D^1417*D^4345[1]:52105:-0.0217597 D^1417*D^4345[2]:52106:-0.0217597 D^1417*D^4345[3]:52107:-0.0194877 -B^105*B^995:52112:-0.013242 +B^105*B^995:52112:-0.0132419 B^105*B^995[1]:52113:-0.0132419 B^105*B^995[2]:52114:-0.0132419 B^105*B^995[3]:52115:-0.011856 @@ -13954,7 +13954,7 @@ D^1266*D^1308:52520:-0.132623 D^1266*D^1308[1]:52521:-0.132623 D^1266*D^1308[2]:52522:-0.132623 D^1266*D^1308[3]:52523:-0.118734 -B^254*B^2495:52548:-0.0298286 +B^254*B^2495:52548:-0.0298285 B^254*B^2495[1]:52549:-0.0298285 B^254*B^2495[2]:52550:-0.0298285 B^254*B^2495[3]:52551:-0.0267056 @@ -14062,11 +14062,11 @@ E^1350*E^3027:53172:-0.0220547 E^1350*E^3027[1]:53173:-0.0220547 E^1350*E^3027[2]:53174:-0.0220547 E^1350*E^3027[3]:53175:-0.0197459 -E^1350*E^3008:53184:-0.0533496 +E^1350*E^3008:53184:-0.0533495 E^1350*E^3008[1]:53185:-0.0533495 E^1350*E^3008[2]:53186:-0.0533495 E^1350*E^3008[3]:53187:-0.04776 -E^2614*E^13810:53240:-0.0128147 +E^2614*E^13810:53240:-0.0128146 E^2614*E^13810[1]:53241:-0.0128146 E^2614*E^13810[2]:53242:-0.0128146 E^2614*E^13810[3]:53243:-0.0114757 @@ -14216,7 +14216,7 @@ B^1796*B^2949:53780:-0.0218335 B^1796*B^2949[1]:53781:-0.0218335 B^1796*B^2949[2]:53782:-0.0218335 B^1796*B^2949[3]:53783:-0.0195499 -E^1081*E^8256:53788:-0.0058911 +E^1081*E^8256:53788:-0.00589109 E^1081*E^8256[1]:53789:-0.00589109 E^1081*E^8256[2]:53790:-0.00589109 E^1081*E^8256[3]:53791:-0.00527608 @@ -14274,7 +14274,7 @@ E^1081*E^8287:53912:-0.00546596 E^1081*E^8287[1]:53913:-0.00546596 E^1081*E^8287[2]:53914:-0.00546596 E^1081*E^8287[3]:53915:-0.00489505 -E^673:53920:-0.00312504 +E^673:53920:-0.00312503 E^673[1]:53921:-0.00312503 E^673[2]:53922:-0.00312503 E^673[3]:53923:-0.00279744 @@ -14326,7 +14326,7 @@ E^45*E^312:54156:-0.152492 E^45*E^312[1]:54157:-0.152492 E^45*E^312[2]:54158:-0.152492 E^45*E^312[3]:54159:-0.136587 -B^106*B^106:54176:-0.0754794 +B^106*B^106:54176:-0.0754793 B^106*B^106[1]:54177:-0.0754793 B^106*B^106[2]:54178:-0.0754793 B^106*B^106[3]:54179:-0.0676023 @@ -14419,7 +14419,7 @@ E^42*E^1038:54488:-0.0617567 E^42*E^1038[1]:54489:-0.0617567 E^42*E^1038[2]:54490:-0.0617567 E^42*E^1038[3]:54491:-0.05529 -E^10752*E^14387:54524:-0.00827053 +E^10752*E^14387:54524:-0.00827052 E^10752*E^14387[1]:54525:-0.00827052 E^10752*E^14387[2]:54526:-0.00827052 E^10752*E^14387[3]:54527:-0.0074048 @@ -14427,7 +14427,7 @@ E^131*E^2166:54540:-0.0260241 E^131*E^2166[1]:54541:-0.0260241 E^131*E^2166[2]:54542:-0.0260241 E^131*E^2166[3]:54543:-0.0233008 -B^497*B^1629:54552:-0.0520048 +B^497*B^1629:54552:-0.0520047 B^497*B^1629[1]:54553:-0.0520047 B^497*B^1629[2]:54554:-0.0520047 B^497*B^1629[3]:54555:-0.0465602 @@ -14467,7 +14467,7 @@ B^254*B^2036:54680:-0.109857 B^254*B^2036[1]:54681:-0.109857 B^254*B^2036[2]:54682:-0.109857 B^254*B^2036[3]:54683:-0.0983811 -E^131*E^2126:54700:-0.0298286 +E^131*E^2126:54700:-0.0298285 E^131*E^2126[1]:54701:-0.0298285 E^131*E^2126[2]:54702:-0.0298285 E^131*E^2126[3]:54703:-0.0267115 @@ -14518,7 +14518,7 @@ B^106*B^305:54940:-0.0100665 B^106*B^305[1]:54941:-0.0100665 B^106*B^305[2]:54942:-0.0100665 B^106*B^305[3]:54943:-0.00901399 -E^276*E^5158:54960:-0.074181 +E^276*E^5158:54960:-0.0741809 E^276*E^5158[1]:54961:-0.0741809 E^276*E^5158[2]:54962:-0.0741809 E^276*E^5158[3]:54963:-0.0664133 @@ -14573,7 +14573,7 @@ C^820*C^3006:55184:-0.0287477 C^820*C^3006[1]:55185:-0.0287477 C^820*C^3006[2]:55186:-0.0287477 C^820*C^3006[3]:55187:-0.0257396 -E^45*E^50:55188:-0.161446 +E^45*E^50:55188:-0.161445 E^45*E^50[1]:55189:-0.161445 E^45*E^50[2]:55190:-0.161445 E^45*E^50[3]:55191:-0.144582 @@ -14604,11 +14604,11 @@ D^2223*D^8472[3]:55239:-0.0378513 E^342*E^const:55244:-0.151936 E^342*E^const[1]:55245:-0.151936 E^342*E^const[2]:55246:-0.151936 -E^342*E^const[3]:55247:-0.136066 +E^342*E^const[3]:55247:-0.136067 B^337*B^4525:55256:-0.00850478 B^337*B^4525[1]:55257:-0.00850478 B^337*B^4525[2]:55258:-0.00850478 -B^337*B^4525[3]:55259:-0.00761635 +B^337*B^4525[3]:55259:-0.00761636 B^106*B^348:55272:-0.00315052 B^106*B^348[1]:55273:-0.00315052 B^106*B^348[2]:55274:-0.00315052 @@ -14873,14 +14873,14 @@ E^50*E^1705:56364:-0.0493151 E^50*E^1705[1]:56365:-0.0493151 E^50*E^1705[2]:56366:-0.0493151 E^50*E^1705[3]:56367:-0.0441548 -E^1038*E^25081:56380:-0.033555 +E^1038*E^25081:56380:-0.0335549 E^1038*E^25081[1]:56381:-0.0335549 E^1038*E^25081[2]:56382:-0.0335549 E^1038*E^25081[3]:56383:-0.0300504 E^1416*E^3845:56388:-0.0459401 E^1416*E^3845[1]:56389:-0.0459401 E^1416*E^3845[2]:56390:-0.0459401 -E^1416*E^3845[3]:56391:-0.041139 +E^1416*E^3845[3]:56391:-0.0411391 A^156*A^16893:56404:-0.0314475 A^156*A^16893[2]:56406:-0.0314475 A^156*A^16893[3]:56407:-0.037758 @@ -14896,7 +14896,7 @@ D^354*D^3483:56420:-0.00116342 D^354*D^3483[1]:56421:-0.00116342 D^354*D^3483[2]:56422:-0.00116342 D^354*D^3483[3]:56423:-0.00104164 -E^50*E^1690:56424:-0.0669104 +E^50*E^1690:56424:-0.0669103 E^50*E^1690[1]:56425:-0.0669103 E^50*E^1690[2]:56426:-0.0669103 E^50*E^1690[3]:56427:-0.0599115 @@ -15022,7 +15022,7 @@ B^9*B^1930:56820:-0.00940493 B^9*B^1930[1]:56821:-0.00940493 B^9*B^1930[2]:56822:-0.00940493 B^9*B^1930[3]:56823:-0.00842317 -B^348*B^518:56840:-0.00912296 +B^348*B^518:56840:-0.00912295 B^348*B^518[1]:56841:-0.00912295 B^348*B^518[2]:56842:-0.00912295 B^348*B^518[3]:56843:-0.0081689 @@ -15034,7 +15034,7 @@ D^12632*D^36203:56876:-0.0710575 D^12632*D^36203[1]:56877:-0.0710575 D^12632*D^36203[2]:56878:-0.0710575 D^12632*D^36203[3]:56879:-0.0636499 -E^131*E^1705:56888:-0.0208488 +E^131*E^1705:56888:-0.0208487 E^131*E^1705[1]:56889:-0.0208487 E^131*E^1705[2]:56890:-0.0208487 E^131*E^1705[3]:56891:-0.0186726 @@ -15148,7 +15148,7 @@ E^131*E^1335[3]:57347:-0.0267056 A^229*A^16893:57352:-0.0432149 A^229*A^16893[2]:57354:-0.0432149 A^229*A^16893[3]:57355:-0.0518888 -E^45*E^3536:57388:-0.0787967 +E^45*E^3536:57388:-0.0787966 E^45*E^3536[1]:57389:-0.0787966 E^45*E^3536[2]:57390:-0.0787966 E^45*E^3536[3]:57391:-0.0705511 @@ -15234,7 +15234,7 @@ E^131*E^1380:57684:-0.0366827 E^131*E^1380[1]:57685:-0.0366827 E^131*E^1380[2]:57686:-0.0366827 E^131*E^1380[3]:57687:-0.0328457 -D^334*D^12632:57736:-0.0872299 +D^334*D^12632:57736:-0.0872298 D^334*D^12632[1]:57737:-0.0872298 D^334*D^12632[2]:57738:-0.0872298 D^334*D^12632[3]:57739:-0.0781315 @@ -15304,7 +15304,7 @@ E^3536*E^5597:57924:-0.0116586 E^3536*E^5597[1]:57925:-0.0116586 E^3536*E^5597[2]:57926:-0.0116586 E^3536*E^5597[3]:57927:-0.0104414 -E^2115*E^18661:57928:-0.00438796 +E^2115*E^18661:57928:-0.00438795 E^2115*E^18661[1]:57929:-0.00438795 E^2115*E^18661[2]:57930:-0.00438795 E^2115*E^18661[3]:57931:-0.00392995 @@ -15448,7 +15448,7 @@ E^131*E^1036:58548:-0.0209702 E^131*E^1036[1]:58549:-0.0209702 E^131*E^1036[2]:58550:-0.0209702 E^131*E^1036[3]:58551:-0.0187758 -E^18668*E^33705:58596:-0.00966305 +E^18668*E^33705:58596:-0.00966304 E^18668*E^33705[1]:58597:-0.00966304 E^18668*E^33705[2]:58598:-0.00966304 E^18668*E^33705[3]:58599:-0.0086521 @@ -15750,7 +15750,7 @@ E^269*E^8269:59552:-0.0567593 E^269*E^8269[1]:59553:-0.0567593 E^269*E^8269[2]:59554:-0.0567593 E^269*E^8269[3]:59555:-0.0508326 -D^41*D^1488:59564:-0.0489594 +D^41*D^1488:59564:-0.0489593 D^41*D^1488[1]:59565:-0.0489593 D^41*D^1488[2]:59566:-0.0489593 D^41*D^1488[3]:59567:-0.0438327 @@ -15972,7 +15972,7 @@ E^131*E^557[3]:60459:-0.0233345 A^188*A^613:60468:-0.317594 A^188*A^613[2]:60470:-0.317594 A^188*A^613[3]:60471:-0.381306 -E^50*E^673:60492:-0.087629 +E^50*E^673:60492:-0.0876289 E^50*E^673[1]:60493:-0.0876289 E^50*E^673[2]:60494:-0.0876289 E^50*E^673[3]:60495:-0.0784702 @@ -16124,7 +16124,7 @@ E^131*E^655[3]:61091:-0.0294062 A^188*A^709:61108:-0.248713 A^188*A^709[2]:61110:-0.248713 A^188*A^709[3]:61111:-0.298573 -E^50*E^516:61120:-0.0860673 +E^50*E^516:61120:-0.0860672 E^50*E^516[1]:61121:-0.0860672 E^50*E^516[2]:61122:-0.0860672 E^50*E^516[3]:61123:-0.0770545 @@ -16140,7 +16140,7 @@ B^337*B^2031:61152:-0.0101619 B^337*B^2031[1]:61153:-0.0101619 B^337*B^2031[2]:61154:-0.0101619 B^337*B^2031[3]:61155:-0.00909869 -E^50*E^510:61176:-0.0941708 +E^50*E^510:61176:-0.0941707 E^50*E^510[1]:61177:-0.0941707 E^50*E^510[2]:61178:-0.0941707 E^50*E^510[3]:61179:-0.0843226 @@ -16155,7 +16155,7 @@ C^1145*C^1529:61216:-0.0294468 C^1145*C^1529[1]:61217:-0.0294468 C^1145*C^1529[2]:61218:-0.0294468 C^1145*C^1529[3]:61219:-0.0263675 -E^233*E^25081:61248:-0.0661122 +E^233*E^25081:61248:-0.0661121 E^233*E^25081[1]:61249:-0.0661121 E^233*E^25081[2]:61250:-0.0661121 E^233*E^25081[3]:61251:-0.0592147 @@ -16212,7 +16212,7 @@ E^101*E^10752:61452:-0.0196384 E^101*E^10752[1]:61453:-0.0196384 E^101*E^10752[2]:61454:-0.0196384 E^101*E^10752[3]:61455:-0.0175871 -B^20*B^50:61464:-0.0121423 +B^20*B^50:61464:-0.0121422 B^20*B^50[1]:61465:-0.0121422 B^20*B^50[2]:61466:-0.0121422 B^20*B^50[3]:61467:-0.0108717 @@ -16328,7 +16328,7 @@ B^910*B^2330:61872:-0.148258 B^910*B^2330[1]:61873:-0.148258 B^910*B^2330[2]:61874:-0.148258 B^910*B^2330[3]:61875:-0.132779 -B^910*B^2344:61928:-0.0283016 +B^910*B^2344:61928:-0.0283015 B^910*B^2344[1]:61929:-0.0283015 B^910*B^2344[2]:61930:-0.0283015 B^910*B^2344[3]:61931:-0.0253401 @@ -16566,7 +16566,7 @@ E^1298*E^20228:62784:-0.0139005 E^1298*E^20228[1]:62785:-0.0139005 E^1298*E^20228[2]:62786:-0.0139005 E^1298*E^20228[3]:62787:-0.0124439 -C^90*C^161:62796:-0.0891213 +C^90*C^161:62796:-0.0891212 C^90*C^161[1]:62797:-0.0891212 C^90*C^161[2]:62798:-0.0891212 C^90*C^161[3]:62799:-0.0798179 @@ -16603,7 +16603,7 @@ E^50*E^52:62976:-0.108781 E^50*E^52[1]:62977:-0.108781 E^50*E^52[2]:62978:-0.108781 E^50*E^52[3]:62979:-0.0974106 -E^50*E^50:62984:-0.0945352 +E^50*E^50:62984:-0.0945351 E^50*E^50[1]:62985:-0.0945351 E^50*E^50[2]:62986:-0.0945351 E^50*E^50[3]:62987:-0.0846791 @@ -16679,7 +16679,7 @@ D^286*D^533:63196:-0.0191699 D^286*D^533[1]:63197:-0.0191699 D^286*D^533[2]:63198:-0.0191699 D^286*D^533[3]:63199:-0.0171648 -E^2362*E^18661:63228:-0.00979319 +E^2362*E^18661:63228:-0.00979318 E^2362*E^18661[1]:63229:-0.00979318 E^2362*E^18661[2]:63230:-0.00979318 E^2362*E^18661[3]:63231:-0.00876897 @@ -17044,7 +17044,7 @@ B^99*B^400:64564:-0.0528984 B^99*B^400[1]:64565:-0.0528984 B^99*B^400[2]:64566:-0.0528984 B^99*B^400[3]:64567:-0.0473798 -E^760*E^8284:64568:-0.00883014 +E^760*E^8284:64568:-0.00883013 E^760*E^8284[1]:64569:-0.00883013 E^760*E^8284[2]:64570:-0.00883013 E^760*E^8284[3]:64571:-0.00790871 @@ -17161,7 +17161,7 @@ B^180*B^655:65036:-0.0465155 B^180*B^655[1]:65037:-0.0465155 B^180*B^655[2]:65038:-0.0465155 B^180*B^655[3]:65039:-0.0416455 -E^533*E^1417:65040:-0.117753 +E^533*E^1417:65040:-0.117752 E^533*E^1417[1]:65041:-0.117752 E^533*E^1417[2]:65042:-0.117752 E^533*E^1417[3]:65043:-0.105446 @@ -17169,7 +17169,7 @@ E^1263*E^2248:65076:-0.0186769 E^1263*E^2248[1]:65077:-0.0186769 E^1263*E^2248[2]:65078:-0.0186769 E^1263*E^2248[3]:65079:-0.0167223 -E^1263*E^2259:65120:-0.0173957 +E^1263*E^2259:65120:-0.0173956 E^1263*E^2259[1]:65121:-0.0173956 E^1263*E^2259[2]:65122:-0.0173956 E^1263*E^2259[3]:65123:-0.0155726 @@ -17219,7 +17219,7 @@ E^760*E^8395:65276:-0.0257508 E^760*E^8395[1]:65277:-0.0257508 E^760*E^8395[2]:65278:-0.0257508 E^760*E^8395[3]:65279:-0.0230608 -D^42*D^76:65288:-0.0173957 +D^42*D^76:65288:-0.0173956 D^42*D^76[1]:65289:-0.0173956 D^42*D^76[2]:65290:-0.0173956 D^42*D^76[3]:65291:-0.0155778 @@ -17426,7 +17426,7 @@ E^276*E^276:66152:-0.165194 E^276*E^276[1]:66153:-0.165194 E^276*E^276[2]:66154:-0.165194 E^276*E^276[3]:66155:-0.147934 -D^1308*D^5651:66172:-0.0134871 +D^1308*D^5651:66172:-0.013487 D^1308*D^5651[1]:66173:-0.013487 D^1308*D^5651[2]:66174:-0.013487 D^1308*D^5651[3]:66175:-0.0120787 @@ -17510,7 +17510,7 @@ E^3825:66528:-0.00103043 E^3825[1]:66529:-0.00103043 E^3825[2]:66530:-0.00103043 E^3825[3]:66531:-0.000922826 -D^494*D^5651:66580:-0.0671621 +D^494*D^5651:66580:-0.067162 D^494*D^5651[1]:66581:-0.067162 D^494*D^5651[2]:66582:-0.067162 D^494*D^5651[3]:66583:-0.0601495 @@ -17534,7 +17534,7 @@ C^584*C^3762:66640:-0.0697735 C^584*C^3762[1]:66641:-0.0697735 C^584*C^3762[2]:66642:-0.0697735 C^584*C^3762[3]:66643:-0.0624886 -E^1335*E^1416:66644:-0.0533496 +E^1335*E^1416:66644:-0.0533495 E^1335*E^1416[1]:66645:-0.0533495 E^1335*E^1416[2]:66646:-0.0533495 E^1335*E^1416[3]:66647:-0.0477837 @@ -17546,7 +17546,7 @@ E^1335*E^1455:66736:-0.0586767 E^1335*E^1455[1]:66737:-0.0586767 E^1335*E^1455[2]:66738:-0.0586767 E^1335*E^1455[3]:66739:-0.0525354 -B^481*B^724:66748:-0.0768706 +B^481*B^724:66748:-0.0768705 B^481*B^724[1]:66749:-0.0768705 B^481*B^724[2]:66750:-0.0768705 B^481*B^724[3]:66751:-0.0688602 @@ -17607,7 +17607,7 @@ B^2039*B^5593[3]:67011:-0.00721267 E^1011*E^2028:67060:-0.0416888 E^1011*E^2028[1]:67061:-0.0416888 E^1011*E^2028[2]:67062:-0.0416888 -E^1011*E^2028[3]:67063:-0.0373215 +E^1011*E^2028[3]:67063:-0.0373216 E^131*E^7351:67072:-0.0216426 E^131*E^7351[1]:67073:-0.0216426 E^131*E^7351[2]:67074:-0.0216426 @@ -17648,7 +17648,7 @@ B^348*B^12332:67216:-0.0526034 B^348*B^12332[1]:67217:-0.0526034 B^348*B^12332[2]:67218:-0.0526034 B^348*B^12332[3]:67219:-0.0470995 -E^1416*E^1454:67232:-0.0779985 +E^1416*E^1454:67232:-0.0779984 E^1416*E^1454[1]:67233:-0.0779984 E^1416*E^1454[2]:67234:-0.0779984 E^1416*E^1454[3]:67235:-0.0698425 @@ -17732,7 +17732,7 @@ B^481*B^768:67564:-0.0463219 B^481*B^768[1]:67565:-0.0463219 B^481*B^768[2]:67566:-0.0463219 B^481*B^768[3]:67567:-0.0414756 -D^85*D^3899:67584:-0.00944876 +D^85*D^3899:67584:-0.00944875 D^85*D^3899[1]:67585:-0.00944875 D^85*D^3899[2]:67586:-0.00944875 D^85*D^3899[3]:67587:-0.00846226 @@ -17784,7 +17784,7 @@ B^2362*B^5401:67772:-0.0289629 B^2362*B^5401[1]:67773:-0.0289629 B^2362*B^5401[2]:67774:-0.0289629 B^2362*B^5401[3]:67775:-0.0259428 -E^193*E^225:67776:-0.0994806 +E^193*E^225:67776:-0.0994805 E^193*E^225[1]:67777:-0.0994805 E^193*E^225[2]:67778:-0.0994805 E^193*E^225[3]:67779:-0.0890758 @@ -18002,7 +18002,7 @@ E^768*E^1865:68948:-0.0389211 E^768*E^1865[1]:68949:-0.0389211 E^768*E^1865[2]:68950:-0.0389211 E^768*E^1865[3]:68951:-0.0348503 -D^3188*D^9041:68964:-0.0765062 +D^3188*D^9041:68964:-0.0765061 D^3188*D^9041[1]:68965:-0.0765061 D^3188*D^9041[2]:68966:-0.0765061 D^3188*D^9041[3]:68967:-0.0685185 @@ -18134,8 +18134,8 @@ A^752*A^757[3]:69479:-0.106784 D^1141*D^4724:69484:-0.0234429 D^1141*D^4724[1]:69485:-0.0234429 D^1141*D^4724[2]:69486:-0.0234429 -D^1141*D^4724[3]:69487:-0.02099 -E^1011*E^2380:69492:-0.0499312 +D^1141*D^4724[3]:69487:-0.0209901 +E^1011*E^2380:69492:-0.0499311 E^1011*E^2380[1]:69493:-0.0499311 E^1011*E^2380[2]:69494:-0.0499311 E^1011*E^2380[3]:69495:-0.0447051 @@ -18201,11 +18201,11 @@ E^193*E^702:69748:-0.124572 E^193*E^702[1]:69749:-0.124572 E^193*E^702[2]:69750:-0.124572 E^193*E^702[3]:69751:-0.11155 -D^1213*D^1237:69768:-0.065019 +D^1213*D^1237:69768:-0.0650189 D^1213*D^1237[1]:69769:-0.0650189 D^1213*D^1237[2]:69770:-0.0650189 D^1213*D^1237[3]:69771:-0.0582221 -E^45*E^4594:69780:-0.0590932 +E^45*E^4594:69780:-0.0590931 E^45*E^4594[1]:69781:-0.0590931 E^45*E^4594[2]:69782:-0.0590931 E^45*E^4594[3]:69783:-0.0529229 @@ -18364,7 +18364,7 @@ E^193*E^510:70516:-0.105085 E^193*E^510[1]:70517:-0.105085 E^193*E^510[2]:70518:-0.105085 E^193*E^510[3]:70519:-0.0940997 -E^193*E^564:70540:-0.0810525 +E^193*E^564:70540:-0.0810524 E^193*E^564[1]:70541:-0.0810524 E^193*E^564[2]:70542:-0.0810524 E^193*E^564[3]:70543:-0.0725971 @@ -18416,7 +18416,7 @@ A^434*A^2525[3]:70831:-0.0531215 A^434*A^2526:70832:-0.073093 A^434*A^2526[2]:70834:-0.073093 A^434*A^2526[3]:70835:-0.087762 -E^276*E^1196:70856:-0.110118 +E^276*E^1196:70856:-0.110117 E^276*E^1196[1]:70857:-0.110117 E^276*E^1196[2]:70858:-0.110117 E^276*E^1196[3]:70859:-0.0985941 @@ -18570,7 +18570,7 @@ E^8467*E^25081:71608:-0.017747 E^8467*E^25081[1]:71609:-0.017747 E^8467*E^25081[2]:71610:-0.017747 E^8467*E^25081[3]:71611:-0.015891 -E^276*E^1130:71616:-0.0928347 +E^276*E^1130:71616:-0.0928346 E^276*E^1130[1]:71617:-0.0928346 E^276*E^1130[2]:71618:-0.0928346 E^276*E^1130[3]:71619:-0.0831421 @@ -18714,7 +18714,7 @@ D^973*D^36203:72032:-0.0351036 D^973*D^36203[1]:72033:-0.0351036 D^973*D^36203[2]:72034:-0.0351036 D^973*D^36203[3]:72035:-0.0314277 -A^1122*A^16893:72044:-0.0470708 +A^1122*A^16893:72044:-0.0470707 A^1122*A^16893[1]:72045:-0.0209355 A^1122*A^16893[2]:72046:-0.0470707 A^1122*A^16893[3]:72047:-0.0321527 @@ -18804,7 +18804,7 @@ D^319*D^814:72364:-0.00571432 D^319*D^814[1]:72365:-0.00571432 D^319*D^814[2]:72366:-0.00571432 D^319*D^814[3]:72367:-0.00511733 -E^193*E^1133:72368:-0.0765062 +E^193*E^1133:72368:-0.0765061 E^193*E^1133[1]:72369:-0.0765061 E^193*E^1133[2]:72370:-0.0765061 E^193*E^1133[3]:72371:-0.0685082 @@ -18831,7 +18831,7 @@ E^193*E^1118:72436:-0.0715434 E^193*E^1118[1]:72437:-0.0715434 E^193*E^1118[2]:72438:-0.0715434 E^193*E^1118[3]:72439:-0.0640715 -B^73*B^606:72452:-0.0089595 +B^73*B^606:72452:-0.00895949 B^73*B^606[1]:72453:-0.00895949 B^73*B^606[2]:72454:-0.00895949 B^73*B^606[3]:72455:-0.00802259 @@ -19139,7 +19139,7 @@ D^494*D^4345:73612:-0.0173306 D^494*D^4345[1]:73613:-0.0173306 D^494*D^4345[2]:73614:-0.0173306 D^494*D^4345[3]:73615:-0.0155156 -C^584*C^3394:73616:-0.000852024 +C^584*C^3394:73616:-0.000852023 C^584*C^3394[1]:73617:-0.000852023 C^584*C^3394[2]:73618:-0.000852023 C^584*C^3394[3]:73619:-0.000763031 @@ -19159,7 +19159,7 @@ E^193*E^1305:73696:-0.0147318 E^193*E^1305[1]:73697:-0.0147318 E^193*E^1305[2]:73698:-0.0147318 E^193*E^1305[3]:73699:-0.013193 -E^1705*E^18661:73712:-0.00448882 +E^1705*E^18661:73712:-0.00448881 E^1705*E^18661[1]:73713:-0.00448881 E^1705*E^18661[2]:73714:-0.00448881 E^1705*E^18661[3]:73715:-0.00402019 @@ -19167,7 +19167,7 @@ C^179*C^2066:73756:-0.0242498 C^179*C^2066[1]:73757:-0.0242498 C^179*C^2066[2]:73758:-0.0242498 C^179*C^2066[3]:73759:-0.0217105 -E^276*E^2433:73764:-0.0867961 +E^276*E^2433:73764:-0.086796 E^276*E^2433[1]:73765:-0.086796 E^276*E^2433[2]:73766:-0.086796 E^276*E^2433[3]:73767:-0.0777039 @@ -19189,7 +19189,7 @@ A^438*A^2848[3]:73867:-0.112311 A^2524*A^24303:73868:-0.0172007 A^2524*A^24303[2]:73870:-0.0172007 A^2524*A^24303[3]:73871:-0.0206523 -D^314*D^483:73892:-0.0281194 +D^314*D^483:73892:-0.0281193 D^314*D^483[1]:73893:-0.0281193 D^314*D^483[2]:73894:-0.0281193 D^314*D^483[3]:73895:-0.0251804 @@ -19209,7 +19209,7 @@ E^2323*E^4045:73960:-0.0188359 E^2323*E^4045[1]:73961:-0.0188359 E^2323*E^4045[2]:73962:-0.0188359 E^2323*E^4045[3]:73963:-0.0168696 -B^234*B^1340:74088:-0.00607113 +B^234*B^1340:74088:-0.00607112 B^234*B^1340[1]:74089:-0.00607112 B^234*B^1340[2]:74090:-0.00607112 B^234*B^1340[3]:74091:-0.00543585 @@ -19415,7 +19415,7 @@ E^8269*E^8467:74904:-0.0172113 E^8269*E^8467[1]:74905:-0.0172113 E^8269*E^8467[2]:74906:-0.0172113 E^8269*E^8467[3]:74907:-0.0154073 -B^69*B^106:74916:-0.0253387 +B^69*B^106:74916:-0.0253386 B^69*B^106[1]:74917:-0.0253386 B^69*B^106[2]:74918:-0.0253386 B^69*B^106[3]:74919:-0.0226832 @@ -19462,7 +19462,7 @@ B^70*B^434[3]:75059:-0.0186363 E^1335*E^3536:75060:-0.0344052 E^1335*E^3536[1]:75061:-0.0344052 E^1335*E^3536[2]:75062:-0.0344052 -E^1335*E^3536[3]:75063:-0.030813 +E^1335*E^3536[3]:75063:-0.0308131 E^276*E^2248:75064:-0.0957498 E^276*E^2248[1]:75065:-0.0957498 E^276*E^2248[2]:75066:-0.0957498 @@ -19594,7 +19594,7 @@ E^1416*E^3536:75560:-0.0349041 E^1416*E^3536[1]:75561:-0.0349041 E^1416*E^3536[2]:75562:-0.0349041 E^1416*E^3536[3]:75563:-0.0312613 -B^2362*B^2493:75564:-0.0637089 +B^2362*B^2493:75564:-0.0637088 B^2362*B^2493[1]:75565:-0.0637088 B^2362*B^2493[2]:75566:-0.0637088 B^2362*B^2493[3]:75567:-0.0570534 @@ -19920,7 +19920,7 @@ B^106*B^11017:77372:-0.0153936 B^106*B^11017[1]:77373:-0.0153936 B^106*B^11017[2]:77374:-0.0153936 B^106*B^11017[3]:77375:-0.0137814 -E^478*E^18661:77388:-0.00800157 +E^478*E^18661:77388:-0.00800156 E^478*E^18661[1]:77389:-0.00800156 E^478*E^18661[2]:77390:-0.00800156 E^478*E^18661[3]:77391:-0.00716274 @@ -20216,7 +20216,7 @@ E^768*E^1298:78960:-0.0300064 E^768*E^1298[1]:78961:-0.0300064 E^768*E^1298[2]:78962:-0.0300064 E^768*E^1298[3]:78963:-0.0268698 -E^193*E^3008:78972:-0.0845403 +E^193*E^3008:78972:-0.0845402 E^193*E^3008[1]:78973:-0.0845402 E^193*E^3008[2]:78974:-0.0845402 E^193*E^3008[3]:78975:-0.0756846 @@ -20340,7 +20340,7 @@ E^1036*E^8256:79544:-0.00802651 E^1036*E^8256[1]:79545:-0.00802651 E^1036*E^8256[2]:79546:-0.00802651 E^1036*E^8256[3]:79547:-0.007189 -B^69*B^995:79552:-0.0948302 +B^69*B^995:79552:-0.0948301 B^69*B^995[1]:79553:-0.0948301 B^69*B^995[2]:79554:-0.0948301 B^69*B^995[3]:79555:-0.0848937 @@ -20802,7 +20802,7 @@ E^131*E^3825:81688:-0.0122203 E^131*E^3825[1]:81689:-0.0122203 E^131*E^3825[2]:81690:-0.0122203 E^131*E^3825[3]:81691:-0.0109414 -D^1059*D^3899:81704:-0.0315725 +D^1059*D^3899:81704:-0.0315724 D^1059*D^3899[1]:81705:-0.0315724 D^1059*D^3899[2]:81706:-0.0315724 D^1059*D^3899[3]:81707:-0.0282663 @@ -20842,7 +20842,7 @@ B^2036*B^2334:81864:-0.0512846 B^2036*B^2334[1]:81865:-0.0512846 B^2036*B^2334[2]:81866:-0.0512846 B^2036*B^2334[3]:81867:-0.0459234 -D^802*D^1029:81868:-0.0522564 +D^802*D^1029:81868:-0.0522563 D^802*D^1029[1]:81869:-0.0522563 D^802*D^1029[2]:81870:-0.0522563 D^802*D^1029[3]:81871:-0.0467954 @@ -21112,10 +21112,10 @@ E^5597*E^18666:83060:-0.0108755 E^5597*E^18666[1]:83061:-0.0108755 E^5597*E^18666[2]:83062:-0.0108755 E^5597*E^18666[3]:83063:-0.00974091 -E^52*E^4599:83068:-0.00982732 +E^52*E^4599:83068:-0.00982731 E^52*E^4599[1]:83069:-0.00982731 E^52*E^4599[2]:83070:-0.00982731 -E^52*E^4599[3]:83071:-0.00879764 +E^52*E^4599[3]:83071:-0.00879765 E^23*E^533:83096:-0.0942228 E^23*E^533[1]:83097:-0.0942228 E^23*E^533[2]:83098:-0.0942228 @@ -21152,7 +21152,7 @@ E^42*E^8287:83220:-0.0115219 E^42*E^8287[1]:83221:-0.0115219 E^42*E^8287[2]:83222:-0.0115219 E^42*E^8287[3]:83223:-0.0103173 -E^1081*E^1416:83260:-0.0275381 +E^1081*E^1416:83260:-0.027538 E^1081*E^1416[1]:83261:-0.027538 E^1081*E^1416[2]:83262:-0.027538 E^1081*E^1416[3]:83263:-0.0246596 @@ -21250,7 +21250,7 @@ E^188*E^1890[3]:83523:-0.143845 E^23*E^673:83528:-0.0538441 E^23*E^673[1]:83529:-0.0538441 E^23*E^673[2]:83530:-0.0538441 -E^23*E^673[3]:83531:-0.04822 +E^23*E^673[3]:83531:-0.0482201 B^234*B^2949:83532:-0.0951945 B^234*B^2949[1]:83533:-0.0951945 B^234*B^2949[2]:83534:-0.0951945 @@ -21277,7 +21277,7 @@ E^23*E^655[3]:83635:-0.0603005 A^13*A^1189:83656:-0.225392 A^13*A^1189[2]:83658:-0.225392 A^13*A^1189[3]:83659:-0.270645 -E^23*E^642:83660:-0.0425507 +E^23*E^642:83660:-0.0425506 E^23*E^642[1]:83661:-0.0425506 E^23*E^642[2]:83662:-0.0425506 E^23*E^642[3]:83663:-0.0380945 @@ -21450,7 +21450,7 @@ D^389*D^483[3]:84195:-0.0301999 E^52*E^3800:84216:-0.0558198 E^52*E^3800[1]:84217:-0.0558198 E^52*E^3800[2]:84218:-0.0558198 -E^52*E^3800[3]:84219:-0.0499716 +E^52*E^3800[3]:84219:-0.0499717 E^2166*E^3825:84220:-0.0188099 E^2166*E^3825[1]:84221:-0.0188099 E^2166*E^3825[2]:84222:-0.0188099 @@ -21491,7 +21491,7 @@ E^8272:84316:-0.00136568 E^8272[1]:84317:-0.00136568 E^8272[2]:84318:-0.00136568 E^8272[3]:84319:-0.00122265 -E^8273:84320:-0.000738963 +E^8273:84320:-0.000738962 E^8273[1]:84321:-0.000738962 E^8273[2]:84322:-0.000738962 E^8273[3]:84323:-0.000661752 @@ -21509,7 +21509,7 @@ D^334*D^1881[3]:84367:-0.000406863 A^188*A^7052:84368:-0.178438 A^188*A^7052[2]:84370:-0.178438 A^188*A^7052[3]:84371:-0.214315 -D^52*D^const:84372:-0.0596745 +D^52*D^const:84372:-0.0596744 D^52*D^const[1]:84373:-0.0596744 D^52*D^const[2]:84374:-0.0596744 D^52*D^const[3]:84375:-0.0534364 @@ -21525,7 +21525,7 @@ B^233*B^2520:84396:-0.0459575 B^233*B^2520[1]:84397:-0.0459575 B^233*B^2520[2]:84398:-0.0459575 B^233*B^2520[3]:84399:-0.0411568 -E^673*E^1038:84404:-0.0705457 +E^673*E^1038:84404:-0.0705456 E^673*E^1038[1]:84405:-0.0705456 E^673*E^1038[2]:84406:-0.0705456 E^673*E^1038[3]:84407:-0.0631632 @@ -21720,7 +21720,7 @@ E^836*E^1036:85000:-0.0597037 E^836*E^1036[1]:85001:-0.0597037 E^836*E^1036[2]:85002:-0.0597037 E^836*E^1036[3]:85003:-0.0534612 -E^836*E^1038:85008:-0.0648368 +E^836*E^1038:85008:-0.0648367 E^836*E^1038[1]:85009:-0.0648367 E^836*E^1038[2]:85010:-0.0648367 E^836*E^1038[3]:85011:-0.0580564 @@ -21802,7 +21802,7 @@ E^23*E^1130[3]:85295:-0.0386456 E^836*E^1110:85296:-0.0684026 E^836*E^1110[1]:85297:-0.0684026 E^836*E^1110[2]:85298:-0.0684026 -E^836*E^1110[3]:85299:-0.061237 +E^836*E^1110[3]:85299:-0.0612371 B^400*B^5401:85300:-0.0922273 B^400*B^5401[1]:85301:-0.0922273 B^400*B^5401[2]:85302:-0.0922273 @@ -21838,7 +21838,7 @@ E^23*E^1118:85372:-0.0393983 E^23*E^1118[1]:85373:-0.0393983 E^23*E^1118[2]:85374:-0.0393983 E^23*E^1118[3]:85375:-0.0352756 -E^836*E^1130:85376:-0.0478217 +E^836*E^1130:85376:-0.0478216 E^836*E^1130[1]:85377:-0.0478216 E^836*E^1130[2]:85378:-0.0478216 E^836*E^1130[3]:85379:-0.0428154 @@ -22013,7 +22013,7 @@ E^269*E^2433:85968:-0.0921753 E^269*E^2433[1]:85969:-0.0921753 E^269*E^2433[2]:85970:-0.0921753 E^269*E^2433[3]:85971:-0.0825725 -E^23*E^1221:85976:-0.0413158 +E^23*E^1221:85976:-0.0413157 E^23*E^1221[1]:85977:-0.0413157 E^23*E^1221[2]:85978:-0.0413157 E^23*E^1221[3]:85979:-0.0369902 @@ -22291,7 +22291,7 @@ E^510*E^1305:87036:-0.0789875 E^510*E^1305[1]:87037:-0.0789875 E^510*E^1305[2]:87038:-0.0789875 E^510*E^1305[3]:87039:-0.0707302 -E^101*E^768:87052:-0.0848353 +E^101*E^768:87052:-0.0848352 E^101*E^768[1]:87053:-0.0848352 E^101*E^768[2]:87054:-0.0848352 E^101*E^768[3]:87055:-0.0759553 @@ -22449,7 +22449,7 @@ E^8421*E^10752:87564:-0.00884315 E^8421*E^10752[1]:87565:-0.00884315 E^8421*E^10752[2]:87566:-0.00884315 E^8421*E^10752[3]:87567:-0.00791888 -C^2871*C^2903:87568:-0.03539 +C^2871*C^2903:87568:-0.0353899 C^2871*C^2903[1]:87569:-0.0353899 C^2871*C^2903[2]:87570:-0.0353899 C^2871*C^2903[3]:87571:-0.0316896 @@ -22475,7 +22475,7 @@ A^175*A^394:87676:-0.160519 A^175*A^394[1]:87677:-0.0362402 A^175*A^394[2]:87678:-0.160519 A^175*A^394[3]:87679:-0.15845 -D^3*D^9095:87704:-0.0439707 +D^3*D^9095:87704:-0.0439706 D^3*D^9095[1]:87705:-0.0439706 D^3*D^9095[2]:87706:-0.0439706 D^3*D^9095[3]:87707:-0.0393897 @@ -22539,7 +22539,7 @@ A^932*A^2543:87852:-0.142801 A^932*A^2543[1]:87853:-0.0802542 A^932*A^2543[2]:87854:-0.142801 A^932*A^2543[3]:87855:-0.076296 -E^510*E^1036:87856:-0.0671643 +E^510*E^1036:87856:-0.0671642 E^510*E^1036[1]:87857:-0.0671642 E^510*E^1036[2]:87858:-0.0671642 E^510*E^1036[3]:87859:-0.0601447 @@ -22607,7 +22607,7 @@ D^1308*D^8318[3]:88075:-0.0435393 A^175*A^757:88080:-0.137354 A^175*A^757[2]:88082:-0.137354 A^175*A^757[3]:88083:-0.164914 -E^23*E^312:88084:-0.0548766 +E^23*E^312:88084:-0.0548765 E^23*E^312[1]:88085:-0.0548765 E^23*E^312[2]:88086:-0.0548765 E^23*E^312[3]:88087:-0.0491432 @@ -22615,7 +22615,7 @@ A^660*A^1664:88096:-0.14217 A^660*A^1664[1]:88097:-0.00638481 A^660*A^1664[2]:88098:-0.14217 A^660*A^1664[3]:88099:-0.0461819 -E^13810*E^18666:88104:-0.00806881 +E^13810*E^18666:88104:-0.0080688 E^13810*E^18666[1]:88105:-0.0080688 E^13810*E^18666[2]:88106:-0.0080688 E^13810*E^18666[3]:88107:-0.0072245 @@ -22754,7 +22754,7 @@ E^760*E^2323[3]:88543:-0.03611 A^175*A^752:88548:-0.0882506 A^175*A^752[2]:88550:-0.0882506 A^175*A^752[3]:88551:-0.105966 -E^2115*E^25081:88568:-0.0212479 +E^2115*E^25081:88568:-0.0212478 E^2115*E^25081[1]:88569:-0.0212478 E^2115*E^25081[2]:88570:-0.0212478 E^2115*E^25081[3]:88571:-0.0190243 @@ -22784,7 +22784,7 @@ E^510*E^836:88656:-0.0297167 E^510*E^836[1]:88657:-0.0297167 E^510*E^836[2]:88658:-0.0297167 E^510*E^836[3]:88659:-0.0266128 -D^389*D^1342:88660:-0.0104768 +D^389*D^1342:88660:-0.0104767 D^389*D^1342[1]:88661:-0.0104767 D^389*D^1342[2]:88662:-0.0104767 D^389*D^1342[3]:88663:-0.00937781 @@ -22880,7 +22880,7 @@ E^516*E^2477[3]:88983:-0.0460159 B^476*B^2036:89008:-0.0186971 B^476*B^2036[1]:89009:-0.0186971 B^476*B^2036[2]:89010:-0.0186971 -B^476*B^2036[3]:89011:-0.0167431 +B^476*B^2036[3]:89011:-0.0167432 E^23*E^464:89012:-0.0910994 E^23*E^464[1]:89013:-0.0910994 E^23*E^464[2]:89014:-0.0910994 @@ -22950,7 +22950,7 @@ E^8256*E^8395:89180:-0.00587049 E^8256*E^8395[1]:89181:-0.00587049 E^8256*E^8395[2]:89182:-0.00587049 E^8256*E^8395[3]:89183:-0.00525685 -E^510*E^702:89192:-0.0095387 +E^510*E^702:89192:-0.00953869 E^510*E^702[1]:89193:-0.00953869 E^510*E^702[2]:89194:-0.00953869 E^510*E^702[3]:89195:-0.00853954 @@ -22958,7 +22958,7 @@ E^516*E^2115:89196:-0.0438752 E^516*E^2115[1]:89197:-0.0438752 E^516*E^2115[2]:89198:-0.0438752 E^516*E^2115[3]:89199:-0.0392899 -E^101*E^1298:89204:-0.037458 +E^101*E^1298:89204:-0.0374579 E^101*E^1298[1]:89205:-0.0374579 E^101*E^1298[2]:89206:-0.0374579 E^101*E^1298[3]:89207:-0.0335318 @@ -23048,7 +23048,7 @@ E^69*E^18666:89428:-0.102361 E^69*E^18666[1]:89429:-0.102361 E^69*E^18666[2]:89430:-0.102361 E^69*E^18666[3]:89431:-0.0916869 -B^476*B^2362:89432:-0.0574724 +B^476*B^2362:89432:-0.0574723 B^476*B^2362[1]:89433:-0.0574723 B^476*B^2362[2]:89434:-0.0574723 B^476*B^2362[3]:89435:-0.0514663 @@ -23101,7 +23101,7 @@ E^23*E^169:89640:-0.0688712 E^23*E^169[1]:89641:-0.0688712 E^23*E^169[2]:89642:-0.0688712 E^23*E^169[3]:89643:-0.0616734 -E^101*E^1416:89644:-0.0500702 +E^101*E^1416:89644:-0.0500701 E^101*E^1416[1]:89645:-0.0500701 E^101*E^1416[2]:89646:-0.0500701 E^101*E^1416[3]:89647:-0.0448283 @@ -23113,7 +23113,7 @@ E^8256*E^8269:89668:-0.00654397 E^8256*E^8269[1]:89669:-0.00654397 E^8256*E^8269[2]:89670:-0.00654397 E^8256*E^8269[3]:89671:-0.00585978 -B^82*B^1340:89672:-0.00669379 +B^82*B^1340:89672:-0.00669378 B^82*B^1340[1]:89673:-0.00669378 B^82*B^1340[2]:89674:-0.00669378 B^82*B^1340[3]:89675:-0.00599379 @@ -23341,7 +23341,7 @@ E^464*E^4599:90668:-0.0597005 E^464*E^4599[1]:90669:-0.0597005 E^464*E^4599[2]:90670:-0.0597005 E^464*E^4599[3]:90671:-0.0534689 -B^400*B^2520:90672:-0.0752238 +B^400*B^2520:90672:-0.0752237 B^400*B^2520[1]:90673:-0.0752237 B^400*B^2520[2]:90674:-0.0752237 B^400*B^2520[3]:90675:-0.0673665 @@ -23360,7 +23360,7 @@ D^3*D^6319[3]:90747:-0.0959327 B^476*B^4525:90772:-0.0652619 B^476*B^4525[1]:90773:-0.0652619 B^476*B^4525[2]:90774:-0.0652619 -B^476*B^4525[3]:90775:-0.0584454 +B^476*B^4525[3]:90775:-0.0584455 E^516*E^1011:90796:-0.0804278 E^516*E^1011[1]:90797:-0.0804278 E^516*E^1011[2]:90798:-0.0804278 @@ -23376,7 +23376,7 @@ D^170*D^4003:90852:-0.0417149 D^170*D^4003[1]:90853:-0.0417149 D^170*D^4003[2]:90854:-0.0417149 D^170*D^4003[3]:90855:-0.0373571 -B^495*B^12340:90900:-0.0565511 +B^495*B^12340:90900:-0.056551 B^495*B^12340[1]:90901:-0.056551 B^495*B^12340[2]:90902:-0.056551 B^495*B^12340[3]:90903:-0.0506433 @@ -23387,7 +23387,7 @@ E^836*E^3027[3]:90927:-0.0238903 E^836*E^3008:90968:-0.0645244 E^836*E^3008[1]:90969:-0.0645244 E^836*E^3008[2]:90970:-0.0645244 -E^836*E^3008[3]:90971:-0.0577738 +E^836*E^3008[3]:90971:-0.0577739 E^760*E^33705:90996:-0.01953 E^760*E^33705[1]:90997:-0.01953 E^760*E^33705[2]:90998:-0.01953 @@ -23608,7 +23608,7 @@ E^225*E^18666:91940:-0.0405696 E^225*E^18666[1]:91941:-0.0405696 E^225*E^18666[2]:91942:-0.0405696 E^225*E^18666[3]:91943:-0.0363282 -D^196:91944:-0.0718905 +D^196:91944:-0.0718904 D^196[1]:91945:-0.0718904 D^196[2]:91946:-0.0718904 D^196[3]:91947:-0.0643807 @@ -23704,7 +23704,7 @@ E^188*E^225:92228:-0.204375 E^188*E^225[1]:92229:-0.204375 E^188*E^225[2]:92230:-0.204375 E^188*E^225[3]:92231:-0.182983 -D^269:92236:-0.00583579 +D^269:92236:-0.00583578 D^269[1]:92237:-0.00583578 D^269[2]:92238:-0.00583578 D^269[3]:92239:-0.00522597 @@ -23860,7 +23860,7 @@ E^101*E^2166:92644:-0.011651 E^101*E^2166[1]:92645:-0.011651 E^101*E^2166[2]:92646:-0.011651 E^101*E^2166[3]:92647:-0.0104342 -B^510*B^5593:92652:-0.00728145 +B^510*B^5593:92652:-0.00728144 B^510*B^5593[1]:92653:-0.00728144 B^510*B^5593[2]:92654:-0.00728144 B^510*B^5593[3]:92655:-0.00652106 @@ -24023,7 +24023,7 @@ E^760*E^1417:93172:-0.0354081 E^760*E^1417[1]:93173:-0.0354081 E^760*E^1417[2]:93174:-0.0354081 E^760*E^1417[3]:93175:-0.0317033 -E^2323*E^8421:93192:-0.0249396 +E^2323*E^8421:93192:-0.0249395 E^2323*E^8421[1]:93193:-0.0249395 E^2323*E^8421[2]:93194:-0.0249395 E^2323*E^8421[3]:93195:-0.0223322 @@ -24055,11 +24055,11 @@ E^789*E^3577:93264:-0.0259677 E^789*E^3577[1]:93265:-0.0259677 E^789*E^3577[2]:93266:-0.0259677 E^789*E^3577[3]:93267:-0.0232594 -E^1196*E^3800:93272:-0.0520742 +E^1196*E^3800:93272:-0.0520741 E^1196*E^3800[1]:93273:-0.0520741 E^1196*E^3800[2]:93274:-0.0520741 E^1196*E^3800[3]:93275:-0.0466246 -C^350*C^584:93280:-0.0395719 +C^350*C^584:93280:-0.0395718 C^350*C^584[1]:93281:-0.0395718 C^350*C^584[2]:93282:-0.0395718 C^350*C^584[3]:93283:-0.0354413 @@ -24206,7 +24206,7 @@ E^2323*E^8287:93728:-0.00853081 E^2323*E^8287[1]:93729:-0.00853081 E^2323*E^8287[2]:93730:-0.00853081 E^2323*E^8287[3]:93731:-0.00763946 -E^101*E^2433:93744:-0.0640386 +E^101*E^2433:93744:-0.0640385 E^101*E^2433[1]:93745:-0.0640385 E^101*E^2433[2]:93746:-0.0640385 E^101*E^2433[3]:93747:-0.0573375 @@ -24373,7 +24373,7 @@ E^188*E^760:94232:-0.153984 E^188*E^760[1]:94233:-0.153984 E^188*E^760[2]:94234:-0.153984 E^188*E^760[3]:94235:-0.137848 -E^10752:94236:-0.000717815 +E^10752:94236:-0.000717814 E^10752[1]:94237:-0.000717814 E^10752[2]:94238:-0.000717814 E^10752[3]:94239:-0.000642798 @@ -24477,7 +24477,7 @@ E^188*E^642:94656:-0.0119267 E^188*E^642[1]:94657:-0.0119267 E^188*E^642[2]:94658:-0.0119267 E^188*E^642[3]:94659:-0.0106815 -E^1890*E^2614:94680:-0.05066 +E^1890*E^2614:94680:-0.0506599 E^1890*E^2614[1]:94681:-0.0506599 E^1890*E^2614[2]:94682:-0.0506599 E^1890*E^2614[3]:94683:-0.0453664 @@ -24485,7 +24485,7 @@ E^269*E^516:94684:-0.120859 E^269*E^516[1]:94685:-0.120859 E^269*E^516[2]:94686:-0.120859 E^269*E^516[3]:94687:-0.10824 -E^836*E^1890:94688:-0.0596745 +E^836*E^1890:94688:-0.0596744 E^836*E^1890[1]:94689:-0.0596744 E^836*E^1890[2]:94690:-0.0596744 E^836*E^1890[3]:94691:-0.0534334 @@ -24636,7 +24636,7 @@ D^1723*D^2234:95308:-0.00819461 D^1723*D^2234[1]:95309:-0.00819461 D^1723*D^2234[2]:95310:-0.00819461 D^1723*D^2234[3]:95311:-0.00733842 -E^1455*E^const:95320:-0.280552 +E^1455*E^const:95320:-0.280551 E^1455*E^const[1]:95321:-0.280551 E^1455*E^const[2]:95322:-0.280551 E^1455*E^const[3]:95323:-0.251191 @@ -24714,11 +24714,11 @@ B^400*B^1796:95616:-0.102257 B^400*B^1796[1]:95617:-0.102257 B^400*B^1796[2]:95618:-0.102257 B^400*B^1796[3]:95619:-0.0915642 -E^2248*E^7351:95628:-0.0392075 +E^2248*E^7351:95628:-0.0392074 E^2248*E^7351[1]:95629:-0.0392074 E^2248*E^7351[2]:95630:-0.0392074 E^2248*E^7351[3]:95631:-0.0351047 -E^73*E^const:95632:-0.623988 +E^73*E^const:95632:-0.623987 E^73*E^const[1]:95633:-0.623987 E^73*E^const[2]:95634:-0.623987 E^73*E^const[3]:95635:-0.55874 @@ -24805,7 +24805,7 @@ E^836*E^1705[3]:95879:-0.0376115 E^269*E^973:95904:-0.08251 E^269*E^973[1]:95905:-0.08251 E^269*E^973[2]:95906:-0.08251 -E^269*E^973[3]:95907:-0.0738841 +E^269*E^973[3]:95907:-0.0738842 E^2024*E^const:95908:-0.241717 E^2024*E^const[1]:95909:-0.241717 E^2024*E^const[2]:95910:-0.241717 @@ -24842,7 +24842,7 @@ D^1213:96012:-0.0739033 D^1213[1]:96013:-0.0739033 D^1213[2]:96014:-0.0739033 D^1213[3]:96015:-0.0661766 -D^1881*D^3899:96048:-0.0386262 +D^1881*D^3899:96048:-0.0386261 D^1881*D^3899[1]:96049:-0.0386261 D^1881*D^3899[2]:96050:-0.0386261 D^1881*D^3899[3]:96051:-0.0345847 @@ -24922,7 +24922,7 @@ B^481*B^2031:96288:-0.0963051 B^481*B^2031[1]:96289:-0.0963051 B^481*B^2031[2]:96290:-0.0963051 B^481*B^2031[3]:96291:-0.0862265 -E^836*E^2323:96300:-0.0483695 +E^836*E^2323:96300:-0.0483694 E^836*E^2323[1]:96301:-0.0483694 E^836*E^2323[2]:96302:-0.0483694 E^836*E^2323[3]:96303:-0.0433211 @@ -24981,7 +24981,7 @@ E^1036*E^4045:96500:-0.0164521 E^1036*E^4045[1]:96501:-0.0164521 E^1036*E^4045[2]:96502:-0.0164521 E^1036*E^4045[3]:96503:-0.0147297 -E^836*E^2380:96520:-0.0454977 +E^836*E^2380:96520:-0.0454976 E^836*E^2380[1]:96521:-0.0454976 E^836*E^2380[2]:96522:-0.0454976 E^836*E^2380[3]:96523:-0.0407433 @@ -25009,7 +25009,7 @@ E^1081*E^1690:96644:-0.0257508 E^1081*E^1690[1]:96645:-0.0257508 E^1081*E^1690[2]:96646:-0.0257508 E^1081*E^1690[3]:96647:-0.0230608 -E^23*E^2380:96676:-0.0410109 +E^23*E^2380:96676:-0.0410108 E^23*E^2380[1]:96677:-0.0410108 E^23*E^2380[2]:96678:-0.0410108 E^23*E^2380[3]:96679:-0.0367273 @@ -25211,7 +25211,7 @@ E^2259*E^18535:97536:-0.0170942 E^2259*E^18535[1]:97537:-0.0170942 E^2259*E^18535[2]:97538:-0.0170942 E^2259*E^18535[3]:97539:-0.0153085 -E^269*E^1335:97544:-0.0990468 +E^269*E^1335:97544:-0.0990467 E^269*E^1335[1]:97545:-0.0990467 E^269*E^1335[2]:97546:-0.0990467 E^269*E^1335[3]:97547:-0.0887059 @@ -25425,7 +25425,7 @@ E^269*E^1417:98224:-0.101979 E^269*E^1417[1]:98225:-0.101979 E^269*E^1417[2]:98226:-0.101979 E^269*E^1417[3]:98227:-0.091323 -E^673*E^2433:98240:-0.0608371 +E^673*E^2433:98240:-0.060837 E^673*E^2433[1]:98241:-0.060837 E^673*E^2433[2]:98242:-0.060837 E^673*E^2433[3]:98243:-0.0544852 @@ -25506,7 +25506,7 @@ D^389*D^36831:98512:-0.0240763 D^389*D^36831[1]:98513:-0.0240763 D^389*D^36831[2]:98514:-0.0240763 D^389*D^36831[3]:98515:-0.0215574 -E^301*E^18656:98540:-0.0121423 +E^301*E^18656:98540:-0.0121422 E^301*E^18656[1]:98541:-0.0121422 E^301*E^18656[2]:98542:-0.0121422 E^301*E^18656[3]:98543:-0.0108717 @@ -25541,7 +25541,7 @@ B^217*B^514[3]:98647:-0.00959945 A^534*A^2536:98664:-0.0118255 A^534*A^2536[2]:98666:-0.0118255 A^534*A^2536[3]:98667:-0.0141999 -B^1100*B^12340:98672:-0.00694525 +B^1100*B^12340:98672:-0.00694524 B^1100*B^12340[1]:98673:-0.00694524 B^1100*B^12340[2]:98674:-0.00694524 B^1100*B^12340[3]:98675:-0.00621983 @@ -25588,7 +25588,7 @@ E^1455*E^8467:98912:-0.0334986 E^1455*E^8467[1]:98913:-0.0334986 E^1455*E^8467[2]:98914:-0.0334986 E^1455*E^8467[3]:98915:-0.029992 -D^76*D^9095:98924:-0.0480051 +D^76*D^9095:98924:-0.048005 D^76*D^9095[1]:98925:-0.048005 D^76*D^9095[2]:98926:-0.048005 D^76*D^9095[3]:98927:-0.0429846 @@ -25648,7 +25648,7 @@ E^342*E^2477:99148:-0.0181244 E^342*E^2477[1]:99149:-0.0181244 E^342*E^2477[2]:99150:-0.0181244 E^342*E^2477[3]:99151:-0.0162279 -B^305*B^3370:99156:-0.00786492 +B^305*B^3370:99156:-0.00786491 B^305*B^3370[1]:99157:-0.00786491 B^305*B^3370[2]:99158:-0.00786491 B^305*B^3370[3]:99159:-0.00704291 @@ -25826,7 +25826,7 @@ E^1130*E^8284:99840:-0.00923791 E^1130*E^8284[1]:99841:-0.00923791 E^1130*E^8284[2]:99842:-0.00923791 E^1130*E^8284[3]:99843:-0.00827208 -E^464*E^18666:99856:-0.0674917 +E^464*E^18666:99856:-0.0674916 E^464*E^18666[1]:99857:-0.0674916 E^464*E^18666[2]:99858:-0.0674916 E^464*E^18666[3]:99859:-0.0604441 @@ -25994,7 +25994,7 @@ E^1479*E^2362:100396:-0.0724631 E^1479*E^2362[1]:100397:-0.0724631 E^1479*E^2362[2]:100398:-0.0724631 E^1479*E^2362[3]:100399:-0.0648881 -E^342*E^1890:100408:-0.0315725 +E^342*E^1890:100408:-0.0315724 E^342*E^1890[1]:100409:-0.0315724 E^342*E^1890[2]:100410:-0.0315724 E^342*E^1890[3]:100411:-0.0282663 @@ -26066,11 +26066,11 @@ E^276*E^24559:100828:-0.00649666 E^276*E^24559[1]:100829:-0.00649666 E^276*E^24559[2]:100830:-0.00649666 E^276*E^24559[3]:100831:-0.00581884 -B^465*B^1712:100844:-0.0331559 +B^465*B^1712:100844:-0.0331558 B^465*B^1712[1]:100845:-0.0331558 B^465*B^1712[2]:100846:-0.0331558 B^465*B^1712[3]:100847:-0.0296954 -D^89*D^31213:100856:-0.033555 +D^89*D^31213:100856:-0.0335549 D^89*D^31213[1]:100857:-0.0335549 D^89*D^31213[2]:100858:-0.0335549 D^89*D^31213[3]:100859:-0.0300504 @@ -26185,7 +26185,7 @@ E^1118*E^4594:101400:-0.0234646 E^1118*E^4594[1]:101401:-0.0234646 E^1118*E^4594[2]:101402:-0.0234646 E^1118*E^4594[3]:101403:-0.0210126 -C^147*C^166:101420:-0.00476971 +C^147*C^166:101420:-0.0047697 C^147*C^166[1]:101421:-0.0047697 C^147*C^166[2]:101422:-0.0047697 C^147*C^166[3]:101423:-0.00427048 @@ -26255,7 +26255,7 @@ E^4045*E^4594:101652:-0.0114807 E^4045*E^4594[1]:101653:-0.0114807 E^4045*E^4594[2]:101654:-0.0114807 E^4045*E^4594[3]:101655:-0.0102776 -E^751*E^2563:101664:-0.0397888 +E^751*E^2563:101664:-0.0397887 E^751*E^2563[1]:101665:-0.0397887 E^751*E^2563[2]:101666:-0.0397887 E^751*E^2563[3]:101667:-0.0356373 @@ -26321,7 +26321,7 @@ E^1417*E^25081:102080:-0.0320063 E^1417*E^25081[1]:102081:-0.0320063 E^1417*E^25081[2]:102082:-0.0320063 E^1417*E^25081[3]:102083:-0.0286613 -E^2538*E^3536:102096:-0.0321191 +E^2538*E^3536:102096:-0.032119 E^2538*E^3536[1]:102097:-0.032119 E^2538*E^3536[2]:102098:-0.032119 E^2538*E^3536[3]:102099:-0.0287567 @@ -26652,7 +26652,7 @@ E^8287*E^8615:103664:-0.00656783 E^8287*E^8615[1]:103665:-0.00656783 E^8287*E^8615[2]:103666:-0.00656783 E^8287*E^8615[3]:103667:-0.00588152 -B^2334*B^2493:103676:-0.0453155 +B^2334*B^2493:103676:-0.0453154 B^2334*B^2493[1]:103677:-0.0453154 B^2334*B^2493[2]:103678:-0.0453154 B^2334*B^2493[3]:103679:-0.0405732 @@ -26691,7 +26691,7 @@ D^1122*D^5845:103820:-0.0248354 D^1122*D^5845[1]:103821:-0.0248354 D^1122*D^5845[2]:103822:-0.0248354 D^1122*D^5845[3]:103823:-0.0222397 -B^1574*B^5593:103884:-0.0336093 +B^1574*B^5593:103884:-0.0336092 B^1574*B^5593[1]:103885:-0.0336092 B^1574*B^5593[2]:103886:-0.0336092 B^1574*B^5593[3]:103887:-0.0300927 @@ -26872,7 +26872,7 @@ B^55*B^254:104812:-0.025061 B^55*B^254[1]:104813:-0.025061 B^55*B^254[2]:104814:-0.025061 B^55*B^254[3]:104815:-0.0224354 -E^6578*E^8287:104820:-0.00637371 +E^6578*E^8287:104820:-0.0063737 E^6578*E^8287[1]:104821:-0.0063737 E^6578*E^8287[2]:104822:-0.0063737 E^6578*E^8287[3]:104823:-0.00570769 @@ -26907,7 +26907,7 @@ E^233*E^6051:104936:-0.0317286 E^233*E^6051[1]:104937:-0.0317286 E^233*E^6051[2]:104938:-0.0317286 E^233*E^6051[3]:104939:-0.0284194 -C^357*C^545:104944:-0.00686825 +C^357*C^545:104944:-0.00686824 C^357*C^545[1]:104945:-0.00686824 C^357*C^545[2]:104946:-0.00686824 C^357*C^545[3]:104947:-0.0061503 @@ -26922,7 +26922,7 @@ E^533*E^const[3]:104963:-0.0292811 A^85*A^465:104984:-0.0456556 A^85*A^465[2]:104986:-0.0456556 A^85*A^465[3]:104987:-0.0548119 -E^1690*E^4594:105000:-0.0244841 +E^1690*E^4594:105000:-0.024484 E^1690*E^4594[1]:105001:-0.024484 E^1690*E^4594[2]:105002:-0.024484 E^1690*E^4594[3]:105003:-0.0219261 @@ -26948,7 +26948,7 @@ B^55*B^233[3]:105091:-0.0305697 A^534*A^3960:105128:-0.0436507 A^534*A^3960[2]:105130:-0.0436507 A^534*A^3960[3]:105131:-0.0524082 -B^133*B^934:105140:-0.0499312 +B^133*B^934:105140:-0.0499311 B^133*B^934[1]:105141:-0.0499311 B^133*B^934[2]:105142:-0.0499311 B^133*B^934[3]:105143:-0.0447028 @@ -27111,7 +27111,7 @@ E^2323*E^const:105960:-0.202501 E^2323*E^const[1]:105961:-0.202501 E^2323*E^const[2]:105962:-0.202501 E^2323*E^const[3]:105963:-0.181323 -E^276*E^25081:105988:-0.0550935 +E^276*E^25081:105988:-0.0550934 E^276*E^25081[1]:105989:-0.0550934 E^276*E^25081[2]:105990:-0.0550934 E^276*E^25081[3]:105991:-0.0493311 @@ -27221,7 +27221,7 @@ E^52*E^18668:106568:-0.0382184 E^52*E^18668[1]:106569:-0.0382184 E^52*E^18668[2]:106570:-0.0382184 E^52*E^18668[3]:106571:-0.0342289 -B^2343*B^3370:106588:-0.0411336 +B^2343*B^3370:106588:-0.0411335 B^2343*B^3370[1]:106589:-0.0411335 B^2343*B^3370[2]:106590:-0.0411335 B^2343*B^3370[3]:106591:-0.0368363 @@ -27351,7 +27351,7 @@ D^334*D^428:107064:-0.0160791 D^334*D^428[1]:107065:-0.0160791 D^334*D^428[2]:107066:-0.0160791 D^334*D^428[3]:107067:-0.0144012 -E^52*E^18535:107068:-0.0457146 +E^52*E^18535:107068:-0.0457145 E^52*E^18535[1]:107069:-0.0457145 E^52*E^18535[2]:107070:-0.0457145 E^52*E^18535[3]:107071:-0.0409386 @@ -27427,7 +27427,7 @@ D^334*D^483:107348:-0.0423742 D^334*D^483[1]:107349:-0.0423742 D^334*D^483[2]:107350:-0.0423742 D^334*D^483[3]:107351:-0.0379362 -D^36832:107416:-0.0309001 +D^36832:107416:-0.0309 D^36832[1]:107417:-0.0309 D^36832[2]:107418:-0.0309 D^36832[3]:107419:-0.0276635 @@ -27447,7 +27447,7 @@ E^182*E^1389:107468:-0.012366 E^182*E^1389[1]:107469:-0.012366 E^182*E^1389[2]:107470:-0.012366 E^182*E^1389[3]:107471:-0.0110734 -E^751*E^1455:107472:-0.0579306 +E^751*E^1455:107472:-0.0579305 E^751*E^1455[1]:107473:-0.0579305 E^751*E^1455[2]:107474:-0.0579305 E^751*E^1455[3]:107475:-0.0518741 @@ -27482,7 +27482,7 @@ E^4045*E^6051:107608:-0.00663399 E^4045*E^6051[1]:107609:-0.00663399 E^4045*E^6051[2]:107610:-0.00663399 E^4045*E^6051[3]:107611:-0.00593912 -E^233*E^5313:107616:-0.0648368 +E^233*E^5313:107616:-0.0648367 E^233*E^5313[1]:107617:-0.0648367 E^233*E^5313[2]:107618:-0.0648367 E^233*E^5313[3]:107619:-0.0580564 @@ -27506,7 +27506,7 @@ E^179*E^478:107692:-0.0598046 E^179*E^478[1]:107693:-0.0598046 E^179*E^478[2]:107694:-0.0598046 E^179*E^478[3]:107695:-0.0535295 -E^2248*E^18666:107696:-0.0329737 +E^2248*E^18666:107696:-0.0329736 E^2248*E^18666[1]:107697:-0.0329736 E^2248*E^18666[2]:107698:-0.0329736 E^2248*E^18666[3]:107699:-0.0295194 @@ -27567,7 +27567,7 @@ A^646*A^932[3]:107835:-0.128285 E^179*E^378:107836:-0.0466516 E^179*E^378[1]:107837:-0.0466516 E^179*E^378[2]:107838:-0.0466516 -E^179*E^378[3]:107839:-0.0417781 +E^179*E^378[3]:107839:-0.0417782 B^49*B^678:107844:-0.0365178 B^49*B^678[1]:107845:-0.0365178 B^49*B^678[2]:107846:-0.0365178 @@ -27716,7 +27716,7 @@ B^65*B^2520:108492:-0.0388431 B^65*B^2520[1]:108493:-0.0388431 B^65*B^2520[2]:108494:-0.0388431 B^65*B^2520[3]:108495:-0.0347733 -E^182*E^1130:108504:-0.0397888 +E^182*E^1130:108504:-0.0397887 E^182*E^1130[1]:108505:-0.0397887 E^182*E^1130[2]:108506:-0.0397887 E^182*E^1130[3]:108507:-0.0356255 @@ -27724,7 +27724,7 @@ E^179*E^301:108520:-0.0231783 E^179*E^301[1]:108521:-0.0231783 E^179*E^301[2]:108522:-0.0231783 E^179*E^301[3]:108523:-0.0207533 -E^233*E^5158:108532:-0.0890519 +E^233*E^5158:108532:-0.0890518 E^233*E^5158[1]:108533:-0.0890518 E^233*E^5158[2]:108534:-0.0890518 E^233*E^5158[3]:108535:-0.0797381 @@ -27736,7 +27736,7 @@ E^312*E^5158:108544:-0.0530025 E^312*E^5158[1]:108545:-0.0530025 E^312*E^5158[2]:108546:-0.0530025 E^312*E^5158[3]:108547:-0.0474685 -E^751*E^836:108580:-0.0593101 +E^751*E^836:108580:-0.05931 E^751*E^836[1]:108581:-0.05931 E^751*E^836[2]:108582:-0.05931 E^751*E^836[3]:108583:-0.0531035 @@ -27788,7 +27788,7 @@ E^69*E^5597:108800:-0.0568374 E^69*E^5597[1]:108801:-0.0568374 E^69*E^5597[2]:108802:-0.0568374 E^69*E^5597[3]:108803:-0.0509036 -E^751*E^768:108820:-0.0640386 +E^751*E^768:108820:-0.0640385 E^751*E^768[1]:108821:-0.0640385 E^751*E^768[2]:108822:-0.0640385 E^751*E^768[3]:108823:-0.0573375 @@ -27799,7 +27799,7 @@ B^50*B^1321:108828:-0.0138948 B^50*B^1321[1]:108829:-0.0138948 B^50*B^1321[2]:108830:-0.0138948 B^50*B^1321[3]:108831:-0.0124426 -E^800*E^5158:108832:-0.03615 +E^800*E^5158:108832:-0.0361499 E^800*E^5158[1]:108833:-0.0361499 E^800*E^5158[2]:108834:-0.0361499 E^800*E^5158[3]:108835:-0.0323694 @@ -27851,7 +27851,7 @@ B^768*B^12340:109056:-0.0492718 B^768*B^12340[1]:109057:-0.0492718 B^768*B^12340[2]:109058:-0.0492718 B^768*B^12340[3]:109059:-0.044117 -B^768*B^12338:109064:-0.00955243 +B^768*B^12338:109064:-0.00955242 B^768*B^12338[1]:109065:-0.00955242 B^768*B^12338[2]:109066:-0.00955242 B^768*B^12338[3]:109067:-0.00855335 @@ -27867,7 +27867,7 @@ E^225*E^6051:109128:-0.0188532 E^225*E^6051[1]:109129:-0.0188532 E^225*E^6051[2]:109130:-0.0188532 E^225*E^6051[3]:109131:-0.0168785 -B^9:109132:-0.0069068 +B^9:109132:-0.00690679 B^9[1]:109133:-0.00690679 B^9[2]:109134:-0.00690679 B^9[3]:109135:-0.00618539 @@ -28108,7 +28108,7 @@ E^182*E^557:110284:-0.0349214 E^182*E^557[1]:110285:-0.0349214 E^182*E^557[2]:110286:-0.0349214 E^182*E^557[3]:110287:-0.0312628 -D^336*D^1488:110288:-0.051276 +D^336*D^1488:110288:-0.0512759 D^336*D^1488[1]:110289:-0.0512759 D^336*D^1488[2]:110290:-0.0512759 D^336*D^1488[3]:110291:-0.0459204 @@ -28139,7 +28139,7 @@ B^326:110400:-0.0333684 B^326[1]:110401:-0.0333684 B^326[2]:110402:-0.0333684 B^326[3]:110403:-0.0298685 -E^2028*E^8273:110404:-0.00977584 +E^2028*E^8273:110404:-0.00977583 E^2028*E^8273[1]:110405:-0.00977583 E^2028*E^8273[2]:110406:-0.00977583 E^2028*E^8273[3]:110407:-0.0087527 @@ -28165,7 +28165,7 @@ B^2343*B^2495:110504:-0.007395 B^2343*B^2495[1]:110505:-0.007395 B^2343*B^2495[2]:110506:-0.007395 B^2343*B^2495[3]:110507:-0.00662181 -D^678*D^const:110524:-0.00670015 +D^678*D^const:110524:-0.00670014 D^678*D^const[1]:110525:-0.00670014 D^678*D^const[2]:110526:-0.00670014 D^678*D^const[3]:110527:-0.00599913 @@ -28244,7 +28244,7 @@ E^233*E^4599:110776:-0.0604119 E^233*E^4599[1]:110777:-0.0604119 E^233*E^4599[2]:110778:-0.0604119 E^233*E^4599[3]:110779:-0.0541065 -E^161*E^8272:110780:-0.0487686 +E^161*E^8272:110780:-0.0487685 E^161*E^8272[1]:110781:-0.0487685 E^161*E^8272[2]:110782:-0.0487685 E^161*E^8272[3]:110783:-0.0436651 @@ -28371,7 +28371,7 @@ E^179*E^1133:111336:-0.0457154 E^179*E^1133[1]:111337:-0.0457154 E^179*E^1133[2]:111338:-0.0457154 E^179*E^1133[3]:111339:-0.04093 -C^368*C^5651:111356:-0.0477882 +C^368*C^5651:111356:-0.0477881 C^368*C^5651[1]:111357:-0.0477881 C^368*C^5651[2]:111358:-0.0477881 C^368*C^5651[3]:111359:-0.0428004 @@ -28567,7 +28567,7 @@ E^69*E^3800:112108:-0.0940658 E^69*E^3800[1]:112109:-0.0940658 E^69*E^3800[2]:112110:-0.0940658 E^69*E^3800[3]:112111:-0.0842429 -E^179*E^1350:112140:-0.0453155 +E^179*E^1350:112140:-0.0453154 E^179*E^1350[1]:112141:-0.0453154 E^179*E^1350[2]:112142:-0.0453154 E^179*E^1350[3]:112143:-0.0405732 @@ -28752,7 +28752,7 @@ D^757*D^18778:112900:-0.0346915 D^757*D^18778[1]:112901:-0.0346915 D^757*D^18778[2]:112902:-0.0346915 D^757*D^18778[3]:112903:-0.0310727 -D^4206*D^5845:112924:-0.0281194 +D^4206*D^5845:112924:-0.0281193 D^4206*D^5845[1]:112925:-0.0281193 D^4206*D^5845[2]:112926:-0.0281193 D^4206*D^5845[3]:112927:-0.0251804 @@ -29201,7 +29201,7 @@ D^476*D^2234:114712:-0.0166061 D^476*D^2234[1]:114713:-0.0166061 D^476*D^2234[2]:114714:-0.0166061 D^476*D^2234[3]:114715:-0.0148684 -E^73*E^4594:114724:-0.0640386 +E^73*E^4594:114724:-0.0640385 E^73*E^4594[1]:114725:-0.0640385 E^73*E^4594[2]:114726:-0.0640385 E^73*E^4594[3]:114727:-0.0573581 @@ -29298,7 +29298,7 @@ E^800*E^10752:115080:-0.0111749 E^800*E^10752[1]:115081:-0.0111749 E^800*E^10752[2]:115082:-0.0111749 E^800*E^10752[3]:115083:-0.0100072 -D^85*D^288:115100:-0.0392075 +D^85*D^288:115100:-0.0392074 D^85*D^288[1]:115101:-0.0392074 D^85*D^288[2]:115102:-0.0392074 D^85*D^288[3]:115103:-0.0351047 @@ -29330,7 +29330,7 @@ B^1574*B^11017:115212:-0.054946 B^1574*B^11017[1]:115213:-0.054946 B^1574*B^11017[2]:115214:-0.054946 B^1574*B^11017[3]:115215:-0.0492024 -E^1389*E^2323:115224:-0.0420793 +E^1389*E^2323:115224:-0.0420792 E^1389*E^2323[1]:115225:-0.0420792 E^1389*E^2323[2]:115226:-0.0420792 E^1389*E^2323[3]:115227:-0.0376869 @@ -29391,7 +29391,7 @@ D^354*D^18535[3]:115543:-0.0774435 E^1305*E^1455:115544:-0.0659994 E^1305*E^1455[1]:115545:-0.0659994 E^1305*E^1455[2]:115546:-0.0659994 -E^1305*E^1455[3]:115547:-0.0590845 +E^1305*E^1455[3]:115547:-0.0590846 A^234*A^433:115548:-0.106129 A^234*A^433[2]:115550:-0.106129 A^234*A^433[3]:115551:-0.127425 @@ -29798,7 +29798,7 @@ C^368*C^3006:117152:-0.0395892 C^368*C^3006[1]:117153:-0.0395892 C^368*C^3006[2]:117154:-0.0395892 C^368*C^3006[3]:117155:-0.0354472 -B^130*B^1796:117160:-0.0610193 +B^130*B^1796:117160:-0.0610192 B^130*B^1796[1]:117161:-0.0610192 B^130*B^1796[2]:117162:-0.0610192 B^130*B^1796[3]:117163:-0.0546405 @@ -29837,7 +29837,7 @@ D^6047*D^18618[3]:117247:-0.0122769 A^236*A^1664:117248:-0.218767 A^236*A^1664[2]:117250:-0.218767 A^236*A^1664[3]:117251:-0.26272 -B^2039:117252:-0.03539 +B^2039:117252:-0.0353899 B^2039[1]:117253:-0.0353899 B^2039[2]:117254:-0.0353899 B^2039[3]:117255:-0.0316874 @@ -29876,7 +29876,7 @@ E^1062*E^2115:117364:-0.0383225 E^1062*E^2115[1]:117365:-0.0383225 E^1062*E^2115[2]:117366:-0.0383225 E^1062*E^2115[3]:117367:-0.0343192 -E^225*E^8081:117376:-0.0420793 +E^225*E^8081:117376:-0.0420792 E^225*E^8081[1]:117377:-0.0420792 E^225*E^8081[2]:117378:-0.0420792 E^225*E^8081[3]:117379:-0.0376766 @@ -29971,7 +29971,7 @@ D^84*D^319:117676:-0.00323709 D^84*D^319[1]:117677:-0.00323709 D^84*D^319[2]:117678:-0.00323709 D^84*D^319[3]:117679:-0.0028978 -E^655*E^2248:117684:-0.0594923 +E^655*E^2248:117684:-0.0594922 E^655*E^2248[1]:117685:-0.0594922 E^655*E^2248[2]:117686:-0.0594922 E^655*E^2248[3]:117687:-0.0532677 @@ -30081,7 +30081,7 @@ E^1389*E^1690:118004:-0.032326 E^1389*E^1690[1]:118005:-0.032326 E^1389*E^1690[2]:118006:-0.032326 E^1389*E^1690[3]:118007:-0.028956 -E^1059*E^1305:118008:-0.0705804 +E^1059*E^1305:118008:-0.0705803 E^1059*E^1305[1]:118009:-0.0705803 E^1059*E^1305[2]:118010:-0.0705803 E^1059*E^1305[3]:118011:-0.0632208 @@ -30242,7 +30242,7 @@ D^8277*D^8318[3]:118551:-0.0185539 A^234*A^660:118568:-0.163213 A^234*A^660[2]:118570:-0.163213 A^234*A^660[3]:118571:-0.195973 -D^76*D^5845:118580:-0.0451333 +D^76*D^5845:118580:-0.0451332 D^76*D^5845[1]:118581:-0.0451332 D^76*D^5845[2]:118582:-0.0451332 D^76*D^5845[3]:118583:-0.0404201 @@ -30328,7 +30328,7 @@ E^73*E^5597[3]:118867:-0.0396079 A^140*A^2536:118880:-0.0344305 A^140*A^2536[2]:118882:-0.0344305 A^140*A^2536[3]:118883:-0.0413392 -E^1221*E^1335:118888:-0.0501481 +E^1221*E^1335:118888:-0.050148 E^1221*E^1335[1]:118889:-0.050148 E^1221*E^1335[2]:118890:-0.050148 E^1221*E^1335[3]:118891:-0.0449092 @@ -30548,7 +30548,7 @@ D^476*D^3013[3]:119863:-0.0222956 A^238*A^812:119864:-0.0527644 A^238*A^812[2]:119866:-0.0527644 A^238*A^812[3]:119867:-0.0633664 -E^1062*E^3008:119872:-0.0558088 +E^1062*E^3008:119872:-0.0558087 E^1062*E^3008[1]:119873:-0.0558087 E^1062*E^3008[2]:119874:-0.0558087 E^1062*E^3008[3]:119875:-0.0499736 @@ -30635,14 +30635,14 @@ E^3236*E^8272[3]:120219:-0.0112885 A^238*A^709:120220:-0.132066 A^238*A^709[2]:120222:-0.132066 A^238*A^709[3]:120223:-0.158541 -E^3236*E^8273:120228:-0.0068227 +E^3236*E^8273:120228:-0.00682269 E^3236*E^8273[1]:120229:-0.00682269 E^3236*E^8273[2]:120230:-0.00682269 E^3236*E^8273[3]:120231:-0.00610906 A^236*A^468:120240:-0.165925 A^236*A^468[2]:120242:-0.165925 A^236*A^468[3]:120243:-0.199188 -E^3236*E^8256:120280:-0.00503867 +E^3236*E^8256:120280:-0.00503866 E^3236*E^8256[1]:120281:-0.00503866 E^3236*E^8256[2]:120282:-0.00503866 E^3236*E^8256[3]:120283:-0.00451226 @@ -30982,7 +30982,7 @@ D^69*D^3380:122028:-0.0146648 D^69*D^3380[1]:122029:-0.0146648 D^69*D^3380[2]:122030:-0.0146648 D^69*D^3380[3]:122031:-0.0131309 -E^169*E^2614:122036:-0.0810525 +E^169*E^2614:122036:-0.0810524 E^169*E^2614[1]:122037:-0.0810524 E^169*E^2614[2]:122038:-0.0810524 E^169*E^2614[3]:122039:-0.0725734 @@ -31209,11 +31209,11 @@ A^229*A^534[3]:123063:-0.108876 A^230*A^932:123064:-0.102391 A^230*A^932[2]:123066:-0.102391 A^230*A^932[3]:123067:-0.122937 -B^678*B^12341:123068:-0.0320769 +B^678*B^12341:123068:-0.0320768 B^678*B^12341[1]:123069:-0.0320768 B^678*B^12341[2]:123070:-0.0320768 B^678*B^12341[3]:123071:-0.0287233 -C^1183*C^1183:123088:-0.0245752 +C^1183*C^1183:123088:-0.0245751 C^1183*C^1183[1]:123089:-0.0245751 C^1183*C^1183[2]:123090:-0.0245751 C^1183*C^1183[3]:123091:-0.0220056 @@ -31255,7 +31255,7 @@ D^161*D^170[3]:123223:-0.0639931 A^229*A^465:123224:-0.0377235 A^229*A^465[2]:123226:-0.0377235 A^229*A^465[3]:123227:-0.0452875 -A^140:123240:-0.0512839 +A^140:123240:-0.0512838 A^140[1]:123241:-0.0315117 A^140[2]:123242:-0.0512838 A^140[3]:123243:-0.024029 @@ -32048,7 +32048,7 @@ E^1380*E^5313[3]:126695:-0.0335684 A^468*A^2543:126700:-0.0496846 A^468*A^2543[2]:126702:-0.0496846 A^468*A^2543[3]:126703:-0.0596674 -E^1298*E^5597:126716:-0.00824667 +E^1298*E^5597:126716:-0.00824666 E^1298*E^5597[1]:126717:-0.00824666 E^1298*E^5597[2]:126718:-0.00824666 E^1298*E^5597[3]:126719:-0.00738261 @@ -32137,7 +32137,7 @@ E^802*E^8615:127124:-0.0361187 E^802*E^8615[1]:127125:-0.0361187 E^802*E^8615[2]:127126:-0.0361187 E^802*E^8615[3]:127127:-0.0323442 -E^1389*E^1454:127140:-0.0764021 +E^1389*E^1454:127140:-0.076402 E^1389*E^1454[1]:127141:-0.076402 E^1389*E^1454[2]:127142:-0.076402 E^1389*E^1454[3]:127143:-0.0684017 @@ -32240,11 +32240,11 @@ E^1380*E^3845:127476:-0.0557008 E^1380*E^3845[1]:127477:-0.0557008 E^1380*E^3845[2]:127478:-0.0557008 E^1380*E^3845[3]:127479:-0.0498755 -D^161*D^1173:127480:-0.0693831 +D^161*D^1173:127480:-0.069383 D^161*D^1173[1]:127481:-0.069383 D^161*D^1173[2]:127482:-0.069383 D^161*D^1173[3]:127483:-0.0621144 -E^169*E^1417:127488:-0.0860673 +E^169*E^1417:127488:-0.0860672 E^169*E^1417[1]:127489:-0.0860672 E^169*E^1417[2]:127490:-0.0860672 E^169*E^1417[3]:127491:-0.0770781 @@ -32446,7 +32446,7 @@ B^465*B^995:128176:-0.0282842 B^465*B^995[1]:128177:-0.0282842 B^465*B^995[2]:128178:-0.0282842 B^465*B^995[3]:128179:-0.0253275 -E^45*E^18666:128180:-0.0734869 +E^45*E^18666:128180:-0.0734868 E^45*E^18666[1]:128181:-0.0734868 E^45*E^18666[2]:128182:-0.0734868 E^45*E^18666[3]:128183:-0.0658128 @@ -32478,7 +32478,7 @@ B^465*B^1011:128240:-0.0588665 B^465*B^1011[1]:128241:-0.0588665 B^465*B^1011[2]:128242:-0.0588665 B^465*B^1011[3]:128243:-0.0527143 -E^169*E^1062:128244:-0.0891213 +E^169*E^1062:128244:-0.0891212 E^169*E^1062[1]:128245:-0.0891212 E^169*E^1062[2]:128246:-0.0891212 E^169*E^1062[3]:128247:-0.0797869 @@ -32641,7 +32641,7 @@ E^973*E^3845:128704:-0.0377238 E^973*E^3845[1]:128705:-0.0377238 E^973*E^3845[2]:128706:-0.0377238 E^973*E^3845[3]:128707:-0.0337866 -D^973*D^1173:128712:-0.0678561 +D^973*D^1173:128712:-0.067856 D^973*D^1173[1]:128713:-0.067856 D^973*D^1173[2]:128714:-0.067856 D^973*D^1173[3]:128715:-0.0607621 @@ -32691,7 +32691,7 @@ E^655*E^1011:128864:-0.0725672 E^655*E^1011[1]:128865:-0.0725672 E^655*E^1011[2]:128866:-0.0725672 E^655*E^1011[3]:128867:-0.064974 -E^169*E^1221:128880:-0.0410588 +E^169*E^1221:128880:-0.0410587 E^169*E^1221[1]:128881:-0.0410587 E^169*E^1221[2]:128882:-0.0410587 E^169*E^1221[3]:128883:-0.0367663 @@ -32736,7 +32736,7 @@ C^1910*C^3845:129036:-0.00788335 C^1910*C^3845[1]:129037:-0.00788335 C^1910*C^3845[2]:129038:-0.00788335 C^1910*C^3845[3]:129039:-0.00705919 -E^800*E^8284:129048:-0.00690187 +E^800*E^8284:129048:-0.00690186 E^800*E^8284[1]:129049:-0.00690186 E^800*E^8284[2]:129050:-0.00690186 E^800*E^8284[3]:129051:-0.00618118 @@ -33034,7 +33034,7 @@ B^3370*B^9217:129884:-0.0273992 B^3370*B^9217[1]:129885:-0.0273992 B^3370*B^9217[2]:129886:-0.0273992 B^3370*B^9217[3]:129887:-0.0245413 -E^1865*E^5158:129908:-0.0251218 +E^1865*E^5158:129908:-0.0251217 E^1865*E^5158[1]:129909:-0.0251217 E^1865*E^5158[2]:129910:-0.0251217 E^1865*E^5158[3]:129911:-0.0224916 @@ -33129,11 +33129,11 @@ E^655*E^1335:130160:-0.0579132 E^655*E^1335[1]:130161:-0.0579132 E^655*E^1335[2]:130162:-0.0579132 E^655*E^1335[3]:130163:-0.0518623 -E^8256*E^18656:130184:-0.00545404 +E^8256*E^18656:130184:-0.00545403 E^8256*E^18656[1]:130185:-0.00545403 E^8256*E^18656[2]:130186:-0.00545403 E^8256*E^18656[3]:130187:-0.0048847 -E^169*E^557:130192:-0.0730531 +E^169*E^557:130192:-0.073053 E^169*E^557[1]:130193:-0.073053 E^169*E^557[2]:130194:-0.073053 E^169*E^557[3]:130195:-0.065431 @@ -33348,7 +33348,7 @@ E^1263*E^18656:130708:-0.0185561 E^1263*E^18656[1]:130709:-0.0185561 E^1263*E^18656[2]:130710:-0.0185561 E^1263*E^18656[3]:130711:-0.0166193 -B^55*B^2031:130728:-0.0357891 +B^55*B^2031:130728:-0.035789 B^55*B^2031[1]:130729:-0.035789 B^55*B^2031[2]:130730:-0.035789 B^55*B^2031[3]:130731:-0.032055 @@ -33475,7 +33475,7 @@ E^1062*E^1305:131036:-0.0630668 E^1062*E^1305[1]:131037:-0.0630668 E^1062*E^1305[2]:131038:-0.0630668 E^1062*E^1305[3]:131039:-0.0564617 -C^368:131052:-0.0160227 +C^368:131052:-0.0160226 C^368[1]:131053:-0.0160226 C^368[2]:131054:-0.0160226 C^368[3]:131055:-0.0143484 diff --git a/test/pred-sets/ref/cbe_adf_cover_dr256.predict b/test/pred-sets/ref/cbe_adf_cover_dr256.predict index 4d13ee21c34..087588b0848 100644 --- a/test/pred-sets/ref/cbe_adf_cover_dr256.predict +++ b/test/pred-sets/ref/cbe_adf_cover_dr256.predict @@ -42,11 +42,11 @@ 1:0.666667,0:0.333333 -2:0.753817,0:0.123091,1:0.123091 +2:0.605121,1:0.271788,0:0.123091 1:0.666667,0:0.333333 -1:0.441074,2:0.441074,0:0.117851 +2:0.764298,0:0.117851,1:0.117851 1:0.666667,0:0.333333 diff --git a/test/pred-sets/ref/ccb_implicit_and_explicit_interactions.inv b/test/pred-sets/ref/ccb_implicit_and_explicit_interactions.inv index 6aea8557e65..49137ec8a67 100644 --- a/test/pred-sets/ref/ccb_implicit_and_explicit_interactions.inv +++ b/test/pred-sets/ref/ccb_implicit_and_explicit_interactions.inv @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:-1 Max label:0 diff --git a/test/pred-sets/ref/ccb_implicit_explicit_ignore_interactions.inv b/test/pred-sets/ref/ccb_implicit_explicit_ignore_interactions.inv index a36955f403c..c5325193537 100644 --- a/test/pred-sets/ref/ccb_implicit_explicit_ignore_interactions.inv +++ b/test/pred-sets/ref/ccb_implicit_explicit_ignore_interactions.inv @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:-1 Max label:0 diff --git a/test/pred-sets/ref/ccb_lots_of_interactions.inv b/test/pred-sets/ref/ccb_lots_of_interactions.inv index eae3fbe3f5d..7f368491380 100644 --- a/test/pred-sets/ref/ccb_lots_of_interactions.inv +++ b/test/pred-sets/ref/ccb_lots_of_interactions.inv @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:-1 Max label:0 diff --git a/test/pred-sets/ref/ccb_quad.inv b/test/pred-sets/ref/ccb_quad.inv index c2694ec6c85..3b9df4b4e2b 100644 --- a/test/pred-sets/ref/ccb_quad.inv +++ b/test/pred-sets/ref/ccb_quad.inv @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:-1 Max label:0 diff --git a/test/pred-sets/ref/ccb_quad_save_resume.inv b/test/pred-sets/ref/ccb_quad_save_resume.inv index a91272dcd18..da376108bdc 100644 --- a/test/pred-sets/ref/ccb_quad_save_resume.inv +++ b/test/pred-sets/ref/ccb_quad_save_resume.inv @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:-1 Max label:0 diff --git a/test/pred-sets/ref/ep_dec_spinoff.inv b/test/pred-sets/ref/ep_dec_spinoff.inv index 385928716d5..afc57407f38 100644 --- a/test/pred-sets/ref/ep_dec_spinoff.inv +++ b/test/pred-sets/ref/ep_dec_spinoff.inv @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:-1 Max label:0 diff --git a/test/pred-sets/ref/inv_hash.cmp b/test/pred-sets/ref/inv_hash.cmp index f4937a1da27..ef88af8b55c 100644 --- a/test/pred-sets/ref/inv_hash.cmp +++ b/test/pred-sets/ref/inv_hash.cmp @@ -8,17 +8,17 @@ lda:0 options: --oaa 3 --quadratic :: Checksum: 1543578543 :0 -f^y*f^c:132:-0.0503654 -f^y*f^c[1]:133:0.0655919 +f^y*f^c:132:-0.05178 +f^y*f^c[1]:133:0.0666978 f^y*f^c[2]:134:-0.0666978 e^y:27664:-0.0331786 e^y[1]:27665:-0.0331786 e^y[2]:27666:0.0331786 -e^x*e^x:28736:-0.0611589 -e^x*e^x[1]:28737:0.019265 +e^x*e^x:28736:-0.0619448 +e^x*e^x[1]:28737:0.0201492 e^x*e^x[2]:28738:-0.0201492 -f^a:57420:-0.0611589 -f^a[1]:57421:0.019265 +f^a:57420:-0.0619448 +f^a[1]:57421:0.0201492 f^a[2]:57422:-0.0201492 f^b:62864:-0.0331786 f^b[1]:62865:-0.0331786 @@ -26,8 +26,8 @@ f^b[2]:62866:0.0331786 e^z*f^c:71484:-0.0331786 e^z*f^c[1]:71485:-0.0331786 e^z*f^c[2]:71486:0.0331786 -f^y*f^y:74344:-0.0503654 -f^y*f^y[1]:74345:0.0655919 +f^y*f^y:74344:-0.05178 +f^y*f^y[1]:74345:0.0666978 f^y*f^y[2]:74346:-0.0666978 e^y*f^c:91056:-0.0331786 e^y*f^c[1]:91057:-0.0331786 @@ -35,8 +35,8 @@ e^y*f^c[2]:91058:0.0331786 e^x*f^b:93488:-0.0331786 e^x*f^b[1]:93489:-0.0331786 e^x*f^b[2]:93490:0.0331786 -e^x*f^a:96492:-0.0611589 -e^x*f^a[1]:96493:0.019265 +e^x*f^a:96492:-0.0619448 +e^x*f^a[1]:96493:0.0201492 e^x*f^a[2]:96494:-0.0201492 e^z*e^z:105832:-0.0331786 e^z*e^z[1]:105833:-0.0331786 @@ -44,8 +44,8 @@ e^z*e^z[2]:105834:0.0331786 f^a*f^b:123444:-0.0331786 f^a*f^b[1]:123445:-0.0331786 f^a*f^b[2]:123446:0.0331786 -e^x:125152:-0.0611589 -e^x[1]:125153:0.019265 +e^x:125152:-0.0619448 +e^x[1]:125153:0.0201492 e^x[2]:125154:-0.0201492 e^y*e^z:125412:-0.0331786 e^y*e^z[1]:125413:-0.0331786 @@ -56,17 +56,17 @@ f^b*f^c[2]:126770:0.0331786 e^x*e^y:128176:-0.0331786 e^x*e^y[1]:128177:-0.0331786 e^x*e^y[2]:128178:0.0331786 -f^a*f^a:129000:-0.0611589 -f^a*f^a[1]:129001:0.019265 +f^a*f^a:129000:-0.0619448 +f^a*f^a[1]:129001:0.0201492 f^a*f^a[2]:129002:-0.0201492 -f^y:154732:-0.0503654 -f^y[1]:154733:0.0655919 +f^y:154732:-0.05178 +f^y[1]:154733:0.0666978 f^y[2]:154734:-0.0666978 f^b*f^b:156704:-0.0331786 f^b*f^b[1]:156705:-0.0331786 f^b*f^b[2]:156706:0.0331786 -f^a*f^c:157988:-0.0611589 -f^a*f^c[1]:157989:0.019265 +f^a*f^c:157988:-0.0619448 +f^a*f^c[1]:157989:0.0201492 f^a*f^c[2]:157990:-0.0201492 e^x*e^z:158836:-0.0331786 e^x*e^z[1]:158837:-0.0331786 @@ -80,8 +80,8 @@ e^z*f^a[2]:166386:0.0331786 e^z*f^b:171052:-0.0331786 e^z*f^b[1]:171053:-0.0331786 e^z*f^b[2]:171054:0.0331786 -e^x*f^c:189984:-0.0611589 -e^x*f^c[1]:189985:0.019265 +e^x*f^c:189984:-0.0619448 +e^x*f^c[1]:189985:0.0201492 e^x*f^c[2]:189986:-0.0201492 e^y*f^b:190624:-0.0331786 e^y*f^b[1]:190625:-0.0331786 @@ -89,20 +89,20 @@ e^y*f^b[2]:190626:0.0331786 e^y*f^a:195964:-0.0331786 e^y*f^a[1]:195965:-0.0331786 e^y*f^a[2]:195966:0.0331786 -Constant:202096:-0.0611589 -Constant[1]:202097:0.019265 +Constant:202096:-0.0619448 +Constant[1]:202097:0.0201492 Constant[2]:202098:-0.0201492 -f^a*f^y:216008:-0.0503654 -f^a*f^y[1]:216009:0.0655919 +f^a*f^y:216008:-0.05178 +f^a*f^y[1]:216009:0.0666978 f^a*f^y[2]:216010:-0.0666978 -f^c*f^c:220416:-0.0611589 -f^c*f^c[1]:220417:0.019265 +f^c*f^c:220416:-0.0619448 +f^c*f^c[1]:220417:0.0201492 f^c*f^c[2]:220418:-0.0201492 -f^c:228992:-0.0611589 -f^c[1]:228993:0.019265 +f^c:228992:-0.0619448 +f^c[1]:228993:0.0201492 f^c[2]:228994:-0.0201492 -e^x*f^y:246988:-0.0503654 -e^x*f^y[1]:246989:0.0655919 +e^x*f^y:246988:-0.05178 +e^x*f^y[1]:246989:0.0666978 e^x*f^y[2]:246990:-0.0666978 e^z:259284:-0.0331786 e^z[1]:259285:-0.0331786 diff --git a/test/pred-sets/ref/lrqfa_nn_preds.predict b/test/pred-sets/ref/lrqfa_nn_preds.predict index 8ee2db52981..63b24dbba40 100644 --- a/test/pred-sets/ref/lrqfa_nn_preds.predict +++ b/test/pred-sets/ref/lrqfa_nn_preds.predict @@ -1,4 +1,4 @@ -0.655581 A -0.424233 B -0.639708 C -0.481847 D +0.579906 A +0.305311 B +0.574215 C +0.339098 D diff --git a/test/pred-sets/ref/lrqfa_nn_preds_rev_order.predict b/test/pred-sets/ref/lrqfa_nn_preds_rev_order.predict index 8317a880572..91491e9edd7 100644 --- a/test/pred-sets/ref/lrqfa_nn_preds_rev_order.predict +++ b/test/pred-sets/ref/lrqfa_nn_preds_rev_order.predict @@ -1,4 +1,4 @@ -0.481847 D -0.639708 C -0.424233 B -0.655581 A +0.339098 D +0.574215 C +0.305311 B +0.579906 A diff --git a/test/pred-sets/ref/oaa_probabilities.predict b/test/pred-sets/ref/oaa_probabilities.predict index 9fd30b4f47e..45a2624af1f 100644 --- a/test/pred-sets/ref/oaa_probabilities.predict +++ b/test/pred-sets/ref/oaa_probabilities.predict @@ -1,200 +1,200 @@ -1:0.250000 2:0.250000 3:0.250000 4:0.250000 +1:0.25 2:0.25 3:0.25 4:0.25 1:0.313018 2:0.228994 3:0.228994 4:0.228994 -1:0.291035 2:0.295440 3:0.206555 4:0.206970 -1:0.253242 2:0.265272 3:0.263602 4:0.217884 -1:0.227201 2:0.341337 3:0.242245 4:0.189217 -1:0.233697 2:0.425845 3:0.172865 4:0.167593 -1:0.231398 2:0.332778 3:0.244011 4:0.191813 -1:0.150265 2:0.546890 3:0.191829 4:0.111015 -1:0.249041 2:0.304864 3:0.246092 4:0.200002 -1:0.236321 2:0.300744 3:0.299171 4:0.163763 -1:0.170244 2:0.478986 3:0.211653 4:0.139117 -1:0.218702 2:0.419013 3:0.227327 4:0.134957 -1:0.212317 2:0.400260 3:0.269698 4:0.117725 -1:0.144234 2:0.508646 3:0.259484 4:0.087636 -1:0.227397 2:0.313344 3:0.282766 4:0.176493 -1:0.237324 2:0.377140 3:0.232437 4:0.153099 -1:0.168455 2:0.448803 3:0.282184 4:0.100558 -1:0.177929 2:0.441483 3:0.263236 4:0.117353 -1:0.186414 2:0.489048 3:0.270411 4:0.054127 -1:0.209226 2:0.474960 3:0.217122 4:0.098691 -1:0.209805 2:0.340413 3:0.284011 4:0.165771 -1:0.177182 2:0.468967 3:0.290710 4:0.063141 -1:0.157003 2:0.583469 3:0.201546 4:0.057982 -1:0.260926 2:0.265473 3:0.299581 4:0.174019 -1:0.231879 2:0.315958 3:0.268965 4:0.183198 -1:0.167826 2:0.512759 3:0.255346 4:0.064069 -1:0.208737 2:0.454268 3:0.237648 4:0.099346 -1:0.114780 2:0.717824 3:0.140690 4:0.026705 -1:0.261489 2:0.399053 3:0.232460 4:0.106998 -1:0.229501 2:0.370548 3:0.295630 4:0.104321 -1:0.148459 2:0.617557 3:0.204870 4:0.029113 -1:0.240552 2:0.353090 3:0.277184 4:0.129173 -1:0.232986 2:0.345071 3:0.304794 4:0.117149 -1:0.139842 2:0.498307 3:0.318343 4:0.043507 -1:0.254853 2:0.387941 3:0.292233 4:0.064972 -1:0.200911 2:0.587753 3:0.167113 4:0.044224 -1:0.166091 2:0.344739 3:0.446369 4:0.042801 -1:0.183872 2:0.594100 3:0.185264 4:0.036764 -1:0.286486 2:0.327117 3:0.339359 4:0.047039 -1:0.315462 2:0.257793 3:0.364178 4:0.062567 -1:0.248853 2:0.413623 3:0.214349 4:0.123175 -1:0.330257 2:0.315563 3:0.262713 4:0.091467 -1:0.261875 2:0.372799 3:0.316970 4:0.048356 -1:0.259048 2:0.303203 3:0.317156 4:0.120593 -1:0.179271 2:0.521001 3:0.240582 4:0.059146 -1:0.138247 2:0.554524 3:0.247828 4:0.059401 -1:0.300100 2:0.268436 3:0.373255 4:0.058209 -1:0.237704 2:0.280231 3:0.397638 4:0.084427 -1:0.221441 2:0.445015 3:0.249532 4:0.084012 -1:0.188855 2:0.351235 3:0.340917 4:0.118992 -1:0.238713 2:0.401296 3:0.312899 4:0.047092 -1:0.179173 2:0.458758 3:0.302998 4:0.059071 -1:0.195911 2:0.517657 3:0.250783 4:0.035649 -1:0.196512 2:0.507026 3:0.252925 4:0.043537 -1:0.319271 2:0.391476 3:0.227183 4:0.062069 -1:0.272092 2:0.387146 3:0.314424 4:0.026337 -1:0.284953 2:0.339017 3:0.342416 4:0.033614 -1:0.217685 2:0.494065 3:0.246079 4:0.042170 -1:0.237363 2:0.292141 3:0.438256 4:0.032240 -1:0.235240 2:0.350313 3:0.364617 4:0.049829 -1:0.288593 2:0.282516 3:0.321421 4:0.107470 -1:0.220633 2:0.428334 3:0.316838 4:0.034195 -1:0.232493 2:0.248186 3:0.399621 4:0.119701 -1:0.152305 2:0.619572 3:0.199025 4:0.029098 -1:0.340920 2:0.251069 3:0.324683 4:0.083327 -1:0.142751 2:0.546445 3:0.276571 4:0.034233 -1:0.207955 2:0.368125 3:0.267819 4:0.156101 -1:0.244388 2:0.548417 3:0.182086 4:0.025109 -1:0.120632 2:0.586263 3:0.250660 4:0.042445 -1:0.353659 2:0.339474 3:0.228157 4:0.078711 -1:0.242564 2:0.230226 3:0.403460 4:0.123749 -1:0.242474 2:0.249793 3:0.341602 4:0.166131 -1:0.238024 2:0.335537 3:0.308017 4:0.118422 -1:0.233738 2:0.359151 3:0.284436 4:0.122675 -1:0.345263 2:0.295647 3:0.214160 4:0.144930 -1:0.299240 2:0.314471 3:0.190503 4:0.195786 -1:0.207511 2:0.350997 3:0.305317 4:0.136174 -1:0.163283 2:0.614122 3:0.190280 4:0.032314 -1:0.229450 2:0.377050 3:0.215341 4:0.178159 -1:0.236848 2:0.375209 3:0.309766 4:0.078177 -1:0.242428 2:0.537258 3:0.130699 4:0.089616 -1:0.298422 2:0.416350 3:0.167858 4:0.117369 -1:0.407449 2:0.216467 3:0.229607 4:0.146477 -1:0.243223 2:0.406400 3:0.236642 4:0.113735 -1:0.183218 2:0.600387 3:0.139488 4:0.076907 -1:0.202612 2:0.489377 3:0.175887 4:0.132124 -1:0.296572 2:0.457303 3:0.131663 4:0.114463 -1:0.328838 2:0.335136 3:0.241981 4:0.094046 -1:0.474776 2:0.309583 3:0.162399 4:0.053241 -1:0.069747 2:0.838180 3:0.064913 4:0.027160 -1:0.201569 2:0.410564 3:0.193064 4:0.194803 -1:0.182910 2:0.388337 3:0.244970 4:0.183783 -1:0.500399 2:0.330396 3:0.089847 4:0.079357 -1:0.273812 2:0.347579 3:0.203222 4:0.175388 -1:0.393020 2:0.294736 3:0.177783 4:0.134460 -1:0.128893 2:0.683318 3:0.129243 4:0.058547 -1:0.220163 2:0.393052 3:0.227522 4:0.159262 -1:0.223136 2:0.332249 3:0.284433 4:0.160182 -1:0.216372 2:0.481355 3:0.179317 4:0.122957 -1:0.251567 2:0.484669 3:0.150804 4:0.112961 -1:0.465755 2:0.310368 3:0.168069 4:0.055807 -1:0.340516 2:0.361368 3:0.148848 4:0.149269 -1:0.313034 2:0.437375 3:0.100300 4:0.149291 -1:0.290660 2:0.351069 3:0.247759 4:0.110512 -1:0.314403 2:0.505852 3:0.095448 4:0.084297 -1:0.400978 2:0.339099 3:0.131155 4:0.128768 -1:0.168565 2:0.446113 3:0.259337 4:0.125984 -1:0.226527 2:0.631779 3:0.067846 4:0.073848 -1:0.226760 2:0.381642 3:0.204949 4:0.186649 -1:0.309706 2:0.493359 3:0.109701 4:0.087234 -1:0.360581 2:0.256700 3:0.218418 4:0.164302 -1:0.301843 2:0.405495 3:0.179256 4:0.113406 -1:0.315438 2:0.378281 3:0.175327 4:0.130954 -1:0.239832 2:0.391982 3:0.258193 4:0.109994 -1:0.394134 2:0.293784 3:0.163035 4:0.149047 -1:0.307266 2:0.577592 3:0.059544 4:0.055598 -1:0.314410 2:0.385529 3:0.187062 4:0.112999 -1:0.279566 2:0.531693 3:0.094299 4:0.094441 -1:0.226256 2:0.555424 3:0.159907 4:0.058413 -1:0.209327 2:0.726276 3:0.043982 4:0.020415 -1:0.289289 2:0.481139 3:0.149656 4:0.079916 -1:0.319581 2:0.301502 3:0.203017 4:0.175900 -1:0.275145 2:0.469714 3:0.154897 4:0.100245 -1:0.421640 2:0.337368 3:0.169400 4:0.071593 -1:0.449820 2:0.401021 3:0.096328 4:0.052831 -1:0.413314 2:0.232537 3:0.180140 4:0.174009 -1:0.282435 2:0.503026 3:0.135334 4:0.079205 -1:0.525945 2:0.348137 3:0.056035 4:0.069883 -1:0.191952 2:0.678243 3:0.083772 4:0.046033 -1:0.371177 2:0.400085 3:0.141723 4:0.087016 -1:0.426765 2:0.337436 3:0.097234 4:0.138566 -1:0.374739 2:0.337914 3:0.162478 4:0.124869 -1:0.244543 2:0.323566 3:0.216215 4:0.215677 -1:0.315904 2:0.478177 3:0.145465 4:0.060454 -1:0.329752 2:0.573241 3:0.066403 4:0.030603 -1:0.157686 2:0.690136 3:0.121434 4:0.030744 -1:0.472587 2:0.304341 3:0.150844 4:0.072227 -1:0.359558 2:0.386440 3:0.148343 4:0.105659 -1:0.426155 2:0.190288 3:0.149565 4:0.233992 -1:0.294433 2:0.533932 3:0.086855 4:0.084781 -1:0.370380 2:0.478400 3:0.096107 4:0.055113 -1:0.213291 2:0.635899 3:0.081270 4:0.069540 -1:0.448615 2:0.159224 3:0.224987 4:0.167174 -1:0.517613 2:0.157906 3:0.158374 4:0.166107 -1:0.371968 2:0.407296 3:0.164692 4:0.056043 -1:0.386412 2:0.230707 3:0.180157 4:0.202724 -1:0.332751 2:0.505209 3:0.102353 4:0.059686 -1:0.605299 2:0.186628 3:0.130294 4:0.077778 -1:0.144558 2:0.748802 3:0.089549 4:0.017091 -1:0.268203 2:0.450590 3:0.219686 4:0.061521 -1:0.380449 2:0.322395 3:0.222840 4:0.074316 -1:0.334443 2:0.413650 3:0.173151 4:0.078756 -1:0.484383 2:0.171258 3:0.224159 4:0.120201 -1:0.244288 2:0.401481 3:0.277923 4:0.076307 -1:0.306557 2:0.637356 3:0.051215 4:0.004871 -1:0.555868 2:0.220080 3:0.151744 4:0.072309 -1:0.555974 2:0.127818 3:0.221978 4:0.094230 -1:0.393626 2:0.420199 3:0.161136 4:0.025039 -1:0.473740 2:0.138051 3:0.249093 4:0.139115 -1:0.445366 2:0.274710 3:0.224229 4:0.055696 -1:0.466317 2:0.165515 3:0.226762 4:0.141405 -1:0.223733 2:0.680319 3:0.084391 4:0.011557 -1:0.249748 2:0.550395 3:0.187512 4:0.012345 -1:0.471695 2:0.239600 3:0.217766 4:0.070939 -1:0.364686 2:0.343646 3:0.212296 4:0.079373 -1:0.402037 2:0.264612 3:0.198450 4:0.134901 -1:0.559318 2:0.136405 3:0.229446 4:0.074831 -1:0.257295 2:0.392430 3:0.292749 4:0.057527 -1:0.386718 2:0.382574 3:0.170794 4:0.059915 -1:0.570606 2:0.188533 3:0.200860 4:0.040001 -1:0.395443 2:0.229419 3:0.259062 4:0.116075 -1:0.309538 2:0.386329 3:0.179265 4:0.124869 -1:0.203160 2:0.610816 3:0.102084 4:0.083940 -1:0.295988 2:0.286876 3:0.211180 4:0.205957 -1:0.225063 2:0.380618 3:0.268151 4:0.126169 -1:0.482279 2:0.361734 3:0.096599 4:0.059388 -1:0.266276 2:0.485864 3:0.157208 4:0.090653 -1:0.692552 2:0.096594 3:0.131223 4:0.079630 -1:0.462469 2:0.322006 3:0.136155 4:0.079370 -1:0.587404 2:0.164549 3:0.155770 4:0.092277 -1:0.406820 2:0.312962 3:0.171238 4:0.108980 -1:0.293866 2:0.341444 3:0.238028 4:0.126662 -1:0.358386 2:0.237113 3:0.222920 4:0.181580 -1:0.222670 2:0.642500 3:0.097586 4:0.037244 -1:0.205485 2:0.544518 3:0.157501 4:0.092496 -1:0.361098 2:0.499780 3:0.118358 4:0.020765 -1:0.510910 2:0.312992 3:0.119489 4:0.056608 -1:0.342952 2:0.516456 3:0.119272 4:0.021320 -1:0.327584 2:0.443874 3:0.156989 4:0.071553 -1:0.444934 2:0.291602 3:0.141311 4:0.122152 -1:0.522691 2:0.201155 3:0.173864 4:0.102290 -1:0.357829 2:0.399492 3:0.113835 4:0.128844 -1:0.275094 2:0.349717 3:0.206985 4:0.168204 -1:0.375564 2:0.348207 3:0.186400 4:0.089828 -1:0.316582 2:0.471081 3:0.128840 4:0.083497 -1:0.404557 2:0.433093 3:0.115948 4:0.046401 -1:0.373883 2:0.205712 3:0.231101 4:0.189303 -1:0.296787 2:0.586767 3:0.078279 4:0.038167 -1:0.328620 2:0.555588 3:0.089240 4:0.026552 -1:0.431454 2:0.414268 3:0.104233 4:0.050044 +1:0.292896 2:0.293785 3:0.20666 4:0.20666 +1:0.253968 2:0.264322 3:0.26389 4:0.21782 +1:0.229096 2:0.339124 3:0.242756 4:0.189023 +1:0.237677 2:0.421962 3:0.173249 4:0.167112 +1:0.232665 2:0.331115 3:0.244515 4:0.191705 +1:0.153281 2:0.542563 3:0.193212 4:0.110944 +1:0.24974 2:0.303776 3:0.24652 4:0.199963 +1:0.236352 2:0.29962 3:0.300125 4:0.163902 +1:0.172813 2:0.475663 3:0.212553 4:0.138971 +1:0.220493 2:0.416412 3:0.228174 4:0.134921 +1:0.213806 2:0.398462 3:0.270086 4:0.117646 +1:0.146578 2:0.505934 3:0.259935 4:0.0875526 +1:0.228377 2:0.312438 3:0.282801 4:0.176384 +1:0.238692 2:0.375405 3:0.232842 4:0.15306 +1:0.170336 2:0.44645 3:0.282693 4:0.100521 +1:0.179429 2:0.438819 3:0.264394 4:0.117358 +1:0.188199 2:0.486086 3:0.271585 4:0.0541296 +1:0.210677 2:0.473321 3:0.217376 4:0.0986256 +1:0.210124 2:0.33964 3:0.284431 4:0.165804 +1:0.177978 2:0.466856 3:0.291972 4:0.0631942 +1:0.158742 2:0.581078 3:0.20221 4:0.0579709 +1:0.261199 2:0.264752 3:0.299985 4:0.174065 +1:0.232218 2:0.315572 3:0.269038 4:0.183173 +1:0.168694 2:0.5108 3:0.256402 4:0.0641049 +1:0.209427 2:0.453269 3:0.237973 4:0.0993312 +1:0.116952 2:0.714906 3:0.141432 4:0.0267092 +1:0.262195 2:0.39806 3:0.23274 4:0.107006 +1:0.230284 2:0.369558 3:0.295852 4:0.104307 +1:0.15102 2:0.61416 3:0.205703 4:0.0291171 +1:0.241222 2:0.352027 3:0.277593 4:0.129158 +1:0.233282 2:0.344757 3:0.304838 4:0.117123 +1:0.140696 2:0.49711 3:0.318699 4:0.0434958 +1:0.255566 2:0.386561 3:0.29288 4:0.0649931 +1:0.201996 2:0.58627 3:0.167508 4:0.0442267 +1:0.166752 2:0.343594 3:0.446855 4:0.0427995 +1:0.185771 2:0.592118 3:0.185372 4:0.0367385 +1:0.286874 2:0.326332 3:0.339739 4:0.0470547 +1:0.317704 2:0.255621 3:0.364207 4:0.0624677 +1:0.24863 2:0.413686 3:0.214481 4:0.123203 +1:0.330877 2:0.314913 3:0.262757 4:0.0914529 +1:0.262772 2:0.372127 3:0.316776 4:0.048325 +1:0.258846 2:0.302906 3:0.317581 4:0.120667 +1:0.179287 2:0.520748 3:0.240816 4:0.0591481 +1:0.138684 2:0.553779 3:0.248153 4:0.059385 +1:0.300173 2:0.267368 3:0.374202 4:0.058258 +1:0.237293 2:0.28002 3:0.398176 4:0.0845108 +1:0.222424 2:0.443738 3:0.24986 4:0.0839782 +1:0.189211 2:0.350639 3:0.341163 4:0.118987 +1:0.238995 2:0.400992 3:0.31294 4:0.0470729 +1:0.180202 2:0.457729 3:0.303039 4:0.0590297 +1:0.196701 2:0.516693 3:0.250974 4:0.0356324 +1:0.196963 2:0.506427 3:0.253075 4:0.0435355 +1:0.319621 2:0.390289 3:0.227969 4:0.0621218 +1:0.272665 2:0.386041 3:0.314943 4:0.0263517 +1:0.285656 2:0.338322 3:0.342417 4:0.0336059 +1:0.217902 2:0.493925 3:0.246017 4:0.0421559 +1:0.238472 2:0.291037 3:0.438277 4:0.0322136 +1:0.235991 2:0.349728 3:0.364493 4:0.0497878 +1:0.288839 2:0.282238 3:0.321453 4:0.10747 +1:0.221085 2:0.427551 3:0.317173 4:0.0341909 +1:0.231811 2:0.248323 3:0.400032 4:0.119833 +1:0.152399 2:0.619269 3:0.19924 4:0.0290919 +1:0.341529 2:0.25048 3:0.324642 4:0.0833488 +1:0.142494 2:0.546613 3:0.276656 4:0.0342365 +1:0.208031 2:0.367978 3:0.267846 4:0.156145 +1:0.245117 2:0.547416 3:0.182342 4:0.025125 +1:0.121155 2:0.585478 3:0.250918 4:0.042448 +1:0.353966 2:0.339298 3:0.228018 4:0.0787179 +1:0.241892 2:0.23038 3:0.40385 4:0.123878 +1:0.241925 2:0.249528 3:0.342213 4:0.166334 +1:0.237823 2:0.335557 3:0.30818 4:0.11844 +1:0.233831 2:0.358557 3:0.284903 4:0.12271 +1:0.345952 2:0.29521 3:0.213977 4:0.14486 +1:0.299648 2:0.314102 3:0.190482 4:0.195769 +1:0.207406 2:0.350752 3:0.305607 4:0.136235 +1:0.164055 2:0.613262 3:0.190371 4:0.0323123 +1:0.229137 2:0.377278 3:0.215372 4:0.178212 +1:0.237206 2:0.375309 3:0.309366 4:0.0781199 +1:0.243208 2:0.536353 3:0.130858 4:0.0895808 +1:0.298699 2:0.416382 3:0.167647 4:0.117271 +1:0.408759 2:0.215428 3:0.229434 4:0.146378 +1:0.243553 2:0.406062 3:0.236691 4:0.113694 +1:0.183896 2:0.599759 3:0.139447 4:0.0768976 +1:0.202686 2:0.489141 3:0.176026 4:0.132147 +1:0.296525 2:0.457356 3:0.131658 4:0.11446 +1:0.328469 2:0.335108 3:0.242333 4:0.0940901 +1:0.475703 2:0.308888 3:0.162192 4:0.0532169 +1:0.0704053 2:0.837243 3:0.065184 4:0.0271682 +1:0.201791 2:0.410369 3:0.193075 4:0.194765 +1:0.183885 2:0.387498 3:0.244918 4:0.183698 +1:0.502533 2:0.328416 3:0.0897919 4:0.0792593 +1:0.273819 2:0.347523 3:0.203253 4:0.175404 +1:0.393043 2:0.294483 3:0.177938 4:0.134535 +1:0.129305 2:0.682749 3:0.129395 4:0.0585511 +1:0.220164 2:0.39305 3:0.227476 4:0.15931 +1:0.223189 2:0.332392 3:0.284282 4:0.160137 +1:0.216788 2:0.480867 3:0.179408 4:0.122938 +1:0.251439 2:0.484659 3:0.150905 4:0.112998 +1:0.466343 2:0.309891 3:0.167982 4:0.0557838 +1:0.340387 2:0.361615 3:0.148765 4:0.149233 +1:0.313943 2:0.436902 3:0.100121 4:0.149034 +1:0.290221 2:0.3513 3:0.247891 4:0.110588 +1:0.314496 2:0.505745 3:0.0954789 4:0.0842801 +1:0.401614 2:0.338487 3:0.131127 4:0.128772 +1:0.168685 2:0.446472 3:0.25894 4:0.125902 +1:0.226761 2:0.631549 3:0.0678447 4:0.0738454 +1:0.226743 2:0.381658 3:0.204946 4:0.186654 +1:0.310712 2:0.492373 3:0.109727 4:0.0871879 +1:0.36138 2:0.256109 3:0.218293 4:0.164219 +1:0.302085 2:0.405023 3:0.179453 4:0.113439 +1:0.315677 2:0.377985 3:0.175361 4:0.130977 +1:0.239506 2:0.392166 3:0.258282 4:0.110045 +1:0.394673 2:0.293404 3:0.162901 4:0.149022 +1:0.307548 2:0.57713 3:0.0596827 4:0.0556398 +1:0.314085 2:0.385555 3:0.187302 4:0.113057 +1:0.279712 2:0.5313 3:0.0944946 4:0.0944927 +1:0.225893 2:0.555752 3:0.159939 4:0.058417 +1:0.209693 2:0.725936 3:0.0439738 4:0.0203973 +1:0.289566 2:0.480817 3:0.1497 4:0.0799165 +1:0.3197 2:0.301292 3:0.203053 4:0.175955 +1:0.274853 2:0.469948 3:0.154932 4:0.100267 +1:0.42143 2:0.337345 3:0.169587 4:0.0716377 +1:0.450278 2:0.400716 3:0.096228 4:0.052778 +1:0.413455 2:0.232376 3:0.180139 4:0.17403 +1:0.282658 2:0.502598 3:0.135536 4:0.0792087 +1:0.526157 2:0.347995 3:0.0560087 4:0.0698396 +1:0.191873 2:0.678378 3:0.0837452 4:0.046004 +1:0.371729 2:0.399404 3:0.141855 4:0.0870113 +1:0.426748 2:0.337491 3:0.097227 4:0.138534 +1:0.374447 2:0.338154 3:0.162477 4:0.124921 +1:0.244386 2:0.323598 3:0.216292 4:0.215724 +1:0.3157 2:0.478331 3:0.145522 4:0.0604461 +1:0.330049 2:0.572994 3:0.0663776 4:0.0305794 +1:0.158403 2:0.689217 3:0.121632 4:0.0307477 +1:0.473456 2:0.303339 3:0.150981 4:0.0722247 +1:0.359109 2:0.386813 3:0.148368 4:0.10571 +1:0.426377 2:0.190054 3:0.149512 4:0.234057 +1:0.294777 2:0.533685 3:0.0868172 4:0.08472 +1:0.370187 2:0.478508 3:0.0961764 4:0.0551286 +1:0.21357 2:0.635491 3:0.0814006 4:0.0695385 +1:0.448559 2:0.158978 3:0.22517 4:0.167294 +1:0.517873 2:0.157614 3:0.158377 4:0.166136 +1:0.37215 2:0.407108 3:0.1647 4:0.0560422 +1:0.386694 2:0.230431 3:0.180123 4:0.202752 +1:0.33305 2:0.504891 3:0.102384 4:0.0596752 +1:0.605128 2:0.186571 3:0.130425 4:0.0778765 +1:0.145837 2:0.747383 3:0.0896978 4:0.0170822 +1:0.268027 2:0.450601 3:0.219824 4:0.0615486 +1:0.380471 2:0.322381 3:0.222816 4:0.0743319 +1:0.334579 2:0.413646 3:0.173045 4:0.0787296 +1:0.484873 2:0.170854 3:0.224093 4:0.12018 +1:0.243809 2:0.401097 3:0.278679 4:0.0764149 +1:0.306017 2:0.63786 3:0.0512515 4:0.00487213 +1:0.55597 2:0.219847 3:0.151841 4:0.0723423 +1:0.55635 2:0.127418 3:0.221988 4:0.0942438 +1:0.39379 2:0.419918 3:0.16125 4:0.0250425 +1:0.474071 2:0.137998 3:0.248838 4:0.139093 +1:0.44545 2:0.27442 3:0.224424 4:0.0557066 +1:0.466578 2:0.165312 3:0.226707 4:0.141403 +1:0.22387 2:0.680152 3:0.0844247 4:0.0115535 +1:0.250021 2:0.549994 3:0.187637 4:0.0123481 +1:0.471566 2:0.239544 3:0.217906 4:0.0709838 +1:0.364405 2:0.343574 3:0.212603 4:0.0794185 +1:0.401939 2:0.264627 3:0.19845 4:0.134984 +1:0.559524 2:0.136177 3:0.229456 4:0.0748424 +1:0.256551 2:0.39281 3:0.293065 4:0.0575744 +1:0.387097 2:0.382129 3:0.170846 4:0.0599278 +1:0.57118 2:0.188057 3:0.200788 4:0.0399749 +1:0.395243 2:0.229136 3:0.25943 4:0.11619 +1:0.310212 2:0.385609 3:0.179358 4:0.124821 +1:0.203637 2:0.610609 3:0.101903 4:0.0838513 +1:0.295613 2:0.287068 3:0.211265 4:0.206054 +1:0.224475 2:0.381087 3:0.268224 4:0.126214 +1:0.48271 2:0.36139 3:0.0965415 4:0.059359 +1:0.265289 2:0.48685 3:0.157142 4:0.0907189 +1:0.692884 2:0.0961933 3:0.131291 4:0.0796317 +1:0.462985 2:0.321546 3:0.136091 4:0.0793776 +1:0.587816 2:0.164364 3:0.155543 4:0.0922778 +1:0.406828 2:0.312667 3:0.171464 4:0.109041 +1:0.294052 2:0.341386 3:0.237928 4:0.126634 +1:0.358487 2:0.236939 3:0.222981 4:0.181594 +1:0.222577 2:0.642572 3:0.09761 4:0.0372417 +1:0.205481 2:0.544466 3:0.157554 4:0.0924986 +1:0.361582 2:0.499233 3:0.118425 4:0.0207605 +1:0.511804 2:0.312372 3:0.119297 4:0.0565275 +1:0.343518 2:0.51587 3:0.119294 4:0.0213172 +1:0.327281 2:0.444203 3:0.156942 4:0.071574 +1:0.444013 2:0.292411 3:0.1413 4:0.122276 +1:0.523091 2:0.200965 3:0.17366 4:0.102284 +1:0.358354 2:0.398924 3:0.113883 4:0.128839 +1:0.275161 2:0.349721 3:0.206926 4:0.168192 +1:0.375458 2:0.348449 3:0.186289 4:0.0898039 +1:0.316544 2:0.471193 3:0.128775 4:0.0834884 +1:0.404512 2:0.433144 3:0.11594 4:0.0464045 +1:0.374182 2:0.205462 3:0.231049 4:0.189307 +1:0.296695 2:0.586875 3:0.0782689 4:0.038161 +1:0.328606 2:0.555615 3:0.089233 4:0.0265465 +1:0.430969 2:0.414781 3:0.104192 4:0.0500578 diff --git a/test/pred-sets/ref/oaa_scores.predict b/test/pred-sets/ref/oaa_scores.predict index 7315dd6efa9..3ee210fccf7 100644 --- a/test/pred-sets/ref/oaa_scores.predict +++ b/test/pred-sets/ref/oaa_scores.predict @@ -1,200 +1,200 @@ -1:0.000000 2:0.000000 3:0.000000 4:0.000000 +1:0 2:0 3:0 4:0 1:0.165033 2:-0.165033 3:-0.165033 4:-0.165033 -1:0.014014 2:-0.014218 3:-0.277262 4:-0.277244 -1:-0.100552 2:-0.077114 3:-0.024543 4:-0.206828 -1:-0.237961 2:0.049008 3:-0.148985 4:-0.340655 -1:-0.237673 2:0.229521 3:-0.432823 4:-0.443868 -1:-0.208910 2:0.024743 3:-0.131143 4:-0.324136 -1:-0.634424 2:0.269960 3:-0.361374 4:-0.728540 -1:-0.127719 2:-0.041121 3:-0.106503 4:-0.281444 -1:-0.277530 2:-0.205950 3:-0.003651 4:-0.488988 -1:-0.439897 2:0.201790 3:-0.266798 4:-0.522220 -1:-0.324062 2:0.015752 3:-0.278112 4:-0.603481 -1:-0.330527 2:-0.065047 3:-0.232640 4:-0.653271 -1:-0.514139 2:0.098036 3:-0.289179 4:-0.729253 -1:-0.186779 2:-0.052829 3:-0.070612 4:-0.335952 -1:-0.216998 2:-0.014883 3:-0.244259 4:-0.507331 -1:-0.466418 2:-0.004534 3:-0.218238 4:-0.729665 -1:-0.459212 2:0.014947 3:-0.116202 4:-0.601347 -1:-0.490440 2:-0.202351 3:-0.225219 4:-0.965778 -1:-0.333342 2:0.086579 3:-0.411171 4:-0.692238 -1:-0.326197 2:-0.033497 3:-0.045484 4:-0.436764 -1:-0.567463 2:-0.156859 3:-0.147185 4:-0.936162 -1:-0.482172 2:0.090143 3:-0.433135 4:-0.884982 -1:-0.130199 2:-0.234565 3:0.003765 4:-0.402801 -1:-0.161537 2:-0.016115 3:-0.076526 4:-0.296066 -1:-0.544579 2:-0.053640 3:-0.249651 4:-0.929202 -1:-0.333219 2:0.028023 3:-0.289565 4:-0.627866 -1:-0.694652 2:0.320544 3:-0.709846 4:-1.000000 -1:-0.171422 2:-0.062873 3:-0.341459 4:-0.646528 -1:-0.288425 2:-0.145001 3:-0.151658 4:-0.689542 -1:-0.567308 2:-0.019843 3:-0.470116 4:-1.000000 -1:-0.273904 2:-0.081186 3:-0.128116 4:-0.577656 -1:-0.222157 2:-0.123990 3:-0.107015 4:-0.501859 -1:-0.634690 2:-0.030252 3:-0.217350 4:-0.961771 -1:-0.262853 2:-0.268826 3:-0.207100 4:-0.881779 -1:-0.330320 2:0.126173 3:-0.603194 4:-0.943824 -1:-0.527559 2:-0.407404 3:0.037192 4:-1.000000 -1:-0.329307 2:0.070661 3:-0.543255 4:-0.975262 -1:-0.185883 2:-0.391699 3:-0.225542 4:-0.921976 -1:-0.081450 2:-0.567352 3:-0.062464 4:-0.843022 -1:-0.189568 2:0.153142 3:-0.361072 4:-0.486067 -1:-0.034231 2:-0.251754 3:-0.320189 4:-0.672913 -1:-0.210227 2:-0.290982 3:-0.264487 4:-0.883643 -1:-0.243261 2:-0.194211 3:-0.055604 4:-0.566826 -1:-0.552833 2:0.210345 3:-0.340392 4:-0.854308 -1:-0.599403 2:0.220652 3:-0.187214 4:-0.702519 -1:-0.321076 2:-0.498264 3:-0.070382 4:-1.000000 -1:-0.443934 2:-0.364370 3:0.059019 4:-0.861287 -1:-0.426193 2:0.113272 3:-0.322913 4:-0.805391 -1:-0.437843 2:-0.087002 3:0.026157 4:-0.534453 -1:-0.311979 2:-0.135685 3:-0.215970 4:-0.841622 -1:-0.454365 2:0.000191 3:-0.241935 4:-0.760023 -1:-0.427762 2:-0.024461 3:-0.374918 4:-0.956297 -1:-0.418238 2:-0.019102 3:-0.361900 4:-0.893634 -1:-0.103199 2:-0.167332 3:-0.393722 4:-0.878256 -1:-0.289980 2:-0.466361 3:-0.260487 4:-1.000000 -1:-0.283610 2:-0.390402 3:-0.255060 4:-1.000000 -1:-0.352710 2:0.090757 3:-0.410904 4:-0.859304 -1:-0.349351 2:-0.468990 3:0.024825 4:-0.959303 -1:-0.343382 2:-0.262993 3:-0.126511 4:-0.797804 -1:-0.142960 2:-0.253010 3:-0.022001 4:-0.553933 -1:-0.435859 2:-0.238726 3:-0.252534 4:-0.920418 -1:-0.395547 2:-0.532232 3:0.023103 4:-0.512396 -1:-0.634724 2:0.255381 3:-0.509500 4:-0.919378 -1:-0.036593 2:-0.486352 3:-0.211336 4:-0.718730 -1:-0.642336 2:0.136220 3:-0.327500 4:-0.820708 -1:-0.538528 2:-0.079521 3:-0.357698 4:-0.391004 -1:-0.491688 2:-0.115805 3:-0.840342 4:-0.977267 -1:-0.734181 2:0.117603 3:-0.410270 4:-0.742374 -1:-0.071615 2:-0.235050 3:-0.576961 4:-0.640694 -1:-0.393680 2:-0.397150 3:0.110934 4:-0.509632 -1:-0.388654 2:-0.377153 3:0.036793 4:-0.446740 -1:-0.393316 2:-0.183872 3:-0.252627 4:-0.458852 -1:-0.498664 2:-0.223354 3:-0.334624 4:-0.483098 -1:0.148435 2:-0.316197 3:-0.575931 4:-0.381921 -1:-0.105045 2:-0.204303 3:-0.550912 4:-0.231692 -1:-0.465551 2:-0.149774 3:-0.172083 4:-0.479065 -1:-0.682957 2:0.024618 3:-0.780557 4:-0.844633 -1:-0.291440 2:0.211547 3:-0.387828 4:-0.287321 -1:-0.310073 2:-0.147019 3:-0.358933 4:-0.543364 -1:-0.305613 2:0.148518 3:-0.780869 4:-0.489400 -1:-0.172113 2:-0.020725 3:-0.698023 4:-0.485816 -1:0.060085 2:-0.629523 3:-0.594807 4:-0.331333 -1:-0.282999 2:-0.023183 3:-0.412319 4:-0.497281 -1:-0.544881 2:0.035015 3:-0.950198 4:-0.575918 -1:-0.444483 2:0.223747 3:-0.427064 4:-0.420915 -1:-0.216019 2:0.077697 3:-0.748300 4:-0.524190 -1:-0.339042 2:-0.641771 3:-0.584583 4:-0.375816 -1:0.162006 2:-0.547647 3:-0.887796 4:-0.745044 -1:-0.954680 2:0.520461 3:-1.000000 4:-0.846443 -1:-0.339775 2:0.175008 3:-0.452204 4:-0.271185 -1:-0.442154 2:-0.071500 3:-0.361288 4:-0.213653 -1:0.328829 2:-0.588106 3:-1.000000 4:-0.740753 -1:-0.122558 2:0.070400 3:-0.409071 4:-0.351685 -1:0.021519 2:-0.371106 3:-0.588581 4:-0.344316 -1:-0.650688 2:0.242542 3:-0.840283 4:-0.796923 -1:-0.459287 2:-0.159417 3:-0.532553 4:-0.383006 -1:-0.309679 2:-0.081208 3:-0.203367 4:-0.276418 -1:-0.520177 2:0.179090 3:-0.698433 4:-0.659443 -1:-0.308079 2:0.156327 3:-0.709431 4:-0.591219 -1:0.004015 2:-0.459981 3:-0.857700 4:-0.963708 -1:-0.084657 2:-0.156245 3:-0.663298 4:-0.332595 -1:-0.084474 2:-0.120486 3:-0.974670 4:-0.394380 -1:-0.262083 2:-0.316387 3:-0.483373 4:-0.569942 -1:-0.118489 2:-0.113060 3:-0.987112 4:-0.687724 -1:0.049966 2:-0.268915 3:-0.873912 4:-0.560742 -1:-0.553186 2:0.103068 3:-0.412821 4:-0.443268 -1:-0.423596 2:0.119662 3:-1.000000 4:-0.575190 -1:-0.378586 2:0.015875 3:-0.458831 4:-0.326730 -1:-0.261389 2:-0.004914 3:-0.981267 4:-0.787860 -1:-0.081632 2:-0.532507 3:-0.635223 4:-0.408948 -1:-0.255751 2:-0.264827 3:-0.620580 4:-0.584371 -1:-0.143708 2:-0.192972 3:-0.589953 4:-0.543852 -1:-0.382524 2:-0.035388 3:-0.337131 4:-0.736692 -1:0.279634 2:-0.265309 3:-0.592006 4:-0.554717 -1:-0.304525 2:-0.004023 3:-1.000000 4:-0.836357 -1:-0.249831 2:-0.030991 3:-0.532602 4:-0.459926 -1:-0.344338 2:-0.100866 3:-0.901237 4:-0.579732 -1:-0.546708 2:-0.013688 3:-0.602777 4:-0.809588 -1:-0.315470 2:0.391793 3:-1.000000 4:-1.000000 -1:-0.278909 2:-0.169023 3:-0.668075 4:-0.689440 -1:-0.147444 2:-0.255970 3:-0.446477 4:-0.473912 -1:-0.296627 2:0.006774 3:-0.718892 4:-0.754023 -1:-0.072438 2:-0.521184 3:-0.479264 4:-0.804734 -1:0.123857 2:-0.319633 3:-0.975151 4:-0.841588 -1:0.314938 2:-0.386030 3:-0.545792 4:-0.531683 -1:-0.181425 2:0.061573 3:-0.577824 4:-0.748705 -1:0.107378 2:-0.569366 3:-1.000000 4:-0.648570 -1:-0.489473 2:0.267934 3:-0.867828 4:-0.942292 -1:-0.006833 2:-0.125891 3:-0.723956 4:-0.835844 -1:0.197900 2:-0.114853 3:-0.770394 4:-0.425630 -1:-0.457019 2:-0.575205 3:-0.760585 4:-0.485611 -1:-0.221081 2:0.035375 3:-0.292301 4:-0.325979 -1:-0.207192 2:0.165635 3:-0.610310 4:-0.816867 -1:-0.247453 2:0.193665 3:-0.925637 4:-1.000000 -1:-0.582320 2:0.187017 3:-0.564550 4:-0.949288 -1:0.099028 2:-0.537103 3:-0.504563 4:-0.763874 -1:-0.129115 2:-0.172640 3:-0.630351 4:-0.545193 -1:0.182463 2:-0.702862 3:-0.694009 4:-0.104832 -1:-0.261685 2:0.057794 3:-0.841810 4:-0.667952 -1:-0.221790 2:-0.194108 3:-0.691783 4:-0.893710 -1:-0.496939 2:0.397944 3:-0.834448 4:-0.878142 -1:0.348306 2:-0.810567 3:-0.350632 4:-0.435247 -1:0.329711 2:-0.894882 3:-0.693108 4:-0.424762 -1:-0.134671 2:-0.293474 3:-0.520628 4:-0.889176 -1:0.114318 2:-0.491239 3:-0.521531 4:-0.394363 -1:-0.026260 2:0.101026 3:-0.811948 4:-0.978949 -1:0.457388 2:-0.677836 3:-0.706687 4:-0.847990 -1:-0.677046 2:0.309736 3:-0.654299 4:-1.000000 -1:-0.381578 2:-0.164198 3:-0.383949 4:-0.936934 -1:-0.131504 2:-0.503868 3:-0.466923 4:-0.868285 -1:-0.169386 2:-0.261256 3:-0.543678 4:-0.595911 -1:0.133361 2:-0.848333 3:-0.413713 4:-0.486445 -1:-0.560511 2:-0.297142 3:-0.148651 4:-0.803172 -1:-0.264418 2:0.004004 3:-0.987077 4:-1.000000 -1:0.296929 2:-0.587778 3:-0.584499 4:-0.838483 -1:0.516969 2:-0.915049 3:-0.310782 4:-0.823707 -1:-0.196296 2:-0.341678 3:-0.569235 4:-1.000000 -1:0.212239 2:-0.811734 3:-0.330311 4:-0.619989 -1:0.170809 2:-0.483177 3:-0.265834 4:-0.904692 -1:0.446511 2:-0.633501 3:-0.380121 4:-0.703243 -1:-0.427339 2:0.180244 3:-0.809270 4:-1.000000 -1:-0.572278 2:-0.279921 3:-0.584076 4:-1.000000 -1:-0.052679 2:-0.654545 3:-0.364615 4:-0.786170 -1:-0.011195 2:-0.065438 3:-0.332390 4:-0.831178 -1:0.115332 2:-0.321409 3:-0.517592 4:-0.565938 -1:0.321014 2:-0.943862 3:-0.448742 4:-0.750792 -1:-0.573661 2:-0.034663 3:-0.254450 4:-1.000000 -1:-0.183186 2:-0.328433 3:-0.667900 4:-0.789857 -1:0.356805 2:-0.663029 3:-0.617935 4:-0.981018 -1:0.017046 2:-0.477132 3:-0.208954 4:-0.776223 -1:-0.373321 2:-0.025566 3:-0.522839 4:-0.615731 -1:-0.386782 2:0.232060 3:-0.767353 4:-0.548294 -1:-0.203116 2:-0.173212 3:-0.338459 4:-0.256009 -1:-0.564822 2:-0.019221 3:-0.338863 4:-0.649468 -1:-0.049083 2:-0.267982 3:-1.000000 4:-0.770395 -1:-0.442585 2:0.349514 3:-0.790406 4:-0.823208 -1:0.709507 2:-1.000000 3:-0.784466 4:-0.702878 -1:0.100846 2:-0.298757 3:-0.729002 4:-0.706330 -1:0.448124 2:-0.656802 3:-0.765663 4:-0.750561 -1:-0.104877 2:-0.314886 3:-0.550668 4:-0.670417 -1:-0.243058 2:-0.164493 3:-0.421371 4:-0.631601 -1:-0.136793 2:-0.442189 3:-0.375185 4:-0.411077 -1:-0.439311 2:0.411510 3:-0.906417 4:-0.992632 -1:-0.599076 2:0.181094 3:-0.659804 4:-0.517412 -1:-0.199520 2:-0.046990 3:-0.749666 4:-1.000000 -1:0.020424 2:-0.703981 3:-0.913015 4:-0.730092 -1:-0.264157 2:-0.186849 3:-0.801602 4:-1.000000 -1:-0.498568 2:-0.267191 3:-0.726043 4:-0.685827 -1:-0.152982 2:-0.376060 3:-0.751850 4:-0.505532 -1:0.219328 2:-0.658084 3:-0.705900 4:-0.752297 -1:-0.029431 2:0.044700 3:-0.787241 4:-0.466444 -1:-0.158657 2:-0.043375 3:-0.423978 4:-0.428055 -1:-0.011708 2:-0.066888 3:-0.490769 4:-0.586908 -1:-0.132649 2:0.305792 3:-0.756729 4:-0.723643 -1:-0.317105 2:-0.397754 3:-0.861786 4:-0.844650 -1:0.032671 2:-0.421563 3:-0.350164 4:-0.388255 -1:-0.293587 2:0.150061 3:-0.943251 4:-0.845711 -1:-0.099770 2:0.047339 3:-0.821364 4:-0.994731 -1:-0.036325 2:-0.209728 3:-0.783416 4:-0.857111 +1:0.0145053 2:-0.0145053 3:-0.277244 4:-0.277244 +1:-0.100736 2:-0.0774836 3:-0.0243054 4:-0.206828 +1:-0.238103 2:0.0485268 3:-0.148737 4:-0.340655 +1:-0.236928 2:0.229033 3:-0.432776 4:-0.443868 +1:-0.209348 2:0.0240656 3:-0.130685 4:-0.324136 +1:-0.635499 2:0.267393 3:-0.359988 4:-0.72854 +1:-0.128256 2:-0.041902 3:-0.105999 4:-0.281444 +1:-0.278941 2:-0.20772 3:-0.0025024 4:-0.488988 +1:-0.439843 2:0.19998 3:-0.266045 4:-0.52222 +1:-0.324378 2:0.0142079 3:-0.277407 4:-0.603481 +1:-0.330692 2:-0.0654223 3:-0.232411 4:-0.653271 +1:-0.513595 2:0.0968041 3:-0.28883 4:-0.729253 +1:-0.18648 2:-0.0537346 3:-0.0702701 4:-0.335952 +1:-0.216308 2:-0.0162897 3:-0.243968 4:-0.507331 +1:-0.465558 2:-0.00631808 3:-0.217824 4:-0.729665 +1:-0.460794 2:0.0125231 3:-0.114818 4:-0.601347 +1:-0.491317 2:-0.204103 3:-0.224385 4:-0.965778 +1:-0.332724 2:0.0861393 3:-0.41134 4:-0.692238 +1:-0.32647 2:-0.0343124 3:-0.0450237 4:-0.436764 +1:-0.568499 2:-0.158593 3:-0.146307 4:-0.936162 +1:-0.482001 2:0.0888018 3:-0.432725 4:-0.884982 +1:-0.130324 2:-0.236026 3:0.00438575 4:-0.402801 +1:-0.16113 2:-0.0164637 3:-0.0764866 4:-0.296066 +1:-0.545273 2:-0.0547799 3:-0.24914 4:-0.929202 +1:-0.333158 2:0.0277946 3:-0.289518 4:-0.627866 +1:-0.694601 2:0.32041 3:-0.709972 4:-1 +1:-0.171412 2:-0.0639559 3:-0.341306 4:-0.646528 +1:-0.288119 2:-0.145931 3:-0.151453 4:-0.689543 +1:-0.567485 2:-0.0214655 3:-0.469706 4:-1 +1:-0.274497 2:-0.0817959 3:-0.127715 4:-0.57766 +1:-0.222165 2:-0.123938 3:-0.107052 4:-0.501883 +1:-0.63556 2:-0.0304996 3:-0.21709 4:-0.96185 +1:-0.263907 2:-0.270086 3:-0.206346 4:-0.881827 +1:-0.329857 2:0.125672 3:-0.603152 4:-0.94389 +1:-0.52841 2:-0.408594 3:0.0378039 4:-1 +1:-0.3283 2:0.0703687 3:-0.543791 4:-0.975277 +1:-0.185736 2:-0.39274 3:-0.225261 4:-0.922014 +1:-0.0812954 2:-0.569375 3:-0.0618974 4:-0.843028 +1:-0.18971 2:0.153292 3:-0.360989 4:-0.486108 +1:-0.0332743 2:-0.251767 3:-0.320512 4:-0.67299 +1:-0.209613 2:-0.291007 3:-0.264777 4:-0.883726 +1:-0.244126 2:-0.194934 3:-0.0551019 4:-0.566902 +1:-0.554073 2:0.21071 3:-0.340283 4:-0.854422 +1:-0.600034 2:0.221186 3:-0.187315 4:-0.702594 +1:-0.322965 2:-0.499154 3:-0.0695594 4:-1 +1:-0.444327 2:-0.365423 3:0.0593379 4:-0.86139 +1:-0.426601 2:0.113287 3:-0.322773 4:-0.805558 +1:-0.438127 2:-0.0873971 3:0.0263828 4:-0.534468 +1:-0.312432 2:-0.135545 3:-0.215903 4:-0.841644 +1:-0.453883 2:0.000582457 3:-0.242186 4:-0.760078 +1:-0.427638 2:-0.0240033 3:-0.375181 4:-0.956359 +1:-0.417841 2:-0.0190905 3:-0.362242 4:-0.893682 +1:-0.104433 2:-0.168838 3:-0.392533 4:-0.87841 +1:-0.289564 2:-0.466871 3:-0.260609 4:-1 +1:-0.282569 2:-0.390085 3:-0.255628 4:-1 +1:-0.352591 2:0.0911422 3:-0.411165 4:-0.859423 +1:-0.349019 2:-0.468962 3:0.0246219 4:-0.959282 +1:-0.343305 2:-0.262675 3:-0.12669 4:-0.79782 +1:-0.142304 2:-0.253144 3:-0.0221618 4:-0.553995 +1:-0.436125 2:-0.238993 3:-0.252318 4:-0.920347 +1:-0.396367 2:-0.534014 3:0.0236693 4:-0.51212 +1:-0.635288 2:0.255871 3:-0.509447 4:-0.919517 +1:-0.0343658 2:-0.487103 3:-0.211834 4:-0.718621 +1:-0.643266 2:0.136335 3:-0.327353 4:-0.820622 +1:-0.538525 2:-0.0791299 3:-0.358196 4:-0.390718 +1:-0.491511 2:-0.11597 3:-0.840466 4:-0.976941 +1:-0.733782 2:0.117297 3:-0.410326 4:-0.742164 +1:-0.0704363 2:-0.235146 3:-0.577349 4:-0.640505 +1:-0.394503 2:-0.397932 3:0.11125 4:-0.509553 +1:-0.389828 2:-0.378549 3:0.037514 4:-0.446551 +1:-0.393829 2:-0.183593 3:-0.252629 4:-0.458683 +1:-0.499239 2:-0.223742 3:-0.334154 4:-0.48273 +1:0.150296 2:-0.316282 3:-0.576374 4:-0.381701 +1:-0.104522 2:-0.204463 3:-0.551027 4:-0.231344 +1:-0.466754 2:-0.15082 3:-0.171447 4:-0.478935 +1:-0.683468 2:0.0246187 3:-0.780573 4:-0.844379 +1:-0.291463 2:0.211748 3:-0.387846 4:-0.287327 +1:-0.310068 2:-0.14654 3:-0.359097 4:-0.543201 +1:-0.305579 2:0.14851 3:-0.780848 4:-0.489189 +1:-0.171441 2:-0.020183 3:-0.698418 4:-0.485641 +1:0.0615353 2:-0.630009 3:-0.594986 4:-0.330867 +1:-0.282962 2:-0.022968 3:-0.412351 4:-0.497216 +1:-0.543731 2:0.0350326 3:-0.950588 4:-0.575527 +1:-0.444571 2:0.223595 3:-0.427098 4:-0.420705 +1:-0.215748 2:0.0779516 3:-0.748491 4:-0.524304 +1:-0.340038 2:-0.641593 3:-0.584261 4:-0.37522 +1:0.163617 2:-0.548212 3:-0.888108 4:-0.744835 +1:-0.954954 2:0.520781 3:-1 4:-0.846116 +1:-0.339677 2:0.174906 3:-0.45213 4:-0.271053 +1:-0.441654 2:-0.071763 3:-0.361458 4:-0.213256 +1:0.330701 2:-0.589393 3:-1 4:-0.740704 +1:-0.122424 2:0.0702136 3:-0.409015 4:-0.35168 +1:0.0219236 2:-0.37177 3:-0.588506 4:-0.34401 +1:-0.650582 2:0.242361 3:-0.840379 4:-0.796889 +1:-0.459116 2:-0.159763 3:-0.532548 4:-0.382664 +1:-0.309893 2:-0.0808202 3:-0.203439 4:-0.276299 +1:-0.520991 2:0.179469 3:-0.698337 4:-0.659458 +1:-0.308625 2:0.156155 3:-0.70922 4:-0.591169 +1:0.00409851 2:-0.46017 3:-0.857547 4:-0.963621 +1:-0.0844761 2:-0.155517 3:-0.663523 4:-0.332291 +1:-0.0841 2:-0.11971 3:-0.974918 4:-0.394208 +1:-0.262351 2:-0.316753 3:-0.483164 4:-0.569823 +1:-0.118437 2:-0.113335 3:-0.986905 4:-0.68765 +1:0.0508571 2:-0.269225 3:-0.874132 4:-0.560584 +1:-0.553126 2:0.103179 3:-0.41274 4:-0.443214 +1:-0.423614 2:0.11974 3:-1 4:-0.574738 +1:-0.378519 2:0.015876 3:-0.458785 4:-0.326637 +1:-0.261499 2:-0.00493839 3:-0.98122 4:-0.787822 +1:-0.0810447 2:-0.532863 3:-0.635194 4:-0.408665 +1:-0.256255 2:-0.265361 3:-0.620176 4:-0.584284 +1:-0.143259 2:-0.193391 3:-0.58992 4:-0.54372 +1:-0.382521 2:-0.0355449 3:-0.33708 4:-0.736856 +1:0.281726 2:-0.265694 3:-0.59252 4:-0.55474 +1:-0.304934 2:-0.00441446 3:-1 4:-0.836136 +1:-0.250816 2:-0.0317174 3:-0.532062 4:-0.459771 +1:-0.344937 2:-0.100792 3:-0.900969 4:-0.579398 +1:-0.547763 2:-0.0133462 3:-0.60266 4:-0.809541 +1:-0.315335 2:0.392429 3:-1 4:-1 +1:-0.278889 2:-0.169191 3:-0.667962 4:-0.689428 +1:-0.146792 2:-0.256358 3:-0.446656 4:-0.47379 +1:-0.29712 2:0.00718694 3:-0.718843 4:-0.754011 +1:-0.0724388 2:-0.521581 3:-0.479028 4:-0.804766 +1:0.123569 2:-0.318798 3:-0.97531 4:-0.841628 +1:0.315502 2:-0.385833 3:-0.545819 4:-0.531976 +1:-0.182074 2:0.0612207 3:-0.577368 4:-0.748809 +1:0.107323 2:-0.568775 3:-1 4:-0.648099 +1:-0.489962 2:0.268536 3:-0.86785 4:-0.942582 +1:-0.00641538 2:-0.126046 3:-0.723849 4:-0.836023 +1:0.198346 2:-0.114265 3:-0.770654 4:-0.425431 +1:-0.457326 2:-0.574967 3:-0.760497 4:-0.485031 +1:-0.221215 2:0.0350131 3:-0.292022 4:-0.32608 +1:-0.206935 2:0.166136 3:-0.610547 4:-0.817009 +1:-0.2475 2:0.194735 3:-0.926025 4:-1 +1:-0.58266 2:0.186739 3:-0.564285 4:-0.949349 +1:0.0993801 2:-0.537592 3:-0.504456 4:-0.763863 +1:-0.129347 2:-0.172437 3:-0.630351 4:-0.545082 +1:0.182889 2:-0.703352 3:-0.694029 4:-0.103985 +1:-0.262172 2:0.0583705 3:-0.842069 4:-0.667894 +1:-0.222111 2:-0.194075 3:-0.6915 4:-0.893776 +1:-0.497969 2:0.398013 3:-0.8343 4:-0.878466 +1:0.348605 2:-0.811584 3:-0.35032 4:-0.43514 +1:0.330014 2:-0.895142 3:-0.693135 4:-0.424315 +1:-0.133905 2:-0.29384 3:-0.520816 4:-0.889265 +1:0.11481 2:-0.491473 3:-0.521661 4:-0.39452 +1:-0.0259395 2:0.101144 3:-0.811918 4:-0.97929 +1:0.456729 2:-0.67853 3:-0.706026 4:-0.848252 +1:-0.676612 2:0.309699 3:-0.654152 4:-1 +1:-0.381953 2:-0.164115 3:-0.383862 4:-0.937237 +1:-0.131852 2:-0.50377 3:-0.46675 4:-0.868593 +1:-0.169531 2:-0.261368 3:-0.543621 4:-0.595914 +1:0.133297 2:-0.84865 3:-0.413621 4:-0.486257 +1:-0.561325 2:-0.29794 3:-0.148097 4:-0.803002 +1:-0.265047 2:0.00378962 3:-0.986875 4:-1 +1:0.297124 2:-0.587671 3:-0.584551 4:-0.838548 +1:0.517689 2:-0.915565 3:-0.310668 4:-0.824054 +1:-0.196412 2:-0.341848 3:-0.569104 4:-1 +1:0.212784 2:-0.811347 3:-0.33041 4:-0.620113 +1:0.171136 2:-0.483529 3:-0.26566 4:-0.904956 +1:0.446919 2:-0.633689 3:-0.380045 4:-0.70385 +1:-0.42785 2:0.180281 3:-0.809184 4:-1 +1:-0.572782 2:-0.279698 3:-0.583997 4:-1 +1:-0.0525168 2:-0.654494 3:-0.36459 4:-0.785851 +1:-0.0115758 2:-0.0661589 3:-0.331925 4:-0.831282 +1:0.11582 2:-0.321707 3:-0.517671 4:-0.565778 +1:0.320847 2:-0.944211 3:-0.448408 4:-0.749803 +1:-0.574683 2:-0.03468 3:-0.254091 4:-1 +1:-0.182987 2:-0.329121 3:-0.667682 4:-0.789233 +1:0.356917 2:-0.663049 3:-0.617956 4:-0.980681 +1:0.0166878 2:-0.478273 3:-0.208384 4:-0.776117 +1:-0.373244 2:-0.0256195 3:-0.522616 4:-0.615158 +1:-0.386465 2:0.232348 3:-0.767628 4:-0.547699 +1:-0.203187 2:-0.173653 3:-0.338196 4:-0.255522 +1:-0.56532 2:-0.0192809 3:-0.338688 4:-0.649031 +1:-0.0495954 2:-0.268217 3:-1 4:-0.770285 +1:-0.444623 2:0.350846 3:-0.790835 4:-0.823203 +1:0.709609 2:-1 3:-0.784039 4:-0.701989 +1:0.102197 2:-0.299138 3:-0.729306 4:-0.705823 +1:0.449622 2:-0.657347 3:-0.765816 4:-0.749573 +1:-0.104932 2:-0.315354 3:-0.550304 4:-0.67023 +1:-0.242888 2:-0.164289 3:-0.421334 4:-0.631106 +1:-0.136763 2:-0.442525 3:-0.374938 4:-0.410609 +1:-0.439448 2:0.411794 3:-0.906512 4:-0.992516 +1:-0.599213 2:0.180969 3:-0.659563 4:-0.516868 +1:-0.199883 2:-0.0474712 3:-0.749272 4:-1 +1:0.020369 2:-0.703609 3:-0.913104 4:-0.729336 +1:-0.264112 2:-0.187519 3:-0.801564 4:-1 +1:-0.498707 2:-0.267089 3:-0.726099 4:-0.684924 +1:-0.153991 2:-0.375729 3:-0.751574 4:-0.504108 +1:0.21962 2:-0.658282 3:-0.70589 4:-0.751924 +1:-0.0292164 2:0.044782 3:-0.787562 4:-0.4663 +1:-0.158567 2:-0.0433856 3:-0.423977 4:-0.427981 +1:-0.0114879 2:-0.0667031 3:-0.490902 4:-0.587065 +1:-0.132182 2:0.305692 3:-0.756668 4:-0.723379 +1:-0.316974 2:-0.397921 3:-0.861741 4:-0.844322 +1:0.0335618 2:-0.422043 3:-0.35025 4:-0.38795 +1:-0.293708 2:0.149864 3:-0.943168 4:-0.845312 +1:-0.099963 2:0.0474417 3:-0.821293 4:-0.99489 +1:-0.0361955 2:-0.209806 3:-0.783435 4:-0.85738 diff --git a/test/pred-sets/ref/readable_model_privacy.txt b/test/pred-sets/ref/readable_model_privacy.txt index fdf0da58096..c26b82a2949 100644 --- a/test/pred-sets/ref/readable_model_privacy.txt +++ b/test/pred-sets/ref/readable_model_privacy.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:0 Max label:1 diff --git a/test/pred-sets/ref/readable_model_privacy_no_tags.txt b/test/pred-sets/ref/readable_model_privacy_no_tags.txt index 9be0914d342..ca033ca229a 100644 --- a/test/pred-sets/ref/readable_model_privacy_no_tags.txt +++ b/test/pred-sets/ref/readable_model_privacy_no_tags.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:0 Max label:1 diff --git a/test/pred-sets/ref/sender.predict b/test/pred-sets/ref/sender.predict index 06c6a61af1d..c97c94567df 100644 --- a/test/pred-sets/ref/sender.predict +++ b/test/pred-sets/ref/sender.predict @@ -1,400 +1,400 @@ 0 1 -0.288508 +0.288482 0 -0.250514 +0.250493 0 -0.151695 +0.151673 0.002511 -0.134052 +0.134034 0 -0.177498 -0.957237 -0.280578 +0.177472 +0.957236 +0.280554 0 -0.320239 -0.021493 -0.306340 +0.320235 +0.021485 +0.306320 0 -0.360428 -0.956512 -0.352446 -0.011309 -0.398080 +0.360417 +0.956504 +0.352442 +0.011304 +0.398068 0.005108 -0.331043 +0.331047 0 -0.293828 -0.028434 -0.312265 -0.973408 -0.512433 -0.971227 -0.444528 -1 -0.493354 +0.293829 +0.028433 +0.312264 +0.973404 +0.512425 +0.971219 +0.444524 +1 +0.493357 0 -0.467740 -0.018804 +0.467731 +0.018800 0.338733 -0.011109 -0.286673 -0.999976 -0.576985 -0.997922 -0.441825 +0.011108 +0.286654 +0.999978 +0.576995 +0.997913 +0.441811 0.017785 -0.514455 +0.514435 1 -0.414803 -0.057054 -0.489435 +0.414789 +0.057045 +0.489434 0 -0.359511 +0.359506 0.005750 -0.234042 -0.013142 -0.512847 +0.234048 +0.013139 +0.512838 1 -0.470552 -0.004585 -0.432580 +0.470541 +0.004584 +0.432584 1 -0.359317 -0.030877 -0.356782 +0.359313 +0.030873 +0.356773 0 -0.326604 -0.019714 -0.507774 -0.985433 +0.326608 +0.019715 +0.507784 +0.985428 0.374016 0.046952 -0.356173 +0.356187 1 -0.364036 +0.364034 0.009751 -0.498992 +0.498996 1 -0.646861 -0.978727 -0.438940 -0.052686 -0.482200 +0.646868 +0.978726 +0.438925 +0.052682 +0.482204 1 -0.505152 +0.505158 0 -0.440954 -0.012698 -0.276929 +0.440955 +0.012696 +0.276928 0.032812 -0.238195 -0.029079 -0.553104 +0.238187 +0.029081 +0.553100 0 -0.489563 +0.489579 0 -0.200131 -0.986413 -0.333675 +0.200139 +0.986409 +0.333667 0.025545 -0.344618 +0.344601 1 -0.348417 +0.348431 1 -0.389359 -0.015633 -0.381550 +0.389364 +0.015632 +0.381547 0 -0.565336 -0.928408 -0.714866 +0.565352 +0.928394 +0.714859 0 -0.471844 -0.002430 -0.232933 -0.025921 -0.474027 +0.471852 +0.002428 +0.232935 +0.025920 +0.474036 1 -0.385437 +0.385427 0 -0.445614 -0.960198 -0.409115 -0.048473 -0.674530 -0.851986 -0.179469 -0.039299 -0.578360 -1 -0.324287 -0.030912 -0.528002 -0.031910 -0.604348 +0.445596 +0.960193 +0.409131 +0.048471 +0.674505 +0.851988 +0.179476 +0.039298 +0.578353 +1 +0.324293 +0.030911 +0.528006 +0.031911 +0.604358 0 -0.212094 -0.015319 -0.513387 +0.212106 +0.015320 +0.513400 1 -0.489408 +0.489422 0 -0.549949 -0.928918 -0.403647 -0.986906 -0.370360 -0.026897 -0.648234 -0.994358 -0.595642 -1 -0.533795 -0.035393 -0.230522 -0.020537 -0.276292 -0.006500 -0.296054 +0.549956 +0.928920 +0.403661 +0.986905 +0.370362 +0.026893 +0.648237 +0.994359 +0.595641 +1 +0.533790 +0.035392 +0.230512 +0.020533 +0.276283 +0.006497 +0.296061 0 -0.383285 +0.383290 0 -0.403020 +0.403010 0 -0.663961 +0.663962 0.993558 -0.382835 -0.011096 -0.287343 +0.382830 +0.011092 +0.287340 0 -0.354519 -0.028469 -0.289691 -1 -0.668551 -0.911740 -0.708026 -1 -0.108328 -0.008735 -0.311773 -0.017494 -0.449976 -0.941561 -0.985932 -1 -0.446070 -0.026253 -0.771832 -0.919864 -0.336414 -0.008072 -0.482719 -1 -0.400305 -0.064569 -0.316067 -0.995160 -0.413323 -1 -0.794345 -0.014183 -0.569145 -1 -0.497119 -0.024257 -0.611813 -1 -0.629713 -0.042556 -0.694271 -0.991941 -0.232949 +0.354508 +0.028472 +0.289669 +1 +0.668534 +0.911748 +0.708047 +1 +0.108341 +0.008736 +0.311765 +0.017493 +0.449968 +0.941564 +0.985946 +1 +0.446075 +0.026250 +0.771836 +0.919862 +0.336428 +0.008061 +0.482717 +1 +0.400304 +0.064562 +0.316070 +0.995163 +0.413338 +1 +0.794339 +0.014176 +0.569149 +1 +0.497126 +0.024256 +0.611803 +1 +0.629694 +0.042560 +0.694284 +0.991942 +0.232943 0.063728 -0.523770 +0.523783 0 -0.341603 -0.020168 -0.516203 +0.341599 +0.020167 +0.516199 1 -0.707363 -0.999005 -0.619874 +0.707353 +0.999003 +0.619867 0.031757 -0.527772 +0.527768 0 -0.287730 -0.976550 -0.718690 -0.108304 -0.349455 -0.015857 -0.322101 -0.947709 -0.531162 -1 -0.241623 -1 -0.348387 -0.010886 -0.472681 +0.287716 +0.976538 +0.718659 +0.108298 +0.349460 +0.015856 +0.322087 +0.947705 +0.531168 +1 +0.241612 +1 +0.348392 +0.010885 +0.472672 0 -0.626485 +0.626489 0.985987 -0.493342 +0.493333 0.042265 -0.498152 -1 -0.674395 -1 -0.719245 -0.867909 -0.526065 -0.017191 -0.822100 -0.957554 -0.199258 -0.008818 -0.502046 -0.988092 -0.576769 +0.498154 +1 +0.674402 +1 +0.719239 +0.867904 +0.526048 +0.017190 +0.822096 +0.957552 +0.199232 +0.008819 +0.502025 +0.988091 +0.576745 0.024048 -0.760615 -0.975184 -0.402406 -0.117217 -0.302041 -0.997807 -0.405981 +0.760621 +0.975182 +0.402392 +0.117212 +0.302033 +0.997803 +0.405991 0 -0.291791 -0.015396 -0.731404 +0.291779 +0.015397 +0.731399 1 -0.577920 +0.577926 1 -0.998343 -0.996419 -0.342021 -0.075457 -0.415734 +0.998324 +0.996421 +0.342035 +0.075455 +0.415724 0.012389 -0.157192 -0.055317 -0.970037 -0.999351 +0.157194 +0.055313 +0.970023 +0.999352 1 1 -0.492518 -0.996121 -0.732147 -0.971602 -0.607432 +0.492520 +0.996122 +0.732128 +0.971604 +0.607451 1 -0.914888 +0.914919 1 -0.285074 +0.285091 0 -0.422229 +0.422244 1 -0.806581 -0.963280 -0.530082 +0.806578 +0.963283 +0.530077 1 -0.831109 -0.992994 -0.495351 -0.058605 -0.481956 +0.831100 +0.992996 +0.495363 +0.058607 +0.481987 0 -0.731975 +0.731953 0.949364 0.995066 0.994018 -0.489652 -0.004964 +0.489638 +0.004966 1 1 -0.659397 -0.021875 -0.851831 -0.965963 -0.192551 +0.659392 +0.021879 +0.851817 +0.965959 +0.192552 0.002801 -0.291766 -0.029433 -0.698994 +0.291760 +0.029430 +0.699006 1 -0.498490 -0.052753 -0.611128 -0.972270 -0.853332 +0.498487 +0.052746 +0.611118 +0.972271 +0.853321 1 -0.297762 -0.056929 -0.537612 +0.297752 +0.056926 +0.537604 1 -0.712747 +0.712756 1 -0.850294 +0.850312 1 -0.372505 -0.018415 -0.301002 -0.047542 -0.500464 -0.950551 -0.163177 -0.126829 -0.510671 -0.060938 -0.264238 -0.071682 +0.372504 +0.018416 +0.300993 +0.047543 +0.500450 +0.950540 +0.163170 +0.126825 +0.510676 +0.060937 +0.264225 +0.071681 1 1 -0.636008 +0.636011 1 0.717034 -0.988209 -0.502779 +0.988208 +0.502789 1 -0.386972 -0.218363 -0.677763 +0.386929 +0.218354 +0.677748 0.970817 -0.194353 -0.026545 -0.303089 -0.058657 -0.354216 -0.050919 -0.768487 +0.194346 +0.026547 +0.303076 +0.058656 +0.354197 +0.050921 +0.768505 0.999324 -0.313592 -0.002586 -0.300919 +0.313585 +0.002587 +0.300918 0 -0.693304 -0.975533 -0.604771 +0.693315 +0.975528 +0.604738 1 -0.401076 -0.170704 -0.409054 +0.401068 +0.170703 +0.409061 0.015167 -0.375616 -0.012087 -0.256892 -0.095069 -0.479701 -1 -0.640061 -0.979546 -0.243934 +0.375611 +0.012092 +0.256909 +0.095067 +0.479721 +1 +0.640047 +0.979545 +0.243953 0.000343 -0.467800 -0.009208 -0.483232 +0.467795 +0.009209 +0.483257 1 diff --git a/test/pred-sets/ref/slates_w_interactions.inv b/test/pred-sets/ref/slates_w_interactions.inv index aa061863ed3..c848e60caf4 100644 --- a/test/pred-sets/ref/slates_w_interactions.inv +++ b/test/pred-sets/ref/slates_w_interactions.inv @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:0 Max label:0.8 diff --git a/test/pred-sets/ref/t288.readable b/test/pred-sets/ref/t288.readable index 884a8c3cfb9..0c77063ee6e 100644 --- a/test/pred-sets/ref/t288.readable +++ b/test/pred-sets/ref/t288.readable @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:0 Max label:1 diff --git a/test/run_tests.py b/test/run_tests.py index 6694bc2c397..38ac131db1c 100644 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -770,7 +770,7 @@ def convert_tests_for_flatbuffers( ) continue # todo: 300 understand why is it failing - # test 189, 312, 316, 318, 351, 438 and 319 depend on dsjson parser behaviour + # test 189, 312, 316, 318, 351, 438, 394 and 319 depend on dsjson parser behaviour # they can be enabled if we ignore diffing the --extra_metrics # (324-326) deals with corrupted data, so cannot be translated to fb # pdrop is not supported in fb, so 327-331 are excluded @@ -795,6 +795,7 @@ def convert_tests_for_flatbuffers( "337", "338", "351", + "394", "399", "400", "404", @@ -926,10 +927,11 @@ def get_test(test_number: int, tests: List[TestData]) -> Optional[TestData]: def interpret_test_arg(arg: str, *, num_tests: int) -> List[int]: single_number_pattern = re.compile(r"^\d+$") range_pattern = re.compile(r"^(\d+)?\.\.(\d+)?$") + range_pattern_match = range_pattern.match(arg) if single_number_pattern.match(arg): return [int(arg)] - elif range_pattern.match(arg): - start, end = range_pattern.match(arg).groups() + elif range_pattern_match: + start, end = range_pattern_match.groups() start = int(start) if start else 1 end = int(end) if end else num_tests if start > end: diff --git a/test/save_resume_test.py b/test/save_resume_test.py index 6725a6a091b..c7e438e6bde 100644 --- a/test/save_resume_test.py +++ b/test/save_resume_test.py @@ -185,8 +185,8 @@ def do_test(filename, args, verbose=None, repeat_args=None, known_failure=False) errors += do_test(filename, "--max_prediction 0.1") errors += do_test(filename, "--hash_seed 10") errors += do_test(filename, "--loss_function logistic") - errors += do_test(filename, "--boosting 10") - errors += do_test(filename, "--bootstrap 10") + # errors += do_test(filename, "--boosting 10") TODO: save_load params_per_model + # errors += do_test(filename, "--bootstrap 10") TODO: save_load params_per_model errors += do_test(filename, "--l1 1e-04") errors += do_test(filename, "--l2 1e-04") errors += do_test(filename, "--learning_rate 0.1") @@ -199,7 +199,7 @@ def do_test(filename, args, verbose=None, repeat_args=None, known_failure=False) errors += do_test(filename, "--normalized") errors += do_test(filename, "--invariant") errors += do_test(filename, "--loss_function logistic --link logistic") - errors += do_test(filename, "--nn 2") + # errors += do_test(filename, "--nn 2") TODO: save_load params_per_model errors += do_test(filename, "--binary") errors += do_test(filename, "--ftrl") errors += do_test(filename, "--pistol") diff --git a/test/test-sets/ref/active_cover.stderr b/test/test-sets/ref/active_cover.stderr index 02303d08a64..08f0a3a7663 100644 --- a/test/test-sets/ref/active_cover.stderr +++ b/test/test-sets/ref/active_cover.stderr @@ -5,7 +5,7 @@ Reading datafile = test-sets/rcv1_small_test.data num sources = 1 Num weight bits = 18 learning rate = 0.5 -initial_t = 217.709 +initial_t = 222.626 power_t = 0.5 Enabled learners: gd, active_cover, scorer-identity, binary, count_label Input label = SIMPLE @@ -16,11 +16,11 @@ loss last counter weight label predict feat 0.500000 1.000000 2 2.0 1.0000 -1.0000 147 0.250000 0.000000 4 4.0 -1.0000 -1.0000 142 0.250000 0.250000 8 8.0 1.0000 1.0000 22 -0.125000 0.000000 16 16.0 -1.0000 -1.0000 50 -0.156250 0.187500 32 32.0 1.0000 1.0000 85 -0.140625 0.125000 64 64.0 1.0000 -1.0000 190 -0.140625 0.140625 128 128.0 1.0000 1.0000 134 -0.156250 0.171875 256 256.0 1.0000 1.0000 54 +0.187500 0.125000 16 16.0 -1.0000 -1.0000 50 +0.125000 0.062500 32 32.0 1.0000 1.0000 85 +0.109375 0.093750 64 64.0 1.0000 -1.0000 190 +0.132812 0.156250 128 128.0 1.0000 1.0000 134 +0.160156 0.187500 256 256.0 1.0000 1.0000 54 finished run number of examples = 500 diff --git a/test/test-sets/ref/aml_ccb_metrics.json b/test/test-sets/ref/aml_ccb_metrics.json index 0ea3a354613..f4babf6ece2 100644 --- a/test/test-sets/ref/aml_ccb_metrics.json +++ b/test/test-sets/ref/aml_ccb_metrics.json @@ -1 +1 @@ -{"cbea_label_first_action":1,"cbea_label_not_first":3,"cbea_labeled_ex":4,"cbea_max_actions":2,"cbea_min_actions":2,"cbea_non_zero_cost":0,"cbea_predict_in_learn":0,"current_champ":0,"sfm_count_learn_example_with_shared":4,"test_county":4,"total_champ_switches":0,"total_learn_calls":4,"total_log_calls":0,"total_predict_calls":0,"cbea_avg_actions_per_event":2.0,"cbea_avg_feat_per_action":0.0,"cbea_avg_feat_per_event":0.0,"cbea_avg_ns_per_action":0.0,"cbea_avg_ns_per_event":0.0,"cbea_sum_cost":0.0,"cbea_sum_cost_baseline":0.0,"estimator_0":{"config_type":"exclusion","elements":"{}","self":{"conf_idx":0,"upcnt":0,"last_r":0.0,"last_w":0.0,"lb":0.0,"ub":1.0,"interactions":"-q AA -q AS -q AU -q SS -q SU -q UU --cubic=AA[ccbid] -q AA --cubic=AS[ccbid] -q AS --cubic=AU[ccbid] -q AU --cubic=SS[ccbid] -q SS --cubic=SU[ccbid] -q SU --cubic=UU[ccbid] -q UU -q [wild][ccbid] "}},"estimator_1":{"config_type":"exclusion","elements":"{[\"U\", \"U\"]}","self":{"conf_idx":6,"upcnt":4,"last_r":-0.0,"last_w":0.0,"lb":0.0,"ub":1.0,"interactions":"-q AA -q AS -q AU -q SS -q SU --cubic=AA[ccbid] -q AA --cubic=AS[ccbid] -q AS --cubic=AU[ccbid] -q AU --cubic=SS[ccbid] -q SS --cubic=SU[ccbid] -q SU -q [wild][ccbid] "},"sync_champ":{"upcnt":4,"last_r":-0.0,"last_w":0.0,"lb":0.0,"ub":1.0}},"estimator_2":{"config_type":"exclusion","elements":"{[\"S\", \"U\"]}","self":{"conf_idx":5,"upcnt":4,"last_r":-0.0,"last_w":0.0,"lb":0.0,"ub":1.0,"interactions":"-q AA -q AS -q AU -q SS -q UU --cubic=AA[ccbid] -q AA --cubic=AS[ccbid] -q AS --cubic=AU[ccbid] -q AU --cubic=SS[ccbid] -q SS --cubic=UU[ccbid] -q UU -q [wild][ccbid] "},"sync_champ":{"upcnt":4,"last_r":-0.0,"last_w":0.0,"lb":0.0,"ub":1.0}}} \ No newline at end of file +{"cbea_label_first_action":14,"cbea_label_not_first":4,"cbea_labeled_ex":18,"cbea_max_actions":3,"cbea_min_actions":1,"cbea_non_zero_cost":18,"cbea_predict_in_learn":0,"current_champ":0,"dsjson_number_label_equal_baseline_first_slot":0,"dsjson_number_label_not_equal_baseline_first_slot":0,"line_parse_error":0,"number_events_zero_actions":0,"number_skipped_events":0,"sfm_count_learn_example_with_shared":18,"test_county":18,"total_champ_switches":0,"total_learn_calls":9,"total_log_calls":0,"total_predict_calls":0,"cbea_avg_actions_per_event":2.0,"cbea_avg_feat_per_action":0.0,"cbea_avg_feat_per_event":0.0,"cbea_avg_ns_per_action":0.0,"cbea_avg_ns_per_event":0.0,"cbea_sum_cost":-16.09790802001953,"cbea_sum_cost_baseline":-12.356508255004883,"dsjson_sum_cost_original":0.0,"dsjson_sum_cost_original_first_slot":0.0,"dsjson_sum_cost_original_label_equal_baseline_first_slot":0.0,"first_event_id":"","first_event_time":"","last_event_id":"","last_event_time":"","estimator_0":{"config_type":"exclusion","elements":"{}","self":{"conf_idx":0,"upcnt":0,"last_r":0.0,"last_w":0.0,"lb":0.0,"ub":1.0,"interactions":"-q [default][default] -q [default]a -q [default]b -q [default]s -q [default]y -q [default][ccbslot] -q [default][ccbid] -q aa -q ab -q as -q ay -q a[ccbslot] -q a[ccbid] -q bb -q bs -q by -q b[ccbslot] -q b[ccbid] -q ss -q sy -q s[ccbslot] -q s[ccbid] -q yy -q y[ccbslot] -q y[ccbid] -q [ccbslot][ccbslot] -q [ccbslot][ccbid] --cubic=[default][default][ccbid] --cubic=[default]a[ccbid] --cubic=[default]b[ccbid] --cubic=[default]s[ccbid] --cubic=[default]y[ccbid] --cubic=[default][ccbslot][ccbid] --cubic=aa[ccbid] --cubic=ab[ccbid] --cubic=as[ccbid] --cubic=ay[ccbid] --cubic=a[ccbslot][ccbid] --cubic=bb[ccbid] --cubic=bs[ccbid] --cubic=by[ccbid] --cubic=b[ccbslot][ccbid] --cubic=ss[ccbid] --cubic=sy[ccbid] --cubic=s[ccbslot][ccbid] --cubic=yy[ccbid] --cubic=y[ccbslot][ccbid] --cubic=[ccbslot][ccbslot][ccbid] "}},"estimator_1":{"config_type":"exclusion","elements":"{[\"s\", \"s\"]}","self":{"conf_idx":10,"upcnt":18,"last_r":1.0,"last_w":1.0,"lb":0.0,"ub":1.0,"interactions":"-q [default][default] -q [default]a -q [default]b -q [default]s -q [default]y -q [default][ccbslot] -q [default][ccbid] -q aa -q ab -q as -q ay -q a[ccbslot] -q a[ccbid] -q bb -q bs -q by -q b[ccbslot] -q b[ccbid] -q sy -q s[ccbslot] -q s[ccbid] -q yy -q y[ccbslot] -q y[ccbid] -q [ccbslot][ccbslot] -q [ccbslot][ccbid] --cubic=[default][default][ccbid] --cubic=[default]a[ccbid] --cubic=[default]b[ccbid] --cubic=[default]s[ccbid] --cubic=[default]y[ccbid] --cubic=[default][ccbslot][ccbid] --cubic=aa[ccbid] --cubic=ab[ccbid] --cubic=as[ccbid] --cubic=ay[ccbid] --cubic=a[ccbslot][ccbid] --cubic=bb[ccbid] --cubic=bs[ccbid] --cubic=by[ccbid] --cubic=b[ccbslot][ccbid] --cubic=sy[ccbid] --cubic=s[ccbslot][ccbid] --cubic=yy[ccbid] --cubic=y[ccbslot][ccbid] --cubic=[ccbslot][ccbslot][ccbid] "},"sync_champ":{"upcnt":18,"last_r":1.0,"last_w":1.0,"lb":0.0,"ub":1.0}},"estimator_2":{"config_type":"exclusion","elements":"{[\"b\", \"s\"]}","self":{"conf_idx":9,"upcnt":18,"last_r":1.0,"last_w":1.0,"lb":0.0,"ub":1.0,"interactions":"-q [default][default] -q [default]a -q [default]b -q [default]s -q [default]y -q [default][ccbslot] -q [default][ccbid] -q aa -q ab -q as -q ay -q a[ccbslot] -q a[ccbid] -q bb -q by -q b[ccbslot] -q b[ccbid] -q ss -q sy -q s[ccbslot] -q s[ccbid] -q yy -q y[ccbslot] -q y[ccbid] -q [ccbslot][ccbslot] -q [ccbslot][ccbid] --cubic=[default][default][ccbid] --cubic=[default]a[ccbid] --cubic=[default]b[ccbid] --cubic=[default]s[ccbid] --cubic=[default]y[ccbid] --cubic=[default][ccbslot][ccbid] --cubic=aa[ccbid] --cubic=ab[ccbid] --cubic=as[ccbid] --cubic=ay[ccbid] --cubic=a[ccbslot][ccbid] --cubic=bb[ccbid] --cubic=by[ccbid] --cubic=b[ccbslot][ccbid] --cubic=ss[ccbid] --cubic=sy[ccbid] --cubic=s[ccbslot][ccbid] --cubic=yy[ccbid] --cubic=y[ccbslot][ccbid] --cubic=[ccbslot][ccbslot][ccbid] "},"sync_champ":{"upcnt":18,"last_r":1.0,"last_w":1.0,"lb":0.0,"ub":1.0}},"estimator_3":{"config_type":"exclusion","elements":"{[\"b\", \"b\"]}","self":{"conf_idx":8,"upcnt":18,"last_r":1.0,"last_w":1.0,"lb":0.0,"ub":1.0,"interactions":"-q [default][default] -q [default]a -q [default]b -q [default]s -q [default]y -q [default][ccbslot] -q [default][ccbid] -q aa -q ab -q as -q ay -q a[ccbslot] -q a[ccbid] -q bs -q by -q b[ccbslot] -q b[ccbid] -q ss -q sy -q s[ccbslot] -q s[ccbid] -q yy -q y[ccbslot] -q y[ccbid] -q [ccbslot][ccbslot] -q [ccbslot][ccbid] --cubic=[default][default][ccbid] --cubic=[default]a[ccbid] --cubic=[default]b[ccbid] --cubic=[default]s[ccbid] --cubic=[default]y[ccbid] --cubic=[default][ccbslot][ccbid] --cubic=aa[ccbid] --cubic=ab[ccbid] --cubic=as[ccbid] --cubic=ay[ccbid] --cubic=a[ccbslot][ccbid] --cubic=bs[ccbid] --cubic=by[ccbid] --cubic=b[ccbslot][ccbid] --cubic=ss[ccbid] --cubic=sy[ccbid] --cubic=s[ccbslot][ccbid] --cubic=yy[ccbid] --cubic=y[ccbslot][ccbid] --cubic=[ccbslot][ccbslot][ccbid] "},"sync_champ":{"upcnt":18,"last_r":1.0,"last_w":1.0,"lb":0.0,"ub":1.0}}} \ No newline at end of file diff --git a/test/test-sets/ref/backwards.stdout b/test/test-sets/ref/backwards.stdout index c2a324e7052..26351d85b87 100644 --- a/test/test-sets/ref/backwards.stdout +++ b/test/test-sets/ref/backwards.stdout @@ -15,48 +15,48 @@ f^c*e^y[2]:20370:0.0914445 e^y:27664:-0.0540374 e^y[1]:27665:-0.0908917 e^y[2]:27666:0.0914445 -f^a:57420:-0.129173 +f^a:57420:-0.129376 f^a[1]:57421:0.0588574 -f^a[2]:57422:-0.0604068 +f^a[2]:57422:-0.059142 f^b:62864:-0.0540374 f^b[1]:62865:-0.0908917 f^b[2]:62866:0.0914445 -f^a*e^x:65348:-0.129173 +f^a*e^x:65348:-0.129376 f^a*e^x[1]:65349:0.0588574 -f^a*e^x[2]:65350:-0.0604068 +f^a*e^x[2]:65350:-0.059142 f^b*e^z:91492:-0.0540374 f^b*e^z[1]:91493:-0.0908917 f^b*e^z[2]:91494:0.0914445 f^a*e^y:97204:-0.0540374 f^a*e^y[1]:97205:-0.0908917 f^a*e^y[2]:97206:0.0914445 -f^c*e^x:117600:-0.129173 +f^c*e^x:117600:-0.129376 f^c*e^x[1]:117601:0.0588574 -f^c*e^x[2]:117602:-0.0604068 -e^x:125152:-0.129173 +f^c*e^x[2]:117602:-0.059142 +e^x:125152:-0.129376 e^x[1]:125153:0.0588574 -e^x[2]:125154:-0.0604068 -f^y:154732:-0.111391 +e^x[2]:125154:-0.059142 +f^y:154732:-0.111593 f^y[1]:154733:0.185667 -f^y[2]:154734:-0.18919 -f^y*e^x:169700:-0.111391 +f^y[2]:154734:-0.187625 +f^y*e^x:169700:-0.111593 f^y*e^x[1]:169701:0.185667 -f^y*e^x[2]:169702:-0.18919 +f^y*e^x[2]:169702:-0.187625 f^a*e^z:189296:-0.0540374 f^a*e^z[1]:189297:-0.0908917 f^a*e^z[2]:189298:0.0914445 f^b*e^y:196000:-0.0540374 f^b*e^y[1]:196001:-0.0908917 f^b*e^y[2]:196002:0.0914445 -Constant:202096:-0.129173 +Constant:202096:-0.129376 Constant[1]:202097:0.0588574 -Constant[2]:202098:-0.0604068 +Constant[2]:202098:-0.059142 f^b*e^x:227664:-0.0540374 f^b*e^x[1]:227665:-0.0908917 f^b*e^x[2]:227666:0.0914445 -f^c:228992:-0.129173 +f^c:228992:-0.129376 f^c[1]:228993:0.0588574 -f^c[2]:228994:-0.0604068 +f^c[2]:228994:-0.059142 f^c*e^z:251732:-0.0540374 f^c*e^z[1]:251733:-0.0908917 f^c*e^z[2]:251734:0.0914445 diff --git a/test/test-sets/ref/metrics_qc.json b/test/test-sets/ref/metrics_qc.json index 0c796aa083b..19a3aa0a506 100644 --- a/test/test-sets/ref/metrics_qc.json +++ b/test/test-sets/ref/metrics_qc.json @@ -1 +1 @@ -{"cbea_label_first_action":0,"cbea_label_not_first":3,"cbea_labeled_ex":3,"cbea_max_actions":12,"cbea_min_actions":12,"cbea_non_zero_cost":2,"cbea_predict_in_learn":0,"current_champ":0,"line_parse_error":0,"number_events_zero_actions":0,"number_skipped_events":0,"sfm_count_learn_example_with_shared":3,"test_county":3,"total_champ_switches":0,"total_learn_calls":3,"total_log_calls":1,"total_predict_calls":0,"cbea_avg_actions_per_event":12.0,"cbea_avg_feat_per_action":835.0,"cbea_avg_feat_per_event":10030.0,"cbea_avg_ns_per_action":9.0,"cbea_avg_ns_per_event":117.0,"cbea_sum_cost":-2.0,"cbea_sum_cost_baseline":0.0,"dsjson_sum_cost_original":0.0,"dsjson_sum_cost_original_baseline":0.0,"first_event_id":"0074434d3a3a46529f65de8a59631939","first_event_time":"","last_event_id":"9077f996581148978a0ebe2484260dab","last_event_time":"","estimator_0":{"config_type":"inclusion","elements":"{}","self":{"conf_idx":0,"upcnt":0,"last_r":0.0,"last_w":0.0,"lb":0.0,"ub":1.0,"interactions":"-q EE -q EG -q EM -q EO -q ER -q ES -q ET -q Ei -q Ej -q GG -q GM -q GO -q GR -q GS -q GT -q Gi -q Gj -q MM -q MO -q MR -q MS -q MT -q Mi -q Mj -q OO -q OR -q OS -q OT -q Oi -q Oj -q RR -q RS -q RT -q Ri -q Rj -q SS -q ST -q Si -q Sj -q TT -q Ti -q Tj -q ii -q ij -q jj "}},"estimator_1":{"config_type":"inclusion","elements":"{[\"E\", \"j\", \"j\"]}","self":{"conf_idx":99,"upcnt":3,"last_r":1.0,"last_w":0.0,"lb":0.0,"ub":1.0,"interactions":"--cubic=Ejj -q EE -q EG -q EM -q EO -q ER -q ES -q ET -q Ei -q Ej -q GG -q GM -q GO -q GR -q GS -q GT -q Gi -q Gj -q MM -q MO -q MR -q MS -q MT -q Mi -q Mj -q OO -q OR -q OS -q OT -q Oi -q Oj -q RR -q RS -q RT -q Ri -q Rj -q SS -q ST -q Si -q Sj -q TT -q Ti -q Tj -q ii -q ij -q jj "},"sync_champ":{"upcnt":3,"last_r":1.0,"last_w":0.0,"lb":0.0,"ub":1.0}},"estimator_2":{"config_type":"inclusion","elements":"{[\"E\", \"i\", \"j\"]}","self":{"conf_idx":98,"upcnt":3,"last_r":1.0,"last_w":0.0,"lb":0.0,"ub":1.0,"interactions":"--cubic=Eij -q EE -q EG -q EM -q EO -q ER -q ES -q ET -q Ei -q Ej -q GG -q GM -q GO -q GR -q GS -q GT -q Gi -q Gj -q MM -q MO -q MR -q MS -q MT -q Mi -q Mj -q OO -q OR -q OS -q OT -q Oi -q Oj -q RR -q RS -q RT -q Ri -q Rj -q SS -q ST -q Si -q Sj -q TT -q Ti -q Tj -q ii -q ij -q jj "},"sync_champ":{"upcnt":3,"last_r":1.0,"last_w":0.0,"lb":0.0,"ub":1.0}}} \ No newline at end of file +{"cbea_label_first_action":0,"cbea_label_not_first":3,"cbea_labeled_ex":3,"cbea_max_actions":12,"cbea_min_actions":12,"cbea_non_zero_cost":2,"cbea_predict_in_learn":0,"current_champ":0,"line_parse_error":0,"number_events_zero_actions":0,"number_skipped_events":0,"sfm_count_learn_example_with_shared":3,"test_county":3,"total_champ_switches":0,"total_learn_calls":3,"total_log_calls":1,"total_predict_calls":0,"cbea_avg_actions_per_event":12.0,"cbea_avg_feat_per_action":835.0,"cbea_avg_feat_per_event":10030.0,"cbea_avg_ns_per_action":9.0,"cbea_avg_ns_per_event":117.0,"cbea_sum_cost":-2.0,"cbea_sum_cost_baseline":0.0,"dsjson_sum_cost_original":0.0,"dsjson_sum_cost_original_baseline":0.0,"first_event_id":"0074434d3a3a46529f65de8a59631939","first_event_time":"","last_event_id":"9077f996581148978a0ebe2484260dab","last_event_time":"","estimator_0":{"config_type":"inclusion","elements":"{}","self":{"conf_idx":0,"upcnt":0,"last_r":0.0,"last_w":0.0,"lb":0.0,"ub":1.0,"interactions":"-q EE -q EG -q EM -q EO -q ER -q ES -q ET -q Ei -q Ej -q GG -q GM -q GO -q GR -q GS -q GT -q Gi -q Gj -q MM -q MO -q MR -q MS -q MT -q Mi -q Mj -q OO -q OR -q OS -q OT -q Oi -q Oj -q RR -q RS -q RT -q Ri -q Rj -q SS -q ST -q Si -q Sj -q TT -q Ti -q Tj -q ii -q ij -q jj "}},"estimator_1":{"config_type":"inclusion","elements":"{[\"E\", \"j\", \"j\"]}","self":{"conf_idx":99,"upcnt":3,"last_r":1.0,"last_w":0.0,"lb":0.0,"ub":1.0,"interactions":"-q EE -q EG -q EM -q EO -q ER -q ES -q ET -q Ei -q Ej -q GG -q GM -q GO -q GR -q GS -q GT -q Gi -q Gj -q MM -q MO -q MR -q MS -q MT -q Mi -q Mj -q OO -q OR -q OS -q OT -q Oi -q Oj -q RR -q RS -q RT -q Ri -q Rj -q SS -q ST -q Si -q Sj -q TT -q Ti -q Tj -q ii -q ij -q jj --cubic=Ejj "},"sync_champ":{"upcnt":3,"last_r":1.0,"last_w":0.0,"lb":0.0,"ub":1.0}},"estimator_2":{"config_type":"inclusion","elements":"{[\"E\", \"i\", \"j\"]}","self":{"conf_idx":98,"upcnt":3,"last_r":1.0,"last_w":0.0,"lb":0.0,"ub":1.0,"interactions":"-q EE -q EG -q EM -q EO -q ER -q ES -q ET -q Ei -q Ej -q GG -q GM -q GO -q GR -q GS -q GT -q Gi -q Gj -q MM -q MO -q MR -q MS -q MT -q Mi -q Mj -q OO -q OR -q OS -q OT -q Oi -q Oj -q RR -q RS -q RT -q Ri -q Rj -q SS -q ST -q Si -q Sj -q TT -q Ti -q Tj -q ii -q ij -q jj --cubic=Eij "},"sync_champ":{"upcnt":3,"last_r":1.0,"last_w":0.0,"lb":0.0,"ub":1.0}}} \ No newline at end of file diff --git a/test/test-sets/ref/sender.stdout b/test/test-sets/ref/sender.stdout index 1d10ed31a62..6164792d867 100644 --- a/test/test-sets/ref/sender.stdout +++ b/test/test-sets/ref/sender.stdout @@ -18,19 +18,19 @@ average since example example current current cur loss last counter weight label predict features 1.000000 1.000000 1 1.0 1.0000 0.0000 51 0.500000 0.000000 2 2.0 1.0000 1.0000 51 -0.270809 0.041618 4 4.0 0.0000 0.0000 104 -0.146127 0.021444 8 8.0 0.0000 0.0025 135 -0.127941 0.109756 16 16.0 0.0000 0.0215 146 +0.270805 0.041611 4 4.0 0.0000 0.0000 104 +0.146122 0.021439 8 8.0 0.0000 0.0025 135 +0.127941 0.109759 16 16.0 0.0000 0.0215 146 0.116989 0.106037 32 32.0 1.0000 0.9712 24 0.113415 0.109840 64 64.0 0.0000 0.0309 32 -0.112083 0.110752 128 128.0 0.0000 0.0393 61 -0.108814 0.105544 256 256.0 1.0000 0.9576 106 +0.112083 0.110751 128 128.0 0.0000 0.0393 61 +0.108813 0.105544 256 256.0 1.0000 0.9576 106 finished run number of examples = 400 weighted example sum = 400.000000 weighted label sum = 182.000000 -average loss = 0.093560 +average loss = 0.093559 best constant = 0.455000 best constant's loss = 0.247975 total feature number = 30964 diff --git a/test/train-sets/ccb_automl.dsjson b/test/train-sets/ccb_automl.dsjson new file mode 100644 index 00000000000..2a1f2e12957 --- /dev/null +++ b/test/train-sets/ccb_automl.dsjson @@ -0,0 +1,9 @@ +{"c":{"shared":{"name":"Anna","pig":0.7579544029403025,"goat":0.04048437818077755},"_multi":[{"a":{"topic":"sports"},"brown":0.3580493746949883},{"a":{"topic":"politics"},"white":0.48592769656281265},{"blue":{"dog":0.6666}}],"_slots":[{"_id":0,"orange":0.9827854760376531},{"_id":1,"brown":0.6183689966753316}]},"_outcomes":[{"_label_cost":-1,"_a":[1,0],"_p":[0.5,0.5]},{"_label_cost":-0.7302805219553081,"_a":[0],"_p":[1.0]}]} +{"c":{"shared":{"name":"Tom","cow":0.7298317482601286,"dog":0.47214271545271336},"_multi":[{"a":{"topic":"sports"},"brown":0.6404234015420902},{"a":{"topic":"politics"},"gray":0.35379132951924075},{"brown":{"horse":0.6666}}],"_slots":[{"_id":0,"green":0.06227958802154521},{"_id":1,"red":0.7197046864039541}]},"_outcomes":[{"_label_cost":-1,"_a":[0,1],"_p":[0.5,0.5]},{"_label_cost":-1,"_a":[1],"_p":[1.0]}]} +{"c":{"shared":{"name":"Anna","cow":0.24391087688713198},"_multi":[{"a":{"topic":"sports"},"yellow":0.9170189691149498},{"a":{"topic":"politics"}}],"_slots":[{"_id":0},{"_id":1,"green":0.14249439390882446,"brown":0.08044581855253541}]},"_outcomes":[{"_label_cost":-0.5483017265475156,"_a":[0,1],"_p":[0.5,0.5]},{"_label_cost":-0.8133723789341119,"_a":[1],"_p":[1.0]}]} +{"c":{"shared":{"name":"Anna","goat":0.10905784593110368,"dog":0.2910906725120649},"_multi":[{"a":{"topic":"sports"},"gray":0.5474409113284238},{"a":{"topic":"politics"},"white":0.5472301064999182},{"black":{"cow":0.6666}}],"_slots":[{"_id":0,"red":0.18939132855435614},{"_id":1,"blue":0.5756510141648885}]},"_outcomes":[{"_label_cost":-0.8896781148206443,"_a":[1,0],"_p":[0.5,0.5]},{"_label_cost":-0.7410598501938372,"_a":[0],"_p":[1.0]}]} +{"c":{"shared":{"name":"Anna","dog":0.47653099200938076,"bird":0.6787239593088918},"_multi":[{"a":{"topic":"sports"},"green":0.8767703708227748},{"a":{"topic":"politics"},"brown":0.898173121357879},{"yellow":{"fish":0.6666}}],"_slots":[{"_id":0},{"_id":1,"gray":0.8950389674266752,"purple":0.27521081604483055}]},"_outcomes":[{"_label_cost":-0.8912326932760299,"_a":[1,0],"_p":[0.5,0.5]},{"_label_cost":-0.8366879366926776,"_a":[0],"_p":[1.0]}]} +{"c":{"shared":{"name":"Tom","dog":0.3243248433827095,"cow":0.4864442019691668},"_multi":[{"a":{"topic":"sports"},"red":0.8450775756715152,"yellow":0.7314892207908478},{"a":{"topic":"politics"}},{"blue":{"cow":0.6666}}],"_slots":[{"_id":0,"green":0.4261311353438578,"red":0.7831185098466164},{"_id":1}]},"_outcomes":[{"_label_cost":-1,"_a":[0,1],"_p":[0.5,0.5]},{"_label_cost":-0.9579140478706729,"_a":[1],"_p":[1.0]}]} +{"c":{"shared":{"name":"Tom","fish":0.026696794662205203,"chicken":0.6063384177542189},"_multi":[{"a":{"topic":"sports"},"purple":0.11969547265179015,"red":0.8338444305994857},{"a":{"topic":"politics"}}],"_slots":[{"_id":0,"green":0.19459095568233187,"red":0.47923365392220396},{"_id":1}]},"_outcomes":[{"_label_cost":-1,"_a":[0,1],"_p":[0.5,0.5]},{"_label_cost":-1,"_a":[1],"_p":[1.0]}]} +{"c":{"shared":{"name":"Anna","horse":0.620599970977755,"dog":0.07000430092833387},"_multi":[{"a":{"topic":"sports"}},{"a":{"topic":"politics"},"black":0.6469257198353225,"red":0.18031790152968785}],"_slots":[{"_id":0},{"_id":1,"yellow":0.6993231250959273,"gray":0.2601332542192585}]},"_outcomes":[{"_label_cost":-0.8494079073157605,"_a":[0,1],"_p":[0.5,0.5]},{"_label_cost":-0.8794841736234199,"_a":[1],"_p":[1.0]}]} +{"c":{"shared":{"name":"Tom","fish":0.6976970707821888,"cat":0.9665489030431832},"_multi":[{"a":{"topic":"sports"},"blue":0.7886482900098098},{"a":{"topic":"politics"},"green":0.342312541078584}],"_slots":[{"_id":0,"red":0.44231403369907896},{"_id":1,"gray":0.47162541509628797}]},"_outcomes":[{"_label_cost":-0.9604893734642754,"_a":[1,0],"_p":[0.5,0.5]},{"_label_cost":-1,"_a":[0],"_p":[1.0]}]} diff --git a/test/train-sets/ref/active_cover.stderr b/test/train-sets/ref/active_cover.stderr index a9e36a783e5..ec2abfc6809 100644 --- a/test/train-sets/ref/active_cover.stderr +++ b/test/train-sets/ref/active_cover.stderr @@ -17,15 +17,15 @@ loss last counter weight label predict feat 0.322371 0.442694 6 10.3 -1.0000 -1.0000 140 0.280696 0.240000 28 20.8 1.0000 -1.0000 76 0.289183 0.297097 56 43.1 -1.0000 -1.0000 25 -0.342784 0.395369 124 87.0 1.0000 1.0000 22 -0.287217 0.232000 213 174.5 -1.0000 1.0000 34 +0.360856 0.431588 124 86.7 1.0000 1.0000 22 +0.281479 0.203259 211 174.7 -1.0000 -1.0000 119 finished run number of examples = 250 -weighted example sum = 217.709346 -weighted label sum = 15.483050 -average loss = 0.264386 -best constant = 0.142477 -best constant's loss = 0.690616 +weighted example sum = 222.625671 +weighted label sum = 8.451557 +average loss = 0.244591 +best constant = 0.075963 +best constant's loss = 0.692426 total feature number = 19393 -total queries = 98 +total queries = 95 diff --git a/test/train-sets/ref/automl_ccb.stderr b/test/train-sets/ref/automl_ccb.stderr new file mode 100644 index 00000000000..685b7becfb9 --- /dev/null +++ b/test/train-sets/ref/automl_ccb.stderr @@ -0,0 +1,24 @@ +using no cache +Reading datafile = train-sets/ccb_automl.dsjson +num sources = 1 +Num weight bits = 18 +learning rate = 0.5 +initial_t = 0 +power_t = 0.5 +cb_type = mtr +Enabled learners: gd, scorer-identity, csoaa_ldf-rank, cb_adf, automl, cb_explore_adf_greedy, cb_sample, shared_feature_merger, ccb_explore_adf, extra_metrics +Input label = CCB +Output pred = DECISION_PROBS +average since example example current current current +loss last counter weight label predict features +-0.66666 -0.66666 1 1.0 1:-1,0:-0.7... 1,0 252 +-1.30000 -1.93333 2 2.0 0:-1,1:-1 0,1 252 +-1.08686 -0.87373 4 4.0 1:-0.889678... 1,0 252 +-0.80037 -0.51387 8 8.0 0:-0.849408... 0,1 176 + +finished run +number of examples = 9 +weighted example sum = 9.000000 +weighted label sum = 0.000000 +average loss = -0.919547 +total feature number = 1872 diff --git a/test/train-sets/ref/automl_readable.txt b/test/train-sets/ref/automl_readable.txt index 60266fea182..1a7fc6f5abc 100644 --- a/test/train-sets/ref/automl_readable.txt +++ b/test/train-sets/ref/automl_readable.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:-1 Max label:0 @@ -535,28 +535,11 @@ _automl_config_manager_index_queue[41].first = -12 _automl_config_manager_index_queue[41].second = 5 _automl_config_manager_index_queue[42].first = -12 _automl_config_manager_index_queue[42].second = 2 -_automl_config_manager_per_live_model_state_double.size() = 9 -_automl_config_manager_per_live_model_state_double[0] = 98.98610879850912 -_automl_config_manager_per_live_model_state_double[1] = 0.30612243711948395 -_automl_config_manager_per_live_model_state_double[2] = 0 -_automl_config_manager_per_live_model_state_double[3] = 96.53712930155325 -_automl_config_manager_per_live_model_state_double[4] = 0.30612243711948395 -_automl_config_manager_per_live_model_state_double[5] = 0 -_automl_config_manager_per_live_model_state_double[6] = 94.08814980459738 -_automl_config_manager_per_live_model_state_double[7] = 0.30612243711948395 -_automl_config_manager_per_live_model_state_double[8] = 0 -_automl_config_manager_per_live_model_state_uint64.size() = 6 -_automl_config_manager_per_live_model_state_uint64[0] = 3 -_automl_config_manager_per_live_model_state_uint64[1] = 36 -_automl_config_manager_per_live_model_state_uint64[2] = 3 -_automl_config_manager_per_live_model_state_uint64[3] = 36 -_automl_config_manager_per_live_model_state_uint64[4] = 3 -_automl_config_manager_per_live_model_state_uint64[5] = 36 -event_sum 0 -action_sum 0 +event_sum 3 +action_sum 36 :1 initial_t 0 -norm normalizer 0 +norm normalizer 98.9861 t 3 sum_loss -0.122449 sum_loss_since_last_dump 0 @@ -568,7 +551,7 @@ weighted_labels 0 weighted_unlabeled_examples 0 example_number 3 total_features 30090 -total_weight 0 +total_weight 0.306122 sd::oec.weighted_labeled_examples 3 current_pass 1 l1_state 0 diff --git a/test/train-sets/ref/automl_readable_cubic.txt b/test/train-sets/ref/automl_readable_cubic.txt index 5faadc2030d..8277cd8c727 100644 --- a/test/train-sets/ref/automl_readable_cubic.txt +++ b/test/train-sets/ref/automl_readable_cubic.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:-1 Max label:0 @@ -1780,28 +1780,11 @@ _automl_config_manager_index_queue[161].first = -12 _automl_config_manager_index_queue[161].second = 3 _automl_config_manager_index_queue[162].first = -12 _automl_config_manager_index_queue[162].second = 2 -_automl_config_manager_per_live_model_state_double.size() = 9 -_automl_config_manager_per_live_model_state_double[0] = 527.0563162750805 -_automl_config_manager_per_live_model_state_double[1] = 0.20408162474632263 -_automl_config_manager_per_live_model_state_double[2] = 0 -_automl_config_manager_per_live_model_state_double[3] = 525.5256293525599 -_automl_config_manager_per_live_model_state_double[4] = 0.20408162474632263 -_automl_config_manager_per_live_model_state_double[5] = 0 -_automl_config_manager_per_live_model_state_double[6] = 523.9874438254192 -_automl_config_manager_per_live_model_state_double[7] = 0.20408162474632263 -_automl_config_manager_per_live_model_state_double[8] = 0 -_automl_config_manager_per_live_model_state_uint64.size() = 6 -_automl_config_manager_per_live_model_state_uint64[0] = 3 -_automl_config_manager_per_live_model_state_uint64[1] = 36 -_automl_config_manager_per_live_model_state_uint64[2] = 3 -_automl_config_manager_per_live_model_state_uint64[3] = 36 -_automl_config_manager_per_live_model_state_uint64[4] = 3 -_automl_config_manager_per_live_model_state_uint64[5] = 36 -event_sum 0 -action_sum 0 +event_sum 3 +action_sum 36 :1 initial_t 0 -norm normalizer 0 +norm normalizer 527.056 t 3 sum_loss -0.122449 sum_loss_since_last_dump 0 @@ -1813,7 +1796,7 @@ weighted_labels 0 weighted_unlabeled_examples 0 example_number 3 total_features 508674 -total_weight 0 +total_weight 0.204082 sd::oec.weighted_labeled_examples 3 current_pass 1 l1_state 0 diff --git a/test/train-sets/ref/automl_readable_qc.txt b/test/train-sets/ref/automl_readable_qc.txt index 431a5b88aa5..57be641704c 100644 --- a/test/train-sets/ref/automl_readable_qc.txt +++ b/test/train-sets/ref/automl_readable_qc.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:-1 Max label:0 @@ -1780,28 +1780,11 @@ _automl_config_manager_index_queue[161].first = -12 _automl_config_manager_index_queue[161].second = 3 _automl_config_manager_index_queue[162].first = -12 _automl_config_manager_index_queue[162].second = 2 -_automl_config_manager_per_live_model_state_double.size() = 9 -_automl_config_manager_per_live_model_state_double[0] = 98.98610879850912 -_automl_config_manager_per_live_model_state_double[1] = 0.30612243711948395 -_automl_config_manager_per_live_model_state_double[2] = 0 -_automl_config_manager_per_live_model_state_double[3] = 101.435088295465 -_automl_config_manager_per_live_model_state_double[4] = 0.30612243711948395 -_automl_config_manager_per_live_model_state_double[5] = 0 -_automl_config_manager_per_live_model_state_double[6] = 103.88406779242086 -_automl_config_manager_per_live_model_state_double[7] = 0.30612243711948395 -_automl_config_manager_per_live_model_state_double[8] = 0 -_automl_config_manager_per_live_model_state_uint64.size() = 6 -_automl_config_manager_per_live_model_state_uint64[0] = 3 -_automl_config_manager_per_live_model_state_uint64[1] = 36 -_automl_config_manager_per_live_model_state_uint64[2] = 3 -_automl_config_manager_per_live_model_state_uint64[3] = 36 -_automl_config_manager_per_live_model_state_uint64[4] = 3 -_automl_config_manager_per_live_model_state_uint64[5] = 36 -event_sum 0 -action_sum 0 +event_sum 3 +action_sum 36 :1 initial_t 0 -norm normalizer 0 +norm normalizer 98.9861 t 3 sum_loss -0.122449 sum_loss_since_last_dump 0 @@ -1813,7 +1796,7 @@ weighted_labels 0 weighted_unlabeled_examples 0 example_number 3 total_features 30090 -total_weight 0 +total_weight 0.306122 sd::oec.weighted_labeled_examples 3 current_pass 1 l1_state 0 diff --git a/test/train-sets/ref/bs.prreg.predict b/test/train-sets/ref/bs.prreg.predict index fefd385bbe3..ac9ea0cbb4b 100644 --- a/test/train-sets/ref/bs.prreg.predict +++ b/test/train-sets/ref/bs.prreg.predict @@ -1,1000 +1,1000 @@ -0.451448 PFF/20091028 0.440796 0.467907 -0.543874 WIP/20091028 0.523515 0.563895 -0.514261 GCC/20091028 0.510565 0.518184 -0.522985 AAXJ/20091028 0.496590 0.562852 -0.533711 VWO/20091028 0.501632 0.567931 -0.331418 EEV/20091028 0.266092 0.409853 -0.651597 GDX/20091028 0.630919 0.690686 -0.505671 RTH/20091028 0.448527 0.530291 -0.571682 MXI/20091028 0.552736 0.608507 -0.528176 EWU/20091028 0.506156 0.549882 -0.451897 SH/20091028 0.426980 0.481060 -0.558286 EDC/20091028 0.509391 0.614216 -0.424634 ERY/20091028 0.406306 0.453355 -0.400657 SDS/20091028 0.368008 0.434602 -0.555669 OEF/20091028 0.527085 0.612924 -0.577313 IYT/20091028 0.543524 0.609254 -0.551570 BIL/20091028 0.533853 0.569964 -0.271069 GLL/20091028 0.212728 0.352205 -0.237743 EDZ/20091028 0.203613 0.280001 -0.493546 IWM/20091028 0.466238 0.535096 -0.488646 VXF/20091028 0.475941 0.499214 -0.488192 IJJ/20091028 0.453542 0.505972 -0.575284 PIN/20091028 0.534800 0.595017 -0.572365 XLB/20091028 0.552892 0.592670 -0.521320 ECH/20091028 0.503846 0.541930 -0.467539 TYH/20091028 0.425795 0.490850 -0.575610 VAW/20091028 0.544317 0.596215 -0.604148 DBP/20091028 0.575216 0.626468 -0.617634 XME/20091028 0.581171 0.647658 -0.499461 VO/20091028 0.465761 0.525383 -0.530152 RSX/20091028 0.486001 0.565991 -0.565171 EWC/20091028 0.537158 0.589445 -0.430180 TUR/20091028 0.404639 0.464043 -0.534602 VYM/20091028 0.514459 0.554282 -0.504611 FCG/20091028 0.468840 0.526611 -0.531585 VGT/20091028 0.512162 0.563033 -0.539582 EWQ/20091028 0.514271 0.583130 -0.528724 IEV/20091028 0.511814 0.553396 -0.523968 XLK/20091028 0.510701 0.544713 -0.527935 EFG/20091028 0.512592 0.550582 -0.543853 BKF/20091028 0.516755 0.570196 -0.536898 KIE/20091028 0.523807 0.560645 -0.570898 EEB/20091028 0.550325 0.598356 -0.506208 IJK/20091028 0.477646 0.538128 -0.450149 DUG/20091028 0.412317 0.479660 -0.422025 TWM/20091028 0.390914 0.446223 -0.513625 MDY/20091028 0.487397 0.537721 -0.544049 ACWI/20091028 0.516454 0.577587 -0.586577 BSV/20091028 0.576495 0.605457 -0.616874 DDM/20091028 0.602176 0.633380 -0.559693 DIA/20091028 0.549533 0.576120 -0.573502 TLT/20091028 0.564827 0.588942 -0.398985 DXD/20091028 0.379306 0.413103 -0.523258 XHB/20091028 0.497512 0.553652 -0.528676 VDE/20091028 0.479234 0.572623 -0.556584 BND/20091028 0.513133 0.585520 -0.503420 EMB/20091028 0.483330 0.520500 -0.540480 SCO/20091028 0.523672 0.555557 -0.538184 AMJ/20091028 0.521334 0.568976 -0.471162 OIL/20091028 0.435295 0.508355 -0.483351 PZA/20091028 0.465612 0.506488 -0.533989 VGK/20091028 0.510740 0.561056 -0.507697 RWX/20091028 0.477525 0.537316 -0.531900 JJA/20091028 0.502749 0.553518 -0.529823 FXD/20091028 0.501993 0.557748 -0.487677 XES/20091028 0.457469 0.516244 -0.536316 VIG/20091028 0.514939 0.552066 -0.206163 DZZ/20091028 0.193256 0.219339 -0.497127 VFH/20091028 0.441662 0.548553 -0.555412 DTO/20091028 0.545103 0.572779 -0.548736 EWP/20091028 0.504169 0.574144 -0.516733 FDN/20091028 0.486088 0.544935 -0.594229 INP/20091028 0.576722 0.618416 -0.363333 TYP/20091028 0.320938 0.405182 -0.538068 RWR/20091028 0.515197 0.563470 -0.534125 KBE/20091028 0.508241 0.573663 -0.482967 EUO/20091028 0.461735 0.505887 -0.525328 IWF/20091028 0.502069 0.548732 -0.296824 SMN/20091028 0.250345 0.345833 -0.533184 SMH/20091028 0.509459 0.562981 -0.492286 XRT/20091028 0.442822 0.549940 -0.466162 USO/20091028 0.453280 0.481712 -0.495648 DJP/20091028 0.476570 0.511449 -0.573226 CFT/20091028 0.556450 0.585841 -0.357514 SRS/20091028 0.311725 0.414922 -0.580546 MOO/20091028 0.569680 0.603773 -0.563519 BIV/20091028 0.541633 0.578519 -0.370216 VXX/20091028 0.329606 0.427222 -0.576818 IYM/20091028 0.541249 0.610989 -0.560686 IFN/20091028 0.544960 0.586019 -0.612061 SLV/20091028 0.576164 0.633514 -0.489881 TAO/20091028 0.477759 0.518295 -0.460662 PGF/20091028 0.446063 0.470191 -0.535217 IYR/20091028 0.515202 0.552391 -0.350301 QID/20091028 0.318798 0.393663 -0.539134 THD/20091028 0.520946 0.554447 -0.513384 IJS/20091028 0.484958 0.549212 -0.502692 VB/20091028 0.476310 0.532238 -0.562987 EDV/20091028 0.554812 0.576606 -0.498754 IEZ/20091028 0.471515 0.533698 -0.546602 VTV/20091028 0.524071 0.576366 -0.505770 IJR/20091028 0.491949 0.534301 -0.435286 UCO/20091028 0.414911 0.459150 -0.453027 JNK/20091028 0.434531 0.474070 -0.517394 IWN/20091028 0.497353 0.544376 -0.534031 VV/20091028 0.510032 0.569133 -0.711270 UGL/20091028 0.639356 0.775919 -0.527721 UWM/20091028 0.487407 0.547210 -0.473804 IWC/20091028 0.439651 0.520664 -0.508197 EWA/20091028 0.468744 0.537712 -0.523816 IVV/20091028 0.500932 0.542937 -0.523860 SPY/20091028 0.507342 0.541174 -0.511518 TFI/20091028 0.503714 0.523354 -0.514785 VEA/20091028 0.488767 0.540944 -0.508145 QQQQ/20091028 0.492699 0.529375 -0.522840 UYG/20091028 0.497695 0.568929 -0.468417 OIH/20091028 0.437608 0.490195 -0.492827 GXC/20091028 0.474767 0.519069 -0.587096 SSO/20091028 0.573606 0.608121 -0.543709 XLI/20091028 0.529461 0.560237 -0.560391 GML/20091028 0.526748 0.599454 -0.562898 ROM/20091028 0.539362 0.588026 -0.497807 FXC/20091028 0.485396 0.516491 -0.454530 DOG/20091028 0.417151 0.475092 -0.480404 IYE/20091028 0.463917 0.502350 -0.418029 SKF/20091028 0.361662 0.471073 -0.563174 SHY/20091028 0.539939 0.584236 -0.503489 DBA/20091028 0.477255 0.540728 -0.515237 RSP/20091028 0.488206 0.548092 -0.613246 DBS/20091028 0.576984 0.650132 -0.532460 IBB/20091028 0.514035 0.561264 -0.453523 KCE/20091028 0.443106 0.477670 -0.500230 PKN/20091028 0.472942 0.535746 -0.508026 TNA/20091028 0.453614 0.564682 -0.589547 FAS/20091028 0.521847 0.657468 -0.502481 FXE/20091028 0.494034 0.508741 -0.457176 HYG/20091028 0.439741 0.470343 -0.520711 IWS/20091028 0.477085 0.546735 -0.429973 FXP/20091028 0.410055 0.466481 -0.563836 MBB/20091028 0.535241 0.588602 -0.512846 RFG/20091028 0.500904 0.535546 -0.546380 EPU/20091028 0.509732 0.565350 -0.479352 UUP/20091028 0.454446 0.496374 -0.706309 AGQ/20091028 0.648535 0.751521 -0.537786 SOXX/20091028 0.518851 0.564067 -0.346973 FAZ/20091028 0.315007 0.374611 -0.493692 VBK/20091028 0.475696 0.523284 -0.516287 RPG/20091028 0.499362 0.540583 -0.507248 EWH/20091028 0.489856 0.527812 -0.405729 TZA/20091028 0.345066 0.458236 -0.502521 SGG/20091028 0.478152 0.532591 -0.581573 KOL/20091028 0.554454 0.608599 -0.516939 EWY/20091028 0.491513 0.540964 -0.529531 PRF/20091028 0.504368 0.559370 -0.569615 TLH/20091028 0.556221 0.582449 -0.511481 EPP/20091028 0.483370 0.545868 -0.496410 XLE/20091028 0.476774 0.523357 -0.546550 EWN/20091028 0.531056 0.557685 -0.503488 SHM/20091028 0.490377 0.524759 -0.496467 FXI/20091028 0.469668 0.517761 -0.553622 EWS/20091028 0.523115 0.581439 -0.541252 IDU/20091028 0.520078 0.571600 -0.540403 VXZ/20091028 0.524366 0.554553 -0.535662 IVE/20091028 0.519155 0.551934 -0.713279 DGP/20091028 0.684701 0.730079 -0.514797 GMF/20091028 0.487211 0.536772 -0.483093 IWR/20091028 0.443460 0.525391 -0.481665 RKH/20091028 0.461825 0.501487 -0.572088 TIP/20091028 0.565375 0.577298 -0.587455 URE/20091028 0.568697 0.603752 -0.454053 DBO/20091028 0.447178 0.463151 -0.500766 IOO/20091028 0.472070 0.526380 -0.442793 DBV/20091028 0.421612 0.472927 -0.506272 EFA/20091028 0.482596 0.549171 -0.544891 BGU/20091028 0.510157 0.589635 -0.504297 EFV/20091028 0.498893 0.515105 -0.525767 IWB/20091028 0.505133 0.537079 -0.502415 IYF/20091028 0.494419 0.515271 -0.320184 YCS/20091028 0.306286 0.330952 -0.502795 DXJ/20091028 0.494647 0.510638 -0.478232 IWO/20091028 0.464413 0.495627 -0.486597 DBC/20091028 0.478696 0.497194 -0.458463 RWM/20091028 0.385909 0.510380 -0.500448 VBR/20091028 0.487303 0.525214 -0.409842 MZZ/20091028 0.327335 0.452596 -0.530802 IWD/20091028 0.521718 0.546647 -0.455899 PCY/20091028 0.432409 0.491469 -0.516952 EWI/20091028 0.506739 0.530480 -0.513646 IJH/20091028 0.493724 0.554480 -0.537298 EEM/20091028 0.522237 0.552646 -0.503504 EWM/20091028 0.484145 0.523775 -0.521152 SDY/20091028 0.506635 0.544119 -0.594751 ILF/20091028 0.575586 0.611439 -0.547066 JJG/20091028 0.531120 0.563164 -0.387882 TBT/20091028 0.375570 0.405894 -0.504885 XLF/20091028 0.497615 0.513139 -0.518053 ERX/20091028 0.469367 0.574754 -0.494998 SHV/20091028 0.478515 0.510473 -0.546962 EWX/20091028 0.512543 0.600251 -0.445223 EFZ/20091028 0.385644 0.484536 -0.492452 FXB/20091028 0.479510 0.503275 -0.492356 PHO/20091028 0.472180 0.512681 -0.548215 IGE/20091028 0.523019 0.598295 -0.291636 BGZ/20091028 0.252912 0.332855 -0.505445 UDN/20091028 0.492803 0.518574 -0.464770 CSJ/20091028 0.457635 0.469035 -0.570196 GXG/20091028 0.548093 0.634124 -0.583746 USD/20091028 0.550778 0.605388 -0.533399 EWD/20091028 0.508606 0.566139 -0.507472 EWJ/20091028 0.501324 0.519299 -0.634334 BRF/20091028 0.590131 0.682936 -0.529409 VEU/20091028 0.511348 0.547846 -0.535242 XLU/20091028 0.499836 0.556990 -0.502439 JJC/20091028 0.492686 0.518259 -0.543800 FGD/20091028 0.527480 0.563949 -0.493406 FXF/20091028 0.489336 0.497648 -0.497229 LQD/20091028 0.478271 0.505026 -0.493023 SCZ/20091028 0.469921 0.504629 -0.519248 IYW/20091028 0.505518 0.538117 -0.482299 VPL/20091028 0.460554 0.511948 -0.512270 DGS/20091028 0.475206 0.540982 -0.524944 ICF/20091028 0.480738 0.554457 -0.514963 DVY/20091028 0.493696 0.545951 -0.497014 IEO/20091028 0.457916 0.568550 -0.499581 VOT/20091028 0.482053 0.526205 -0.517111 CIU/20091028 0.497116 0.526293 -0.538254 EWG/20091028 0.501445 0.568025 -0.521434 EWT/20091028 0.498002 0.539044 -0.452724 GSG/20091028 0.443921 0.463845 -0.497608 KRE/20091028 0.466180 0.510659 -0.540288 LVL/20091028 0.525962 0.564034 -0.394351 UNG/20091028 0.370676 0.437387 -0.464842 MUB/20091028 0.456126 0.475461 -0.526182 VT/20091028 0.509175 0.551666 -0.582286 DAG/20091028 0.552120 0.608167 -0.576990 PPH/20091028 0.560567 0.589387 -0.511336 VSS/20091028 0.483963 0.543917 -0.470110 DBB/20091028 0.463354 0.476670 -0.557002 XLP/20091028 0.531623 0.576333 -0.479333 IJT/20091028 0.456947 0.516311 -0.581076 EWZ/20091028 0.545633 0.627149 -0.487639 PBW/20091028 0.454472 0.524630 -0.610530 FXY/20091028 0.584168 0.632781 -0.578601 IYZ/20091028 0.567807 0.597957 -0.543695 MVV/20091028 0.514899 0.571773 -0.515496 VUG/20091028 0.489012 0.533946 -0.377773 PST/20091028 0.367786 0.386632 -0.454832 PSQ/20091028 0.431709 0.487740 -0.519808 VNQ/20091028 0.481337 0.545352 -0.561795 IEI/20091028 0.551355 0.573772 -0.554628 EWW/20091028 0.497385 0.590295 -0.505267 IWP/20091028 0.471417 0.536240 -0.516385 IWV/20091028 0.484265 0.548845 -0.523498 DIG/20091028 0.499607 0.556362 -0.521055 VTI/20091028 0.501618 0.539452 -0.474664 FXA/20091028 0.452106 0.496375 -0.486183 NLR/20091028 0.454733 0.518333 -0.541990 AGG/20091028 0.520334 0.557531 -0.531066 BWX/20091028 0.516688 0.542774 -0.608355 IAU/20091028 0.602428 0.618826 -0.571027 XLV/20091028 0.558290 0.582612 -0.482916 XOP/20091028 0.443478 0.532549 -0.525339 EZU/20091028 0.501909 0.554626 -0.531411 JXI/20091028 0.513717 0.541520 -0.521266 XBI/20091028 0.499698 0.549071 -0.504320 IYG/20091028 0.478512 0.530976 -0.595637 SLX/20091028 0.561838 0.633384 -0.513119 HAO/20091028 0.473114 0.535134 -0.540203 EZA/20091028 0.512863 0.574501 -0.517458 XLY/20091028 0.494250 0.547167 -0.543223 IEF/20091028 0.524311 0.567502 -0.513728 DEM/20091028 0.481871 0.557107 -0.505244 IVW/20091028 0.493017 0.526682 -0.658889 UYM/20091028 0.625872 0.703789 -0.480174 IXC/20091028 0.461462 0.511862 -0.493026 PFF/20091029 0.481401 0.501223 -0.583658 WIP/20091029 0.570349 0.610268 -0.547000 GCC/20091029 0.536589 0.560545 -0.580480 AAXJ/20091029 0.562533 0.605025 -0.594849 VWO/20091029 0.586528 0.611595 -0.293981 EEV/20091029 0.253258 0.345125 -0.701847 GDX/20091029 0.679344 0.724662 -0.544061 RTH/20091029 0.498549 0.581758 -0.624527 MXI/20091029 0.607351 0.645605 -0.556666 EWU/20091029 0.540123 0.591319 -0.389863 SH/20091029 0.369527 0.407589 -0.577255 EDC/20091029 0.548256 0.604187 -0.392138 ERY/20091029 0.369768 0.403559 -0.348276 SDS/20091029 0.331452 0.362757 -0.605014 OEF/20091029 0.582449 0.649222 -0.624881 IYT/20091029 0.604398 0.643626 -0.540508 BIL/20091029 0.526577 0.571345 -0.239457 GLL/20091029 0.195599 0.299686 -0.220301 EDZ/20091029 0.211220 0.233275 -0.558603 IWM/20091029 0.547320 0.588956 -0.559969 VXF/20091029 0.552269 0.573146 -0.556108 IJJ/20091029 0.539949 0.571603 -0.643812 PIN/20091029 0.622121 0.666796 -0.628363 XLB/20091029 0.623380 0.635393 -0.572721 ECH/20091029 0.559439 0.577804 -0.503103 TYH/20091029 0.482552 0.527436 -0.637372 VAW/20091029 0.625612 0.647895 -0.656838 DBP/20091029 0.647619 0.665442 -0.671254 XME/20091029 0.655365 0.684598 -0.570210 VO/20091029 0.556609 0.582464 -0.571809 RSX/20091029 0.554609 0.588537 -0.631224 EWC/20091029 0.623262 0.642063 -0.488132 TUR/20091029 0.476154 0.502656 -0.591652 VYM/20091029 0.585099 0.598573 -0.557652 FCG/20091029 0.538557 0.585740 -0.587593 VGT/20091029 0.562250 0.605837 -0.595365 EWQ/20091029 0.580308 0.625251 -0.576742 IEV/20091029 0.565110 0.585596 -0.568302 XLK/20091029 0.555854 0.574465 -0.571358 EFG/20091029 0.558903 0.576961 -0.588461 BKF/20091029 0.579220 0.594947 -0.582209 KIE/20091029 0.562785 0.592077 -0.617209 EEB/20091029 0.601065 0.633458 -0.573948 IJK/20091029 0.557178 0.593351 -0.417739 DUG/20091029 0.394744 0.434051 -0.367958 TWM/20091029 0.350934 0.380378 -0.580690 MDY/20091029 0.572292 0.590572 -0.604866 ACWI/20091029 0.594818 0.622703 -0.558970 BSV/20091029 0.545362 0.571841 -0.649869 DDM/20091029 0.619081 0.665675 -0.594003 DIA/20091029 0.581620 0.601931 -0.540428 TLT/20091029 0.534487 0.543554 -0.367166 DXD/20091029 0.327199 0.388060 -0.567678 XHB/20091029 0.556260 0.583448 -0.575331 VDE/20091029 0.544876 0.630101 -0.548841 BND/20091029 0.505178 0.577541 -0.539435 EMB/20091029 0.523726 0.552501 -0.514197 SCO/20091029 0.483731 0.532999 -0.561465 AMJ/20091029 0.549474 0.579552 -0.510217 OIL/20091029 0.491028 0.558612 -0.519708 PZA/20091029 0.508629 0.538581 -0.582659 VGK/20091029 0.572656 0.593052 -0.570142 RWX/20091029 0.553841 0.582236 -0.564573 JJA/20091029 0.553358 0.572214 -0.580572 FXD/20091029 0.562999 0.596193 -0.542697 XES/20091029 0.524806 0.556890 -0.591136 VIG/20091029 0.580244 0.600205 -0.177306 DZZ/20091029 0.170141 0.183016 -0.558321 VFH/20091029 0.526078 0.592961 -0.553762 DTO/20091029 0.533400 0.565430 -0.589730 EWP/20091029 0.566662 0.599096 -0.568179 FDN/20091029 0.553869 0.585056 -0.655516 INP/20091029 0.650157 0.663128 -0.331040 TYP/20091029 0.307644 0.350624 -0.587319 RWR/20091029 0.580857 0.592314 -0.585812 KBE/20091029 0.571331 0.611077 -0.446309 EUO/20091029 0.435402 0.454870 -0.589106 IWF/20091029 0.582890 0.600552 -0.260138 SMN/20091029 0.234216 0.286832 -0.581557 SMH/20091029 0.567583 0.599307 -0.551465 XRT/20091029 0.519716 0.597649 -0.505426 USO/20091029 0.486998 0.538100 -0.536226 DJP/20091029 0.535806 0.536562 -0.558315 CFT/20091029 0.543851 0.569369 -0.325635 SRS/20091029 0.304723 0.356929 -0.618122 MOO/20091029 0.594083 0.627480 -0.543226 BIV/20091029 0.527760 0.551077 -0.343190 VXX/20091029 0.319603 0.376536 -0.632977 IYM/20091029 0.617569 0.651715 -0.609440 IFN/20091029 0.589824 0.621931 -0.659961 SLV/20091029 0.641621 0.673454 -0.538481 TAO/20091029 0.512812 0.552504 -0.514079 PGF/20091029 0.509036 0.519122 -0.585910 IYR/20091029 0.578917 0.591841 -0.297763 QID/20091029 0.279919 0.323464 -0.588020 THD/20091029 0.576005 0.601607 -0.575012 IJS/20091029 0.558169 0.600031 -0.570216 VB/20091029 0.554176 0.587445 -0.521609 EDV/20091029 0.496838 0.532554 -0.549250 IEZ/20091029 0.526996 0.569418 -0.606004 VTV/20091029 0.581172 0.622053 -0.571218 IJR/20091029 0.540570 0.588785 -0.463880 UCO/20091029 0.426923 0.501102 -0.509726 JNK/20091029 0.494051 0.519856 -0.581093 IWN/20091029 0.561993 0.596194 -0.597664 VV/20091029 0.573408 0.619505 -0.743846 UGL/20091029 0.681181 0.792744 -0.581483 UWM/20091029 0.559897 0.601270 -0.545205 IWC/20091029 0.516371 0.581049 -0.570332 EWA/20091029 0.547655 0.589510 -0.582820 IVV/20091029 0.563431 0.599526 -0.584965 SPY/20091029 0.563072 0.599389 -0.528415 TFI/20091029 0.517803 0.542454 -0.568552 VEA/20091029 0.553477 0.579067 -0.568572 QQQQ/20091029 0.552096 0.577717 -0.566074 UYG/20091029 0.522103 0.590494 -0.517066 OIH/20091029 0.504128 0.527841 -0.540455 GXC/20091029 0.529314 0.552835 -0.640459 SSO/20091029 0.619014 0.649164 -0.608934 XLI/20091029 0.594794 0.621779 -0.614076 GML/20091029 0.589125 0.632464 -0.608301 ROM/20091029 0.595365 0.618115 -0.556150 FXC/20091029 0.532156 0.575436 -0.415701 DOG/20091029 0.392678 0.428547 -0.526157 IYE/20091029 0.512181 0.535280 -0.378751 SKF/20091029 0.345221 0.405600 -0.541736 SHY/20091029 0.526316 0.551868 -0.532558 DBA/20091029 0.519861 0.555965 -0.581680 RSP/20091029 0.561584 0.601235 -0.661436 DBS/20091029 0.642200 0.685412 -0.607478 IBB/20091029 0.589616 0.626050 -0.527392 KCE/20091029 0.502043 0.541038 -0.547632 PKN/20091029 0.531323 0.569540 -0.550192 TNA/20091029 0.497102 0.603977 -0.616918 FAS/20091029 0.576867 0.658421 -0.549555 FXE/20091029 0.527556 0.568221 -0.510843 HYG/20091029 0.507198 0.516014 -0.587917 IWS/20091029 0.564548 0.618567 -0.396992 FXP/20091029 0.375904 0.412646 -0.526425 MBB/20091029 0.507885 0.539287 -0.577758 RFG/20091029 0.557988 0.587879 -0.610367 EPU/20091029 0.593016 0.634543 -0.440106 UUP/20091029 0.408531 0.454863 -0.740296 AGQ/20091029 0.701468 0.768433 -0.599393 SOXX/20091029 0.565879 0.620226 -0.323602 FAZ/20091029 0.319256 0.330810 -0.561864 VBK/20091029 0.529735 0.579626 -0.577526 RPG/20091029 0.551962 0.590610 -0.550314 EWH/20091029 0.537615 0.557346 -0.361879 TZA/20091029 0.318497 0.399061 -0.537589 SGG/20091029 0.520654 0.550839 -0.621317 KOL/20091029 0.598153 0.641913 -0.572605 EWY/20091029 0.555623 0.585909 -0.591691 PRF/20091029 0.548238 0.607635 -0.535994 TLH/20091029 0.527173 0.549982 -0.574276 EPP/20091029 0.552854 0.595664 -0.538790 XLE/20091029 0.516440 0.552065 -0.599234 EWN/20091029 0.580484 0.618123 -0.488429 SHM/20091029 0.471828 0.513176 -0.537559 FXI/20091029 0.528972 0.547232 -0.603119 EWS/20091029 0.593162 0.614357 -0.578728 IDU/20091029 0.564458 0.601168 -0.515626 VXZ/20091029 0.511517 0.521672 -0.598375 IVE/20091029 0.575239 0.614769 -0.747933 DGP/20091029 0.740425 0.761522 -0.572588 GMF/20091029 0.560768 0.584606 -0.552354 IWR/20091029 0.532066 0.581429 -0.529146 RKH/20091029 0.504246 0.548435 -0.561601 TIP/20091029 0.556478 0.564855 -0.619744 URE/20091029 0.608275 0.632118 -0.499986 DBO/20091029 0.491106 0.509893 -0.545872 IOO/20091029 0.533280 0.557227 -0.493079 DBV/20091029 0.460611 0.544757 -0.557759 EFA/20091029 0.533099 0.585543 -0.590864 BGU/20091029 0.573443 0.617682 -0.557139 EFV/20091029 0.534685 0.575023 -0.577537 IWB/20091029 0.562172 0.597230 -0.560062 IYF/20091029 0.540135 0.574862 -0.344182 YCS/20091029 0.324207 0.354047 -0.512419 DXJ/20091029 0.497641 0.530478 -0.544801 IWO/20091029 0.534573 0.551757 -0.532793 DBC/20091029 0.516873 0.549467 -0.391877 RWM/20091029 0.308310 0.425928 -0.566673 VBR/20091029 0.543090 0.578476 -0.352836 MZZ/20091029 0.285599 0.377275 -0.591391 IWD/20091029 0.569142 0.608762 -0.542150 PCY/20091029 0.497624 0.575891 -0.574260 EWI/20091029 0.554390 0.586064 -0.580910 IJH/20091029 0.553504 0.609533 -0.596743 EEM/20091029 0.579118 0.608321 -0.565692 EWM/20091029 0.555329 0.573152 -0.574975 SDY/20091029 0.562164 0.588218 -0.640832 ILF/20091029 0.625133 0.666024 -0.575113 JJG/20091029 0.566226 0.581199 -0.417966 TBT/20091029 0.398648 0.434637 -0.558891 XLF/20091029 0.536060 0.575057 -0.536902 ERX/20091029 0.509454 0.577225 -0.483731 SHV/20091029 0.471510 0.492802 -0.612068 EWX/20091029 0.589695 0.651501 -0.388864 EFZ/20091029 0.345382 0.409801 -0.485876 FXB/20091029 0.469448 0.497154 -0.548799 PHO/20091029 0.530024 0.561968 -0.601229 IGE/20091029 0.570342 0.636403 -0.247010 BGZ/20091029 0.227223 0.265708 -0.545748 UDN/20091029 0.528134 0.557088 -0.467382 CSJ/20091029 0.459578 0.472585 -0.628149 GXG/20091029 0.590002 0.677032 -0.626092 USD/20091029 0.612219 0.641433 -0.548405 EWD/20091029 0.528925 0.560399 -0.522224 EWJ/20091029 0.504334 0.542170 -0.679519 BRF/20091029 0.650982 0.706792 -0.588949 VEU/20091029 0.567598 0.604935 -0.567899 XLU/20091029 0.543215 0.589046 -0.540131 JJC/20091029 0.514563 0.560736 -0.607071 FGD/20091029 0.591675 0.613830 -0.527702 FXF/20091029 0.512753 0.538088 -0.509595 LQD/20091029 0.494143 0.516051 -0.552989 SCZ/20091029 0.539997 0.575136 -0.571769 IYW/20091029 0.555088 0.578194 -0.532034 VPL/20091029 0.511440 0.548654 -0.579287 DGS/20091029 0.563420 0.592687 -0.574902 ICF/20091029 0.554489 0.592253 -0.568587 DVY/20091029 0.539247 0.588249 -0.551852 IEO/20091029 0.520021 0.608571 -0.571544 VOT/20091029 0.542476 0.585209 -0.502909 CIU/20091029 0.485825 0.510424 -0.596790 EWG/20091029 0.579783 0.614359 -0.573111 EWT/20091029 0.563408 0.582302 -0.482950 GSG/20091029 0.475347 0.490856 -0.506311 KRE/20091029 0.469348 0.524715 -0.591855 LVL/20091029 0.568125 0.602185 -0.431816 UNG/20091029 0.397943 0.483732 -0.483611 MUB/20091029 0.476448 0.492256 -0.585944 VT/20091029 0.560773 0.596810 -0.596461 DAG/20091029 0.577605 0.605705 -0.614313 PPH/20091029 0.589295 0.630655 -0.577580 VSS/20091029 0.563280 0.594050 -0.499762 DBB/20091029 0.480214 0.516637 -0.552898 XLP/20091029 0.517837 0.571723 -0.546539 IJT/20091029 0.524520 0.573092 -0.623337 EWZ/20091029 0.606290 0.645656 -0.551028 PBW/20091029 0.540400 0.570823 -0.565300 FXY/20091029 0.550103 0.580384 -0.612818 IYZ/20091029 0.593310 0.625934 -0.600020 MVV/20091029 0.589968 0.613206 -0.579381 VUG/20091029 0.569312 0.590969 -0.406099 PST/20091029 0.399462 0.409892 -0.393232 PSQ/20091029 0.382658 0.408649 -0.570318 VNQ/20091029 0.555741 0.591138 -0.536350 IEI/20091029 0.533993 0.539331 -0.600012 EWW/20091029 0.564352 0.618358 -0.574079 IWP/20091029 0.558199 0.592997 -0.583708 IWV/20091029 0.571253 0.602608 -0.555212 DIG/20091029 0.541608 0.572917 -0.582320 VTI/20091029 0.564788 0.594720 -0.525464 FXA/20091029 0.512841 0.531115 -0.542582 NLR/20091029 0.527104 0.559729 -0.502745 AGG/20091029 0.488421 0.512436 -0.558271 BWX/20091029 0.536846 0.573580 -0.648180 IAU/20091029 0.629977 0.662963 -0.615657 XLV/20091029 0.589990 0.633497 -0.533673 XOP/20091029 0.514307 0.565506 -0.582654 EZU/20091029 0.562805 0.597091 -0.567913 JXI/20091029 0.556089 0.586764 -0.582829 XBI/20091029 0.571639 0.597303 -0.558237 IYG/20091029 0.513907 0.583624 -0.645154 SLX/20091029 0.633626 0.664366 -0.561628 HAO/20091029 0.540954 0.581264 -0.592131 EZA/20091029 0.574692 0.607433 -0.575433 XLY/20091029 0.558276 0.591040 -0.514145 IEF/20091029 0.491847 0.534731 -0.578866 DEM/20091029 0.555336 0.605544 -0.560477 IVW/20091029 0.541802 0.572122 -0.696338 UYM/20091029 0.682107 0.720600 -0.530213 IXC/20091029 0.510621 0.548145 -0.514960 PFF/20091030 0.498872 0.524749 -0.560775 WIP/20091030 0.547769 0.582497 -0.547430 GCC/20091030 0.542803 0.550022 -0.556565 AAXJ/20091030 0.534464 0.596864 -0.570254 VWO/20091030 0.544710 0.603490 -0.290976 EEV/20091030 0.224988 0.362317 -0.698358 GDX/20091030 0.679257 0.731456 -0.533428 RTH/20091030 0.476828 0.553730 -0.607095 MXI/20091030 0.591225 0.643516 -0.535314 EWU/20091030 0.510444 0.555752 -0.436964 SH/20091030 0.411337 0.465517 -0.602841 EDC/20091030 0.557851 0.660944 -0.396115 ERY/20091030 0.371116 0.422314 -0.383325 SDS/20091030 0.348745 0.415024 -0.555193 OEF/20091030 0.524877 0.616041 -0.592179 IYT/20091030 0.562482 0.618701 -0.516442 BIL/20091030 0.498088 0.561407 -0.241153 GLL/20091030 0.194457 0.306620 -0.195405 EDZ/20091030 0.160522 0.229125 -0.528865 IWM/20091030 0.509082 0.568302 -0.527456 VXF/20091030 0.518910 0.532983 -0.521491 IJJ/20091030 0.493297 0.533255 -0.581676 PIN/20091030 0.538954 0.600005 -0.594969 XLB/20091030 0.580065 0.614850 -0.555093 ECH/20091030 0.535149 0.580986 -0.499194 TYH/20091030 0.460157 0.527547 -0.602761 VAW/20091030 0.576792 0.622637 -0.654834 DBP/20091030 0.642340 0.667680 -0.656619 XME/20091030 0.628984 0.683272 -0.532202 VO/20091030 0.504610 0.556723 -0.580720 RSX/20091030 0.546074 0.613579 -0.607965 EWC/20091030 0.588812 0.629688 -0.458565 TUR/20091030 0.433820 0.494270 -0.543597 VYM/20091030 0.524943 0.564556 -0.522575 FCG/20091030 0.488806 0.546224 -0.552696 VGT/20091030 0.534301 0.584894 -0.548305 EWQ/20091030 0.523326 0.593842 -0.541537 IEV/20091030 0.526050 0.566653 -0.536076 XLK/20091030 0.522296 0.559612 -0.545252 EFG/20091030 0.532578 0.567557 -0.589818 BKF/20091030 0.573049 0.611820 -0.551948 KIE/20091030 0.544387 0.572254 -0.612696 EEB/20091030 0.596532 0.637345 -0.543287 IJK/20091030 0.518635 0.573397 -0.423455 DUG/20091030 0.379504 0.454136 -0.388192 TWM/20091030 0.358144 0.404178 -0.547559 MDY/20091030 0.527596 0.568360 -0.564463 ACWI/20091030 0.540019 0.597918 -0.531970 BSV/20091030 0.519426 0.550007 -0.622655 DDM/20091030 0.607341 0.643504 -0.562328 DIA/20091030 0.550688 0.581928 -0.548052 TLT/20091030 0.539934 0.562813 -0.393342 DXD/20091030 0.374851 0.409459 -0.549660 XHB/20091030 0.529252 0.577683 -0.550008 VDE/20091030 0.500650 0.594697 -0.534411 BND/20091030 0.485631 0.567551 -0.559106 EMB/20091030 0.545861 0.568031 -0.518488 SCO/20091030 0.501655 0.539107 -0.567335 AMJ/20091030 0.555062 0.589104 -0.489074 OIL/20091030 0.450191 0.525987 -0.505142 PZA/20091030 0.488540 0.528655 -0.545819 VGK/20091030 0.523304 0.574014 -0.548053 RWX/20091030 0.524782 0.567510 -0.566385 JJA/20091030 0.544653 0.584848 -0.577323 FXD/20091030 0.554885 0.601376 -0.514238 XES/20091030 0.486407 0.544525 -0.546775 VIG/20091030 0.525205 0.563439 -0.178434 DZZ/20091030 0.175554 0.181895 -0.516323 VFH/20091030 0.465886 0.566726 -0.532514 DTO/20091030 0.512836 0.558850 -0.561225 EWP/20091030 0.518186 0.586517 -0.556626 FDN/20091030 0.532253 0.584604 -0.604969 INP/20091030 0.586874 0.626441 -0.333102 TYP/20091030 0.285843 0.372391 -0.585669 RWR/20091030 0.579366 0.600101 -0.543065 KBE/20091030 0.517544 0.582889 -0.463896 EUO/20091030 0.441921 0.485707 -0.544254 IWF/20091030 0.523289 0.568452 -0.267077 SMN/20091030 0.221719 0.309497 -0.556505 SMH/20091030 0.534646 0.584935 -0.536588 XRT/20091030 0.495658 0.592737 -0.483145 USO/20091030 0.469610 0.495159 -0.529256 DJP/20091030 0.512385 0.545524 -0.529805 CFT/20091030 0.518755 0.539140 -0.317068 SRS/20091030 0.282089 0.358343 -0.609500 MOO/20091030 0.600994 0.626739 -0.538180 BIV/20091030 0.512294 0.556697 -0.332484 VXX/20091030 0.288350 0.391795 -0.604489 IYM/20091030 0.574527 0.637330 -0.587875 IFN/20091030 0.570332 0.619353 -0.655908 SLV/20091030 0.630194 0.672427 -0.530591 TAO/20091030 0.510039 0.554262 -0.509226 PGF/20091030 0.505948 0.512389 -0.583863 IYR/20091030 0.579162 0.591243 -0.320680 QID/20091030 0.286611 0.361108 -0.556493 THD/20091030 0.537741 0.566899 -0.540596 IJS/20091030 0.518010 0.575523 -0.538321 VB/20091030 0.519532 0.565627 -0.534354 EDV/20091030 0.522133 0.548037 -0.524104 IEZ/20091030 0.499707 0.559863 -0.558186 VTV/20091030 0.537379 0.589096 -0.538606 IJR/20091030 0.516450 0.565869 -0.459669 UCO/20091030 0.446510 0.483172 -0.515477 JNK/20091030 0.500648 0.530343 -0.547274 IWN/20091030 0.533809 0.572593 -0.549737 VV/20091030 0.527845 0.585766 -0.741339 UGL/20091030 0.676239 0.793313 -0.564072 UWM/20091030 0.532261 0.582579 -0.507178 IWC/20091030 0.480253 0.552182 -0.540486 EWA/20091030 0.505809 0.571240 -0.537681 IVV/20091030 0.516526 0.557957 -0.536853 SPY/20091030 0.521812 0.555518 -0.518191 TFI/20091030 0.509360 0.531661 -0.530377 VEA/20091030 0.506509 0.556824 -0.535905 QQQQ/20091030 0.523584 0.557874 -0.542650 UYG/20091030 0.511451 0.589287 -0.494925 OIH/20091030 0.466443 0.517809 -0.537218 GXC/20091030 0.522592 0.566470 -0.602579 SSO/20091030 0.590273 0.626600 -0.562452 XLI/20091030 0.549114 0.580302 -0.613184 GML/20091030 0.592195 0.646917 -0.589918 ROM/20091030 0.569314 0.617717 -0.524109 FXC/20091030 0.510299 0.541689 -0.452234 DOG/20091030 0.412133 0.474414 -0.502322 IYE/20091030 0.483917 0.529043 -0.397109 SKF/20091030 0.341724 0.443676 -0.510365 SHY/20091030 0.494648 0.525796 -0.533476 DBA/20091030 0.511356 0.569960 -0.538771 RSP/20091030 0.516006 0.571005 -0.656764 DBS/20091030 0.630423 0.688953 -0.581261 IBB/20091030 0.569047 0.603917 -0.491904 KCE/20091030 0.477167 0.513623 -0.511369 PKN/20091030 0.484193 0.549495 -0.542674 TNA/20091030 0.491721 0.600253 -0.613075 FAS/20091030 0.551317 0.683407 -0.529163 FXE/20091030 0.515516 0.535343 -0.502656 HYG/20091030 0.494550 0.508112 -0.549866 IWS/20091030 0.512233 0.574108 -0.384165 FXP/20091030 0.358754 0.415982 -0.513268 MBB/20091030 0.489527 0.537793 -0.558158 RFG/20091030 0.545113 0.578325 -0.587034 EPU/20091030 0.557195 0.605123 -0.466028 UUP/20091030 0.440347 0.479604 -0.749603 AGQ/20091030 0.702482 0.789622 -0.565580 SOXX/20091030 0.541299 0.590432 -0.325069 FAZ/20091030 0.293671 0.345278 -0.536582 VBK/20091030 0.511027 0.562975 -0.542782 RPG/20091030 0.528855 0.567659 -0.542087 EWH/20091030 0.529462 0.560933 -0.365889 TZA/20091030 0.303963 0.412778 -0.534125 SGG/20091030 0.519048 0.551472 -0.615936 KOL/20091030 0.595213 0.642115 -0.545909 EWY/20091030 0.526325 0.569581 -0.550845 PRF/20091030 0.519651 0.579276 -0.539382 TLH/20091030 0.528617 0.551129 -0.547603 EPP/20091030 0.524691 0.582158 -0.518140 XLE/20091030 0.498116 0.548487 -0.569293 EWN/20091030 0.559087 0.579384 -0.475907 SHM/20091030 0.449077 0.507454 -0.538185 FXI/20091030 0.515360 0.561846 -0.579746 EWS/20091030 0.553919 0.606735 -0.540839 IDU/20091030 0.517720 0.574540 -0.504716 VXZ/20091030 0.480503 0.527693 -0.545940 IVE/20091030 0.531489 0.563048 -0.745734 DGP/20091030 0.734141 0.755949 -0.538570 GMF/20091030 0.511348 0.565382 -0.516223 IWR/20091030 0.482964 0.556773 -0.490097 RKH/20091030 0.472633 0.511084 -0.524569 TIP/20091030 0.519480 0.531442 -0.632115 URE/20091030 0.622791 0.638605 -0.479661 DBO/20091030 0.471666 0.488098 -0.507888 IOO/20091030 0.478460 0.536157 -0.477412 DBV/20091030 0.456877 0.510949 -0.521461 EFA/20091030 0.499858 0.564686 -0.564631 BGU/20091030 0.531613 0.613522 -0.520123 EFV/20091030 0.510707 0.530847 -0.532589 IWB/20091030 0.512803 0.545448 -0.518121 IYF/20091030 0.512820 0.530645 -0.378637 YCS/20091030 0.373955 0.382780 -0.490886 DXJ/20091030 0.481274 0.502169 -0.522369 IWO/20091030 0.513535 0.536702 -0.530019 DBC/20091030 0.525633 0.532744 -0.426505 RWM/20091030 0.350669 0.471639 -0.530513 VBR/20091030 0.516773 0.553624 -0.373352 MZZ/20091030 0.291413 0.408723 -0.542499 IWD/20091030 0.533732 0.559354 -0.526973 PCY/20091030 0.485430 0.558188 -0.533143 EWI/20091030 0.525306 0.546834 -0.548883 IJH/20091030 0.530183 0.588061 -0.575024 EEM/20091030 0.566490 0.589436 -0.538609 EWM/20091030 0.523685 0.552540 -0.531276 SDY/20091030 0.515775 0.557017 -0.636091 ILF/20091030 0.621931 0.648927 -0.581143 JJG/20091030 0.565775 0.597047 -0.421984 TBT/20091030 0.408138 0.436292 -0.520198 XLF/20091030 0.511928 0.528406 -0.547567 ERX/20091030 0.497556 0.597391 -0.489390 SHV/20091030 0.481831 0.493313 -0.591522 EWX/20091030 0.562110 0.642180 -0.432104 EFZ/20091030 0.372676 0.469426 -0.435003 FXB/20091030 0.420393 0.451566 -0.514801 PHO/20091030 0.500395 0.532434 -0.573990 IGE/20091030 0.552364 0.624741 -0.268803 BGZ/20091030 0.228300 0.306322 -0.520813 UDN/20091030 0.512047 0.533014 -0.488994 CSJ/20091030 0.485014 0.496986 -0.619524 GXG/20091030 0.584743 0.674658 -0.611933 USD/20091030 0.588071 0.631703 -0.538647 EWD/20091030 0.515925 0.570525 -0.498792 EWJ/20091030 0.490716 0.511135 -0.680206 BRF/20091030 0.649775 0.721833 -0.555749 VEU/20091030 0.542328 0.573021 -0.531912 XLU/20091030 0.493463 0.557552 -0.531350 JJC/20091030 0.514277 0.544771 -0.568065 FGD/20091030 0.555937 0.587735 -0.513001 FXF/20091030 0.504721 0.518799 -0.519111 LQD/20091030 0.512326 0.532162 -0.542289 SCZ/20091030 0.531951 0.558927 -0.542282 IYW/20091030 0.531017 0.562274 -0.502615 VPL/20091030 0.483166 0.530948 -0.555752 DGS/20091030 0.527636 0.581438 -0.572128 ICF/20091030 0.543905 0.591265 -0.528673 DVY/20091030 0.508598 0.561135 -0.522873 IEO/20091030 0.486685 0.595797 -0.533991 VOT/20091030 0.515465 0.559021 -0.499709 CIU/20091030 0.479692 0.509413 -0.558022 EWG/20091030 0.523502 0.588903 -0.553458 EWT/20091030 0.533430 0.572493 -0.495545 GSG/20091030 0.487708 0.509722 -0.497112 KRE/20091030 0.474080 0.522363 -0.571389 LVL/20091030 0.559703 0.594324 -0.415438 UNG/20091030 0.385837 0.459103 -0.498236 MUB/20091030 0.488293 0.506344 -0.551693 VT/20091030 0.539227 0.576240 -0.616534 DAG/20091030 0.591462 0.640133 -0.567856 PPH/20091030 0.551504 0.579965 -0.550234 VSS/20091030 0.531217 0.579415 -0.521336 DBB/20091030 0.505663 0.530891 -0.528572 XLP/20091030 0.508077 0.552847 -0.519860 IJT/20091030 0.504652 0.554806 -0.622211 EWZ/20091030 0.598459 0.662119 -0.532717 PBW/20091030 0.508936 0.565779 -0.572829 FXY/20091030 0.549017 0.596580 -0.557642 IYZ/20091030 0.540400 0.582477 -0.579947 MVV/20091030 0.558208 0.608038 -0.535573 VUG/20091030 0.511798 0.554482 -0.414458 PST/20091030 0.406852 0.419466 -0.421807 PSQ/20091030 0.399233 0.450014 -0.568351 VNQ/20091030 0.546569 0.583942 -0.518633 IEI/20091030 0.511640 0.529370 -0.587998 EWW/20091030 0.538282 0.620980 -0.537051 IWP/20091030 0.508871 0.566882 -0.537592 IWV/20091030 0.509143 0.569877 -0.549796 DIG/20091030 0.523773 0.589821 -0.541434 VTI/20091030 0.525326 0.560029 -0.509066 FXA/20091030 0.492363 0.528553 -0.510983 NLR/20091030 0.482680 0.536924 -0.515897 AGG/20091030 0.490519 0.531344 -0.544547 BWX/20091030 0.524382 0.558167 -0.642637 IAU/20091030 0.626603 0.654483 -0.572591 XLV/20091030 0.558703 0.585733 -0.514209 XOP/20091030 0.477715 0.556594 -0.542950 EZU/20091030 0.522323 0.572200 -0.524838 JXI/20091030 0.504975 0.535285 -0.556283 XBI/20091030 0.540518 0.578212 -0.514085 IYG/20091030 0.485485 0.541768 -0.643325 SLX/20091030 0.620622 0.676170 -0.558159 HAO/20091030 0.520113 0.573340 -0.576630 EZA/20091030 0.558317 0.606793 -0.551124 XLY/20091030 0.534504 0.578900 -0.508844 IEF/20091030 0.491751 0.533356 -0.544083 DEM/20091030 0.517311 0.587346 -0.522903 IVW/20091030 0.510988 0.546200 -0.688590 UYM/20091030 0.661972 0.732614 -0.512091 IXC/20091030 0.495350 0.545917 -0.490976 PFF/20091102 0.482427 0.507378 -0.541520 WIP/20091102 0.526962 0.561376 -0.554828 GCC/20091102 0.544578 0.565491 -0.582937 AAXJ/20091102 0.565540 0.613672 -0.592711 VWO/20091102 0.579199 0.616356 -0.293790 EEV/20091102 0.241147 0.354041 -0.685825 GDX/20091102 0.665539 0.711871 -0.556444 RTH/20091102 0.506665 0.589075 -0.616597 MXI/20091102 0.601751 0.645215 -0.557885 EWU/20091102 0.543512 0.584571 -0.410310 SH/20091102 0.390497 0.433985 -0.588908 EDC/20091102 0.553911 0.632281 -0.392238 ERY/20091102 0.374149 0.412285 -0.366042 SDS/20091102 0.339406 0.387423 -0.588336 OEF/20091102 0.564226 0.641804 -0.599647 IYT/20091102 0.574112 0.622569 -0.490230 BIL/20091102 0.475029 0.529876 -0.278836 GLL/20091102 0.230447 0.335500 -0.214475 EDZ/20091102 0.196036 0.237178 -0.545500 IWM/20091102 0.531820 0.581572 -0.544891 VXF/20091102 0.539574 0.553617 -0.547285 IJJ/20091102 0.527344 0.558901 -0.618459 PIN/20091102 0.589340 0.637195 -0.612629 XLB/20091102 0.602876 0.626028 -0.564838 ECH/20091102 0.551037 0.585513 -0.511228 TYH/20091102 0.481449 0.531383 -0.620118 VAW/20091102 0.603162 0.633619 -0.632313 DBP/20091102 0.623552 0.639387 -0.653888 XME/20091102 0.630792 0.676295 -0.553595 VO/20091102 0.534186 0.573071 -0.579444 RSX/20091102 0.554053 0.602102 -0.610125 EWC/20091102 0.596787 0.627147 -0.475527 TUR/20091102 0.457412 0.503280 -0.567263 VYM/20091102 0.557365 0.582915 -0.542721 FCG/20091102 0.518734 0.563958 -0.576552 VGT/20091102 0.556030 0.604199 -0.572035 EWQ/20091102 0.557645 0.609599 -0.559151 IEV/20091102 0.551963 0.578466 -0.565708 XLK/20091102 0.558526 0.582930 -0.560823 EFG/20091102 0.553976 0.576842 -0.593025 BKF/20091102 0.584891 0.607103 -0.540478 KIE/20091102 0.532337 0.556708 -0.615096 EEB/20091102 0.604044 0.631177 -0.568665 IJK/20091102 0.548897 0.593273 -0.407187 DUG/20091102 0.370092 0.427863 -0.380434 TWM/20091102 0.355260 0.392046 -0.574194 MDY/20091102 0.562500 0.589582 -0.591973 ACWI/20091102 0.576602 0.617990 -0.529346 BSV/20091102 0.511244 0.536839 -0.642232 DDM/20091102 0.617412 0.653543 -0.583233 DIA/20091102 0.577636 0.595459 -0.518620 TLT/20091102 0.512948 0.522003 -0.376655 DXD/20091102 0.346481 0.391000 -0.562461 XHB/20091102 0.546483 0.584208 -0.573339 VDE/20091102 0.532927 0.622706 -0.506446 BND/20091102 0.470168 0.530619 -0.522478 EMB/20091102 0.503808 0.538555 -0.502611 SCO/20091102 0.477449 0.513507 -0.570485 AMJ/20091102 0.557812 0.591773 -0.513868 OIL/20091102 0.485717 0.556904 -0.522302 PZA/20091102 0.507967 0.546212 -0.567056 VGK/20091102 0.550483 0.587695 -0.550170 RWX/20091102 0.530635 0.564584 -0.542427 JJA/20091102 0.537216 0.545106 -0.582751 FXD/20091102 0.563225 0.602938 -0.532736 XES/20091102 0.510183 0.556865 -0.579543 VIG/20091102 0.563716 0.589280 -0.219820 DZZ/20091102 0.209430 0.233641 -0.532575 VFH/20091102 0.490466 0.576772 -0.532758 DTO/20091102 0.519637 0.548115 -0.567353 EWP/20091102 0.530457 0.587261 -0.576636 FDN/20091102 0.557335 0.601515 -0.634378 INP/20091102 0.628996 0.645292 -0.326778 TYP/20091102 0.286710 0.358161 -0.575270 RWR/20091102 0.570585 0.586792 -0.552934 KBE/20091102 0.533049 0.584230 -0.464307 EUO/20091102 0.445511 0.481861 -0.576661 IWF/20091102 0.565869 0.594801 -0.271194 SMN/20091102 0.234833 0.305178 -0.582907 SMH/20091102 0.564923 0.607350 -0.553442 XRT/20091102 0.520358 0.604250 -0.508791 USO/20091102 0.494067 0.532339 -0.547536 DJP/20091102 0.539483 0.557802 -0.515382 CFT/20091102 0.507202 0.523373 -0.333676 SRS/20091102 0.303909 0.372903 -0.628693 MOO/20091102 0.613109 0.636956 -0.516439 BIV/20091102 0.501104 0.522795 -0.324660 VXX/20091102 0.284997 0.378421 -0.618210 IYM/20091102 0.596603 0.644906 -0.606304 IFN/20091102 0.593352 0.632103 -0.639218 SLV/20091102 0.614899 0.652355 -0.523934 TAO/20091102 0.500281 0.542677 -0.516031 PGF/20091102 0.513212 0.518840 -0.575973 IYR/20091102 0.572259 0.580497 -0.301060 QID/20091102 0.273383 0.333306 -0.580304 THD/20091102 0.570358 0.590206 -0.558440 IJS/20091102 0.541936 0.590151 -0.555272 VB/20091102 0.541417 0.578538 -0.522678 EDV/20091102 0.504519 0.533258 -0.538697 IEZ/20091102 0.520233 0.569054 -0.582260 VTV/20091102 0.564610 0.607543 -0.559547 IJR/20091102 0.530730 0.583055 -0.477048 UCO/20091102 0.452830 0.509210 -0.514496 JNK/20091102 0.501212 0.532994 -0.561003 IWN/20091102 0.545880 0.583644 -0.578266 VV/20091102 0.559881 0.608245 -0.704933 UGL/20091102 0.642168 0.745777 -0.575384 UWM/20091102 0.551019 0.591078 -0.513200 IWC/20091102 0.489962 0.556435 -0.568596 EWA/20091102 0.544808 0.591293 -0.563211 IVV/20091102 0.550022 0.577611 -0.566127 SPY/20091102 0.550397 0.578387 -0.508092 TFI/20091102 0.502411 0.519815 -0.550650 VEA/20091102 0.536447 0.570233 -0.564206 QQQQ/20091102 0.549622 0.581000 -0.548145 UYG/20091102 0.509734 0.584692 -0.513728 OIH/20091102 0.493609 0.530957 -0.553595 GXC/20091102 0.540028 0.569987 -0.624451 SSO/20091102 0.607598 0.639740 -0.595361 XLI/20091102 0.584408 0.605244 -0.609212 GML/20091102 0.588996 0.636314 -0.608774 ROM/20091102 0.596550 0.630314 -0.538970 FXC/20091102 0.519925 0.553350 -0.428083 DOG/20091102 0.394763 0.444990 -0.530239 IYE/20091102 0.520594 0.550572 -0.397896 SKF/20091102 0.352459 0.434976 -0.529152 SHY/20091102 0.520545 0.537347 -0.530100 DBA/20091102 0.517980 0.555031 -0.562619 RSP/20091102 0.546712 0.589297 -0.640063 DBS/20091102 0.615102 0.668940 -0.577860 IBB/20091102 0.564312 0.596028 -0.512732 KCE/20091102 0.490852 0.529495 -0.526859 PKN/20091102 0.507418 0.559023 -0.548571 TNA/20091102 0.492577 0.599533 -0.608428 FAS/20091102 0.559387 0.664846 -0.536542 FXE/20091102 0.518595 0.548552 -0.519363 HYG/20091102 0.514793 0.523320 -0.568345 IWS/20091102 0.538582 0.593791 -0.381404 FXP/20091102 0.361888 0.396738 -0.517636 MBB/20091102 0.505539 0.527689 -0.572306 RFG/20091102 0.553303 0.587747 -0.600616 EPU/20091102 0.578889 0.622095 -0.456978 UUP/20091102 0.429233 0.468122 -0.722736 AGQ/20091102 0.675647 0.757594 -0.589091 SOXX/20091102 0.556877 0.609537 -0.336451 FAZ/20091102 0.318482 0.345453 -0.550576 VBK/20091102 0.519444 0.573190 -0.574267 RPG/20091102 0.551058 0.592583 -0.547701 EWH/20091102 0.537774 0.563924 -0.362527 TZA/20091102 0.306442 0.403653 -0.486581 SGG/20091102 0.475679 0.498526 -0.629692 KOL/20091102 0.604941 0.647930 -0.564270 EWY/20091102 0.550784 0.580477 -0.558583 PRF/20091102 0.522464 0.582251 -0.513997 TLH/20091102 0.505772 0.524302 -0.572336 EPP/20091102 0.550465 0.599707 -0.541955 XLE/20091102 0.524085 0.566590 -0.563855 EWN/20091102 0.552824 0.576032 -0.525016 SHM/20091102 0.511907 0.544387 -0.555021 FXI/20091102 0.543618 0.565668 -0.598805 EWS/20091102 0.584604 0.616838 -0.551908 IDU/20091102 0.532643 0.584089 -0.485919 VXZ/20091102 0.470214 0.497674 -0.569352 IVE/20091102 0.553326 0.580511 -0.706236 DGP/20091102 0.698202 0.719777 -0.585538 GMF/20091102 0.574912 0.600334 -0.537882 IWR/20091102 0.512676 0.573471 -0.510734 RKH/20091102 0.491391 0.524447 -0.518479 TIP/20091102 0.517125 0.519211 -0.618903 URE/20091102 0.608848 0.627444 -0.513162 DBO/20091102 0.507850 0.519252 -0.538491 IOO/20091102 0.520898 0.558949 -0.493080 DBV/20091102 0.468439 0.530330 -0.539792 EFA/20091102 0.520440 0.576515 -0.579101 BGU/20091102 0.558846 0.616827 +0.451538 PFF/20091028 0.440871 0.467956 +0.543911 WIP/20091028 0.523647 0.563905 +0.514364 GCC/20091028 0.510615 0.518241 +0.523054 AAXJ/20091028 0.496725 0.562864 +0.533791 VWO/20091028 0.501782 0.567986 +0.331371 EEV/20091028 0.265972 0.410107 +0.651821 GDX/20091028 0.631098 0.690795 +0.505666 RTH/20091028 0.448661 0.529840 +0.571783 MXI/20091028 0.552894 0.608530 +0.527963 EWU/20091028 0.506018 0.549452 +0.451783 SH/20091028 0.426624 0.480909 +0.558423 EDC/20091028 0.509725 0.614216 +0.424295 ERY/20091028 0.406155 0.452080 +0.400557 SDS/20091028 0.367952 0.434771 +0.555752 OEF/20091028 0.526847 0.613436 +0.577292 IYT/20091028 0.543671 0.609086 +0.551702 BIL/20091028 0.533170 0.570979 +0.271137 GLL/20091028 0.212522 0.353346 +0.237789 EDZ/20091028 0.203784 0.280822 +0.493377 IWM/20091028 0.466087 0.534800 +0.488584 VXF/20091028 0.475796 0.499186 +0.488204 IJJ/20091028 0.453593 0.506045 +0.575258 PIN/20091028 0.534806 0.594913 +0.572375 XLB/20091028 0.552937 0.592635 +0.521250 ECH/20091028 0.503910 0.541805 +0.467600 TYH/20091028 0.425689 0.490814 +0.575675 VAW/20091028 0.544550 0.596210 +0.604181 DBP/20091028 0.575271 0.626508 +0.617639 XME/20091028 0.581136 0.647709 +0.499465 VO/20091028 0.465744 0.525408 +0.530214 RSX/20091028 0.485825 0.566008 +0.565182 EWC/20091028 0.537238 0.589474 +0.430270 TUR/20091028 0.404785 0.464098 +0.534598 VYM/20091028 0.514534 0.554292 +0.504553 FCG/20091028 0.468824 0.526611 +0.531691 VGT/20091028 0.512223 0.563309 +0.539608 EWQ/20091028 0.514344 0.583195 +0.528741 IEV/20091028 0.511908 0.553429 +0.523979 XLK/20091028 0.510800 0.544756 +0.527961 EFG/20091028 0.512698 0.550630 +0.543930 BKF/20091028 0.516779 0.570292 +0.536966 KIE/20091028 0.523967 0.560722 +0.570969 EEB/20091028 0.550392 0.598417 +0.506274 IJK/20091028 0.477662 0.538252 +0.450119 DUG/20091028 0.412262 0.480093 +0.421988 TWM/20091028 0.390913 0.446531 +0.513725 MDY/20091028 0.487642 0.537734 +0.544157 ACWI/20091028 0.516641 0.577741 +0.586551 BSV/20091028 0.576361 0.605858 +0.616974 DDM/20091028 0.602240 0.633561 +0.559758 DIA/20091028 0.549745 0.576202 +0.573525 TLT/20091028 0.564778 0.589181 +0.398877 DXD/20091028 0.379607 0.412580 +0.523297 XHB/20091028 0.497651 0.553727 +0.528661 VDE/20091028 0.479315 0.572397 +0.556615 BND/20091028 0.513113 0.585593 +0.503495 EMB/20091028 0.483377 0.520562 +0.540467 SCO/20091028 0.523286 0.556003 +0.538234 AMJ/20091028 0.521369 0.568991 +0.471139 OIL/20091028 0.435448 0.508175 +0.483408 PZA/20091028 0.465810 0.506485 +0.534086 VGK/20091028 0.510890 0.561114 +0.507788 RWX/20091028 0.477704 0.537365 +0.531943 JJA/20091028 0.502914 0.553629 +0.529912 FXD/20091028 0.502123 0.557786 +0.487777 XES/20091028 0.457691 0.516267 +0.536390 VIG/20091028 0.514973 0.552079 +0.206011 DZZ/20091028 0.193149 0.219135 +0.497369 VFH/20091028 0.442059 0.548590 +0.555333 DTO/20091028 0.545151 0.572846 +0.548822 EWP/20091028 0.504409 0.574209 +0.516821 FDN/20091028 0.486328 0.544987 +0.594335 INP/20091028 0.577039 0.618520 +0.363282 TYP/20091028 0.320917 0.405145 +0.538129 RWR/20091028 0.515383 0.563495 +0.534221 KBE/20091028 0.508278 0.573713 +0.482986 EUO/20091028 0.461721 0.505938 +0.525479 IWF/20091028 0.502283 0.548804 +0.296672 SMN/20091028 0.250217 0.345839 +0.533272 SMH/20091028 0.509525 0.563069 +0.492414 XRT/20091028 0.443016 0.549853 +0.466129 USO/20091028 0.453339 0.481450 +0.495727 DJP/20091028 0.476689 0.511459 +0.573252 CFT/20091028 0.556456 0.585915 +0.357352 SRS/20091028 0.311652 0.414700 +0.580648 MOO/20091028 0.569826 0.603844 +0.563534 BIV/20091028 0.541565 0.578582 +0.370136 VXX/20091028 0.329499 0.427187 +0.576896 IYM/20091028 0.541223 0.611057 +0.560782 IFN/20091028 0.545096 0.586062 +0.612196 SLV/20091028 0.576409 0.633535 +0.489953 TAO/20091028 0.477888 0.518350 +0.460745 PGF/20091028 0.446140 0.470308 +0.535312 IYR/20091028 0.515294 0.552423 +0.350221 QID/20091028 0.318687 0.393710 +0.539231 THD/20091028 0.521031 0.554533 +0.513477 IJS/20091028 0.485016 0.549287 +0.502795 VB/20091028 0.476353 0.532302 +0.563072 EDV/20091028 0.554868 0.576882 +0.498848 IEZ/20091028 0.471571 0.533793 +0.546668 VTV/20091028 0.524090 0.576456 +0.505853 IJR/20091028 0.491948 0.534408 +0.435303 UCO/20091028 0.414876 0.459339 +0.453089 JNK/20091028 0.434623 0.474137 +0.517467 IWN/20091028 0.497349 0.544473 +0.534104 VV/20091028 0.510042 0.569231 +0.711433 UGL/20091028 0.639634 0.776142 +0.527848 UWM/20091028 0.487583 0.547294 +0.473879 IWC/20091028 0.439693 0.520749 +0.508259 EWA/20091028 0.468814 0.537735 +0.523883 IVV/20091028 0.500974 0.543022 +0.523901 SPY/20091028 0.507280 0.541260 +0.511538 TFI/20091028 0.503677 0.523394 +0.514837 VEA/20091028 0.488757 0.541033 +0.508185 QQQQ/20091028 0.492648 0.529451 +0.522897 UYG/20091028 0.497758 0.568991 +0.468481 OIH/20091028 0.437658 0.490247 +0.492851 GXC/20091028 0.474771 0.519114 +0.587153 SSO/20091028 0.573642 0.608179 +0.543746 XLI/20091028 0.529519 0.560310 +0.560479 GML/20091028 0.526852 0.599532 +0.562920 ROM/20091028 0.539373 0.588035 +0.497867 FXC/20091028 0.485451 0.516546 +0.454510 DOG/20091028 0.417135 0.475082 +0.480437 IYE/20091028 0.463946 0.502388 +0.417979 SKF/20091028 0.361639 0.471096 +0.563147 SHY/20091028 0.539903 0.584177 +0.503542 DBA/20091028 0.477285 0.540871 +0.515288 RSP/20091028 0.488253 0.548129 +0.613329 DBS/20091028 0.577113 0.650219 +0.532525 IBB/20091028 0.514104 0.561344 +0.453586 KCE/20091028 0.443200 0.477716 +0.500282 PKN/20091028 0.473006 0.535786 +0.508016 TNA/20091028 0.453800 0.564677 +0.589573 FAS/20091028 0.521768 0.657547 +0.502534 FXE/20091028 0.494029 0.508801 +0.457192 HYG/20091028 0.439765 0.470346 +0.520737 IWS/20091028 0.477104 0.546812 +0.429958 FXP/20091028 0.409997 0.466553 +0.563795 MBB/20091028 0.535207 0.588599 +0.512900 RFG/20091028 0.500936 0.535637 +0.546458 EPU/20091028 0.509846 0.565451 +0.479280 UUP/20091028 0.454346 0.496177 +0.706370 AGQ/20091028 0.648441 0.751662 +0.537814 SOXX/20091028 0.518884 0.564100 +0.346989 FAZ/20091028 0.315046 0.374797 +0.493754 VBK/20091028 0.475683 0.523354 +0.516317 RPG/20091028 0.499357 0.540652 +0.507289 EWH/20091028 0.489865 0.527888 +0.405743 TZA/20091028 0.345037 0.458339 +0.502580 SGG/20091028 0.478173 0.532669 +0.581633 KOL/20091028 0.554389 0.608680 +0.516955 EWY/20091028 0.491458 0.541039 +0.529607 PRF/20091028 0.504442 0.559431 +0.569607 TLH/20091028 0.556230 0.582456 +0.511533 EPP/20091028 0.483359 0.545938 +0.496449 XLE/20091028 0.476749 0.523419 +0.546607 EWN/20091028 0.531030 0.557731 +0.503487 SHM/20091028 0.490384 0.524725 +0.496517 FXI/20091028 0.469628 0.517832 +0.553660 EWS/20091028 0.523084 0.581464 +0.541299 IDU/20091028 0.520063 0.571623 +0.540403 VXZ/20091028 0.524403 0.554568 +0.535698 IVE/20091028 0.519147 0.551906 +0.713338 DGP/20091028 0.684645 0.730259 +0.514870 GMF/20091028 0.487194 0.536814 +0.483172 IWR/20091028 0.443514 0.525436 +0.481730 RKH/20091028 0.461843 0.501540 +0.572089 TIP/20091028 0.565389 0.577318 +0.587514 URE/20091028 0.568642 0.603799 +0.454112 DBO/20091028 0.447255 0.463173 +0.500817 IOO/20091028 0.472093 0.526410 +0.442858 DBV/20091028 0.421700 0.473064 +0.506346 EFA/20091028 0.482621 0.549244 +0.544934 BGU/20091028 0.510080 0.589684 +0.504310 EFV/20091028 0.498942 0.515136 +0.525839 IWB/20091028 0.505264 0.537112 +0.502442 IYF/20091028 0.494451 0.515305 +0.320152 YCS/20091028 0.306290 0.330927 +0.502812 DXJ/20091028 0.494703 0.510613 +0.478245 IWO/20091028 0.464445 0.495678 +0.486612 DBC/20091028 0.478721 0.497217 +0.458400 RWM/20091028 0.385729 0.510320 +0.500464 VBR/20091028 0.487294 0.525279 +0.409747 MZZ/20091028 0.327042 0.452582 +0.530827 IWD/20091028 0.521787 0.546699 +0.455938 PCY/20091028 0.432448 0.491506 +0.516979 EWI/20091028 0.506732 0.530501 +0.513695 IJH/20091028 0.493785 0.554515 +0.537325 EEM/20091028 0.522220 0.552694 +0.503523 EWM/20091028 0.484119 0.523748 +0.521209 SDY/20091028 0.506717 0.544175 +0.594779 ILF/20091028 0.575660 0.611494 +0.547105 JJG/20091028 0.531143 0.563172 +0.387912 TBT/20091028 0.375537 0.405866 +0.504920 XLF/20091028 0.497598 0.513172 +0.518120 ERX/20091028 0.469245 0.574830 +0.494940 SHV/20091028 0.478450 0.510381 +0.547033 EWX/20091028 0.512624 0.600315 +0.445131 EFZ/20091028 0.385385 0.484502 +0.492477 FXB/20091028 0.479523 0.503210 +0.492363 PHO/20091028 0.472178 0.512736 +0.548281 IGE/20091028 0.523039 0.598328 +0.291581 BGZ/20091028 0.252865 0.332842 +0.505495 UDN/20091028 0.492835 0.518631 +0.464754 CSJ/20091028 0.457608 0.469018 +0.570242 GXG/20091028 0.548116 0.634167 +0.583783 USD/20091028 0.550746 0.605432 +0.533439 EWD/20091028 0.508625 0.566144 +0.507507 EWJ/20091028 0.501357 0.519375 +0.634398 BRF/20091028 0.590109 0.682997 +0.529466 VEU/20091028 0.511352 0.547905 +0.535324 XLU/20091028 0.500012 0.557034 +0.502458 JJC/20091028 0.492738 0.518308 +0.543846 FGD/20091028 0.527470 0.564001 +0.493451 FXF/20091028 0.489358 0.497689 +0.497244 LQD/20091028 0.478332 0.505038 +0.493094 SCZ/20091028 0.470019 0.504631 +0.519301 IYW/20091028 0.505536 0.538175 +0.482337 VPL/20091028 0.460584 0.512004 +0.512334 DGS/20091028 0.475291 0.541018 +0.525005 ICF/20091028 0.480809 0.554533 +0.515036 DVY/20091028 0.493816 0.546006 +0.497073 IEO/20091028 0.457943 0.568606 +0.499639 VOT/20091028 0.482112 0.526267 +0.517103 CIU/20091028 0.497092 0.526290 +0.538328 EWG/20091028 0.501558 0.568075 +0.521486 EWT/20091028 0.498065 0.539099 +0.452752 GSG/20091028 0.443990 0.463887 +0.497597 KRE/20091028 0.466172 0.510675 +0.540345 LVL/20091028 0.526021 0.564092 +0.394418 UNG/20091028 0.370776 0.437549 +0.464850 MUB/20091028 0.456188 0.475465 +0.526244 VT/20091028 0.509282 0.551725 +0.582350 DAG/20091028 0.552134 0.608234 +0.577006 PPH/20091028 0.560548 0.589395 +0.511405 VSS/20091028 0.484059 0.543980 +0.470140 DBB/20091028 0.463406 0.476722 +0.557082 XLP/20091028 0.531709 0.576374 +0.479391 IJT/20091028 0.457040 0.516377 +0.581149 EWZ/20091028 0.545676 0.627209 +0.487719 PBW/20091028 0.454631 0.524696 +0.610508 FXY/20091028 0.584177 0.632708 +0.578638 IYZ/20091028 0.567846 0.597993 +0.543748 MVV/20091028 0.514936 0.571836 +0.515574 VUG/20091028 0.489153 0.533980 +0.377760 PST/20091028 0.367750 0.386636 +0.454767 PSQ/20091028 0.431633 0.487652 +0.519859 VNQ/20091028 0.481384 0.545394 +0.561785 IEI/20091028 0.551350 0.573806 +0.554692 EWW/20091028 0.497473 0.590357 +0.505340 IWP/20091028 0.471543 0.536312 +0.516468 IWV/20091028 0.484402 0.548913 +0.523532 DIG/20091028 0.499587 0.556369 +0.521104 VTI/20091028 0.501645 0.539531 +0.474713 FXA/20091028 0.452156 0.496446 +0.486242 NLR/20091028 0.454801 0.518356 +0.541980 AGG/20091028 0.520282 0.557482 +0.531140 BWX/20091028 0.516704 0.542833 +0.608415 IAU/20091028 0.602519 0.618880 +0.571069 XLV/20091028 0.558312 0.582667 +0.482980 XOP/20091028 0.443543 0.532585 +0.525405 EZU/20091028 0.501993 0.554682 +0.531493 JXI/20091028 0.513897 0.541581 +0.521319 XBI/20091028 0.499725 0.549129 +0.504386 IYG/20091028 0.478522 0.531044 +0.595715 SLX/20091028 0.561914 0.633460 +0.513169 HAO/20091028 0.473162 0.535166 +0.540277 EZA/20091028 0.512956 0.574573 +0.517524 XLY/20091028 0.494338 0.547238 +0.543224 IEF/20091028 0.524291 0.567560 +0.513804 DEM/20091028 0.481970 0.557189 +0.505286 IVW/20091028 0.493041 0.526738 +0.658936 UYM/20091028 0.625720 0.703897 +0.480244 IXC/20091028 0.461517 0.511932 +0.493109 PFF/20091029 0.481436 0.501240 +0.583667 WIP/20091029 0.570314 0.610245 +0.547072 GCC/20091029 0.536604 0.560816 +0.580531 AAXJ/20091029 0.562585 0.604973 +0.594914 VWO/20091029 0.586758 0.611579 +0.293936 EEV/20091029 0.253221 0.345151 +0.702062 GDX/20091029 0.679495 0.724715 +0.544014 RTH/20091029 0.498633 0.581954 +0.624609 MXI/20091029 0.607279 0.645568 +0.556410 EWU/20091029 0.540046 0.590825 +0.389776 SH/20091029 0.369666 0.407485 +0.577417 EDC/20091029 0.548509 0.604108 +0.391809 ERY/20091029 0.369693 0.402961 +0.348197 SDS/20091029 0.331461 0.362453 +0.605066 OEF/20091029 0.582295 0.649670 +0.624845 IYT/20091029 0.604565 0.643450 +0.540601 BIL/20091029 0.525867 0.572303 +0.239555 GLL/20091029 0.195419 0.300787 +0.220331 EDZ/20091029 0.211034 0.233725 +0.558424 IWM/20091029 0.547274 0.588616 +0.559896 VXF/20091029 0.552232 0.573118 +0.556109 IJJ/20091029 0.540039 0.571495 +0.643768 PIN/20091029 0.622161 0.666833 +0.628354 XLB/20091029 0.623240 0.635304 +0.572621 ECH/20091029 0.559470 0.577602 +0.503155 TYH/20091029 0.482566 0.527617 +0.637420 VAW/20091029 0.625896 0.647929 +0.656847 DBP/20091029 0.647690 0.665420 +0.671248 XME/20091029 0.655446 0.684593 +0.570199 VO/20091029 0.556633 0.582433 +0.571863 RSX/20091029 0.554621 0.588738 +0.631221 EWC/20091029 0.623401 0.642039 +0.488211 TUR/20091029 0.476170 0.502640 +0.591621 VYM/20091029 0.585125 0.598521 +0.557581 FCG/20091029 0.538574 0.585397 +0.587676 VGT/20091029 0.562234 0.606054 +0.595374 EWQ/20091029 0.580310 0.625264 +0.576731 IEV/20091029 0.565102 0.585564 +0.568279 XLK/20091029 0.555844 0.574436 +0.571347 EFG/20091029 0.558870 0.577072 +0.588520 BKF/20091029 0.579253 0.594966 +0.582255 KIE/20091029 0.562775 0.592329 +0.617259 EEB/20091029 0.601089 0.633660 +0.574002 IJK/20091029 0.557166 0.593426 +0.417725 DUG/20091029 0.394745 0.433980 +0.367922 TWM/20091029 0.350982 0.380372 +0.580777 MDY/20091029 0.572265 0.590662 +0.604949 ACWI/20091029 0.594771 0.622790 +0.558951 BSV/20091029 0.545229 0.572257 +0.649934 DDM/20091029 0.619128 0.665828 +0.594030 DIA/20091029 0.581626 0.602111 +0.540464 TLT/20091029 0.534436 0.543558 +0.367095 DXD/20091029 0.327484 0.387986 +0.567708 XHB/20091029 0.556140 0.583462 +0.575292 VDE/20091029 0.544986 0.629823 +0.548864 BND/20091029 0.505154 0.577604 +0.539503 EMB/20091029 0.523763 0.552542 +0.514195 SCO/20091029 0.483382 0.533059 +0.561492 AMJ/20091029 0.549470 0.579557 +0.510173 OIL/20091029 0.491214 0.558396 +0.519759 PZA/20091029 0.508815 0.538558 +0.582726 VGK/20091029 0.572748 0.593041 +0.570219 RWX/20091029 0.553981 0.582262 +0.564585 JJA/20091029 0.553515 0.572262 +0.580657 FXD/20091029 0.563110 0.596206 +0.542785 XES/20091029 0.524985 0.556859 +0.591180 VIG/20091029 0.580230 0.600387 +0.177181 DZZ/20091029 0.169888 0.182534 +0.558540 VFH/20091029 0.526530 0.592920 +0.553692 DTO/20091029 0.533102 0.565546 +0.589786 EWP/20091029 0.566946 0.599132 +0.568259 FDN/20091029 0.554145 0.585057 +0.655609 INP/20091029 0.650082 0.663219 +0.331003 TYP/20091029 0.307701 0.350471 +0.587361 RWR/20091029 0.580908 0.592243 +0.585893 KBE/20091029 0.571486 0.611065 +0.446351 EUO/20091029 0.435436 0.454937 +0.589235 IWF/20091029 0.583103 0.600570 +0.259992 SMN/20091029 0.234152 0.286653 +0.581631 SMH/20091029 0.567619 0.599343 +0.551585 XRT/20091029 0.519954 0.597511 +0.505371 USO/20091029 0.487004 0.537872 +0.536274 DJP/20091029 0.535917 0.536607 +0.558339 CFT/20091029 0.543853 0.569434 +0.325472 SRS/20091029 0.304747 0.356476 +0.618195 MOO/20091029 0.594071 0.627578 +0.543255 BIV/20091029 0.527727 0.551122 +0.343124 VXX/20091029 0.319538 0.376386 +0.633038 IYM/20091029 0.617608 0.651724 +0.609515 IFN/20091029 0.589881 0.621912 +0.660082 SLV/20091029 0.641925 0.673590 +0.538538 TAO/20091029 0.512807 0.552506 +0.514150 PGF/20091029 0.509150 0.519195 +0.585988 IYR/20091029 0.578965 0.591987 +0.297691 QID/20091029 0.279861 0.323444 +0.588112 THD/20091029 0.575973 0.601805 +0.575092 IJS/20091029 0.558237 0.600060 +0.570309 VB/20091029 0.554285 0.587461 +0.521707 EDV/20091029 0.496873 0.532657 +0.549327 IEZ/20091029 0.527047 0.569459 +0.606043 VTV/20091029 0.581225 0.622078 +0.571289 IJR/20091029 0.540564 0.588847 +0.463888 UCO/20091029 0.426833 0.501260 +0.509769 JNK/20091029 0.494098 0.519875 +0.581153 IWN/20091029 0.562055 0.596243 +0.597709 VV/20091029 0.573466 0.619541 +0.743979 UGL/20091029 0.681396 0.792897 +0.581612 UWM/20091029 0.560203 0.601422 +0.545270 IWC/20091029 0.516414 0.581090 +0.570390 EWA/20091029 0.547789 0.589591 +0.582860 IVV/20091029 0.563460 0.599589 +0.584978 SPY/20091029 0.563100 0.599449 +0.528427 TFI/20091029 0.517817 0.542480 +0.568577 VEA/20091029 0.553504 0.579103 +0.568596 QQQQ/20091029 0.552125 0.577743 +0.566121 UYG/20091029 0.522110 0.590475 +0.517117 OIH/20091029 0.504242 0.527930 +0.540460 GXC/20091029 0.529309 0.552822 +0.640496 SSO/20091029 0.618994 0.649238 +0.608948 XLI/20091029 0.594793 0.621806 +0.614148 GML/20091029 0.589127 0.632458 +0.608309 ROM/20091029 0.595347 0.618059 +0.556186 FXC/20091029 0.532176 0.575545 +0.415722 DOG/20091029 0.392740 0.428530 +0.526165 IYE/20091029 0.512200 0.535262 +0.378714 SKF/20091029 0.345286 0.405451 +0.541735 SHY/20091029 0.526275 0.551853 +0.532579 DBA/20091029 0.519867 0.556043 +0.581712 RSP/20091029 0.561607 0.601216 +0.661506 DBS/20091029 0.642385 0.685447 +0.607530 IBB/20091029 0.589648 0.626086 +0.527442 KCE/20091029 0.502068 0.541173 +0.547664 PKN/20091029 0.531345 0.569522 +0.550195 TNA/20091029 0.496939 0.604076 +0.616947 FAS/20091029 0.577073 0.658416 +0.549581 FXE/20091029 0.527556 0.568328 +0.510831 HYG/20091029 0.507184 0.515970 +0.587927 IWS/20091029 0.564615 0.618593 +0.396984 FXP/20091029 0.375915 0.412600 +0.526407 MBB/20091029 0.507842 0.539322 +0.577802 RFG/20091029 0.557979 0.588021 +0.610439 EPU/20091029 0.593207 0.634611 +0.440060 UUP/20091029 0.408462 0.454878 +0.740354 AGQ/20091029 0.701532 0.768518 +0.599408 SOXX/20091029 0.565837 0.620268 +0.323618 FAZ/20091029 0.319385 0.330989 +0.561917 VBK/20091029 0.529704 0.579651 +0.577535 RPG/20091029 0.551945 0.590639 +0.550331 EWH/20091029 0.537630 0.557368 +0.361890 TZA/20091029 0.318528 0.399247 +0.537619 SGG/20091029 0.520643 0.550846 +0.621363 KOL/20091029 0.598182 0.642016 +0.572609 EWY/20091029 0.555560 0.585961 +0.591749 PRF/20091029 0.548294 0.607760 +0.536009 TLH/20091029 0.527166 0.550016 +0.574318 EPP/20091029 0.552857 0.595685 +0.538803 XLE/20091029 0.516436 0.552068 +0.599275 EWN/20091029 0.580484 0.618244 +0.488432 SHM/20091029 0.471839 0.513139 +0.537587 FXI/20091029 0.528976 0.547346 +0.603125 EWS/20091029 0.593148 0.614305 +0.578754 IDU/20091029 0.564492 0.601152 +0.515655 VXZ/20091029 0.511559 0.521686 +0.598386 IVE/20091029 0.575256 0.614879 +0.747961 DGP/20091029 0.740512 0.761456 +0.572653 GMF/20091029 0.560784 0.584607 +0.552419 IWR/20091029 0.532158 0.581420 +0.529186 RKH/20091029 0.504264 0.548532 +0.561601 TIP/20091029 0.556488 0.564879 +0.619806 URE/20091029 0.608287 0.632231 +0.500026 DBO/20091029 0.491191 0.509916 +0.545888 IOO/20091029 0.533303 0.557189 +0.493110 DBV/20091029 0.460629 0.544873 +0.557805 EFA/20091029 0.533145 0.585551 +0.590899 BGU/20091029 0.573423 0.617669 +0.557126 EFV/20091029 0.534714 0.574984 +0.577585 IWB/20091029 0.562213 0.597265 +0.560065 IYF/20091029 0.540145 0.574945 +0.344147 YCS/20091029 0.324176 0.353949 +0.512412 DXJ/20091029 0.497681 0.530455 +0.544807 IWO/20091029 0.534540 0.551797 +0.532782 DBC/20091029 0.516852 0.549439 +0.391824 RWM/20091029 0.308162 0.425957 +0.566677 VBR/20091029 0.543048 0.578489 +0.352743 MZZ/20091029 0.285361 0.377293 +0.591390 IWD/20091029 0.569160 0.608733 +0.542168 PCY/20091029 0.497661 0.576003 +0.574272 EWI/20091029 0.554361 0.586101 +0.580948 IJH/20091029 0.553506 0.609520 +0.596753 EEM/20091029 0.579092 0.608344 +0.565684 EWM/20091029 0.555312 0.573187 +0.575013 SDY/20091029 0.562168 0.588229 +0.640838 ILF/20091029 0.625135 0.666076 +0.575135 JJG/20091029 0.566230 0.581198 +0.417986 TBT/20091029 0.398693 0.434745 +0.558902 XLF/20091029 0.536056 0.575110 +0.536959 ERX/20091029 0.509520 0.577219 +0.483686 SHV/20091029 0.471476 0.492726 +0.612131 EWX/20091029 0.589745 0.651514 +0.388801 EFZ/20091029 0.345192 0.409832 +0.485885 FXB/20091029 0.469432 0.497067 +0.548777 PHO/20091029 0.530011 0.561900 +0.601274 IGE/20091029 0.570329 0.636378 +0.246965 BGZ/20091029 0.227243 0.265550 +0.545767 UDN/20091029 0.528176 0.557122 +0.467376 CSJ/20091029 0.459553 0.472582 +0.628190 GXG/20091029 0.589988 0.677002 +0.626129 USD/20091029 0.612317 0.641537 +0.548397 EWD/20091029 0.528916 0.560301 +0.522226 EWJ/20091029 0.504307 0.542203 +0.679577 BRF/20091029 0.650994 0.706774 +0.588983 VEU/20091029 0.567608 0.605020 +0.567961 XLU/20091029 0.543372 0.589099 +0.540122 JJC/20091029 0.514590 0.560752 +0.607104 FGD/20091029 0.591680 0.613901 +0.527721 FXF/20091029 0.512776 0.538113 +0.509602 LQD/20091029 0.494193 0.516013 +0.553041 SCZ/20091029 0.539998 0.575209 +0.571798 IYW/20091029 0.555115 0.578269 +0.532048 VPL/20091029 0.511435 0.548655 +0.579341 DGS/20091029 0.563576 0.592659 +0.574946 ICF/20091029 0.554650 0.592272 +0.568637 DVY/20091029 0.539272 0.588252 +0.551897 IEO/20091029 0.520027 0.608573 +0.571588 VOT/20091029 0.542488 0.585265 +0.502903 CIU/20091029 0.485798 0.510441 +0.596848 EWG/20091029 0.579958 0.614421 +0.573157 EWT/20091029 0.563506 0.582320 +0.482950 GSG/20091029 0.475375 0.490933 +0.506287 KRE/20091029 0.469336 0.524726 +0.591893 LVL/20091029 0.568119 0.602347 +0.431875 UNG/20091029 0.397925 0.483928 +0.483614 MUB/20091029 0.476417 0.492254 +0.585976 VT/20091029 0.560764 0.596827 +0.596501 DAG/20091029 0.577574 0.605808 +0.614322 PPH/20091029 0.589279 0.630694 +0.577630 VSS/20091029 0.563286 0.594048 +0.499778 DBB/20091029 0.480243 0.516656 +0.552929 XLP/20091029 0.517916 0.571799 +0.546586 IJT/20091029 0.524520 0.573116 +0.623395 EWZ/20091029 0.606286 0.645627 +0.551099 PBW/20091029 0.540674 0.570826 +0.565290 FXY/20091029 0.550109 0.580315 +0.612841 IYZ/20091029 0.593319 0.625935 +0.600073 MVV/20091029 0.589954 0.613216 +0.579433 VUG/20091029 0.569343 0.591033 +0.406068 PST/20091029 0.399437 0.409846 +0.393183 PSQ/20091029 0.382633 0.408551 +0.570351 VNQ/20091029 0.555874 0.591179 +0.536361 IEI/20091029 0.533983 0.539337 +0.600048 EWW/20091029 0.564489 0.618353 +0.574139 IWP/20091029 0.558356 0.593022 +0.583767 IWV/20091029 0.571275 0.602614 +0.555232 DIG/20091029 0.541608 0.572868 +0.582349 VTI/20091029 0.564814 0.594742 +0.525471 FXA/20091029 0.512852 0.531110 +0.542628 NLR/20091029 0.527205 0.559739 +0.502754 AGG/20091029 0.488406 0.512403 +0.558334 BWX/20091029 0.536862 0.573630 +0.648206 IAU/20091029 0.629999 0.663047 +0.615677 XLV/20091029 0.590015 0.633500 +0.533722 XOP/20091029 0.514446 0.565511 +0.582703 EZU/20091029 0.562818 0.597090 +0.567974 JXI/20091029 0.556115 0.586803 +0.582865 XBI/20091029 0.571641 0.597304 +0.558277 IYG/20091029 0.513902 0.583654 +0.645223 SLX/20091029 0.633842 0.664381 +0.561655 HAO/20091029 0.541016 0.581285 +0.592189 EZA/20091029 0.574685 0.607435 +0.575479 XLY/20091029 0.558306 0.591048 +0.514166 IEF/20091029 0.491849 0.534751 +0.578928 DEM/20091029 0.555353 0.605558 +0.560488 IVW/20091029 0.541832 0.572168 +0.696384 UYM/20091029 0.682135 0.720646 +0.530257 IXC/20091029 0.510665 0.548155 +0.515036 PFF/20091030 0.498914 0.524946 +0.560791 WIP/20091030 0.547882 0.582476 +0.547518 GCC/20091030 0.542825 0.550277 +0.556630 AAXJ/20091030 0.534578 0.596872 +0.570324 VWO/20091030 0.544831 0.603537 +0.290947 EEV/20091030 0.224871 0.362623 +0.698558 GDX/20091030 0.679446 0.731544 +0.533423 RTH/20091030 0.476968 0.553942 +0.607179 MXI/20091030 0.591355 0.643535 +0.535101 EWU/20091030 0.510284 0.555326 +0.436858 SH/20091030 0.410981 0.465381 +0.602959 EDC/20091030 0.558139 0.660943 +0.395816 ERY/20091030 0.370954 0.421132 +0.383239 SDS/20091030 0.348684 0.415239 +0.555277 OEF/20091030 0.524649 0.616566 +0.592140 IYT/20091030 0.562606 0.618533 +0.516485 BIL/20091030 0.498074 0.562285 +0.241259 GLL/20091030 0.194264 0.307781 +0.195474 EDZ/20091030 0.160691 0.229997 +0.528685 IWM/20091030 0.508918 0.567994 +0.527382 VXF/20091030 0.518742 0.532967 +0.521495 IJJ/20091030 0.493331 0.533248 +0.581633 PIN/20091030 0.538875 0.599926 +0.594964 XLB/20091030 0.580081 0.614812 +0.555032 ECH/20091030 0.535192 0.580878 +0.499245 TYH/20091030 0.459984 0.527515 +0.602811 VAW/20091030 0.576996 0.622627 +0.654836 DBP/20091030 0.642372 0.667670 +0.656601 XME/20091030 0.628928 0.683314 +0.532195 VO/20091030 0.504567 0.556739 +0.580762 RSX/20091030 0.545866 0.613584 +0.607957 EWC/20091030 0.588861 0.629705 +0.458643 TUR/20091030 0.433949 0.494325 +0.543585 VYM/20091030 0.524993 0.564569 +0.522503 FCG/20091030 0.488760 0.546232 +0.552798 VGT/20091030 0.534357 0.585169 +0.548319 EWQ/20091030 0.523360 0.593914 +0.541542 IEV/20091030 0.526113 0.566687 +0.536087 XLK/20091030 0.522379 0.559664 +0.545266 EFG/20091030 0.532652 0.567602 +0.589880 BKF/20091030 0.573062 0.611897 +0.551991 KIE/20091030 0.544414 0.572322 +0.612751 EEB/20091030 0.596569 0.637393 +0.543344 IJK/20091030 0.518636 0.573509 +0.423432 DUG/20091030 0.379432 0.454615 +0.388172 TWM/20091030 0.358149 0.404501 +0.547650 MDY/20091030 0.527822 0.568488 +0.564561 ACWI/20091030 0.540178 0.598070 +0.531923 BSV/20091030 0.519265 0.550385 +0.622748 DDM/20091030 0.607416 0.643700 +0.562392 DIA/20091030 0.550884 0.582020 +0.548060 TLT/20091030 0.539878 0.563049 +0.393243 DXD/20091030 0.375194 0.408950 +0.549692 XHB/20091030 0.529147 0.577743 +0.549988 VDE/20091030 0.500708 0.594464 +0.534392 BND/20091030 0.485605 0.567572 +0.559162 EMB/20091030 0.545886 0.568036 +0.518480 SCO/20091030 0.501241 0.539626 +0.567368 AMJ/20091030 0.555077 0.589115 +0.489054 OIL/20091030 0.450316 0.525825 +0.505195 PZA/20091030 0.488718 0.528659 +0.545904 VGK/20091030 0.523435 0.574077 +0.548126 RWX/20091030 0.524922 0.567552 +0.566409 JJA/20091030 0.544774 0.584940 +0.577402 FXD/20091030 0.555000 0.601416 +0.514325 XES/20091030 0.486607 0.544549 +0.546844 VIG/20091030 0.525227 0.563495 +0.178322 DZZ/20091030 0.175368 0.182223 +0.516552 VFH/20091030 0.466258 0.566758 +0.532446 DTO/20091030 0.512852 0.559033 +0.561296 EWP/20091030 0.518383 0.586527 +0.556713 FDN/20091030 0.532477 0.584643 +0.605061 INP/20091030 0.587126 0.626565 +0.333063 TYP/20091030 0.285816 0.372428 +0.585707 RWR/20091030 0.579552 0.600069 +0.543145 KBE/20091030 0.517705 0.582943 +0.463923 EUO/20091030 0.441908 0.485785 +0.544402 IWF/20091030 0.523485 0.568523 +0.266953 SMN/20091030 0.221587 0.309550 +0.556586 SMH/20091030 0.534699 0.585013 +0.536708 XRT/20091030 0.495827 0.592633 +0.483115 USO/20091030 0.469689 0.494872 +0.529331 DJP/20091030 0.512477 0.545535 +0.529822 CFT/20091030 0.518750 0.539188 +0.316926 SRS/20091030 0.282062 0.358085 +0.609591 MOO/20091030 0.601091 0.626818 +0.538171 BIV/20091030 0.512194 0.556736 +0.332398 VXX/20091030 0.288262 0.391816 +0.604551 IYM/20091030 0.574472 0.637393 +0.587965 IFN/20091030 0.570406 0.619415 +0.656019 SLV/20091030 0.630410 0.672424 +0.530650 TAO/20091030 0.510036 0.554300 +0.509303 PGF/20091030 0.506029 0.512410 +0.583937 IYR/20091030 0.579251 0.591226 +0.320607 QID/20091030 0.286500 0.361196 +0.556582 THD/20091030 0.537773 0.566999 +0.540679 IJS/20091030 0.518051 0.575589 +0.538413 VB/20091030 0.519559 0.565679 +0.534428 EDV/20091030 0.522193 0.548287 +0.524182 IEZ/20091030 0.499726 0.559960 +0.558244 VTV/20091030 0.537374 0.589188 +0.538679 IJR/20091030 0.516459 0.565965 +0.459683 UCO/20091030 0.446300 0.483395 +0.515525 JNK/20091030 0.500702 0.530381 +0.547334 IWN/20091030 0.533791 0.572681 +0.549802 VV/20091030 0.527832 0.585864 +0.741419 UGL/20091030 0.676444 0.793313 +0.564184 UWM/20091030 0.532424 0.582655 +0.507238 IWC/20091030 0.480274 0.552257 +0.540541 EWA/20091030 0.505850 0.571257 +0.537741 IVV/20091030 0.516545 0.558043 +0.536887 SPY/20091030 0.521727 0.555606 +0.518202 TFI/20091030 0.509372 0.531682 +0.530418 VEA/20091030 0.506468 0.556913 +0.535942 QQQQ/20091030 0.523510 0.557947 +0.542686 UYG/20091030 0.511532 0.589354 +0.494979 OIH/20091030 0.466464 0.517861 +0.537249 GXC/20091030 0.522589 0.566521 +0.602623 SSO/20091030 0.590260 0.626666 +0.562480 XLI/20091030 0.549158 0.580377 +0.613253 GML/20091030 0.592285 0.646972 +0.589934 ROM/20091030 0.569285 0.617727 +0.524160 FXC/20091030 0.510319 0.541734 +0.452216 DOG/20091030 0.412108 0.474410 +0.502358 IYE/20091030 0.483927 0.529096 +0.397083 SKF/20091030 0.341699 0.443746 +0.510333 SHY/20091030 0.494602 0.525736 +0.533519 DBA/20091030 0.511354 0.570095 +0.538812 RSP/20091030 0.516030 0.571038 +0.656823 DBS/20091030 0.630526 0.689017 +0.581309 IBB/20091030 0.569084 0.603969 +0.491956 KCE/20091030 0.477217 0.513655 +0.511411 PKN/20091030 0.484214 0.549541 +0.542646 TNA/20091030 0.491843 0.600246 +0.613070 FAS/20091030 0.551156 0.683500 +0.529210 FXE/20091030 0.515515 0.535466 +0.502660 HYG/20091030 0.494551 0.508140 +0.549879 IWS/20091030 0.512230 0.574176 +0.384146 FXP/20091030 0.358690 0.416071 +0.513206 MBB/20091030 0.489484 0.537760 +0.558201 RFG/20091030 0.545128 0.578400 +0.587100 EPU/20091030 0.557276 0.605214 +0.465964 UUP/20091030 0.440261 0.479414 +0.749631 AGQ/20091030 0.702360 0.789747 +0.565596 SOXX/20091030 0.541278 0.590457 +0.325118 FAZ/20091030 0.293699 0.345530 +0.536631 VBK/20091030 0.511014 0.563030 +0.542806 RPG/20091030 0.528823 0.567726 +0.542121 EWH/20091030 0.529455 0.561003 +0.365937 TZA/20091030 0.303943 0.412996 +0.534147 SGG/20091030 0.519057 0.551491 +0.615987 KOL/20091030 0.595132 0.642189 +0.545923 EWY/20091030 0.526256 0.569646 +0.550905 PRF/20091030 0.519731 0.579332 +0.539361 TLH/20091030 0.528613 0.551129 +0.547648 EPP/20091030 0.524651 0.582221 +0.518178 XLE/20091030 0.498074 0.548558 +0.569328 EWN/20091030 0.559030 0.579509 +0.475932 SHM/20091030 0.449114 0.507414 +0.538241 FXI/20091030 0.515317 0.561919 +0.579780 EWS/20091030 0.553872 0.606754 +0.540881 IDU/20091030 0.517685 0.574571 +0.504692 VXZ/20091030 0.480428 0.527727 +0.545966 IVE/20091030 0.531455 0.563020 +0.745753 DGP/20091030 0.734064 0.755875 +0.538644 GMF/20091030 0.511301 0.565432 +0.516291 IWR/20091030 0.482996 0.556807 +0.490150 RKH/20091030 0.472619 0.511143 +0.524542 TIP/20091030 0.519455 0.531431 +0.632156 URE/20091030 0.622863 0.638606 +0.479725 DBO/20091030 0.471644 0.488247 +0.507935 IOO/20091030 0.478462 0.536195 +0.477476 DBV/20091030 0.456896 0.511081 +0.521524 EFA/20091030 0.499851 0.564759 +0.564656 BGU/20091030 0.531464 0.613582 +0.520122 EFV/20091030 0.510761 0.530879 +0.532653 IWB/20091030 0.512913 0.545484 +0.518136 IYF/20091030 0.512857 0.530677 +0.378637 YCS/20091030 0.373985 0.382784 +0.490882 DXJ/20091030 0.481317 0.502135 +0.522371 IWO/20091030 0.513543 0.536737 +0.530029 DBC/20091030 0.525670 0.532775 +0.426452 RWM/20091030 0.350514 0.471596 +0.530518 VBR/20091030 0.516751 0.553679 +0.373270 MZZ/20091030 0.291129 0.408733 +0.542514 IWD/20091030 0.533781 0.559407 +0.526985 PCY/20091030 0.485469 0.558276 +0.533151 EWI/20091030 0.525311 0.546859 +0.548924 IJH/20091030 0.530200 0.588085 +0.575040 EEM/20091030 0.566443 0.589476 +0.538628 EWM/20091030 0.523655 0.552518 +0.531335 SDY/20091030 0.515860 0.557080 +0.636100 ILF/20091030 0.621967 0.648969 +0.581175 JJG/20091030 0.565806 0.597121 +0.422029 TBT/20091030 0.408141 0.436293 +0.520219 XLF/20091030 0.511958 0.528439 +0.547613 ERX/20091030 0.497346 0.597480 +0.489361 SHV/20091030 0.481832 0.493261 +0.591585 EWX/20091030 0.562178 0.642230 +0.432024 EFZ/20091030 0.372417 0.469422 +0.435028 FXB/20091030 0.420532 0.451573 +0.514794 PHO/20091030 0.500370 0.532475 +0.574042 IGE/20091030 0.552351 0.624775 +0.268769 BGZ/20091030 0.228245 0.306369 +0.520853 UDN/20091030 0.512069 0.533059 +0.488973 CSJ/20091030 0.484978 0.496952 +0.619546 GXG/20091030 0.584748 0.674652 +0.611958 USD/20091030 0.588030 0.631738 +0.538666 EWD/20091030 0.515889 0.570526 +0.498808 EWJ/20091030 0.490730 0.511178 +0.680253 BRF/20091030 0.649767 0.721869 +0.555793 VEU/20091030 0.542300 0.573075 +0.531990 XLU/20091030 0.493620 0.557607 +0.531358 JJC/20091030 0.514337 0.544804 +0.568101 FGD/20091030 0.555904 0.587786 +0.513038 FXF/20091030 0.504745 0.518842 +0.519078 LQD/20091030 0.512248 0.532094 +0.542336 SCZ/20091030 0.532035 0.558999 +0.542332 IYW/20091030 0.531016 0.562331 +0.502641 VPL/20091030 0.483177 0.530995 +0.555807 DGS/20091030 0.527707 0.581456 +0.572167 ICF/20091030 0.543977 0.591288 +0.528742 DVY/20091030 0.508702 0.561192 +0.522921 IEO/20091030 0.486678 0.595854 +0.534039 VOT/20091030 0.515495 0.559073 +0.499687 CIU/20091030 0.479667 0.509389 +0.558083 EWG/20091030 0.523572 0.588956 +0.553510 EWT/20091030 0.533476 0.572546 +0.495572 GSG/20091030 0.487740 0.509770 +0.497133 KRE/20091030 0.474094 0.522409 +0.571437 LVL/20091030 0.559742 0.594375 +0.415487 UNG/20091030 0.385815 0.459255 +0.498235 MUB/20091030 0.488268 0.506344 +0.551742 VT/20091030 0.539302 0.576294 +0.616575 DAG/20091030 0.591460 0.640185 +0.567868 PPH/20091030 0.551486 0.579966 +0.550285 VSS/20091030 0.531287 0.579464 +0.521379 DBB/20091030 0.505728 0.530942 +0.528646 XLP/20091030 0.508155 0.552898 +0.519907 IJT/20091030 0.504666 0.554858 +0.622265 EWZ/20091030 0.598501 0.662160 +0.532781 PBW/20091030 0.509089 0.565829 +0.572779 FXY/20091030 0.549013 0.596475 +0.557677 IYZ/20091030 0.540424 0.582535 +0.579989 MVV/20091030 0.558223 0.608093 +0.535644 VUG/20091030 0.511917 0.554513 +0.414456 PST/20091030 0.406813 0.419504 +0.421747 PSQ/20091030 0.399163 0.449949 +0.568378 VNQ/20091030 0.546616 0.583983 +0.518607 IEI/20091030 0.511586 0.529390 +0.588038 EWW/20091030 0.538324 0.621034 +0.537115 IWP/20091030 0.508978 0.566945 +0.537666 IWV/20091030 0.509258 0.569941 +0.549825 DIG/20091030 0.523704 0.589847 +0.541476 VTI/20091030 0.525332 0.560104 +0.509106 FXA/20091030 0.492398 0.528614 +0.511030 NLR/20091030 0.482713 0.536954 +0.515872 AGG/20091030 0.490433 0.531288 +0.544604 BWX/20091030 0.524390 0.558209 +0.642660 IAU/20091030 0.626627 0.654545 +0.572630 XLV/20091030 0.558726 0.585788 +0.514264 XOP/20091030 0.477746 0.556637 +0.543000 EZU/20091030 0.522369 0.572257 +0.524912 JXI/20091030 0.505134 0.535341 +0.556321 XBI/20091030 0.540513 0.578246 +0.514138 IYG/20091030 0.485507 0.541840 +0.643381 SLX/20091030 0.620681 0.676230 +0.558215 HAO/20091030 0.520136 0.573385 +0.576682 EZA/20091030 0.558382 0.606852 +0.551180 XLY/20091030 0.534570 0.578957 +0.508828 IEF/20091030 0.491721 0.533402 +0.544150 DEM/20091030 0.517379 0.587423 +0.522944 IVW/20091030 0.511012 0.546260 +0.688613 UYM/20091030 0.661783 0.732726 +0.512157 IXC/20091030 0.495382 0.545990 +0.491068 PFF/20091102 0.482470 0.507426 +0.541536 WIP/20091102 0.527094 0.561347 +0.554900 GCC/20091102 0.544585 0.565759 +0.582990 AAXJ/20091102 0.565589 0.613644 +0.592773 VWO/20091102 0.579401 0.616369 +0.293762 EEV/20091102 0.241053 0.354171 +0.686014 GDX/20091102 0.665690 0.711943 +0.556421 RTH/20091102 0.506771 0.589304 +0.616671 MXI/20091102 0.601678 0.645212 +0.557662 EWU/20091102 0.543393 0.584110 +0.410213 SH/20091102 0.390167 0.433884 +0.589030 EDC/20091102 0.554102 0.632267 +0.391943 ERY/20091102 0.374003 0.411199 +0.365961 SDS/20091102 0.339368 0.387552 +0.588409 OEF/20091102 0.563963 0.642297 +0.599605 IYT/20091102 0.574262 0.622394 +0.490279 BIL/20091102 0.475029 0.530767 +0.278967 GLL/20091102 0.230298 0.336627 +0.214542 EDZ/20091102 0.196223 0.237810 +0.545313 IWM/20091102 0.531687 0.581253 +0.544810 VXF/20091102 0.539512 0.553561 +0.547282 IJJ/20091102 0.527413 0.558775 +0.618411 PIN/20091102 0.589361 0.637203 +0.612615 XLB/20091102 0.602709 0.625968 +0.564774 ECH/20091102 0.551128 0.585389 +0.511279 TYH/20091102 0.481384 0.531332 +0.620160 VAW/20091102 0.603423 0.633588 +0.632300 DBP/20091102 0.623601 0.639343 +0.653863 XME/20091102 0.630800 0.676331 +0.553581 VO/20091102 0.534186 0.573069 +0.579478 RSX/20091102 0.553967 0.602086 +0.610109 EWC/20091102 0.596890 0.627153 +0.475607 TUR/20091102 0.457503 0.503316 +0.567243 VYM/20091102 0.557453 0.582905 +0.542648 FCG/20091102 0.518781 0.563593 +0.576648 VGT/20091102 0.556014 0.604456 +0.572041 EWQ/20091102 0.557755 0.609646 +0.559149 IEV/20091102 0.552077 0.578479 +0.565709 XLK/20091102 0.558515 0.582955 +0.560826 EFG/20091102 0.553942 0.576861 +0.593075 BKF/20091102 0.584971 0.607155 +0.540521 KIE/20091102 0.532337 0.556767 +0.615146 EEB/20091102 0.604047 0.631202 +0.568713 IJK/20091102 0.548859 0.593364 +0.407165 DUG/20091102 0.370037 0.428241 +0.380419 TWM/20091102 0.355273 0.392025 +0.574277 MDY/20091102 0.562451 0.589691 +0.592061 ACWI/20091102 0.576546 0.618112 +0.529338 BSV/20091102 0.511104 0.536793 +0.642317 DDM/20091102 0.617458 0.653704 +0.583285 DIA/20091102 0.577634 0.595518 +0.518640 TLT/20091102 0.512870 0.522235 +0.376564 DXD/20091102 0.346749 0.390929 +0.562487 XHB/20091102 0.546344 0.584250 +0.573314 VDE/20091102 0.533038 0.622433 +0.506469 BND/20091102 0.470138 0.530668 +0.522540 EMB/20091102 0.503826 0.538601 +0.502605 SCO/20091102 0.477083 0.513440 +0.570520 AMJ/20091102 0.557822 0.591775 +0.513839 OIL/20091102 0.485896 0.556709 +0.522350 PZA/20091102 0.508139 0.546218 +0.567132 VGK/20091102 0.550569 0.587730 +0.550229 RWX/20091102 0.530729 0.564601 +0.542406 JJA/20091102 0.537354 0.544883 +0.582823 FXD/20091102 0.563308 0.602966 +0.532817 XES/20091102 0.510335 0.556874 +0.579597 VIG/20091102 0.563701 0.589302 +0.219733 DZZ/20091102 0.209209 0.233200 +0.532800 VFH/20091102 0.490897 0.576782 +0.532684 DTO/20091102 0.519303 0.548123 +0.567421 EWP/20091102 0.530718 0.587254 +0.576719 FDN/20091102 0.557580 0.601544 +0.634464 INP/20091102 0.628950 0.645378 +0.326738 TYP/20091102 0.286697 0.358089 +0.575303 RWR/20091102 0.570605 0.586750 +0.553009 KBE/20091102 0.533168 0.584256 +0.464344 EUO/20091102 0.445514 0.481915 +0.576798 IWF/20091102 0.566090 0.594846 +0.271075 SMN/20091102 0.234713 0.305107 +0.582984 SMH/20091102 0.564958 0.607409 +0.553552 XRT/20091102 0.520567 0.604129 +0.508752 USO/20091102 0.494077 0.532106 +0.547597 DJP/20091102 0.539598 0.557786 +0.515400 CFT/20091102 0.507196 0.523419 +0.333541 SRS/20091102 0.303887 0.372581 +0.628770 MOO/20091102 0.613128 0.637013 +0.516456 BIV/20091102 0.501063 0.522822 +0.324564 VXX/20091102 0.284920 0.378335 +0.618264 IYM/20091102 0.596609 0.644950 +0.606394 IFN/20091102 0.593408 0.632147 +0.639317 SLV/20091102 0.615151 0.652345 +0.523972 TAO/20091102 0.500255 0.542694 +0.516110 PGF/20091102 0.513331 0.518863 +0.576041 IYR/20091102 0.572315 0.580469 +0.300992 QID/20091102 0.273289 0.333330 +0.580392 THD/20091102 0.570457 0.590385 +0.558519 IJS/20091102 0.542006 0.590207 +0.555357 VB/20091102 0.541531 0.578577 +0.522752 EDV/20091102 0.504544 0.533511 +0.538769 IEZ/20091102 0.520293 0.569137 +0.582311 VTV/20091102 0.564661 0.607612 +0.559615 IJR/20091102 0.530729 0.583139 +0.477060 UCO/20091102 0.452782 0.509386 +0.514551 JNK/20091102 0.501262 0.533048 +0.561060 IWN/20091102 0.545948 0.583723 +0.578322 VV/20091102 0.559935 0.608318 +0.705030 UGL/20091102 0.642347 0.745885 +0.575491 UWM/20091102 0.551249 0.591169 +0.513257 IWC/20091102 0.490011 0.556506 +0.568642 EWA/20091102 0.544920 0.591287 +0.563262 IVV/20091102 0.550047 0.577673 +0.566152 SPY/20091102 0.550421 0.578448 +0.508110 TFI/20091102 0.502424 0.519836 +0.550681 VEA/20091102 0.536458 0.570296 +0.564235 QQQQ/20091102 0.549652 0.581054 +0.548183 UYG/20091102 0.509763 0.584736 +0.513774 OIH/20091102 0.493688 0.530993 +0.553599 GXC/20091102 0.540014 0.569988 +0.624491 SSO/20091102 0.607582 0.639780 +0.595377 XLI/20091102 0.584403 0.605287 +0.609274 GML/20091102 0.589005 0.636354 +0.608786 ROM/20091102 0.596585 0.630305 +0.539013 FXC/20091102 0.519937 0.553478 +0.428077 DOG/20091102 0.394771 0.445018 +0.530268 IYE/20091102 0.520615 0.550602 +0.397868 SKF/20091102 0.352456 0.434912 +0.529177 SHY/20091102 0.520506 0.537435 +0.530113 DBA/20091102 0.517974 0.555111 +0.562651 RSP/20091102 0.546734 0.589308 +0.640111 DBS/20091102 0.615241 0.668998 +0.577902 IBB/20091102 0.564333 0.596062 +0.512779 KCE/20091102 0.490886 0.529510 +0.526896 PKN/20091102 0.507504 0.559051 +0.548539 TNA/20091102 0.492443 0.599508 +0.608432 FAS/20091102 0.559448 0.664915 +0.536576 FXE/20091102 0.518585 0.548681 +0.519365 HYG/20091102 0.514803 0.523340 +0.568350 IWS/20091102 0.538624 0.593795 +0.381403 FXP/20091102 0.361932 0.396705 +0.517641 MBB/20091102 0.505500 0.527713 +0.572341 RFG/20091102 0.553298 0.587807 +0.600675 EPU/20091102 0.579039 0.622126 +0.456928 UUP/20091102 0.429170 0.468148 +0.722747 AGQ/20091102 0.675578 0.757727 +0.589100 SOXX/20091102 0.556844 0.609544 +0.336494 FAZ/20091102 0.318532 0.345500 +0.550618 VBK/20091102 0.519416 0.573232 +0.574282 RPG/20091102 0.551042 0.592626 +0.547724 EWH/20091102 0.537782 0.563981 +0.362585 TZA/20091102 0.306431 0.403970 +0.486578 SGG/20091102 0.475654 0.498549 +0.629728 KOL/20091102 0.604971 0.648004 +0.564283 EWY/20091102 0.550739 0.580518 +0.558638 PRF/20091102 0.522521 0.582292 +0.513992 TLH/20091102 0.505740 0.524308 +0.572371 EPP/20091102 0.550468 0.599748 +0.541986 XLE/20091102 0.524083 0.566641 +0.563884 EWN/20091102 0.552823 0.576110 +0.525019 SHM/20091102 0.511901 0.544360 +0.555051 FXI/20091102 0.543618 0.565688 +0.598824 EWS/20091102 0.584602 0.616818 +0.551947 IDU/20091102 0.532628 0.584112 +0.485902 VXZ/20091102 0.470172 0.497635 +0.569369 IVE/20091102 0.553338 0.580459 +0.706228 DGP/20091102 0.698260 0.719677 +0.585600 GMF/20091102 0.574940 0.600342 +0.537942 IWR/20091102 0.512747 0.573487 +0.510778 RKH/20091102 0.491403 0.524524 +0.518467 TIP/20091102 0.517058 0.519217 +0.618937 URE/20091102 0.608881 0.627512 +0.513211 DBO/20091102 0.507855 0.519270 +0.538525 IOO/20091102 0.520937 0.558954 +0.493143 DBV/20091102 0.468450 0.530481 +0.539845 EFA/20091102 0.520483 0.576562 +0.579125 BGU/20091102 0.558845 0.616862 diff --git a/test/train-sets/ref/bs.prvote.predict b/test/train-sets/ref/bs.prvote.predict index cb44e068f96..ce4c6a89eaa 100644 --- a/test/train-sets/ref/bs.prvote.predict +++ b/test/train-sets/ref/bs.prvote.predict @@ -1,200 +1,200 @@ -0.000000 0.458392 1.000000 -1.000000 0.452659 0.619412 -0.000000 0.310507 0.456517 -0.000000 0.173335 0.336450 -0.000000 0.248374 0.432724 -1.000000 0.935125 1.000000 -0.000000 0.283262 0.428031 -0.000000 0.045959 0.367023 -0.000000 0.355603 0.473138 -1.000000 0.949092 1.000000 -0.000000 0.191053 0.379315 -0.000000 0.324044 0.538324 -0.000000 0.191435 0.419371 -0.000000 0.310092 0.535166 -1.000000 0.902808 0.994895 -1.000000 0.592455 0.885632 -1.000000 0.860310 1.000000 -0.000000 0.057171 0.353275 -0.000000 0.190818 0.644261 -0.000000 0.045328 0.169037 -0.000000 0.230530 0.923944 -1.000000 0.611789 1.000000 -0.000000 0.070341 0.310366 -1.000000 0.489735 1.000000 -0.000000 0.000000 0.151716 -0.000000 0.138823 0.475543 -0.000000 0.224302 0.449844 -0.000000 0.044406 0.248419 -1.000000 0.468885 0.996748 -0.000000 0.243104 0.542973 -1.000000 0.605581 1.000000 -0.000000 0.138691 0.428629 -0.000000 0.122978 0.198036 -0.000000 0.272718 0.450171 -1.000000 0.685426 0.997519 -0.000000 0.175658 0.571002 -1.000000 0.654157 1.000000 -0.000000 0.099214 0.276944 -1.000000 0.602038 1.000000 +0.000000 0.461143 1.000000 +1.000000 0.454323 0.623666 +0.000000 0.308435 0.457801 +0.000000 0.172900 0.336010 +0.000000 0.249786 0.431456 +1.000000 0.907605 1.000000 +0.000000 0.279606 0.426237 +0.000000 0.042017 0.357248 +0.000000 0.357054 0.475297 +1.000000 0.950801 1.000000 +0.000000 0.184601 0.377596 +0.000000 0.321348 0.539043 +0.000000 0.188588 0.420924 +0.000000 0.305400 0.532424 +1.000000 0.930153 0.977953 +1.000000 0.593942 0.871659 +1.000000 0.860024 1.000000 +0.000000 0.061745 0.352176 +0.000000 0.188317 0.644698 +0.000000 0.043844 0.168271 +0.000000 0.230575 0.920715 +1.000000 0.612290 1.000000 +0.000000 0.072660 0.311332 +1.000000 0.491655 1.000000 +0.000000 0.000000 0.151854 +0.000000 0.142024 0.477105 +0.000000 0.223774 0.449209 +0.000000 0.043890 0.249818 +1.000000 0.471270 0.998228 +0.000000 0.244106 0.543028 +1.000000 0.605217 1.000000 +0.000000 0.139485 0.430454 +0.000000 0.123310 0.198560 +0.000000 0.274046 0.447987 +1.000000 0.689590 0.996963 +0.000000 0.175736 0.571229 +1.000000 0.654082 1.000000 +0.000000 0.101181 0.274260 +1.000000 0.604198 1.000000 1.000000 1.000000 1.000000 -0.000000 0.109666 0.408097 -1.000000 0.826260 0.946926 -0.000000 0.011032 0.125592 -0.000000 0.147675 0.375113 -0.000000 0.034709 0.309596 -0.000000 0.083989 0.181821 -0.000000 0.119489 0.701049 -0.000000 0.043627 0.557042 -1.000000 0.290648 0.961409 -0.000000 0.222167 0.287372 -1.000000 0.428921 1.000000 -1.000000 0.368639 1.000000 -0.000000 0.000000 0.401585 -0.000000 0.092099 0.481334 -1.000000 0.969464 1.000000 -1.000000 0.222807 1.000000 -0.000000 0.119059 0.317707 -0.000000 0.069827 0.380974 -0.000000 0.371971 0.885179 -0.000000 0.067896 0.437237 -1.000000 0.570298 1.000000 -0.000000 0.016143 0.348595 -1.000000 0.597393 0.885829 -0.000000 0.066054 0.201453 -1.000000 0.621176 1.000000 -0.000000 0.093998 0.313379 -0.000000 0.069193 0.460195 -0.000000 0.095883 0.806659 -0.000000 0.045853 0.196326 -1.000000 0.566345 0.797911 -0.000000 0.056480 0.425846 -1.000000 0.539467 0.902147 -0.000000 0.283821 0.976362 -0.000000 0.000000 0.127377 -1.000000 0.726537 1.000000 -1.000000 0.459946 1.000000 -0.000000 0.029138 0.371215 -0.000000 0.187812 0.601657 -0.000000 0.057713 0.119670 -0.000000 0.059909 0.347788 -0.000000 0.000000 0.192228 -0.000000 0.141604 0.400030 -1.000000 0.787472 1.000000 -0.000000 0.102555 0.489685 -0.000000 0.150986 0.222639 -0.000000 0.117383 0.415650 -0.000000 0.345909 1.000000 -1.000000 0.664915 1.000000 -1.000000 0.772281 1.000000 -0.000000 0.000000 0.300062 -0.000000 0.053952 0.209100 -1.000000 0.534162 0.951390 +0.000000 0.113866 0.408043 +1.000000 0.827012 0.946687 +0.000000 0.011149 0.121813 +0.000000 0.149601 0.378061 +0.000000 0.035188 0.309740 +0.000000 0.087694 0.181494 +0.000000 0.121508 0.700599 +0.000000 0.044575 0.556986 +1.000000 0.291835 0.967559 +0.000000 0.221937 0.288573 +1.000000 0.430156 1.000000 +1.000000 0.368187 1.000000 +0.000000 0.000000 0.402200 +0.000000 0.092237 0.481399 +1.000000 0.969786 1.000000 +1.000000 0.226676 0.999709 +0.000000 0.118357 0.315847 +0.000000 0.070360 0.380899 +0.000000 0.371091 0.885248 +0.000000 0.068514 0.437476 +1.000000 0.569563 1.000000 +0.000000 0.015767 0.348235 +1.000000 0.603747 0.882706 +0.000000 0.065635 0.197630 +1.000000 0.622951 1.000000 +0.000000 0.093818 0.311870 +0.000000 0.068874 0.461730 +0.000000 0.095590 0.807631 +0.000000 0.045792 0.197864 +1.000000 0.568593 0.796978 +0.000000 0.055345 0.427854 +1.000000 0.543749 0.912553 +0.000000 0.284230 0.972667 +0.000000 0.000000 0.124847 +1.000000 0.727956 1.000000 +1.000000 0.465338 1.000000 +0.000000 0.029203 0.370035 +0.000000 0.191575 0.599250 +0.000000 0.057811 0.120291 +0.000000 0.059244 0.347535 +0.000000 0.000000 0.192067 +0.000000 0.142321 0.399597 +1.000000 0.785679 1.000000 +0.000000 0.102917 0.492744 +0.000000 0.149349 0.222598 +0.000000 0.117364 0.413955 +0.000000 0.345273 1.000000 +1.000000 0.662519 1.000000 +1.000000 0.768806 1.000000 +0.000000 0.000000 0.295798 +0.000000 0.054802 0.208882 +1.000000 0.538991 0.951464 1.000000 1.000000 1.000000 -0.000000 0.072308 0.175301 -1.000000 0.681809 1.000000 -0.000000 0.000000 0.213567 -1.000000 0.504172 1.000000 -0.000000 0.027734 0.319510 -1.000000 0.384518 0.950060 -1.000000 0.533635 1.000000 -0.000000 0.000000 0.401651 -1.000000 0.534493 0.939360 -0.000000 0.000000 0.104174 -1.000000 0.493532 1.000000 -0.000000 0.000000 0.147888 -1.000000 0.963237 1.000000 -0.000000 0.000000 0.143924 -0.000000 0.048359 0.415193 -0.000000 0.133686 0.228804 -1.000000 0.917059 1.000000 -1.000000 0.665322 1.000000 -1.000000 0.154025 0.628948 -0.000000 0.000000 0.408536 -0.000000 0.317683 1.000000 -0.000000 0.000000 0.509912 -0.000000 0.000000 0.506966 -1.000000 0.323823 0.987185 -1.000000 0.452777 1.000000 -0.000000 0.327834 1.000000 -0.000000 0.001862 0.496234 -0.000000 0.018963 0.388772 -1.000000 0.927328 0.949857 -0.000000 0.000000 0.404609 -1.000000 0.453133 1.000000 -1.000000 0.495457 0.991451 -1.000000 0.369999 0.885172 -0.000000 0.000000 0.405252 -1.000000 0.449114 0.886408 -0.000000 0.000000 0.400789 -1.000000 0.414330 0.945832 -0.000000 0.000000 0.117269 -1.000000 0.733348 1.000000 -0.000000 0.037800 0.243425 -0.000000 0.237421 0.939561 -0.000000 0.000000 0.367407 -0.000000 0.026962 0.300478 -1.000000 0.496068 1.000000 -0.000000 0.350595 0.995408 -1.000000 0.954148 1.000000 -0.000000 0.029820 0.379896 -0.000000 0.002710 0.456159 -0.000000 0.000000 0.174533 -1.000000 0.683822 0.909649 -1.000000 0.823318 0.990685 -1.000000 0.408713 0.985697 -1.000000 0.611208 0.920679 -1.000000 0.488054 0.945687 -1.000000 0.892140 1.000000 -0.000000 0.000000 0.323413 -1.000000 0.309824 0.909166 -1.000000 0.908086 1.000000 -0.000000 0.215659 0.913836 -1.000000 0.622499 0.926225 -0.000000 0.000000 0.364806 -1.000000 0.000000 0.663355 -1.000000 0.528307 0.920641 -1.000000 0.937751 1.000000 -0.000000 0.000000 0.061637 -1.000000 0.849636 1.000000 -0.000000 0.000000 0.621811 -1.000000 0.548149 0.956069 -0.000000 0.000000 0.203684 -0.000000 0.040461 0.379246 -1.000000 0.678339 0.936413 -0.000000 0.000532 0.331340 -1.000000 0.409028 0.923825 -1.000000 0.933654 1.000000 -0.000000 0.000000 0.086990 -0.000000 0.417785 0.997241 -1.000000 0.705726 1.000000 -1.000000 0.584154 0.913484 -0.000000 0.029493 0.334615 -0.000000 0.000000 0.273973 -0.000000 0.314658 0.826037 -0.000000 0.045142 0.200166 -0.000000 0.031250 0.597995 -0.000000 0.025126 0.275946 -1.000000 0.774048 0.971336 -1.000000 0.677971 1.000000 -1.000000 0.733083 0.959720 -1.000000 0.448381 0.996360 -0.000000 0.083232 0.174466 -1.000000 0.425506 0.908774 -0.000000 0.000000 0.248872 -0.000000 0.068516 0.384860 -0.000000 0.000000 0.513376 -1.000000 0.627244 1.000000 -0.000000 0.000000 0.558645 -0.000000 0.036497 0.363344 -1.000000 0.671142 0.994183 -1.000000 0.661582 0.920171 -0.000000 0.075627 0.462856 -0.000000 0.000000 0.035419 -0.000000 0.000000 0.297109 -0.000000 0.063500 0.297037 -1.000000 0.982524 1.000000 -0.000000 0.450480 0.962684 -0.000000 0.000000 0.019079 -0.000000 0.018797 0.458421 -1.000000 0.532585 1.000000 +0.000000 0.072319 0.175520 +1.000000 0.683836 1.000000 +0.000000 0.000000 0.213366 +1.000000 0.504550 1.000000 +0.000000 0.027464 0.319801 +1.000000 0.385072 0.948512 +1.000000 0.531008 1.000000 +0.000000 0.000000 0.400175 +1.000000 0.535597 0.939290 +0.000000 0.000000 0.103842 +1.000000 0.493747 1.000000 +0.000000 0.000000 0.149937 +1.000000 0.963327 1.000000 +0.000000 0.000000 0.144073 +0.000000 0.048538 0.415294 +0.000000 0.133516 0.228533 +1.000000 0.916381 1.000000 +1.000000 0.670913 1.000000 +1.000000 0.154807 0.629488 +0.000000 0.000000 0.410882 +0.000000 0.319040 1.000000 +0.000000 0.000000 0.511062 +0.000000 0.000000 0.507449 +1.000000 0.322397 0.990013 +1.000000 0.454484 1.000000 +0.000000 0.330018 1.000000 +0.000000 0.001962 0.494417 +0.000000 0.019529 0.388293 +1.000000 0.923936 0.950239 +0.000000 0.000000 0.407068 +1.000000 0.454277 1.000000 +1.000000 0.493289 0.991915 +1.000000 0.369752 0.895276 +0.000000 0.000000 0.403232 +1.000000 0.446424 0.886753 +0.000000 0.000000 0.397994 +1.000000 0.414919 0.945409 +0.000000 0.000000 0.115876 +1.000000 0.731144 1.000000 +0.000000 0.037814 0.245286 +0.000000 0.235990 0.941447 +0.000000 0.000000 0.369359 +0.000000 0.026594 0.300016 +1.000000 0.496750 1.000000 +0.000000 0.350516 0.993364 +1.000000 0.951748 1.000000 +0.000000 0.029824 0.380193 +0.000000 0.003930 0.455034 +0.000000 0.000000 0.176951 +1.000000 0.692829 0.909403 +1.000000 0.823935 0.990112 +1.000000 0.407728 0.985795 +1.000000 0.608834 0.917782 +1.000000 0.492258 0.947096 +1.000000 0.897744 1.000000 +0.000000 0.000000 0.320114 +1.000000 0.308367 0.909533 +1.000000 0.907653 1.000000 +0.000000 0.216536 0.913418 +1.000000 0.623193 0.926460 +0.000000 0.000000 0.366917 +1.000000 0.000000 0.661785 +1.000000 0.528981 0.920058 +1.000000 0.942926 1.000000 +0.000000 0.000000 0.062394 +1.000000 0.848808 1.000000 +0.000000 0.000000 0.619862 +1.000000 0.548329 0.955759 +0.000000 0.000000 0.202468 +0.000000 0.040626 0.376901 +1.000000 0.678915 0.939687 +0.000000 0.000000 0.334237 +1.000000 0.409379 0.925681 +1.000000 0.933672 1.000000 +0.000000 0.000000 0.089187 +0.000000 0.420383 0.996869 +1.000000 0.705073 1.000000 +1.000000 0.589230 0.911277 +0.000000 0.030905 0.334830 +0.000000 0.000000 0.273854 +0.000000 0.316853 0.819736 +0.000000 0.043955 0.200812 +0.000000 0.031169 0.598600 +0.000000 0.025726 0.276507 +1.000000 0.776421 0.971211 +1.000000 0.678084 1.000000 +1.000000 0.731907 0.961202 +1.000000 0.450472 0.994384 +0.000000 0.084867 0.173725 +1.000000 0.425013 0.906104 +0.000000 0.000000 0.250731 +0.000000 0.068779 0.384481 +0.000000 0.000000 0.511504 +1.000000 0.627747 1.000000 +0.000000 0.000000 0.560697 +0.000000 0.036193 0.363160 +1.000000 0.672138 0.993925 +1.000000 0.659953 0.924970 +0.000000 0.077117 0.464498 +0.000000 0.000000 0.035566 +0.000000 0.000000 0.296212 +0.000000 0.064630 0.297574 +1.000000 0.982108 1.000000 +0.000000 0.452194 0.962764 +0.000000 0.000000 0.019144 +0.000000 0.018814 0.458618 +1.000000 0.532506 1.000000 diff --git a/test/train-sets/ref/bs.reg.predict b/test/train-sets/ref/bs.reg.predict index 2874a65b337..8dc78d59053 100644 --- a/test/train-sets/ref/bs.reg.predict +++ b/test/train-sets/ref/bs.reg.predict @@ -4,997 +4,997 @@ 0.439056 AAXJ/20091028 0.000000 0.585408 0.593261 VWO/20091028 0.593261 0.593261 0.000000 EEV/20091028 0.000000 0.000000 -0.290315 GDX/20091028 0.225452 0.442247 -0.378561 RTH/20091028 0.179144 0.475001 -0.585071 MXI/20091028 0.443472 0.655028 -0.444858 EWU/20091028 0.357182 0.484989 +0.290757 GDX/20091028 0.225463 0.444674 +0.379107 RTH/20091028 0.179146 0.475489 +0.585064 MXI/20091028 0.440664 0.656833 +0.444599 EWU/20091028 0.354873 0.485586 0.000000 SH/20091028 0.000000 0.000000 -0.606460 EDC/20091028 0.305231 0.732510 +0.605829 EDC/20091028 0.300935 0.732510 0.000000 ERY/20091028 0.000000 0.000000 -0.012822 SDS/20091028 0.000000 0.051289 -0.255150 OEF/20091028 0.160742 0.374502 -0.431715 IYT/20091028 0.369129 0.534166 -0.255980 BIL/20091028 0.243539 0.274304 -0.011685 GLL/20091028 0.000000 0.035180 -0.001640 EDZ/20091028 0.000000 0.006560 -0.511457 IWM/20091028 0.421974 0.700504 -0.511652 VXF/20091028 0.473648 0.607753 -0.512152 IJJ/20091028 0.482314 0.583431 -0.486665 PIN/20091028 0.449962 0.561017 -0.520507 XLB/20091028 0.460457 0.558732 -0.426171 ECH/20091028 0.377721 0.480919 -0.595816 TYH/20091028 0.565675 0.626044 -0.582131 VAW/20091028 0.543556 0.607910 -0.535925 DBP/20091028 0.525421 0.544788 -0.659723 XME/20091028 0.632536 0.678948 -0.646803 VO/20091028 0.639761 0.661921 -0.638586 RSX/20091028 0.598896 0.682955 -0.607202 EWC/20091028 0.575668 0.644853 -0.640491 TUR/20091028 0.621044 0.669233 -0.499586 VYM/20091028 0.453916 0.524433 -0.544912 FCG/20091028 0.535271 0.552071 -0.523680 VGT/20091028 0.506421 0.535509 -0.544690 EWQ/20091028 0.540586 0.551918 -0.539332 IEV/20091028 0.537697 0.540845 -0.508624 XLK/20091028 0.505189 0.511793 -0.545409 EFG/20091028 0.536857 0.550891 -0.569773 BKF/20091028 0.542368 0.606578 -0.578183 KIE/20091028 0.563761 0.600789 -0.587247 EEB/20091028 0.558164 0.610581 -0.597040 IJK/20091028 0.587901 0.606232 -0.016087 DUG/20091028 0.000000 0.064347 -0.040781 TWM/20091028 0.000000 0.085759 -0.486905 MDY/20091028 0.330986 0.592090 -0.507868 ACWI/20091028 0.415573 0.577034 -0.201354 BSV/20091028 0.164801 0.256020 -0.473924 DDM/20091028 0.430971 0.509127 -0.499726 DIA/20091028 0.453813 0.552051 -0.304825 TLT/20091028 0.225116 0.336724 -0.072223 DXD/20091028 0.000000 0.193034 -0.527745 XHB/20091028 0.471452 0.627395 -0.454011 VDE/20091028 0.414389 0.537884 -0.400462 BND/20091028 0.326779 0.429659 -0.543854 EMB/20091028 0.513390 0.600109 -0.395384 SCO/20091028 0.319927 0.452324 -0.445699 AMJ/20091028 0.418503 0.466262 -0.280794 OIL/20091028 0.250704 0.297346 -0.523219 PZA/20091028 0.510724 0.545216 -0.578244 VGK/20091028 0.537398 0.620463 -0.604781 RWX/20091028 0.558453 0.649337 -0.429231 JJA/20091028 0.416592 0.461275 -0.632446 FXD/20091028 0.562518 0.674074 -0.577492 XES/20091028 0.506916 0.632728 -0.554157 VIG/20091028 0.542968 0.570027 -0.146950 DZZ/20091028 0.065569 0.243146 -0.583121 VFH/20091028 0.555215 0.606088 -0.352428 DTO/20091028 0.260236 0.412303 -0.564788 EWP/20091028 0.549161 0.575633 -0.565963 FDN/20091028 0.553590 0.578099 -0.542757 INP/20091028 0.503393 0.559513 -0.128734 TYP/20091028 0.038642 0.216879 -0.519134 RWR/20091028 0.470821 0.561306 -0.529168 KBE/20091028 0.499253 0.557769 -0.173053 EUO/20091028 0.143568 0.190654 -0.464670 IWF/20091028 0.434536 0.489948 -0.297624 SMN/20091028 0.223947 0.423966 -0.505594 SMH/20091028 0.494006 0.530873 -0.490444 XRT/20091028 0.465844 0.523961 -0.322314 USO/20091028 0.257601 0.356985 -0.447048 DJP/20091028 0.421085 0.475877 -0.315908 CFT/20091028 0.271113 0.357166 -0.187434 SRS/20091028 0.157995 0.207994 -0.520072 MOO/20091028 0.490663 0.560318 -0.388644 BIV/20091028 0.360509 0.407748 -0.264770 VXX/20091028 0.225146 0.279485 -0.554958 IYM/20091028 0.515887 0.601696 -0.490363 IFN/20091028 0.471029 0.517838 -0.568004 SLV/20091028 0.544736 0.593742 -0.537942 TAO/20091028 0.526044 0.551895 -0.593063 PGF/20091028 0.584924 0.601310 -0.610601 IYR/20091028 0.605769 0.616480 -0.166759 QID/20091028 0.143164 0.203528 -0.567624 THD/20091028 0.510985 0.588842 -0.580338 IJS/20091028 0.524899 0.607918 -0.581464 VB/20091028 0.526380 0.619749 -0.388887 EDV/20091028 0.349929 0.427010 -0.526853 IEZ/20091028 0.472705 0.579002 -0.561213 VTV/20091028 0.521536 0.585645 -0.577892 IJR/20091028 0.539075 0.604866 -0.342600 UCO/20091028 0.325039 0.368570 -0.564289 JNK/20091028 0.541004 0.580260 -0.591500 IWN/20091028 0.569734 0.625220 -0.569358 VV/20091028 0.557441 0.578799 -0.509093 UGL/20091028 0.499927 0.519330 -0.720449 UWM/20091028 0.674927 0.748964 -0.668882 IWC/20091028 0.645621 0.714603 -0.610202 EWA/20091028 0.562022 0.685522 -0.593851 IVV/20091028 0.560103 0.637193 -0.594682 SPY/20091028 0.561661 0.639935 -0.467887 TFI/20091028 0.452465 0.486599 -0.600055 VEA/20091028 0.572830 0.647302 -0.596489 QQQQ/20091028 0.574010 0.618548 -0.577196 UYG/20091028 0.550218 0.624624 -0.559377 OIH/20091028 0.546141 0.573196 -0.523627 GXC/20091028 0.504769 0.537132 -0.573358 SSO/20091028 0.562198 0.578965 -0.588913 XLI/20091028 0.578784 0.599544 -0.596136 GML/20091028 0.587825 0.614255 -0.584284 ROM/20091028 0.570055 0.594239 -0.566229 FXC/20091028 0.561401 0.574536 -0.220194 DOG/20091028 0.204259 0.234258 -0.513817 IYE/20091028 0.482993 0.549373 -0.277851 SKF/20091028 0.232956 0.321242 -0.345983 SHY/20091028 0.329633 0.358190 -0.467000 DBA/20091028 0.443392 0.512508 -0.530523 RSP/20091028 0.509174 0.557814 -0.521390 DBS/20091028 0.510440 0.548182 -0.566015 IBB/20091028 0.546101 0.585829 -0.553615 KCE/20091028 0.531553 0.579948 -0.548167 PKN/20091028 0.528272 0.564400 -0.513508 TNA/20091028 0.497776 0.532993 -0.511868 FAS/20091028 0.495443 0.526011 -0.547046 FXE/20091028 0.517483 0.570002 -0.544132 HYG/20091028 0.512773 0.572699 -0.556409 IWS/20091028 0.537699 0.577195 -0.358020 FXP/20091028 0.345056 0.372133 -0.388420 MBB/20091028 0.373599 0.410829 -0.555981 RFG/20091028 0.535716 0.572217 -0.552428 EPU/20091028 0.530279 0.564733 -0.297795 UUP/20091028 0.277170 0.309960 -0.498048 AGQ/20091028 0.439128 0.526424 -0.590786 SOXX/20091028 0.465121 0.673800 -0.286528 FAZ/20091028 0.160846 0.443861 -0.594678 VBK/20091028 0.545429 0.636456 -0.566972 RPG/20091028 0.541444 0.584335 -0.540922 EWH/20091028 0.530660 0.555599 -0.338887 TZA/20091028 0.248310 0.435607 -0.551538 SGG/20091028 0.516193 0.574259 -0.562024 KOL/20091028 0.536093 0.578155 -0.590314 EWY/20091028 0.570657 0.617134 -0.595164 PRF/20091028 0.583868 0.608489 -0.438491 TLH/20091028 0.419104 0.457428 -0.586407 EPP/20091028 0.562372 0.604374 -0.544920 XLE/20091028 0.523634 0.558484 -0.592315 EWN/20091028 0.581835 0.598672 -0.459931 SHM/20091028 0.433867 0.473322 -0.534872 FXI/20091028 0.517586 0.546951 -0.581039 EWS/20091028 0.571749 0.596903 -0.573621 IDU/20091028 0.563443 0.582347 -0.390784 VXZ/20091028 0.370394 0.414998 -0.577952 IVE/20091028 0.565982 0.593154 -0.543430 DGP/20091028 0.524661 0.552346 -0.642388 GMF/20091028 0.554847 0.686935 -0.639087 IWR/20091028 0.573207 0.663319 -0.598932 RKH/20091028 0.568611 0.611759 -0.446548 TIP/20091028 0.419989 0.474800 -0.575971 URE/20091028 0.566555 0.581642 -0.454292 DBO/20091028 0.432584 0.489380 -0.582509 IOO/20091028 0.559622 0.603515 -0.578564 DBV/20091028 0.554390 0.598962 -0.588491 EFA/20091028 0.564837 0.615907 -0.610481 BGU/20091028 0.569215 0.690671 -0.590627 EFV/20091028 0.569299 0.610241 -0.580725 IWB/20091028 0.559574 0.601491 -0.586665 IYF/20091028 0.566122 0.602941 -0.427825 YCS/20091028 0.408396 0.441391 -0.554929 DXJ/20091028 0.548978 0.560719 -0.580258 IWO/20091028 0.570401 0.595480 -0.471576 DBC/20091028 0.448268 0.496037 -0.302318 RWM/20091028 0.263305 0.342059 -0.560671 VBR/20091028 0.542464 0.600012 -0.337066 MZZ/20091028 0.270480 0.370694 -0.544962 IWD/20091028 0.540687 0.548802 -0.557624 PCY/20091028 0.550736 0.562429 -0.552983 EWI/20091028 0.539557 0.565196 -0.547168 IJH/20091028 0.539265 0.556518 -0.549197 EEM/20091028 0.542063 0.556035 -0.523389 EWM/20091028 0.514161 0.529783 -0.534579 SDY/20091028 0.524839 0.542595 -0.552525 ILF/20091028 0.543802 0.564007 -0.486957 JJG/20091028 0.453201 0.526129 -0.432555 TBT/20091028 0.428215 0.442571 -0.583780 XLF/20091028 0.568448 0.590413 -0.560914 ERX/20091028 0.528231 0.580317 -0.367641 SHV/20091028 0.338127 0.396667 -0.544339 EWX/20091028 0.527555 0.561944 -0.378406 EFZ/20091028 0.351506 0.393253 -0.445638 FXB/20091028 0.428421 0.452949 -0.532272 PHO/20091028 0.527920 0.537598 -0.536151 IGE/20091028 0.519693 0.554937 -0.389905 BGZ/20091028 0.382941 0.406058 -0.546017 UDN/20091028 0.530998 0.562312 -0.433064 CSJ/20091028 0.428145 0.438322 -0.563435 GXG/20091028 0.539062 0.594554 -0.564782 USD/20091028 0.551219 0.588533 -0.538043 EWD/20091028 0.526444 0.553168 -0.546322 EWJ/20091028 0.535168 0.553086 -0.581612 BRF/20091028 0.565079 0.601741 -0.591162 VEU/20091028 0.577618 0.597346 -0.566344 XLU/20091028 0.558277 0.573943 -0.513019 JJC/20091028 0.511025 0.516924 -0.607122 FGD/20091028 0.594796 0.612158 -0.566438 FXF/20091028 0.555363 0.574200 -0.524421 LQD/20091028 0.517569 0.530808 -0.596752 SCZ/20091028 0.587011 0.605878 -0.572689 IYW/20091028 0.569403 0.575889 -0.575152 VPL/20091028 0.568673 0.583977 -0.577430 DGS/20091028 0.572517 0.584904 -0.562317 ICF/20091028 0.554102 0.574471 -0.557443 DVY/20091028 0.555405 0.559380 -0.577809 IEO/20091028 0.567937 0.587239 -0.566201 VOT/20091028 0.559445 0.574544 -0.465958 CIU/20091028 0.449043 0.472492 -0.570596 EWG/20091028 0.561906 0.578074 -0.552928 EWT/20091028 0.549699 0.560256 -0.519579 GSG/20091028 0.510068 0.532265 -0.470063 KRE/20091028 0.462458 0.479905 -0.572444 LVL/20091028 0.567631 0.577395 -0.546124 UNG/20091028 0.505885 0.562485 -0.505476 MUB/20091028 0.481203 0.518621 -0.543724 VT/20091028 0.517209 0.564614 -0.530109 DAG/20091028 0.509902 0.569658 -0.512280 PPH/20091028 0.493295 0.523477 -0.581824 VSS/20091028 0.560550 0.597503 -0.513705 DBB/20091028 0.484988 0.538642 -0.544992 XLP/20091028 0.518174 0.558588 -0.580108 IJT/20091028 0.567193 0.587718 -0.594220 EWZ/20091028 0.570214 0.626943 -0.589597 PBW/20091028 0.566345 0.605761 -0.461735 FXY/20091028 0.430587 0.479514 -0.551966 IYZ/20091028 0.541141 0.562319 -0.591421 MVV/20091028 0.578557 0.603190 -0.569175 VUG/20091028 0.551080 0.575663 -0.466971 PST/20091028 0.458482 0.476410 -0.367575 PSQ/20091028 0.349356 0.381341 -0.558876 VNQ/20091028 0.550516 0.572792 -0.464909 IEI/20091028 0.451865 0.475898 -0.576513 EWW/20091028 0.562381 0.589243 -0.578834 IWP/20091028 0.571405 0.584938 -0.570433 IWV/20091028 0.564591 0.574415 -0.570930 DIG/20091028 0.552804 0.589719 -0.568687 VTI/20091028 0.563930 0.573496 -0.557742 FXA/20091028 0.541520 0.564102 -0.572787 NLR/20091028 0.566008 0.579252 -0.481876 AGG/20091028 0.473915 0.488844 -0.548461 BWX/20091028 0.541364 0.555086 -0.544160 IAU/20091028 0.531672 0.552185 -0.555991 XLV/20091028 0.541290 0.571436 -0.591507 XOP/20091028 0.573391 0.605277 -0.581535 EZU/20091028 0.566036 0.603126 -0.567531 JXI/20091028 0.553486 0.586165 -0.559403 XBI/20091028 0.525975 0.586818 -0.571859 IYG/20091028 0.557720 0.586288 -0.592004 SLX/20091028 0.571423 0.604238 -0.566703 HAO/20091028 0.554608 0.581920 -0.617000 EZA/20091028 0.609158 0.624357 -0.599601 XLY/20091028 0.590730 0.617208 -0.479730 IEF/20091028 0.471733 0.490895 -0.609526 DEM/20091028 0.601172 0.619612 -0.582715 IVW/20091028 0.576818 0.592027 -0.628638 UYM/20091028 0.607386 0.645563 -0.604445 IXC/20091028 0.570630 0.622436 -0.580997 PFF/20091029 0.561267 0.608563 -0.543060 WIP/20091029 0.511483 0.570628 -0.491460 GCC/20091029 0.461060 0.545404 -0.539961 AAXJ/20091029 0.515102 0.591190 -0.524865 VWO/20091029 0.517262 0.534217 -0.431757 EEV/20091029 0.420807 0.456958 -0.569912 GDX/20091029 0.519932 0.598976 -0.540826 RTH/20091029 0.483457 0.571155 -0.534582 MXI/20091029 0.517520 0.544460 -0.536155 EWU/20091029 0.487923 0.559391 -0.461425 SH/20091029 0.414407 0.512653 -0.604348 EDC/20091029 0.546671 0.645460 -0.565104 ERY/20091029 0.502251 0.605348 -0.506563 SDS/20091029 0.455115 0.544434 -0.567689 OEF/20091029 0.523177 0.610656 -0.511835 IYT/20091029 0.455776 0.621577 -0.557868 BIL/20091029 0.541109 0.588677 -0.518355 GLL/20091029 0.462049 0.555878 -0.387696 EDZ/20091029 0.246942 0.520015 -0.553044 IWM/20091029 0.437588 0.632746 -0.526681 VXF/20091029 0.475880 0.604261 -0.536007 IJJ/20091029 0.517788 0.561282 -0.535682 PIN/20091029 0.526846 0.550691 -0.538768 XLB/20091029 0.500310 0.579657 -0.519259 ECH/20091029 0.486360 0.547398 -0.488976 TYH/20091029 0.438108 0.530233 -0.508790 VAW/20091029 0.472537 0.529142 -0.556074 DBP/20091029 0.551481 0.562500 -0.510834 XME/20091029 0.493364 0.542705 -0.497404 VO/20091029 0.477450 0.537402 -0.540906 RSX/20091029 0.524872 0.572721 -0.536886 EWC/20091029 0.528108 0.553436 -0.468815 TUR/20091029 0.462442 0.476572 -0.517640 VYM/20091029 0.473200 0.537600 -0.488504 FCG/20091029 0.474723 0.503449 -0.509665 VGT/20091029 0.491763 0.524266 -0.513157 EWQ/20091029 0.497229 0.532091 -0.502128 IEV/20091029 0.491503 0.511613 -0.507344 XLK/20091029 0.477677 0.527801 -0.486506 EFG/20091029 0.479675 0.501332 -0.509599 BKF/20091029 0.482974 0.532673 -0.460325 KIE/20091029 0.445863 0.477764 -0.520362 EEB/20091029 0.503255 0.543753 -0.507691 IJK/20091029 0.469672 0.533691 -0.431587 DUG/20091029 0.348341 0.529895 -0.404288 TWM/20091029 0.359795 0.457012 -0.522437 MDY/20091029 0.446135 0.604820 -0.497728 ACWI/20091029 0.454354 0.534052 -0.541224 BSV/20091029 0.450381 0.589487 -0.469835 DDM/20091029 0.438221 0.528692 -0.516954 DIA/20091029 0.475866 0.540825 -0.559114 TLT/20091029 0.486327 0.598626 -0.500182 DXD/20091029 0.470239 0.523276 -0.496730 XHB/20091029 0.440597 0.561565 -0.515895 VDE/20091029 0.489298 0.552638 -0.493312 BND/20091029 0.453677 0.574747 -0.488684 EMB/20091029 0.457773 0.525777 -0.476999 SCO/20091029 0.438972 0.534340 -0.482680 AMJ/20091029 0.443519 0.549296 -0.499117 OIL/20091029 0.456084 0.520937 -0.492713 PZA/20091029 0.473454 0.513225 -0.485988 VGK/20091029 0.455633 0.508308 -0.496720 RWX/20091029 0.473620 0.523995 -0.505267 JJA/20091029 0.418437 0.561342 -0.500046 FXD/20091029 0.460072 0.524199 -0.468245 XES/20091029 0.444133 0.485427 -0.489377 VIG/20091029 0.468513 0.504059 -0.449945 DZZ/20091029 0.382222 0.487982 -0.491513 VFH/20091029 0.477360 0.507538 -0.446631 DTO/20091029 0.411890 0.497397 -0.527659 EWP/20091029 0.506091 0.542631 -0.535264 FDN/20091029 0.528251 0.543758 -0.556556 INP/20091029 0.518303 0.611484 -0.438196 TYP/20091029 0.407296 0.468879 -0.517500 RWR/20091029 0.505179 0.541916 -0.537675 KBE/20091029 0.514734 0.563409 -0.456415 EUO/20091029 0.403397 0.486224 -0.541560 IWF/20091029 0.522618 0.593638 -0.346930 SMN/20091029 0.301838 0.415177 -0.532715 SMH/20091029 0.507668 0.569047 -0.567692 XRT/20091029 0.544805 0.585860 -0.466949 USO/20091029 0.375562 0.518626 -0.467269 DJP/20091029 0.425262 0.504563 -0.509761 CFT/20091029 0.451133 0.582665 -0.423883 SRS/20091029 0.391073 0.453052 -0.536064 MOO/20091029 0.503172 0.564946 -0.517811 BIV/20091029 0.465717 0.603337 -0.422516 VXX/20091029 0.410814 0.432713 -0.543586 IYM/20091029 0.494313 0.597814 -0.514689 IFN/20091029 0.451173 0.566109 -0.576188 SLV/20091029 0.563819 0.583127 -0.470000 TAO/20091029 0.444211 0.487158 -0.462547 PGF/20091029 0.436783 0.501537 -0.471816 IYR/20091029 0.456612 0.480686 -0.373100 QID/20091029 0.342344 0.451363 -0.524297 THD/20091029 0.484039 0.576576 -0.502802 IJS/20091029 0.463857 0.527332 -0.529311 VB/20091029 0.514226 0.547517 -0.569205 EDV/20091029 0.507777 0.596667 -0.494645 IEZ/20091029 0.454132 0.513869 -0.503111 VTV/20091029 0.485496 0.533914 -0.510816 IJR/20091029 0.476385 0.527046 -0.442937 UCO/20091029 0.350347 0.485196 -0.467917 JNK/20091029 0.437284 0.491255 -0.478807 IWN/20091029 0.459952 0.492961 -0.493501 VV/20091029 0.480379 0.513910 -0.575046 UGL/20091029 0.556854 0.592006 -0.512474 UWM/20091029 0.461899 0.551712 -0.502654 IWC/20091029 0.454797 0.521947 -0.523010 EWA/20091029 0.457661 0.574890 -0.503966 IVV/20091029 0.466957 0.526961 -0.505198 SPY/20091029 0.476255 0.525781 -0.521553 TFI/20091029 0.478021 0.551103 -0.493725 VEA/20091029 0.452801 0.526751 -0.509498 QQQQ/20091029 0.473657 0.529200 -0.511648 UYG/20091029 0.501808 0.526589 -0.490480 OIH/20091029 0.479161 0.498909 -0.493857 GXC/20091029 0.475383 0.517461 -0.528013 SSO/20091029 0.501234 0.561140 -0.516868 XLI/20091029 0.488716 0.529928 -0.535450 GML/20091029 0.500772 0.567776 -0.540942 ROM/20091029 0.525777 0.561539 -0.488773 FXC/20091029 0.469191 0.508756 -0.475247 DOG/20091029 0.403240 0.516234 -0.493006 IYE/20091029 0.486634 0.502487 -0.406590 SKF/20091029 0.338573 0.464525 -0.524749 SHY/20091029 0.460060 0.554171 -0.478868 DBA/20091029 0.445715 0.510500 -0.544049 RSP/20091029 0.508004 0.573171 -0.566756 DBS/20091029 0.536417 0.610498 -0.544620 IBB/20091029 0.514326 0.582267 -0.499950 KCE/20091029 0.473774 0.541986 -0.509135 PKN/20091029 0.490374 0.516471 -0.522123 TNA/20091029 0.501365 0.567256 -0.476559 FAS/20091029 0.450436 0.514630 -0.471284 FXE/20091029 0.433069 0.499481 -0.442766 HYG/20091029 0.427320 0.457487 -0.481393 IWS/20091029 0.455421 0.503949 -0.425149 FXP/20091029 0.396977 0.461434 -0.473907 MBB/20091029 0.430972 0.540678 -0.487833 RFG/20091029 0.462271 0.505474 -0.494217 EPU/20091029 0.442249 0.515188 -0.441146 UUP/20091029 0.388089 0.496107 -0.507313 AGQ/20091029 0.455255 0.542240 -0.496428 SOXX/20091029 0.391713 0.621436 -0.432163 FAZ/20091029 0.353955 0.500897 -0.491799 VBK/20091029 0.440785 0.578989 -0.513168 RPG/20091029 0.487005 0.573398 -0.481635 EWH/20091029 0.449719 0.519236 -0.419795 TZA/20091029 0.370345 0.468068 -0.444159 SGG/20091029 0.413568 0.487921 -0.520627 KOL/20091029 0.485277 0.571281 -0.497050 EWY/20091029 0.448320 0.519846 -0.502283 PRF/20091029 0.460383 0.520663 -0.527125 TLH/20091029 0.480588 0.583514 -0.520072 EPP/20091029 0.509050 0.548850 -0.475168 XLE/20091029 0.461958 0.502755 -0.512337 EWN/20091029 0.480669 0.534071 -0.488272 SHM/20091029 0.460066 0.522782 -0.490118 FXI/20091029 0.460760 0.523898 -0.504555 EWS/20091029 0.483020 0.517114 -0.498329 IDU/20091029 0.477397 0.518211 -0.504247 VXZ/20091029 0.459859 0.545009 -0.512497 IVE/20091029 0.482918 0.524526 -0.562329 DGP/20091029 0.497229 0.597249 -0.525611 GMF/20091029 0.498242 0.576947 -0.529077 IWR/20091029 0.502142 0.564448 -0.494676 RKH/20091029 0.474244 0.508045 -0.493168 TIP/20091029 0.472107 0.536218 -0.576710 URE/20091029 0.532749 0.606833 -0.448654 DBO/20091029 0.417594 0.510711 -0.519007 IOO/20091029 0.504904 0.537533 -0.496377 DBV/20091029 0.464201 0.537117 -0.524945 EFA/20091029 0.510438 0.550121 -0.550187 BGU/20091029 0.518307 0.624032 -0.507840 EFV/20091029 0.472132 0.528148 -0.512816 IWB/20091029 0.482211 0.541114 -0.476241 IYF/20091029 0.423132 0.520497 -0.404929 YCS/20091029 0.384820 0.425476 -0.478275 DXJ/20091029 0.433667 0.519393 -0.518001 IWO/20091029 0.492234 0.545446 -0.493634 DBC/20091029 0.472531 0.507016 -0.440465 RWM/20091029 0.384506 0.470958 -0.512823 VBR/20091029 0.473892 0.540121 -0.421403 MZZ/20091029 0.403203 0.446293 -0.516276 IWD/20091029 0.488501 0.535040 -0.509125 PCY/20091029 0.479691 0.531492 -0.516991 EWI/20091029 0.498618 0.530246 -0.515425 IJH/20091029 0.505861 0.528233 -0.507214 EEM/20091029 0.493618 0.533543 -0.494701 EWM/20091029 0.483193 0.507533 -0.511347 SDY/20091029 0.498725 0.529389 -0.533799 ILF/20091029 0.523149 0.542974 -0.487575 JJG/20091029 0.451159 0.525943 -0.430900 TBT/20091029 0.404302 0.453864 -0.463525 XLF/20091029 0.451470 0.473155 -0.446654 ERX/20091029 0.427615 0.469951 -0.498854 SHV/20091029 0.486675 0.509405 -0.515838 EWX/20091029 0.471378 0.546418 -0.449927 EFZ/20091029 0.424038 0.482966 -0.461065 FXB/20091029 0.413341 0.483766 -0.485414 PHO/20091029 0.462302 0.500813 -0.483757 IGE/20091029 0.443764 0.524832 -0.418916 BGZ/20091029 0.393210 0.447950 -0.488145 UDN/20091029 0.475136 0.508870 -0.483760 CSJ/20091029 0.437652 0.508904 -0.518640 GXG/20091029 0.498446 0.545684 -0.502536 USD/20091029 0.485054 0.531811 -0.483599 EWD/20091029 0.434797 0.509102 -0.455987 EWJ/20091029 0.439420 0.495932 -0.527286 BRF/20091029 0.500556 0.544587 -0.511649 VEU/20091029 0.494468 0.530245 -0.514511 XLU/20091029 0.506223 0.526488 -0.492463 JJC/20091029 0.487249 0.496728 -0.513532 FGD/20091029 0.483694 0.544162 -0.489737 FXF/20091029 0.479290 0.506174 -0.490924 LQD/20091029 0.473932 0.499093 -0.502824 SCZ/20091029 0.487795 0.514094 -0.513352 IYW/20091029 0.489630 0.526822 -0.482679 VPL/20091029 0.466017 0.497656 -0.529057 DGS/20091029 0.505307 0.547796 -0.512704 ICF/20091029 0.503028 0.523998 -0.521426 DVY/20091029 0.508926 0.536556 -0.481572 IEO/20091029 0.455900 0.506931 -0.519890 VOT/20091029 0.506851 0.535186 -0.501121 CIU/20091029 0.469937 0.525608 -0.534345 EWG/20091029 0.515297 0.555738 -0.536234 EWT/20091029 0.525322 0.548766 -0.451925 GSG/20091029 0.440065 0.465599 -0.477829 KRE/20091029 0.458783 0.488623 -0.522713 LVL/20091029 0.507344 0.537565 -0.441530 UNG/20091029 0.410394 0.519430 -0.500821 MUB/20091029 0.487170 0.512900 -0.530095 VT/20091029 0.498590 0.547977 -0.490330 DAG/20091029 0.438276 0.524301 -0.564534 PPH/20091029 0.561055 0.568323 -0.518098 VSS/20091029 0.505456 0.528340 -0.485065 DBB/20091029 0.464150 0.497503 -0.487465 XLP/20091029 0.472865 0.501119 -0.517939 IJT/20091029 0.485510 0.541473 -0.512789 EWZ/20091029 0.469599 0.546975 -0.501540 PBW/20091029 0.486331 0.519799 -0.541887 FXY/20091029 0.504936 0.564103 -0.543717 IYZ/20091029 0.537289 0.550020 -0.532947 MVV/20091029 0.520059 0.553397 -0.530276 VUG/20091029 0.527286 0.533303 -0.433746 PST/20091029 0.431507 0.435176 -0.437461 PSQ/20091029 0.397086 0.458775 -0.528685 VNQ/20091029 0.516945 0.535280 -0.500104 IEI/20091029 0.487276 0.528100 -0.529282 EWW/20091029 0.508524 0.557051 -0.526529 IWP/20091029 0.510230 0.548612 -0.523849 IWV/20091029 0.513772 0.529544 -0.483336 DIG/20091029 0.470148 0.499876 -0.524574 VTI/20091029 0.516357 0.528402 -0.487734 FXA/20091029 0.484631 0.492955 -0.498346 NLR/20091029 0.476684 0.537429 -0.512829 AGG/20091029 0.488120 0.534642 -0.532471 BWX/20091029 0.522318 0.547927 -0.566227 IAU/20091029 0.516222 0.589617 -0.554933 XLV/20091029 0.524349 0.588224 -0.484132 XOP/20091029 0.453986 0.525944 -0.535781 EZU/20091029 0.518899 0.551743 -0.529955 JXI/20091029 0.519366 0.545986 -0.540268 XBI/20091029 0.526996 0.558023 -0.510282 IYG/20091029 0.486800 0.523212 -0.571223 SLX/20091029 0.560084 0.584747 -0.517941 HAO/20091029 0.503330 0.528710 -0.520420 EZA/20091029 0.512037 0.537292 -0.517762 XLY/20091029 0.508036 0.526175 -0.516794 IEF/20091029 0.490498 0.542792 -0.526750 DEM/20091029 0.507716 0.538770 -0.506742 IVW/20091029 0.496929 0.514012 -0.553278 UYM/20091029 0.508487 0.580044 -0.496833 IXC/20091029 0.464100 0.526446 -0.512425 PFF/20091030 0.474631 0.550214 -0.557045 WIP/20091030 0.527545 0.576498 -0.520751 GCC/20091030 0.497472 0.549201 -0.547544 AAXJ/20091030 0.527307 0.565679 -0.546081 VWO/20091030 0.525206 0.561314 -0.406914 EEV/20091030 0.386910 0.418445 -0.615173 GDX/20091030 0.519863 0.668857 -0.597195 RTH/20091030 0.574588 0.630798 -0.618619 MXI/20091030 0.572471 0.638902 -0.595400 EWU/20091030 0.560016 0.637443 -0.446913 SH/20091030 0.415228 0.463004 -0.567439 EDC/20091030 0.524105 0.608664 -0.472282 ERY/20091030 0.436612 0.511440 -0.404000 SDS/20091030 0.349073 0.449852 -0.634760 OEF/20091030 0.609120 0.651081 -0.567343 IYT/20091030 0.528018 0.583711 -0.508750 BIL/20091030 0.492196 0.532203 -0.401948 GLL/20091030 0.353770 0.462276 -0.291394 EDZ/20091030 0.247117 0.310300 -0.541955 IWM/20091030 0.511982 0.559409 -0.553274 VXF/20091030 0.527660 0.570239 -0.559057 IJJ/20091030 0.527983 0.588103 -0.591903 PIN/20091030 0.551666 0.613448 -0.600817 XLB/20091030 0.589861 0.608957 -0.585301 ECH/20091030 0.573845 0.600472 -0.511288 TYH/20091030 0.475288 0.577247 -0.587297 VAW/20091030 0.545571 0.607137 -0.641715 DBP/20091030 0.628595 0.650465 -0.612564 XME/20091030 0.594238 0.629933 -0.516770 VO/20091030 0.504645 0.527700 -0.561955 RSX/20091030 0.542819 0.574598 -0.570615 EWC/20091030 0.559112 0.581373 -0.460927 TUR/20091030 0.449940 0.485814 -0.567094 VYM/20091030 0.538813 0.591764 -0.541989 FCG/20091030 0.510334 0.561704 -0.562975 VGT/20091030 0.553606 0.572205 -0.560168 EWQ/20091030 0.545232 0.570788 -0.555122 IEV/20091030 0.545262 0.573238 -0.558809 XLK/20091030 0.515763 0.582449 -0.539353 EFG/20091030 0.529186 0.551336 -0.591879 BKF/20091030 0.579990 0.596467 -0.528216 KIE/20091030 0.505608 0.552287 -0.605405 EEB/20091030 0.593135 0.610030 -0.533090 IJK/20091030 0.520891 0.539357 -0.444299 DUG/20091030 0.430094 0.461732 -0.407226 TWM/20091030 0.400525 0.416978 -0.562404 MDY/20091030 0.553018 0.571087 -0.576154 ACWI/20091030 0.560042 0.600547 -0.525396 BSV/20091030 0.496978 0.556836 -0.580996 DDM/20091030 0.524780 0.602828 -0.578371 DIA/20091030 0.555482 0.629735 -0.513883 TLT/20091030 0.480828 0.543276 -0.427420 DXD/20091030 0.340540 0.479202 -0.529463 XHB/20091030 0.477640 0.560695 -0.568738 VDE/20091030 0.521468 0.630552 -0.495210 BND/20091030 0.439594 0.539255 -0.542455 EMB/20091030 0.515248 0.560525 -0.500013 SCO/20091030 0.464573 0.520891 -0.569152 AMJ/20091030 0.545489 0.595252 -0.520559 OIL/20091030 0.457175 0.571340 -0.496822 PZA/20091030 0.489315 0.504751 -0.553751 VGK/20091030 0.537529 0.593697 -0.523296 RWX/20091030 0.510421 0.553139 -0.563299 JJA/20091030 0.547229 0.582092 -0.563831 FXD/20091030 0.542824 0.583111 -0.498071 XES/20091030 0.481545 0.526456 -0.527162 VIG/20091030 0.515910 0.554585 -0.270763 DZZ/20091030 0.225463 0.320857 -0.489457 VFH/20091030 0.481722 0.499222 -0.465099 DTO/20091030 0.408734 0.499108 -0.538926 EWP/20091030 0.521495 0.553976 -0.545211 FDN/20091030 0.539289 0.560742 -0.570017 INP/20091030 0.564726 0.577817 -0.344893 TYP/20091030 0.301889 0.361775 -0.556393 RWR/20091030 0.544093 0.582358 -0.557848 KBE/20091030 0.527486 0.588738 -0.436363 EUO/20091030 0.405084 0.458518 -0.553718 IWF/20091030 0.535058 0.582715 -0.260442 SMN/20091030 0.252198 0.274135 -0.534153 SMH/20091030 0.514807 0.551890 -0.572642 XRT/20091030 0.563467 0.580732 -0.518499 USO/20091030 0.482029 0.551115 -0.546831 DJP/20091030 0.532103 0.558812 -0.488048 CFT/20091030 0.427637 0.546068 -0.373240 SRS/20091030 0.357920 0.379109 -0.597337 MOO/20091030 0.579468 0.614547 -0.475040 BIV/20091030 0.459000 0.489223 -0.315671 VXX/20091030 0.299814 0.333023 -0.579916 IYM/20091030 0.558434 0.599562 -0.572396 IFN/20091030 0.518231 0.609009 -0.642881 SLV/20091030 0.630367 0.654207 -0.506272 TAO/20091030 0.491728 0.526135 -0.495498 PGF/20091030 0.459267 0.523186 -0.557331 IYR/20091030 0.543978 0.568086 -0.266585 QID/20091030 0.240030 0.315442 -0.497768 THD/20091030 0.478063 0.515376 -0.500333 IJS/20091030 0.486880 0.513095 -0.535769 VB/20091030 0.509477 0.569542 -0.541463 EDV/20091030 0.507773 0.581020 -0.531030 IEZ/20091030 0.497700 0.553486 -0.540427 VTV/20091030 0.514584 0.556854 -0.519805 IJR/20091030 0.493398 0.541690 -0.483267 UCO/20091030 0.448993 0.519562 -0.522888 JNK/20091030 0.477873 0.541367 -0.514156 IWN/20091030 0.503091 0.526672 -0.524690 VV/20091030 0.505544 0.544069 -0.678100 UGL/20091030 0.630183 0.732554 -0.523929 UWM/20091030 0.503344 0.540598 -0.508716 IWC/20091030 0.482812 0.525053 -0.546217 EWA/20091030 0.527642 0.577128 -0.561498 IVV/20091030 0.554454 0.574345 -0.554132 SPY/20091030 0.543502 0.561686 -0.527167 TFI/20091030 0.521736 0.534622 -0.526347 VEA/20091030 0.487676 0.542163 -0.551222 QQQQ/20091030 0.536458 0.563170 -0.540321 UYG/20091030 0.531076 0.552399 -0.549955 OIH/20091030 0.544349 0.555213 -0.575790 GXC/20091030 0.562864 0.584322 -0.602504 SSO/20091030 0.592716 0.612198 -0.574347 XLI/20091030 0.569274 0.577480 -0.632105 GML/20091030 0.625700 0.636023 -0.587817 ROM/20091030 0.578024 0.594858 -0.525449 FXC/20091030 0.514796 0.536037 -0.453604 DOG/20091030 0.414539 0.480012 -0.538375 IYE/20091030 0.530525 0.547708 -0.368545 SKF/20091030 0.281238 0.415636 -0.486059 SHY/20091030 0.476562 0.499864 -0.548303 DBA/20091030 0.503754 0.572940 -0.559966 RSP/20091030 0.540887 0.585910 -0.646130 DBS/20091030 0.634184 0.664451 -0.570534 IBB/20091030 0.566424 0.578097 -0.495396 KCE/20091030 0.478630 0.505959 -0.510711 PKN/20091030 0.499045 0.522936 -0.556289 TNA/20091030 0.495329 0.600526 -0.561579 FAS/20091030 0.517776 0.631072 -0.553004 FXE/20091030 0.526084 0.586924 -0.506902 HYG/20091030 0.481186 0.523983 -0.572704 IWS/20091030 0.556679 0.601910 -0.386034 FXP/20091030 0.365060 0.408921 -0.472169 MBB/20091030 0.442310 0.499210 -0.569151 RFG/20091030 0.562914 0.581521 -0.583837 EPU/20091030 0.571557 0.598798 -0.463246 UUP/20091030 0.445050 0.487065 -0.647751 AGQ/20091030 0.579973 0.685054 -0.585566 SOXX/20091030 0.559262 0.609393 -0.314624 FAZ/20091030 0.262289 0.397124 -0.553657 VBK/20091030 0.533306 0.578433 -0.541007 RPG/20091030 0.510333 0.578513 -0.523253 EWH/20091030 0.502877 0.544790 -0.390678 TZA/20091030 0.338988 0.416606 -0.548278 SGG/20091030 0.531899 0.580348 -0.587977 KOL/20091030 0.550332 0.615317 -0.538726 EWY/20091030 0.511931 0.571680 -0.534545 PRF/20091030 0.513400 0.564662 -0.519578 TLH/20091030 0.506366 0.534900 -0.544211 EPP/20091030 0.527870 0.560411 -0.536727 XLE/20091030 0.522664 0.556528 -0.557673 EWN/20091030 0.548708 0.571073 -0.499203 SHM/20091030 0.491072 0.504405 -0.541663 FXI/20091030 0.521138 0.553782 -0.583695 EWS/20091030 0.567732 0.590489 -0.520542 IDU/20091030 0.514257 0.529770 -0.480232 VXZ/20091030 0.467993 0.498977 -0.528749 IVE/20091030 0.502805 0.540552 -0.704430 DGP/20091030 0.656343 0.729111 -0.491185 GMF/20091030 0.476432 0.510343 -0.520021 IWR/20091030 0.515447 0.527439 -0.489056 RKH/20091030 0.485692 0.495895 -0.466376 TIP/20091030 0.442964 0.487818 -0.612127 URE/20091030 0.585889 0.647897 -0.497467 DBO/20091030 0.467061 0.513937 -0.556752 IOO/20091030 0.544509 0.573388 -0.539194 DBV/20091030 0.496544 0.579670 -0.552629 EFA/20091030 0.524950 0.574747 -0.620897 BGU/20091030 0.598337 0.651020 -0.553929 EFV/20091030 0.539940 0.569661 -0.548773 IWB/20091030 0.530403 0.560578 -0.521358 IYF/20091030 0.503558 0.529909 -0.425214 YCS/20091030 0.407598 0.442305 -0.500161 DXJ/20091030 0.491618 0.513264 -0.565143 IWO/20091030 0.534266 0.598332 -0.548683 DBC/20091030 0.539226 0.554658 -0.419613 RWM/20091030 0.347882 0.452798 -0.543064 VBR/20091030 0.538069 0.548289 -0.373441 MZZ/20091030 0.337352 0.396801 -0.555862 IWD/20091030 0.552435 0.560515 -0.549910 PCY/20091030 0.538758 0.581120 -0.543147 EWI/20091030 0.527025 0.557042 -0.554119 IJH/20091030 0.542022 0.580874 -0.584520 EEM/20091030 0.574015 0.597820 -0.543858 EWM/20091030 0.527898 0.563238 -0.539570 SDY/20091030 0.522925 0.554502 -0.646937 ILF/20091030 0.633152 0.668421 -0.545981 JJG/20091030 0.481922 0.607611 -0.456892 TBT/20091030 0.442608 0.470781 -0.514657 XLF/20091030 0.505803 0.522721 -0.538100 ERX/20091030 0.501250 0.584361 -0.505585 SHV/20091030 0.499229 0.511468 -0.595204 EWX/20091030 0.578931 0.611677 -0.427597 EFZ/20091030 0.412039 0.434135 -0.461436 FXB/20091030 0.447179 0.472561 -0.538604 PHO/20091030 0.533152 0.542131 -0.591923 IGE/20091030 0.569837 0.624262 -0.293642 BGZ/20091030 0.285557 0.301842 -0.533127 UDN/20091030 0.503149 0.547913 -0.496927 CSJ/20091030 0.491804 0.500399 -0.611728 GXG/20091030 0.585550 0.623559 -0.582709 USD/20091030 0.566064 0.605728 -0.555879 EWD/20091030 0.551952 0.558717 -0.479198 EWJ/20091030 0.465191 0.498627 -0.653867 BRF/20091030 0.629793 0.670947 -0.527401 VEU/20091030 0.514862 0.535594 -0.510262 XLU/20091030 0.504494 0.526633 -0.523954 JJC/20091030 0.507933 0.533953 -0.538504 FGD/20091030 0.521050 0.557371 -0.523246 FXF/20091030 0.512808 0.542604 -0.499745 LQD/20091030 0.485084 0.513352 -0.537289 SCZ/20091030 0.525076 0.544425 -0.538349 IYW/20091030 0.525305 0.550874 -0.496691 VPL/20091030 0.484820 0.503278 -0.553911 DGS/20091030 0.540014 0.565225 -0.555729 ICF/20091030 0.543751 0.579376 -0.536696 DVY/20091030 0.526756 0.548001 -0.518846 IEO/20091030 0.496732 0.567153 -0.534574 VOT/20091030 0.526512 0.545977 -0.475854 CIU/20091030 0.444248 0.488271 -0.549660 EWG/20091030 0.543952 0.560148 -0.558043 EWT/20091030 0.542370 0.571934 -0.515456 GSG/20091030 0.509392 0.523318 -0.497990 KRE/20091030 0.485820 0.513663 -0.572047 LVL/20091030 0.557797 0.589674 -0.453905 UNG/20091030 0.431126 0.469470 -0.504213 MUB/20091030 0.498408 0.509995 -0.552119 VT/20091030 0.542729 0.565034 -0.600295 DAG/20091030 0.524990 0.638775 -0.555419 PPH/20091030 0.550330 0.563318 -0.562336 VSS/20091030 0.548111 0.579116 -0.547551 DBB/20091030 0.528193 0.575054 -0.529937 XLP/20091030 0.513130 0.548717 -0.540198 IJT/20091030 0.507312 0.566816 -0.618551 EWZ/20091030 0.610070 0.635596 -0.519892 PBW/20091030 0.515019 0.525571 -0.552124 FXY/20091030 0.538807 0.562598 -0.537142 IYZ/20091030 0.527829 0.545743 -0.561713 MVV/20091030 0.556666 0.570382 -0.542344 VUG/20091030 0.537344 0.551140 -0.448188 PST/20091030 0.435976 0.467739 -0.433679 PSQ/20091030 0.424153 0.443626 -0.564633 VNQ/20091030 0.550745 0.576041 -0.486237 IEI/20091030 0.467930 0.504219 -0.578016 EWW/20091030 0.552100 0.607762 -0.541064 IWP/20091030 0.529468 0.557573 -0.540479 IWV/20091030 0.536104 0.544970 -0.549947 DIG/20091030 0.537668 0.564676 -0.543748 VTI/20091030 0.540721 0.546364 -0.525131 FXA/20091030 0.518276 0.532207 -0.517521 NLR/20091030 0.497939 0.552489 -0.505042 AGG/20091030 0.486115 0.514474 -0.537070 BWX/20091030 0.529530 0.541036 -0.635946 IAU/20091030 0.620560 0.645029 -0.574935 XLV/20091030 0.571856 0.576068 -0.525621 XOP/20091030 0.503506 0.563899 -0.525983 EZU/20091030 0.518646 0.531536 -0.528124 JXI/20091030 0.522509 0.535792 -0.554677 XBI/20091030 0.538057 0.573755 -0.510281 IYG/20091030 0.492480 0.541315 -0.638757 SLX/20091030 0.619929 0.662520 -0.569455 HAO/20091030 0.562219 0.573387 -0.586801 EZA/20091030 0.581303 0.596825 -0.570705 XLY/20091030 0.563696 0.578513 -0.489661 IEF/20091030 0.466560 0.504771 -0.558643 DEM/20091030 0.541292 0.573025 -0.547050 IVW/20091030 0.539505 0.553310 -0.672248 UYM/20091030 0.658060 0.686454 -0.528167 IXC/20091030 0.502643 0.543932 -0.520399 PFF/20091102 0.508829 0.542538 -0.543887 WIP/20091102 0.521868 0.561157 -0.551710 GCC/20091102 0.538059 0.562726 -0.585851 AAXJ/20091102 0.576494 0.598103 -0.591266 VWO/20091102 0.582881 0.598907 -0.362248 EEV/20091102 0.345914 0.390854 -0.651734 GDX/20091102 0.633995 0.680352 -0.603423 RTH/20091102 0.583841 0.619459 -0.637036 MXI/20091102 0.610520 0.658254 -0.584423 EWU/20091102 0.554871 0.638145 -0.407630 SH/20091102 0.398344 0.415856 -0.600773 EDC/20091102 0.569252 0.656516 -0.404892 ERY/20091102 0.377006 0.418191 -0.377168 SDS/20091102 0.345442 0.396886 -0.619787 OEF/20091102 0.596588 0.647708 -0.629280 IYT/20091102 0.623742 0.638350 -0.480395 BIL/20091102 0.454126 0.520670 -0.379440 GLL/20091102 0.296737 0.487713 -0.242799 EDZ/20091102 0.181783 0.308892 -0.564986 IWM/20091102 0.542776 0.591645 -0.574378 VXF/20091102 0.561307 0.585619 -0.579557 IJJ/20091102 0.572149 0.585468 -0.639902 PIN/20091102 0.622661 0.649615 -0.629034 XLB/20091102 0.612638 0.649022 -0.579103 ECH/20091102 0.556793 0.598132 -0.525641 TYH/20091102 0.507697 0.544488 -0.630616 VAW/20091102 0.626020 0.635883 -0.632307 DBP/20091102 0.612123 0.661329 -0.652452 XME/20091102 0.645154 0.657487 -0.556731 VO/20091102 0.539661 0.575618 -0.596321 RSX/20091102 0.565114 0.622347 -0.597092 EWC/20091102 0.583464 0.614001 -0.479146 TUR/20091102 0.462470 0.500550 -0.587433 VYM/20091102 0.573766 0.601957 -0.571564 FCG/20091102 0.553661 0.590148 -0.578124 VGT/20091102 0.563134 0.594010 -0.577040 EWQ/20091102 0.557467 0.589412 -0.562536 IEV/20091102 0.550757 0.568809 -0.577812 XLK/20091102 0.568603 0.582206 -0.555491 EFG/20091102 0.546504 0.561447 -0.592219 BKF/20091102 0.572985 0.610137 -0.526870 KIE/20091102 0.512586 0.537370 -0.615954 EEB/20091102 0.605368 0.625226 -0.569307 IJK/20091102 0.561113 0.574747 -0.400077 DUG/20091102 0.388385 0.407327 -0.383927 TWM/20091102 0.372508 0.397667 -0.588397 MDY/20091102 0.581522 0.599421 -0.596339 ACWI/20091102 0.578516 0.631113 -0.522718 BSV/20091102 0.496165 0.536527 -0.624776 DDM/20091102 0.606424 0.640204 -0.572463 DIA/20091102 0.567498 0.580414 -0.507449 TLT/20091102 0.491129 0.518571 -0.409782 DXD/20091102 0.342387 0.462692 -0.531627 XHB/20091102 0.472059 0.575842 -0.593371 VDE/20091102 0.556882 0.640558 -0.494427 BND/20091102 0.456057 0.524315 -0.515284 EMB/20091102 0.480705 0.539334 -0.475346 SCO/20091102 0.436174 0.520067 -0.564753 AMJ/20091102 0.547120 0.575976 -0.509504 OIL/20091102 0.463117 0.568218 -0.515816 PZA/20091102 0.490880 0.525199 -0.563094 VGK/20091102 0.538425 0.576339 -0.537994 RWX/20091102 0.534517 0.541200 -0.535359 JJA/20091102 0.516364 0.544497 -0.574170 FXD/20091102 0.550421 0.599747 -0.533316 XES/20091102 0.513375 0.542181 -0.567650 VIG/20091102 0.549349 0.587893 -0.231847 DZZ/20091102 0.223879 0.244694 -0.513234 VFH/20091102 0.483114 0.543416 -0.486833 DTO/20091102 0.388145 0.536072 -0.542770 EWP/20091102 0.524558 0.554507 -0.557830 FDN/20091102 0.554226 0.567155 -0.607401 INP/20091102 0.585903 0.628803 -0.336858 TYP/20091102 0.288863 0.379293 -0.557261 RWR/20091102 0.544142 0.563442 -0.563257 KBE/20091102 0.533573 0.586199 -0.447593 EUO/20091102 0.446278 0.448514 -0.576537 IWF/20091102 0.570657 0.589574 -0.239945 SMN/20091102 0.230158 0.247156 -0.547057 SMH/20091102 0.527650 0.569478 -0.559613 XRT/20091102 0.548654 0.574090 -0.518265 USO/20091102 0.498843 0.559701 -0.544547 DJP/20091102 0.533032 0.552770 -0.478784 CFT/20091102 0.425412 0.517209 -0.331412 SRS/20091102 0.326389 0.336911 -0.611551 MOO/20091102 0.585697 0.625821 -0.497978 BIV/20091102 0.477335 0.516732 -0.303910 VXX/20091102 0.292706 0.318422 -0.612112 IYM/20091102 0.604874 0.620541 -0.571576 IFN/20091102 0.532060 0.588907 -0.633825 SLV/20091102 0.623022 0.650524 -0.499477 TAO/20091102 0.494365 0.505451 -0.513844 PGF/20091102 0.498728 0.524701 -0.566315 IYR/20091102 0.558889 0.577641 -0.276441 QID/20091102 0.249516 0.293224 -0.565190 THD/20091102 0.543287 0.588358 -0.548599 IJS/20091102 0.539243 0.564869 -0.555809 VB/20091102 0.544979 0.569463 -0.546836 EDV/20091102 0.520998 0.576821 -0.558068 IEZ/20091102 0.535738 0.584806 -0.568868 VTV/20091102 0.564302 0.572146 -0.537066 IJR/20091102 0.528357 0.558791 -0.475742 UCO/20091102 0.437968 0.501665 -0.497004 JNK/20091102 0.490356 0.500923 -0.527732 IWN/20091102 0.517202 0.536145 -0.557170 VV/20091102 0.538617 0.577339 -0.674956 UGL/20091102 0.621504 0.707626 -0.545210 UWM/20091102 0.526346 0.554368 -0.507323 IWC/20091102 0.503140 0.511025 -0.570478 EWA/20091102 0.539322 0.589576 -0.568896 IVV/20091102 0.564007 0.572578 -0.569376 SPY/20091102 0.560316 0.575210 -0.497800 TFI/20091102 0.491816 0.505776 -0.540357 VEA/20091102 0.523169 0.551437 -0.568180 QQQQ/20091102 0.563692 0.575858 -0.541717 UYG/20091102 0.528411 0.556538 -0.540609 OIH/20091102 0.527532 0.547605 -0.555073 GXC/20091102 0.542253 0.570301 -0.625441 SSO/20091102 0.604466 0.644275 -0.603539 XLI/20091102 0.583606 0.617821 -0.604824 GML/20091102 0.580060 0.618047 -0.612203 ROM/20091102 0.598942 0.617913 -0.542117 FXC/20091102 0.527025 0.562928 -0.408611 DOG/20091102 0.388160 0.437539 -0.552562 IYE/20091102 0.549757 0.558509 -0.374257 SKF/20091102 0.351383 0.402320 -0.528215 SHY/20091102 0.524851 0.530898 -0.519696 DBA/20091102 0.496387 0.544504 -0.578264 RSP/20091102 0.558751 0.605492 -0.620422 DBS/20091102 0.612974 0.628773 -0.573433 IBB/20091102 0.556930 0.583573 -0.520999 KCE/20091102 0.509809 0.534386 -0.521953 PKN/20091102 0.516642 0.531298 -0.568226 TNA/20091102 0.538952 0.627641 -0.612079 FAS/20091102 0.557143 0.636069 -0.549427 FXE/20091102 0.522268 0.592089 -0.520684 HYG/20091102 0.512185 0.526007 -0.585105 IWS/20091102 0.572118 0.601922 -0.362026 FXP/20091102 0.337532 0.392899 -0.519866 MBB/20091102 0.509332 0.537398 -0.577722 RFG/20091102 0.553826 0.602849 -0.595552 EPU/20091102 0.588269 0.603868 -0.449076 UUP/20091102 0.428856 0.466952 -0.688912 AGQ/20091102 0.674028 0.708127 -0.594299 SOXX/20091102 0.566105 0.616395 -0.287211 FAZ/20091102 0.254014 0.314214 -0.552292 VBK/20091102 0.542950 0.559337 -0.592809 RPG/20091102 0.578490 0.620101 -0.531131 EWH/20091102 0.513900 0.548721 -0.368422 TZA/20091102 0.339190 0.402533 -0.484816 SGG/20091102 0.472619 0.501932 -0.622873 KOL/20091102 0.617742 0.627313 -0.539355 EWY/20091102 0.508507 0.564928 -0.542718 PRF/20091102 0.529287 0.556354 -0.506972 TLH/20091102 0.491893 0.519516 -0.569409 EPP/20091102 0.538509 0.595650 -0.546112 XLE/20091102 0.539201 0.560444 -0.541973 EWN/20091102 0.529128 0.565620 -0.539954 SHM/20091102 0.529579 0.551343 -0.550070 FXI/20091102 0.543993 0.552652 -0.594678 EWS/20091102 0.589434 0.600073 -0.538226 IDU/20091102 0.527169 0.553690 -0.480252 VXZ/20091102 0.472182 0.498769 -0.551335 IVE/20091102 0.528816 0.566990 -0.694118 DGP/20091102 0.674669 0.707900 -0.580853 GMF/20091102 0.574118 0.592834 -0.522502 IWR/20091102 0.516940 0.533795 -0.480924 RKH/20091102 0.462937 0.499553 -0.497052 TIP/20091102 0.440106 0.527780 -0.599009 URE/20091102 0.571439 0.618311 -0.519924 DBO/20091102 0.510033 0.533929 -0.556169 IOO/20091102 0.542041 0.576312 -0.506694 DBV/20091102 0.481867 0.551494 -0.555422 EFA/20091102 0.517382 0.576515 -0.638445 BGU/20091102 0.616827 0.668940 +0.012872 SDS/20091028 0.000000 0.051487 +0.255330 OEF/20091028 0.161245 0.374525 +0.431825 IYT/20091028 0.367783 0.535036 +0.255740 BIL/20091028 0.242556 0.274973 +0.011447 GLL/20091028 0.000000 0.033965 +0.001259 EDZ/20091028 0.000000 0.005034 +0.511729 IWM/20091028 0.423371 0.702291 +0.511213 VXF/20091028 0.472621 0.607597 +0.511801 IJJ/20091028 0.482650 0.583365 +0.486540 PIN/20091028 0.450133 0.560680 +0.520321 XLB/20091028 0.460708 0.558662 +0.426303 ECH/20091028 0.377798 0.481869 +0.595674 TYH/20091028 0.565093 0.625840 +0.582002 VAW/20091028 0.543561 0.607416 +0.535900 DBP/20091028 0.525790 0.544326 +0.659744 XME/20091028 0.632614 0.679220 +0.646768 VO/20091028 0.639552 0.661721 +0.638775 RSX/20091028 0.598522 0.682649 +0.607116 EWC/20091028 0.575810 0.644459 +0.640435 TUR/20091028 0.621148 0.669090 +0.499607 VYM/20091028 0.454127 0.523964 +0.545100 FCG/20091028 0.535191 0.551897 +0.523701 VGT/20091028 0.506839 0.535415 +0.544705 EWQ/20091028 0.540765 0.551806 +0.539351 IEV/20091028 0.537798 0.540967 +0.508638 XLK/20091028 0.505569 0.512008 +0.545419 EFG/20091028 0.537025 0.550873 +0.569936 BKF/20091028 0.542382 0.606353 +0.578041 KIE/20091028 0.563788 0.600446 +0.587352 EEB/20091028 0.558177 0.610487 +0.596973 IJK/20091028 0.588001 0.605906 +0.016241 DUG/20091028 0.000000 0.064964 +0.041409 TWM/20091028 0.000000 0.084541 +0.486638 MDY/20091028 0.330491 0.591862 +0.507738 ACWI/20091028 0.415217 0.576824 +0.201397 BSV/20091028 0.163601 0.256040 +0.473876 DDM/20091028 0.430860 0.509080 +0.499721 DIA/20091028 0.453548 0.552619 +0.304853 TLT/20091028 0.223952 0.336807 +0.072859 DXD/20091028 0.000000 0.192828 +0.527872 XHB/20091028 0.471470 0.627500 +0.454252 VDE/20091028 0.414362 0.538044 +0.400398 BND/20091028 0.325593 0.429572 +0.543833 EMB/20091028 0.513472 0.600479 +0.395100 SCO/20091028 0.316768 0.452563 +0.445868 AMJ/20091028 0.418563 0.466951 +0.281101 OIL/20091028 0.250780 0.297209 +0.523070 PZA/20091028 0.510803 0.545451 +0.578176 VGK/20091028 0.537321 0.620644 +0.604691 RWX/20091028 0.558451 0.649510 +0.429354 JJA/20091028 0.416579 0.461200 +0.632332 FXD/20091028 0.562477 0.673463 +0.577642 XES/20091028 0.507069 0.633542 +0.554086 VIG/20091028 0.542793 0.569560 +0.146870 DZZ/20091028 0.063182 0.242834 +0.583008 VFH/20091028 0.555189 0.605109 +0.351889 DTO/20091028 0.257813 0.412006 +0.564765 EWP/20091028 0.549114 0.575142 +0.565971 FDN/20091028 0.554101 0.577657 +0.542769 INP/20091028 0.503668 0.559522 +0.128570 TYP/20091028 0.036516 0.216529 +0.519085 RWR/20091028 0.471878 0.561242 +0.529070 KBE/20091028 0.499893 0.557721 +0.173080 EUO/20091028 0.142925 0.190644 +0.464556 IWF/20091028 0.433254 0.490904 +0.297456 SMN/20091028 0.224720 0.423139 +0.505495 SMH/20091028 0.493116 0.530437 +0.490422 XRT/20091028 0.465087 0.523501 +0.322622 USO/20091028 0.259246 0.356794 +0.447049 DJP/20091028 0.420830 0.476226 +0.315963 CFT/20091028 0.270404 0.357317 +0.187646 SRS/20091028 0.159321 0.207268 +0.520066 MOO/20091028 0.490085 0.560361 +0.388674 BIV/20091028 0.360048 0.407837 +0.264538 VXX/20091028 0.225270 0.279294 +0.555016 IYM/20091028 0.516613 0.601655 +0.490424 IFN/20091028 0.470928 0.517791 +0.567966 SLV/20091028 0.544432 0.593694 +0.537976 TAO/20091028 0.525800 0.552023 +0.592931 PGF/20091028 0.584993 0.601487 +0.610454 IYR/20091028 0.605694 0.615959 +0.166797 QID/20091028 0.143918 0.203523 +0.567687 THD/20091028 0.511525 0.588879 +0.580361 IJS/20091028 0.525140 0.607876 +0.581531 VB/20091028 0.526667 0.619689 +0.388863 EDV/20091028 0.349526 0.427262 +0.527064 IEZ/20091028 0.473173 0.578853 +0.561255 VTV/20091028 0.521806 0.585689 +0.577920 IJR/20091028 0.539194 0.604890 +0.342833 UCO/20091028 0.325766 0.368492 +0.564206 JNK/20091028 0.541103 0.579954 +0.591456 IWN/20091028 0.569611 0.625152 +0.569329 VV/20091028 0.557467 0.578705 +0.509183 UGL/20091028 0.499850 0.519457 +0.720563 UWM/20091028 0.675237 0.748327 +0.668927 IWC/20091028 0.645543 0.714701 +0.610346 EWA/20091028 0.562064 0.685770 +0.593822 IVV/20091028 0.560226 0.637029 +0.594645 SPY/20091028 0.561784 0.639763 +0.467924 TFI/20091028 0.452575 0.486538 +0.600015 VEA/20091028 0.572897 0.647184 +0.596448 QQQQ/20091028 0.574130 0.618404 +0.577255 UYG/20091028 0.550055 0.624871 +0.559491 OIH/20091028 0.546145 0.573303 +0.523601 GXC/20091028 0.504858 0.537196 +0.573380 SSO/20091028 0.562262 0.578994 +0.588863 XLI/20091028 0.578848 0.599556 +0.596200 GML/20091028 0.587781 0.614315 +0.584283 ROM/20091028 0.569942 0.594373 +0.566192 FXC/20091028 0.561310 0.574538 +0.220244 DOG/20091028 0.203726 0.234382 +0.513806 IYE/20091028 0.482740 0.549321 +0.277869 SKF/20091028 0.232728 0.321354 +0.346119 SHY/20091028 0.329609 0.358272 +0.466964 DBA/20091028 0.443417 0.512625 +0.530476 RSP/20091028 0.508746 0.558135 +0.521468 DBS/20091028 0.510418 0.548499 +0.565935 IBB/20091028 0.546092 0.585820 +0.553611 KCE/20091028 0.531479 0.580056 +0.548130 PKN/20091028 0.528209 0.564368 +0.513616 TNA/20091028 0.498258 0.533449 +0.511894 FAS/20091028 0.495223 0.526332 +0.546944 FXE/20091028 0.517179 0.569691 +0.543997 HYG/20091028 0.512303 0.572344 +0.556323 IWS/20091028 0.537422 0.577040 +0.358021 FXP/20091028 0.345309 0.371660 +0.388557 MBB/20091028 0.373978 0.410882 +0.555923 RFG/20091028 0.535445 0.572073 +0.552437 EPU/20091028 0.530192 0.564645 +0.297889 UUP/20091028 0.277542 0.310114 +0.498179 AGQ/20091028 0.439341 0.526661 +0.590733 SOXX/20091028 0.465269 0.673985 +0.286445 FAZ/20091028 0.161083 0.443985 +0.594702 VBK/20091028 0.545649 0.636242 +0.567000 RPG/20091028 0.541756 0.584456 +0.540920 EWH/20091028 0.530822 0.555721 +0.338639 TZA/20091028 0.247054 0.435124 +0.551427 SGG/20091028 0.516285 0.574338 +0.561956 KOL/20091028 0.536366 0.577890 +0.590206 EWY/20091028 0.570261 0.616953 +0.595084 PRF/20091028 0.583603 0.608398 +0.438650 TLH/20091028 0.419247 0.457601 +0.586381 EPP/20091028 0.562160 0.604303 +0.544896 XLE/20091028 0.523469 0.558537 +0.592289 EWN/20091028 0.581761 0.598655 +0.460009 SHM/20091028 0.433815 0.473334 +0.534798 FXI/20091028 0.517296 0.546882 +0.580981 EWS/20091028 0.571589 0.596795 +0.573565 IDU/20091028 0.563397 0.582251 +0.390846 VXZ/20091028 0.370253 0.415205 +0.577912 IVE/20091028 0.565964 0.593085 +0.543487 DGP/20091028 0.524723 0.552390 +0.642398 GMF/20091028 0.554782 0.687093 +0.639043 IWR/20091028 0.573082 0.663349 +0.598893 RKH/20091028 0.568571 0.611746 +0.446636 TIP/20091028 0.420024 0.474886 +0.575917 URE/20091028 0.566329 0.581768 +0.454281 DBO/20091028 0.432416 0.489377 +0.582510 IOO/20091028 0.559659 0.603578 +0.578552 DBV/20091028 0.554426 0.599015 +0.588503 EFA/20091028 0.564834 0.615935 +0.610514 BGU/20091028 0.569351 0.690519 +0.590633 EFV/20091028 0.569293 0.610232 +0.580728 IWB/20091028 0.559586 0.601476 +0.586659 IYF/20091028 0.566114 0.602911 +0.427740 YCS/20091028 0.408418 0.441031 +0.554895 DXJ/20091028 0.549005 0.560542 +0.580280 IWO/20091028 0.570373 0.595552 +0.471601 DBC/20091028 0.448338 0.496026 +0.302380 RWM/20091028 0.263052 0.342157 +0.560720 VBR/20091028 0.542520 0.600050 +0.337024 MZZ/20091028 0.270155 0.370703 +0.544991 IWD/20091028 0.540791 0.548850 +0.557630 PCY/20091028 0.550824 0.562321 +0.553043 EWI/20091028 0.539668 0.565300 +0.547172 IJH/20091028 0.539316 0.556440 +0.549238 EEM/20091028 0.542096 0.556050 +0.523407 EWM/20091028 0.514229 0.529728 +0.534576 SDY/20091028 0.524910 0.542513 +0.552598 ILF/20091028 0.543814 0.564092 +0.487006 JJG/20091028 0.453281 0.526047 +0.432489 TBT/20091028 0.428159 0.442680 +0.583748 XLF/20091028 0.568412 0.590382 +0.561112 ERX/20091028 0.528667 0.580533 +0.367730 SHV/20091028 0.338274 0.396764 +0.544368 EWX/20091028 0.527553 0.561961 +0.378460 EFZ/20091028 0.351577 0.393432 +0.445602 FXB/20091028 0.428446 0.452773 +0.532248 PHO/20091028 0.527792 0.537589 +0.536219 IGE/20091028 0.519785 0.554992 +0.389863 BGZ/20091028 0.382857 0.406184 +0.545988 UDN/20091028 0.531085 0.562219 +0.433090 CSJ/20091028 0.428177 0.438420 +0.563476 GXG/20091028 0.539129 0.594615 +0.564748 USD/20091028 0.551238 0.588409 +0.538004 EWD/20091028 0.526517 0.553084 +0.546280 EWJ/20091028 0.535071 0.553120 +0.581679 BRF/20091028 0.565063 0.601910 +0.591144 VEU/20091028 0.577600 0.597378 +0.566314 XLU/20091028 0.558238 0.573976 +0.512942 JJC/20091028 0.511069 0.516858 +0.607135 FGD/20091028 0.594819 0.612163 +0.566438 FXF/20091028 0.555373 0.574189 +0.524419 LQD/20091028 0.517478 0.530781 +0.596746 SCZ/20091028 0.587032 0.605857 +0.572685 IYW/20091028 0.569430 0.575847 +0.575135 VPL/20091028 0.568640 0.583945 +0.577438 DGS/20091028 0.572555 0.584883 +0.562284 ICF/20091028 0.554107 0.574308 +0.557443 DVY/20091028 0.555418 0.559361 +0.577871 IEO/20091028 0.568013 0.587297 +0.566184 VOT/20091028 0.559425 0.574539 +0.466006 CIU/20091028 0.449021 0.472573 +0.570593 EWG/20091028 0.561938 0.578057 +0.552924 EWT/20091028 0.549654 0.560275 +0.519592 GSG/20091028 0.510107 0.532216 +0.470052 KRE/20091028 0.462474 0.479989 +0.572436 LVL/20091028 0.567616 0.577368 +0.546153 UNG/20091028 0.506073 0.562227 +0.505479 MUB/20091028 0.481099 0.518574 +0.543663 VT/20091028 0.517076 0.564605 +0.530136 DAG/20091028 0.510084 0.569574 +0.512274 PPH/20091028 0.493235 0.523548 +0.581810 VSS/20091028 0.560530 0.597544 +0.513662 DBB/20091028 0.484744 0.538549 +0.544992 XLP/20091028 0.518240 0.558569 +0.580079 IJT/20091028 0.567196 0.587691 +0.594278 EWZ/20091028 0.570354 0.627032 +0.589573 PBW/20091028 0.566398 0.605710 +0.461858 FXY/20091028 0.430685 0.479731 +0.551941 IYZ/20091028 0.541174 0.562329 +0.591441 MVV/20091028 0.578592 0.603217 +0.569168 VUG/20091028 0.551059 0.575705 +0.466942 PST/20091028 0.458350 0.476487 +0.367627 PSQ/20091028 0.349468 0.381452 +0.558848 VNQ/20091028 0.550360 0.572823 +0.464977 IEI/20091028 0.451888 0.476034 +0.576545 EWW/20091028 0.562488 0.589243 +0.578829 IWP/20091028 0.571367 0.584957 +0.570418 IWV/20091028 0.564544 0.574435 +0.571031 DIG/20091028 0.553009 0.589864 +0.568656 VTI/20091028 0.563893 0.573516 +0.557744 FXA/20091028 0.541487 0.564061 +0.572767 NLR/20091028 0.566030 0.579237 +0.481916 AGG/20091028 0.473934 0.488911 +0.548430 BWX/20091028 0.541369 0.555045 +0.544168 IAU/20091028 0.531678 0.552190 +0.555954 XLV/20091028 0.541318 0.571390 +0.591547 XOP/20091028 0.573391 0.605311 +0.581510 EZU/20091028 0.565955 0.603143 +0.567478 JXI/20091028 0.553365 0.586143 +0.559303 XBI/20091028 0.525728 0.586667 +0.571849 IYG/20091028 0.557712 0.586296 +0.592055 SLX/20091028 0.571385 0.604333 +0.566696 HAO/20091028 0.554543 0.581878 +0.616994 EZA/20091028 0.609143 0.624337 +0.599582 XLY/20091028 0.590756 0.617142 +0.479792 IEF/20091028 0.471776 0.491009 +0.609519 DEM/20091028 0.601183 0.619594 +0.582710 IVW/20091028 0.576734 0.591972 +0.628669 UYM/20091028 0.607516 0.645583 +0.604467 IXC/20091028 0.570627 0.622363 +0.580949 PFF/20091029 0.561203 0.608446 +0.543065 WIP/20091029 0.511478 0.570588 +0.491635 GCC/20091029 0.461193 0.545906 +0.539921 AAXJ/20091029 0.514952 0.591151 +0.524841 VWO/20091029 0.517235 0.534177 +0.431774 EEV/20091029 0.420937 0.457058 +0.569998 GDX/20091029 0.519963 0.599073 +0.540620 RTH/20091029 0.483408 0.570157 +0.534269 MXI/20091029 0.517755 0.544116 +0.535776 EWU/20091029 0.487971 0.558237 +0.461290 SH/20091029 0.414340 0.512398 +0.604573 EDC/20091029 0.546714 0.646060 +0.564769 ERY/20091029 0.502298 0.605556 +0.506637 SDS/20091029 0.455271 0.544487 +0.567464 OEF/20091029 0.523658 0.609204 +0.511772 IYT/20091029 0.455623 0.621254 +0.558271 BIL/20091029 0.541948 0.590385 +0.518446 GLL/20091029 0.462009 0.557316 +0.387541 EDZ/20091029 0.247112 0.520055 +0.552880 IWM/20091029 0.437010 0.632485 +0.526868 VXF/20091029 0.475997 0.604701 +0.536233 IJJ/20091029 0.517938 0.561520 +0.535682 PIN/20091029 0.526303 0.550652 +0.538855 XLB/20091029 0.500201 0.579683 +0.519069 ECH/20091029 0.486308 0.546968 +0.489179 TYH/20091029 0.438371 0.530069 +0.508878 VAW/20091029 0.472504 0.529262 +0.556063 DBP/20091029 0.551111 0.562294 +0.510670 XME/20091029 0.493380 0.542325 +0.497345 VO/20091029 0.477120 0.537391 +0.540790 RSX/20091029 0.524273 0.572971 +0.536854 EWC/20091029 0.528101 0.553355 +0.468893 TUR/20091029 0.462459 0.476483 +0.517591 VYM/20091029 0.473203 0.537839 +0.488359 FCG/20091029 0.474825 0.503369 +0.509648 VGT/20091029 0.491534 0.524105 +0.513101 EWQ/20091029 0.497176 0.531826 +0.502094 IEV/20091029 0.491306 0.511736 +0.507337 XLK/20091029 0.477488 0.527893 +0.486486 EFG/20091029 0.479493 0.501274 +0.509456 BKF/20091029 0.483028 0.532495 +0.460339 KIE/20091029 0.445948 0.477601 +0.520289 EEB/20091029 0.503366 0.543556 +0.507699 IJK/20091029 0.469785 0.533698 +0.431674 DUG/20091029 0.348048 0.530158 +0.404429 TWM/20091029 0.360148 0.457218 +0.522483 MDY/20091029 0.445833 0.604961 +0.497792 ACWI/20091029 0.454626 0.534192 +0.541376 BSV/20091029 0.450478 0.590453 +0.469829 DDM/20091029 0.438192 0.528763 +0.516979 DIA/20091029 0.475732 0.540974 +0.559237 TLT/20091029 0.486298 0.599006 +0.500110 DXD/20091029 0.470364 0.523403 +0.496574 XHB/20091029 0.440459 0.561165 +0.515789 VDE/20091029 0.489067 0.552458 +0.493315 BND/20091029 0.453622 0.574743 +0.488727 EMB/20091029 0.457620 0.525698 +0.476920 SCO/20091029 0.438932 0.534059 +0.482600 AMJ/20091029 0.443536 0.548948 +0.499066 OIL/20091029 0.455979 0.520960 +0.492898 PZA/20091029 0.473535 0.513346 +0.485970 VGK/20091029 0.455435 0.508384 +0.496719 RWX/20091029 0.473559 0.523960 +0.505303 JJA/20091029 0.418512 0.561221 +0.500076 FXD/20091029 0.460337 0.524113 +0.468219 XES/20091029 0.443989 0.485434 +0.489323 VIG/20091029 0.468478 0.503947 +0.449840 DZZ/20091029 0.381956 0.489289 +0.491487 VFH/20091029 0.477359 0.506908 +0.446648 DTO/20091029 0.411937 0.497439 +0.527684 EWP/20091029 0.506141 0.543278 +0.535264 FDN/20091029 0.528274 0.543777 +0.556567 INP/20091029 0.518199 0.611783 +0.438251 TYP/20091029 0.407347 0.469124 +0.517430 RWR/20091029 0.505100 0.541866 +0.537770 KBE/20091029 0.514626 0.562792 +0.456550 EUO/20091029 0.403835 0.486338 +0.541617 IWF/20091029 0.522539 0.593951 +0.346974 SMN/20091029 0.302692 0.414814 +0.532614 SMH/20091029 0.507484 0.569018 +0.567771 XRT/20091029 0.544660 0.586049 +0.467003 USO/20091029 0.375646 0.518374 +0.467236 DJP/20091029 0.425202 0.504730 +0.509725 CFT/20091029 0.451097 0.582552 +0.424053 SRS/20091029 0.390913 0.453160 +0.536034 MOO/20091029 0.503120 0.564898 +0.517788 BIV/20091029 0.465540 0.603188 +0.422707 VXX/20091029 0.411070 0.432668 +0.543466 IYM/20091029 0.494159 0.597462 +0.514698 IFN/20091029 0.451187 0.565965 +0.576181 SLV/20091029 0.563897 0.583072 +0.469939 TAO/20091029 0.443989 0.486924 +0.462530 PGF/20091029 0.436664 0.501672 +0.471866 IYR/20091029 0.456708 0.480823 +0.373087 QID/20091029 0.342253 0.451454 +0.524158 THD/20091029 0.483896 0.576241 +0.502785 IJS/20091029 0.463640 0.527345 +0.529227 VB/20091029 0.514135 0.547473 +0.569304 EDV/20091029 0.508020 0.596997 +0.494566 IEZ/20091029 0.454139 0.514100 +0.503116 VTV/20091029 0.485393 0.533928 +0.510806 IJR/20091029 0.476336 0.527062 +0.442934 UCO/20091029 0.350086 0.485387 +0.467918 JNK/20091029 0.437410 0.491425 +0.478849 IWN/20091029 0.459815 0.493018 +0.493559 VV/20091029 0.480470 0.513907 +0.575228 UGL/20091029 0.557038 0.592193 +0.512471 UWM/20091029 0.462297 0.551651 +0.502710 IWC/20091029 0.455144 0.521865 +0.522975 EWA/20091029 0.457511 0.574862 +0.503979 IVV/20091029 0.467105 0.526943 +0.505237 SPY/20091029 0.476389 0.525763 +0.521513 TFI/20091029 0.477998 0.551059 +0.493786 VEA/20091029 0.452831 0.526724 +0.509542 QQQQ/20091029 0.473697 0.529313 +0.511700 UYG/20091029 0.501630 0.526842 +0.490447 OIH/20091029 0.479122 0.498934 +0.493842 GXC/20091029 0.475401 0.517404 +0.527999 SSO/20091029 0.501229 0.561112 +0.516888 XLI/20091029 0.488696 0.530064 +0.535450 GML/20091029 0.500821 0.567784 +0.540944 ROM/20091029 0.525736 0.561590 +0.488822 FXC/20091029 0.469274 0.508780 +0.475277 DOG/20091029 0.403221 0.516030 +0.492936 IYE/20091029 0.486621 0.502189 +0.406630 SKF/20091029 0.338440 0.464645 +0.524887 SHY/20091029 0.460332 0.554141 +0.478926 DBA/20091029 0.445798 0.510528 +0.544115 RSP/20091029 0.508065 0.573345 +0.566715 DBS/20091029 0.536461 0.610583 +0.544687 IBB/20091029 0.514384 0.582385 +0.499971 KCE/20091029 0.473745 0.542025 +0.509154 PKN/20091029 0.490378 0.516555 +0.522032 TNA/20091029 0.500989 0.567318 +0.476566 FAS/20091029 0.450232 0.514693 +0.471252 FXE/20091029 0.433038 0.499510 +0.442791 HYG/20091029 0.427588 0.457400 +0.481452 IWS/20091029 0.455645 0.503974 +0.425096 FXP/20091029 0.397071 0.461367 +0.473927 MBB/20091029 0.430868 0.540668 +0.487865 RFG/20091029 0.462460 0.505442 +0.494229 EPU/20091029 0.442293 0.515216 +0.441161 UUP/20091029 0.388059 0.496224 +0.507359 AGQ/20091029 0.455338 0.542245 +0.496458 SOXX/20091029 0.391799 0.621335 +0.432217 FAZ/20091029 0.354130 0.500808 +0.491685 VBK/20091029 0.440640 0.578798 +0.513120 RPG/20091029 0.486989 0.573230 +0.481588 EWH/20091029 0.449564 0.519190 +0.419718 TZA/20091029 0.370381 0.467916 +0.444181 SGG/20091029 0.413512 0.487916 +0.520612 KOL/20091029 0.485283 0.571296 +0.497151 EWY/20091029 0.448405 0.519999 +0.502341 PRF/20091029 0.460412 0.520675 +0.527066 TLH/20091029 0.480397 0.583550 +0.520106 EPP/20091029 0.509062 0.548948 +0.475185 XLE/20091029 0.461933 0.502841 +0.512364 EWN/20091029 0.480661 0.533989 +0.488232 SHM/20091029 0.460004 0.522813 +0.490126 FXI/20091029 0.460734 0.523968 +0.504553 EWS/20091029 0.483086 0.517204 +0.498352 IDU/20091029 0.477426 0.518294 +0.504204 VXZ/20091029 0.459811 0.545080 +0.512485 IVE/20091029 0.482913 0.524360 +0.562361 DGP/20091029 0.497208 0.597246 +0.525570 GMF/20091029 0.498182 0.576733 +0.529085 IWR/20091029 0.502136 0.564359 +0.494669 RKH/20091029 0.474200 0.507916 +0.493153 TIP/20091029 0.472021 0.536291 +0.576640 URE/20091029 0.532746 0.606633 +0.448660 DBO/20091029 0.417390 0.510806 +0.518998 IOO/20091029 0.504886 0.537580 +0.496374 DBV/20091029 0.464232 0.537135 +0.524951 EFA/20091029 0.510447 0.550185 +0.550135 BGU/20091029 0.518200 0.623800 +0.507879 EFV/20091029 0.472265 0.528201 +0.512849 IWB/20091029 0.482312 0.541145 +0.476282 IYF/20091029 0.423288 0.520537 +0.404927 YCS/20091029 0.384542 0.425424 +0.478298 DXJ/20091029 0.433832 0.519441 +0.518013 IWO/20091029 0.492237 0.545483 +0.493636 DBC/20091029 0.472629 0.507046 +0.440419 RWM/20091029 0.384363 0.470943 +0.512802 VBR/20091029 0.473835 0.540108 +0.421302 MZZ/20091029 0.403128 0.446102 +0.516276 IWD/20091029 0.488580 0.534965 +0.509126 PCY/20091029 0.479725 0.531513 +0.517001 EWI/20091029 0.498714 0.530231 +0.515445 IJH/20091029 0.505874 0.528205 +0.507249 EEM/20091029 0.493716 0.533542 +0.494728 EWM/20091029 0.483257 0.507575 +0.511351 SDY/20091029 0.498772 0.529342 +0.533826 ILF/20091029 0.523217 0.542964 +0.487607 JJG/20091029 0.451266 0.526042 +0.430928 TBT/20091029 0.404187 0.453998 +0.463549 XLF/20091029 0.451493 0.473227 +0.446766 ERX/20091029 0.427480 0.470249 +0.498844 SHV/20091029 0.486543 0.509358 +0.515800 EWX/20091029 0.471234 0.546424 +0.449911 EFZ/20091029 0.424022 0.482961 +0.461068 FXB/20091029 0.413261 0.483804 +0.485390 PHO/20091029 0.462134 0.500821 +0.483722 IGE/20091029 0.443664 0.524813 +0.418932 BGZ/20091029 0.393172 0.447869 +0.488166 UDN/20091029 0.475140 0.508921 +0.483711 CSJ/20091029 0.437576 0.508925 +0.518634 GXG/20091029 0.498442 0.545658 +0.502528 USD/20091029 0.485048 0.531811 +0.483590 EWD/20091029 0.434847 0.509056 +0.455965 EWJ/20091029 0.439343 0.495923 +0.527258 BRF/20091029 0.500559 0.544581 +0.511636 VEU/20091029 0.494487 0.530237 +0.514519 XLU/20091029 0.506284 0.526508 +0.492447 JJC/20091029 0.487244 0.496751 +0.513526 FGD/20091029 0.483626 0.544148 +0.489718 FXF/20091029 0.479176 0.506182 +0.490928 LQD/20091029 0.473870 0.499092 +0.502815 SCZ/20091029 0.487859 0.514051 +0.513361 IYW/20091029 0.489599 0.526841 +0.482673 VPL/20091029 0.465952 0.497628 +0.529060 DGS/20091029 0.505261 0.547782 +0.512708 ICF/20091029 0.503064 0.524018 +0.521411 DVY/20091029 0.508848 0.536553 +0.481550 IEO/20091029 0.455837 0.506938 +0.519897 VOT/20091029 0.506833 0.535179 +0.501101 CIU/20091029 0.470046 0.525551 +0.534345 EWG/20091029 0.515260 0.555712 +0.536238 EWT/20091029 0.525289 0.548780 +0.451899 GSG/20091029 0.440110 0.465561 +0.477810 KRE/20091029 0.458782 0.488651 +0.522705 LVL/20091029 0.507366 0.537541 +0.441471 UNG/20091029 0.410451 0.519284 +0.500827 MUB/20091029 0.487118 0.512903 +0.530141 VT/20091029 0.498602 0.548048 +0.490384 DAG/20091029 0.438368 0.524372 +0.564540 PPH/20091029 0.560997 0.568340 +0.518105 VSS/20091029 0.505450 0.528338 +0.485011 DBB/20091029 0.464102 0.497423 +0.487493 XLP/20091029 0.472813 0.501156 +0.517946 IJT/20091029 0.485459 0.541460 +0.512811 EWZ/20091029 0.469624 0.546996 +0.501563 PBW/20091029 0.486388 0.519797 +0.541837 FXY/20091029 0.504933 0.563976 +0.543732 IYZ/20091029 0.537295 0.549931 +0.532924 MVV/20091029 0.520001 0.553392 +0.530287 VUG/20091029 0.527355 0.533294 +0.433754 PST/20091029 0.431542 0.435230 +0.437456 PSQ/20091029 0.397004 0.458764 +0.528673 VNQ/20091029 0.516930 0.535267 +0.500129 IEI/20091029 0.487268 0.528063 +0.529262 EWW/20091029 0.508518 0.557062 +0.526507 IWP/20091029 0.510242 0.548610 +0.523857 IWV/20091029 0.513743 0.529537 +0.483295 DIG/20091029 0.470162 0.499882 +0.524581 VTI/20091029 0.516284 0.528439 +0.487729 FXA/20091029 0.484610 0.492948 +0.498366 NLR/20091029 0.476672 0.537440 +0.512822 AGG/20091029 0.488110 0.534610 +0.532469 BWX/20091029 0.522253 0.547918 +0.566227 IAU/20091029 0.516251 0.589564 +0.554940 XLV/20091029 0.524348 0.588246 +0.484137 XOP/20091029 0.453972 0.525936 +0.535799 EZU/20091029 0.518863 0.551841 +0.529956 JXI/20091029 0.519363 0.546020 +0.540281 XBI/20091029 0.527052 0.558011 +0.510258 IYG/20091029 0.486758 0.523200 +0.571214 SLX/20091029 0.560025 0.584739 +0.517920 HAO/20091029 0.503326 0.528653 +0.520423 EZA/20091029 0.512029 0.537311 +0.517775 XLY/20091029 0.508069 0.526158 +0.516767 IEF/20091029 0.490465 0.542746 +0.526737 DEM/20091029 0.507703 0.538783 +0.506709 IVW/20091029 0.496858 0.513947 +0.553275 UYM/20091029 0.508489 0.580067 +0.496836 IXC/20091029 0.464107 0.526423 +0.512417 PFF/20091030 0.474597 0.550199 +0.557037 WIP/20091030 0.527567 0.576470 +0.520848 GCC/20091030 0.497426 0.549686 +0.547505 AAXJ/20091030 0.527356 0.565585 +0.546073 VWO/20091030 0.525265 0.561276 +0.406903 EEV/20091030 0.386774 0.418289 +0.615258 GDX/20091030 0.519914 0.668971 +0.597021 RTH/20091030 0.574688 0.630113 +0.618424 MXI/20091030 0.571886 0.638828 +0.594982 EWU/20091030 0.560210 0.636163 +0.446746 SH/20091030 0.415324 0.462710 +0.567677 EDC/20091030 0.524324 0.608789 +0.471862 ERY/20091030 0.436788 0.509946 +0.404056 SDS/20091030 0.348819 0.449774 +0.634642 OEF/20091030 0.607745 0.651681 +0.567249 IYT/20091030 0.527941 0.583519 +0.509078 BIL/20091030 0.491435 0.533923 +0.402178 GLL/20091030 0.353631 0.463530 +0.291337 EDZ/20091030 0.246779 0.309815 +0.541740 IWM/20091030 0.512172 0.559487 +0.553404 VXF/20091030 0.527636 0.570278 +0.559298 IJJ/20091030 0.527979 0.588557 +0.591788 PIN/20091030 0.551594 0.613622 +0.600913 XLB/20091030 0.589915 0.609159 +0.585112 ECH/20091030 0.574176 0.600315 +0.511341 TYH/20091030 0.475381 0.577148 +0.587517 VAW/20091030 0.546079 0.607307 +0.641675 DBP/20091030 0.628593 0.650263 +0.612548 XME/20091030 0.594173 0.630039 +0.516762 VO/20091030 0.504546 0.527706 +0.561913 RSX/20091030 0.542586 0.574654 +0.570602 EWC/20091030 0.559231 0.581243 +0.461010 TUR/20091030 0.449993 0.486085 +0.567123 VYM/20091030 0.538862 0.592056 +0.541811 FCG/20091030 0.510149 0.561636 +0.563118 VGT/20091030 0.553584 0.572583 +0.560130 EWQ/20091030 0.545228 0.570699 +0.555172 IEV/20091030 0.545216 0.573330 +0.558805 XLK/20091030 0.515748 0.582372 +0.539364 EFG/20091030 0.529220 0.551271 +0.591767 BKF/20091030 0.579840 0.596595 +0.528336 KIE/20091030 0.505777 0.552279 +0.605367 EEB/20091030 0.593040 0.609987 +0.533134 IJK/20091030 0.520861 0.539497 +0.444497 DUG/20091030 0.430276 0.461804 +0.407292 TWM/20091030 0.400583 0.417205 +0.562499 MDY/20091030 0.553183 0.571081 +0.576221 ACWI/20091030 0.560168 0.600651 +0.525515 BSV/20091030 0.497049 0.557553 +0.581017 DDM/20091030 0.524742 0.602969 +0.578397 DIA/20091030 0.555447 0.629743 +0.513885 TLT/20091030 0.480990 0.543325 +0.427308 DXD/20091030 0.340226 0.478757 +0.529433 XHB/20091030 0.477477 0.560756 +0.568680 VDE/20091030 0.521466 0.630612 +0.495201 BND/20091030 0.439591 0.539464 +0.542474 EMB/20091030 0.514885 0.560656 +0.500097 SCO/20091030 0.464769 0.521170 +0.569125 AMJ/20091030 0.545352 0.595236 +0.520450 OIL/20091030 0.456984 0.571411 +0.496899 PZA/20091030 0.489322 0.504955 +0.553761 VGK/20091030 0.537578 0.593732 +0.523298 RWX/20091030 0.510310 0.553146 +0.563238 JJA/20091030 0.547110 0.582230 +0.563886 FXD/20091030 0.542768 0.583175 +0.498017 XES/20091030 0.481429 0.526450 +0.527169 VIG/20091030 0.515879 0.554683 +0.270775 DZZ/20091030 0.224927 0.322309 +0.489665 VFH/20091030 0.481667 0.499353 +0.465134 DTO/20091030 0.408574 0.499268 +0.538969 EWP/20091030 0.521538 0.554024 +0.545289 FDN/20091030 0.539403 0.560942 +0.570045 INP/20091030 0.564668 0.577844 +0.344873 TYP/20091030 0.301939 0.361591 +0.556404 RWR/20091030 0.543966 0.582296 +0.557993 KBE/20091030 0.527558 0.588204 +0.436313 EUO/20091030 0.404857 0.458658 +0.553832 IWF/20091030 0.535104 0.583084 +0.260578 SMN/20091030 0.252268 0.274190 +0.534179 SMH/20091030 0.514850 0.551998 +0.572722 XRT/20091030 0.563363 0.581114 +0.518377 USO/20091030 0.481716 0.550803 +0.546754 DJP/20091030 0.532071 0.558721 +0.488061 CFT/20091030 0.427623 0.546205 +0.373272 SRS/20091030 0.357971 0.378909 +0.597314 MOO/20091030 0.579448 0.614620 +0.475005 BIV/20091030 0.459117 0.489204 +0.315755 VXX/20091030 0.300091 0.332967 +0.579829 IYM/20091030 0.558327 0.599354 +0.572328 IFN/20091030 0.518160 0.609002 +0.642821 SLV/20091030 0.630029 0.654334 +0.506188 TAO/20091030 0.491611 0.526142 +0.495486 PGF/20091030 0.459155 0.523286 +0.557368 IYR/20091030 0.543985 0.568327 +0.266491 QID/20091030 0.239956 0.315291 +0.497738 THD/20091030 0.478036 0.515166 +0.500309 IJS/20091030 0.486867 0.513153 +0.535677 VB/20091030 0.509479 0.569375 +0.541566 EDV/20091030 0.507543 0.581361 +0.530896 IEZ/20091030 0.497762 0.553076 +0.540404 VTV/20091030 0.514560 0.556780 +0.519811 IJR/20091030 0.493344 0.541808 +0.483126 UCO/20091030 0.448927 0.519458 +0.522913 JNK/20091030 0.477931 0.541434 +0.514124 IWN/20091030 0.503053 0.526600 +0.524721 VV/20091030 0.505477 0.544024 +0.678189 UGL/20091030 0.629691 0.732657 +0.523901 UWM/20091030 0.503111 0.540532 +0.508803 IWC/20091030 0.482811 0.525018 +0.546165 EWA/20091030 0.527655 0.577062 +0.561484 IVV/20091030 0.554396 0.574389 +0.554146 SPY/20091030 0.543501 0.561790 +0.527161 TFI/20091030 0.521674 0.534710 +0.526340 VEA/20091030 0.487636 0.542195 +0.551267 QQQQ/20091030 0.536569 0.563144 +0.540256 UYG/20091030 0.531066 0.552100 +0.549907 OIH/20091030 0.544288 0.555089 +0.575806 GXC/20091030 0.562835 0.584395 +0.602507 SSO/20091030 0.592689 0.612198 +0.574375 XLI/20091030 0.569262 0.577493 +0.632072 GML/20091030 0.625701 0.636015 +0.587818 ROM/20091030 0.577955 0.594918 +0.525485 FXC/20091030 0.514740 0.536128 +0.453573 DOG/20091030 0.414546 0.480155 +0.538363 IYE/20091030 0.530464 0.547678 +0.368574 SKF/20091030 0.281154 0.415936 +0.486047 SHY/20091030 0.476483 0.499950 +0.548300 DBA/20091030 0.503753 0.573032 +0.560018 RSP/20091030 0.540906 0.585915 +0.646091 DBS/20091030 0.634284 0.664215 +0.570579 IBB/20091030 0.566391 0.578255 +0.495425 KCE/20091030 0.478668 0.505992 +0.510726 PKN/20091030 0.499054 0.522969 +0.556143 TNA/20091030 0.495084 0.600540 +0.561506 FAS/20091030 0.517834 0.631113 +0.552965 FXE/20091030 0.525936 0.586898 +0.506929 HYG/20091030 0.481233 0.524106 +0.572732 IWS/20091030 0.556660 0.601965 +0.386031 FXP/20091030 0.364890 0.409006 +0.472114 MBB/20091030 0.442307 0.499158 +0.569213 RFG/20091030 0.563185 0.581528 +0.583822 EPU/20091030 0.571549 0.598796 +0.463271 UUP/20091030 0.445149 0.487258 +0.647702 AGQ/20091030 0.580011 0.684742 +0.585566 SOXX/20091030 0.559297 0.609431 +0.314816 FAZ/20091030 0.262397 0.397202 +0.553632 VBK/20091030 0.533413 0.578471 +0.541035 RPG/20091030 0.510326 0.578569 +0.523272 EWH/20091030 0.502860 0.544807 +0.390837 TZA/20091030 0.339046 0.416920 +0.548307 SGG/20091030 0.531855 0.580336 +0.588004 KOL/20091030 0.550312 0.615221 +0.538836 EWY/20091030 0.511983 0.571691 +0.534631 PRF/20091030 0.513485 0.564673 +0.519570 TLH/20091030 0.506283 0.534799 +0.544222 EPP/20091030 0.527936 0.560442 +0.536733 XLE/20091030 0.522685 0.556523 +0.557662 EWN/20091030 0.548733 0.571059 +0.499245 SHM/20091030 0.491076 0.504570 +0.541741 FXI/20091030 0.521304 0.553826 +0.583732 EWS/20091030 0.567733 0.590572 +0.520605 IDU/20091030 0.514278 0.529719 +0.480236 VXZ/20091030 0.467937 0.499063 +0.528724 IVE/20091030 0.502838 0.540672 +0.704397 DGP/20091030 0.656171 0.729189 +0.491210 GMF/20091030 0.476450 0.510396 +0.520052 IWR/20091030 0.515427 0.527463 +0.489048 RKH/20091030 0.485611 0.495986 +0.466329 TIP/20091030 0.442904 0.487784 +0.612080 URE/20091030 0.585897 0.647822 +0.497467 DBO/20091030 0.467089 0.514040 +0.556717 IOO/20091030 0.544501 0.573227 +0.539182 DBV/20091030 0.496485 0.579777 +0.552625 EFA/20091030 0.524909 0.574782 +0.620811 BGU/20091030 0.598397 0.650722 +0.553951 EFV/20091030 0.539935 0.569616 +0.548817 IWB/20091030 0.530626 0.560611 +0.521362 IYF/20091030 0.503503 0.530028 +0.425186 YCS/20091030 0.407579 0.442294 +0.500215 DXJ/20091030 0.491568 0.513308 +0.565175 IWO/20091030 0.534385 0.598382 +0.548649 DBC/20091030 0.539161 0.554643 +0.419572 RWM/20091030 0.347704 0.452879 +0.543074 VBR/20091030 0.538001 0.548323 +0.373390 MZZ/20091030 0.337093 0.396683 +0.555884 IWD/20091030 0.552543 0.560540 +0.549899 PCY/20091030 0.538661 0.581169 +0.543139 EWI/20091030 0.527022 0.557064 +0.554139 IJH/20091030 0.542057 0.580869 +0.584532 EEM/20091030 0.574029 0.597860 +0.543891 EWM/20091030 0.528032 0.563195 +0.539612 SDY/20091030 0.522942 0.554518 +0.646931 ILF/20091030 0.633169 0.668468 +0.546002 JJG/20091030 0.481953 0.607584 +0.456942 TBT/20091030 0.442667 0.470949 +0.514681 XLF/20091030 0.505777 0.522749 +0.538098 ERX/20091030 0.501190 0.584409 +0.505595 SHV/20091030 0.499235 0.511503 +0.595234 EWX/20091030 0.578936 0.611666 +0.427476 EFZ/20091030 0.411829 0.434063 +0.461462 FXB/20091030 0.447440 0.472387 +0.538611 PHO/20091030 0.533038 0.542301 +0.591926 IGE/20091030 0.569837 0.624310 +0.293655 BGZ/20091030 0.285796 0.301834 +0.533189 UDN/20091030 0.503191 0.547978 +0.496914 CSJ/20091030 0.491794 0.500372 +0.611717 GXG/20091030 0.585534 0.623623 +0.582745 USD/20091030 0.566119 0.605749 +0.555883 EWD/20091030 0.551956 0.558669 +0.479230 EWJ/20091030 0.465191 0.498759 +0.653839 BRF/20091030 0.629821 0.670930 +0.527434 VEU/20091030 0.514881 0.535708 +0.510288 XLU/20091030 0.504626 0.526665 +0.523936 JJC/20091030 0.507844 0.533955 +0.538519 FGD/20091030 0.521028 0.557369 +0.523278 FXF/20091030 0.512799 0.542647 +0.499753 LQD/20091030 0.485077 0.513313 +0.537303 SCZ/20091030 0.525107 0.544454 +0.538378 IYW/20091030 0.525318 0.550851 +0.496710 VPL/20091030 0.484767 0.503296 +0.553913 DGS/20091030 0.540037 0.565221 +0.555745 ICF/20091030 0.543759 0.579361 +0.536711 DVY/20091030 0.526748 0.547976 +0.518817 IEO/20091030 0.496710 0.567151 +0.534589 VOT/20091030 0.526506 0.545953 +0.475847 CIU/20091030 0.444377 0.488258 +0.549658 EWG/20091030 0.544004 0.560136 +0.558067 EWT/20091030 0.542406 0.571934 +0.515420 GSG/20091030 0.509362 0.523313 +0.497981 KRE/20091030 0.485736 0.513599 +0.572042 LVL/20091030 0.557793 0.589670 +0.453889 UNG/20091030 0.431046 0.469562 +0.504219 MUB/20091030 0.498393 0.510015 +0.552137 VT/20091030 0.542701 0.565055 +0.600297 DAG/20091030 0.525064 0.638642 +0.555422 PPH/20091030 0.550301 0.563389 +0.562336 VSS/20091030 0.548093 0.579119 +0.547542 DBB/20091030 0.528210 0.575044 +0.529951 XLP/20091030 0.513063 0.548804 +0.540200 IJT/20091030 0.507299 0.566845 +0.618525 EWZ/20091030 0.610082 0.635634 +0.519920 PBW/20091030 0.515018 0.525556 +0.552070 FXY/20091030 0.538777 0.562558 +0.537143 IYZ/20091030 0.527831 0.545632 +0.561718 MVV/20091030 0.556664 0.570436 +0.542385 VUG/20091030 0.537404 0.551182 +0.448196 PST/20091030 0.435957 0.467653 +0.433640 PSQ/20091030 0.424118 0.443528 +0.564627 VNQ/20091030 0.550732 0.576024 +0.486250 IEI/20091030 0.468048 0.504206 +0.577999 EWW/20091030 0.552099 0.607746 +0.541089 IWP/20091030 0.529463 0.557558 +0.540512 IWV/20091030 0.536095 0.544984 +0.549903 DIG/20091030 0.537659 0.564686 +0.543792 VTI/20091030 0.540740 0.546398 +0.525134 FXA/20091030 0.518236 0.532220 +0.517562 NLR/20091030 0.497927 0.552489 +0.505014 AGG/20091030 0.486083 0.514435 +0.537086 BWX/20091030 0.529497 0.541088 +0.635935 IAU/20091030 0.620546 0.645064 +0.574961 XLV/20091030 0.571872 0.576160 +0.525620 XOP/20091030 0.503523 0.563891 +0.525995 EZU/20091030 0.518648 0.531565 +0.528147 JXI/20091030 0.522501 0.535902 +0.554725 XBI/20091030 0.538114 0.573885 +0.510272 IYG/20091030 0.492481 0.541244 +0.638734 SLX/20091030 0.619939 0.662544 +0.569460 HAO/20091030 0.562223 0.573395 +0.586798 EZA/20091030 0.581308 0.596803 +0.570721 XLY/20091030 0.563692 0.578555 +0.489651 IEF/20091030 0.466579 0.504757 +0.558642 DEM/20091030 0.541303 0.573025 +0.547048 IVW/20091030 0.539454 0.553377 +0.672218 UYM/20091030 0.658064 0.686500 +0.528167 IXC/20091030 0.502650 0.543928 +0.520434 PFF/20091102 0.508803 0.542717 +0.543841 WIP/20091102 0.521831 0.561083 +0.551716 GCC/20091102 0.537977 0.562894 +0.585859 AAXJ/20091102 0.576606 0.598040 +0.591270 VWO/20091102 0.582858 0.598958 +0.362282 EEV/20091102 0.345833 0.391128 +0.651845 GDX/20091102 0.633992 0.680477 +0.603278 RTH/20091102 0.583914 0.618759 +0.636887 MXI/20091102 0.610538 0.658217 +0.584071 EWU/20091102 0.554816 0.637592 +0.407566 SH/20091102 0.398270 0.415574 +0.600776 EDC/20091102 0.569018 0.656373 +0.404738 ERY/20091102 0.377193 0.418103 +0.377214 SDS/20091102 0.345612 0.396700 +0.619740 OEF/20091102 0.596333 0.648186 +0.629238 IYT/20091102 0.623600 0.638585 +0.480570 BIL/20091102 0.454185 0.522013 +0.379622 GLL/20091102 0.296687 0.489202 +0.242714 EDZ/20091102 0.181355 0.308967 +0.564813 IWM/20091102 0.542621 0.591353 +0.574344 VXF/20091102 0.561361 0.585575 +0.579574 IJJ/20091102 0.572244 0.585450 +0.639755 PIN/20091102 0.622603 0.649024 +0.629141 XLB/20091102 0.612839 0.649020 +0.578868 ECH/20091102 0.556539 0.597800 +0.525826 TYH/20091102 0.507892 0.544827 +0.630790 VAW/20091102 0.626199 0.635957 +0.632350 DBP/20091102 0.612031 0.661476 +0.652559 XME/20091102 0.645301 0.657585 +0.556813 VO/20091102 0.539482 0.575801 +0.596465 RSX/20091102 0.565173 0.622601 +0.597116 EWC/20091102 0.583421 0.614050 +0.479297 TUR/20091102 0.462598 0.500693 +0.587444 VYM/20091102 0.573872 0.602034 +0.571562 FCG/20091102 0.553778 0.590126 +0.578254 VGT/20091102 0.563190 0.593999 +0.577119 EWQ/20091102 0.557511 0.589445 +0.562612 IEV/20091102 0.550806 0.568790 +0.577864 XLK/20091102 0.568649 0.582411 +0.555540 EFG/20091102 0.546532 0.561636 +0.592274 BKF/20091102 0.572997 0.610265 +0.526959 KIE/20091102 0.512797 0.537382 +0.615997 EEB/20091102 0.605362 0.625248 +0.569351 IJK/20091102 0.561126 0.574673 +0.400037 DUG/20091102 0.388317 0.407443 +0.383908 TWM/20091102 0.372484 0.397585 +0.588436 MDY/20091102 0.581525 0.599409 +0.596376 ACWI/20091102 0.578508 0.631282 +0.522754 BSV/20091102 0.496198 0.536956 +0.624766 DDM/20091102 0.606439 0.640192 +0.572414 DIA/20091102 0.567461 0.580356 +0.507393 TLT/20091102 0.491033 0.518559 +0.409619 DXD/20091102 0.342318 0.462142 +0.531536 XHB/20091102 0.471858 0.575855 +0.593296 VDE/20091102 0.556957 0.640164 +0.494437 BND/20091102 0.456074 0.524474 +0.515291 EMB/20091102 0.480769 0.539345 +0.475310 SCO/20091102 0.436267 0.520260 +0.564717 AMJ/20091102 0.547064 0.575958 +0.509381 OIL/20091102 0.462975 0.568092 +0.515791 PZA/20091102 0.491060 0.525060 +0.563076 VGK/20091102 0.538412 0.576236 +0.537964 RWX/20091102 0.534510 0.541074 +0.535291 JJA/20091102 0.516272 0.544481 +0.574172 FXD/20091102 0.550461 0.599735 +0.533269 XES/20091102 0.513397 0.542158 +0.567614 VIG/20091102 0.549237 0.587815 +0.231802 DZZ/20091102 0.222943 0.244693 +0.513412 VFH/20091102 0.483372 0.543404 +0.486760 DTO/20091102 0.388094 0.535668 +0.542752 EWP/20091102 0.524662 0.554477 +0.557824 FDN/20091102 0.554319 0.566991 +0.607392 INP/20091102 0.585945 0.628620 +0.336860 TYP/20091102 0.288916 0.379123 +0.557219 RWR/20091102 0.544100 0.563365 +0.563371 KBE/20091102 0.533500 0.585672 +0.447636 EUO/20091102 0.446296 0.448783 +0.576582 IWF/20091102 0.570678 0.589662 +0.240069 SMN/20091102 0.229865 0.247240 +0.547047 SMH/20091102 0.527707 0.569455 +0.559673 XRT/20091102 0.548788 0.573903 +0.518185 USO/20091102 0.499004 0.559273 +0.544527 DJP/20091102 0.533022 0.552719 +0.478801 CFT/20091102 0.425392 0.517323 +0.331433 SRS/20091102 0.326350 0.337044 +0.611578 MOO/20091102 0.585682 0.625873 +0.497911 BIV/20091102 0.477214 0.516780 +0.303927 VXX/20091102 0.292687 0.318398 +0.612066 IYM/20091102 0.604732 0.620571 +0.571571 IFN/20091102 0.532005 0.588906 +0.633834 SLV/20091102 0.622986 0.650479 +0.499446 TAO/20091102 0.494345 0.505396 +0.513860 PGF/20091102 0.498715 0.524774 +0.566344 IYR/20091102 0.558842 0.577675 +0.276482 QID/20091102 0.249586 0.293332 +0.565138 THD/20091102 0.542993 0.588436 +0.548611 IJS/20091102 0.539258 0.564884 +0.555797 VB/20091102 0.545018 0.569471 +0.546904 EDV/20091102 0.520929 0.577105 +0.557947 IEZ/20091102 0.535743 0.584790 +0.568857 VTV/20091102 0.564404 0.572027 +0.537078 IJR/20091102 0.528308 0.558877 +0.475671 UCO/20091102 0.437922 0.501832 +0.497029 JNK/20091102 0.490332 0.500995 +0.527725 IWN/20091102 0.517211 0.536124 +0.557212 VV/20091102 0.538628 0.577339 +0.675019 UGL/20091102 0.621657 0.707719 +0.545242 UWM/20091102 0.526631 0.554268 +0.507294 IWC/20091102 0.503143 0.510997 +0.570424 EWA/20091102 0.539294 0.589523 +0.568827 IVV/20091102 0.563967 0.572570 +0.569344 SPY/20091102 0.560306 0.575201 +0.497749 TFI/20091102 0.491757 0.505750 +0.540322 VEA/20091102 0.523120 0.551381 +0.568134 QQQQ/20091102 0.563581 0.575741 +0.541680 UYG/20091102 0.528389 0.556440 +0.540570 OIH/20091102 0.527506 0.547604 +0.555045 GXC/20091102 0.542227 0.570275 +0.625419 SSO/20091102 0.604418 0.644275 +0.603508 XLI/20091102 0.583566 0.617787 +0.604817 GML/20091102 0.579956 0.618057 +0.612171 ROM/20091102 0.598934 0.617871 +0.542111 FXC/20091102 0.526971 0.563008 +0.408620 DOG/20091102 0.388166 0.437668 +0.552565 IYE/20091102 0.549764 0.558496 +0.374266 SKF/20091102 0.351382 0.402365 +0.528225 SHY/20091102 0.524919 0.530928 +0.519714 DBA/20091102 0.496461 0.544560 +0.578270 RSP/20091102 0.558743 0.605486 +0.620413 DBS/20091102 0.612914 0.628841 +0.573457 IBB/20091102 0.556846 0.583624 +0.521014 KCE/20091102 0.509815 0.534416 +0.521960 PKN/20091102 0.516648 0.531249 +0.568087 TNA/20091102 0.538936 0.627479 +0.612044 FAS/20091102 0.557053 0.636113 +0.549464 FXE/20091102 0.522287 0.592061 +0.520719 HYG/20091102 0.512235 0.526067 +0.585153 IWS/20091102 0.572153 0.601981 +0.361987 FXP/20091102 0.337471 0.392869 +0.519877 MBB/20091102 0.509313 0.537371 +0.577776 RFG/20091102 0.553875 0.602879 +0.595610 EPU/20091102 0.588283 0.603963 +0.449062 UUP/20091102 0.428796 0.466951 +0.688911 AGQ/20091102 0.674053 0.707987 +0.594342 SOXX/20091102 0.566222 0.616392 +0.287278 FAZ/20091102 0.254277 0.314216 +0.552338 VBK/20091102 0.543067 0.559346 +0.592850 RPG/20091102 0.578502 0.620151 +0.531159 EWH/20091102 0.513910 0.548741 +0.368490 TZA/20091102 0.339336 0.402899 +0.484804 SGG/20091102 0.472656 0.501824 +0.622914 KOL/20091102 0.617728 0.627348 +0.539424 EWY/20091102 0.508512 0.564877 +0.542777 PRF/20091102 0.529422 0.556358 +0.506927 TLH/20091102 0.491843 0.519403 +0.569427 EPP/20091102 0.538630 0.595623 +0.546117 XLE/20091102 0.539142 0.560424 +0.541978 EWN/20091102 0.529128 0.565639 +0.539955 SHM/20091102 0.529582 0.551296 +0.550075 FXI/20091102 0.543977 0.552706 +0.594693 EWS/20091102 0.589474 0.600059 +0.538233 IDU/20091102 0.527133 0.553718 +0.480239 VXZ/20091102 0.472225 0.498692 +0.551367 IVE/20091102 0.528869 0.567014 +0.694092 DGP/20091102 0.674514 0.707954 +0.580882 GMF/20091102 0.574069 0.592882 +0.522530 IWR/20091102 0.516985 0.533792 +0.480960 RKH/20091102 0.463088 0.499607 +0.497029 TIP/20091102 0.440025 0.527809 +0.598967 URE/20091102 0.571346 0.618307 +0.519978 DBO/20091102 0.510042 0.534018 +0.556191 IOO/20091102 0.542044 0.576340 +0.506740 DBV/20091102 0.481907 0.551608 +0.555422 EFA/20091102 0.517422 0.576562 +0.638394 BGU/20091102 0.616862 0.668940 diff --git a/test/train-sets/ref/bs.reg.stderr b/test/train-sets/ref/bs.reg.stderr index 698508d8886..00e50f0a48a 100644 --- a/test/train-sets/ref/bs.reg.stderr +++ b/test/train-sets/ref/bs.reg.stderr @@ -15,18 +15,18 @@ loss last counter weight label predict feat 0.271591 0.271591 1 1.0 0.5211 0.0000 15 0.218951 0.166311 2 2.0 0.5353 0.1274 15 0.129025 0.039098 4 4.0 0.5854 0.4391 15 -0.101940 0.074856 8 8.0 0.5575 0.3786 15 -0.087679 0.073417 16 16.0 0.5878 0.4317 15 -0.049631 0.011583 32 32.0 0.6038 0.6072 15 -0.035617 0.021602 64 64.0 0.5683 0.4292 15 -0.022610 0.009603 128 128.0 0.5351 0.4670 15 -0.013289 0.003968 256 256.0 0.5385 0.5577 15 -0.007966 0.002642 512 512.0 0.5053 0.5181 15 +0.101867 0.074710 8 8.0 0.5575 0.3791 15 +0.087637 0.073406 16 16.0 0.5878 0.4318 15 +0.049622 0.011606 32 32.0 0.6038 0.6071 15 +0.035594 0.021566 64 64.0 0.5683 0.4294 15 +0.022597 0.009601 128 128.0 0.5351 0.4670 15 +0.013281 0.003965 256 256.0 0.5385 0.5577 15 +0.007962 0.002643 512 512.0 0.5053 0.5181 15 finished run number of examples = 1000 weighted example sum = 1000.000000 weighted label sum = 526.517528 -average loss = 0.004498 +average loss = 0.004496 best constant = 0.526518 total feature number = 14996 diff --git a/test/train-sets/ref/bs.vote.predict b/test/train-sets/ref/bs.vote.predict index 5a125d835a2..1c1c9b16a3a 100644 --- a/test/train-sets/ref/bs.vote.predict +++ b/test/train-sets/ref/bs.vote.predict @@ -4,197 +4,197 @@ 0.000000 0.000000 0.056914 0.000000 0.000000 0.055368 0.000000 0.000000 0.108588 -0.000000 0.060035 0.099882 -0.000000 0.185384 0.214994 -0.000000 0.076365 0.133533 -0.000000 0.153715 0.229805 -0.000000 0.151824 0.281808 -0.000000 0.124296 0.267574 -0.000000 0.052844 0.236524 -0.000000 0.146848 0.297702 -0.000000 0.095344 0.189743 -0.000000 0.263105 0.362490 -0.000000 0.188637 0.294482 -0.000000 0.314464 0.363651 -0.000000 0.391540 0.490716 -0.000000 0.129572 0.261093 -0.000000 0.069807 0.134286 -0.000000 0.280419 0.559181 -0.000000 0.203773 0.384708 -0.000000 0.212224 0.261401 -0.000000 0.110674 0.136587 -0.000000 0.292162 0.489801 -0.000000 0.138171 0.244588 -0.000000 0.214734 0.363004 -0.000000 0.244205 0.385861 -0.000000 0.231640 0.377310 -0.000000 0.346767 0.538611 -0.000000 0.116643 0.256667 -0.000000 0.135187 0.213753 -0.000000 0.266950 0.357525 -0.000000 0.326916 0.541221 -0.000000 0.232655 0.516905 -0.000000 0.223680 0.378183 -0.000000 0.279042 0.426095 -0.000000 0.358019 0.456708 -1.000000 0.444014 0.563562 -0.000000 0.208084 0.394655 -0.000000 0.265202 0.434631 -0.000000 0.330610 0.459316 -0.000000 0.204864 0.334043 -0.000000 0.264325 0.352090 -0.000000 0.122603 0.278541 -0.000000 0.337852 0.663552 -0.000000 0.389132 0.541451 -0.000000 0.065278 0.283866 -0.000000 0.137934 0.193465 -0.000000 0.223702 0.296520 -0.000000 0.273999 0.323600 -0.000000 0.339832 0.371714 -0.000000 0.293943 0.427828 -0.000000 0.427890 0.529465 -1.000000 0.535167 0.703842 -0.000000 0.393061 0.459777 -0.000000 0.255470 0.342168 -0.000000 0.382347 0.476401 -0.000000 0.279962 0.380802 -0.000000 0.266670 0.319516 -0.000000 0.354260 0.441987 -1.000000 0.400923 0.619577 -0.000000 0.178755 0.332358 -0.000000 0.434999 0.491165 -0.000000 0.225289 0.315372 -0.000000 0.300479 0.432123 -1.000000 0.609933 0.803870 -0.000000 0.305149 0.435861 -0.000000 0.348280 0.558524 -0.000000 0.382903 0.416826 -0.000000 0.409965 0.560847 -0.000000 0.181245 0.388152 -0.000000 0.206158 0.387712 -0.000000 0.206952 0.514329 -0.000000 0.258637 0.408113 -0.000000 0.175173 0.466962 -1.000000 0.328643 0.587575 -0.000000 0.125153 0.153853 -0.000000 0.140399 0.238694 -0.000000 0.210751 0.259890 -0.000000 0.220530 0.347812 -1.000000 0.537810 0.595470 -0.000000 0.246706 0.371924 -0.000000 0.167971 0.435135 -0.000000 0.231589 0.307771 -0.000000 0.214186 0.235897 -1.000000 0.368226 0.654591 -1.000000 0.539059 0.683148 -0.000000 0.200537 0.395829 -0.000000 0.072891 0.139037 -0.000000 0.255444 0.503878 -1.000000 0.718731 0.908420 -0.000000 0.148134 0.234985 -1.000000 0.419400 0.526940 -0.000000 0.281590 0.449685 -0.000000 0.416415 0.524149 -0.000000 0.181038 0.257179 -0.000000 0.288000 0.435358 -0.000000 0.241666 0.383677 -1.000000 0.542362 0.815245 -1.000000 0.406909 0.555072 -0.000000 0.307712 0.434762 -0.000000 0.413690 0.456132 -1.000000 0.394815 0.766386 -0.000000 0.465605 0.558527 -0.000000 0.123853 0.193798 -0.000000 0.322638 0.425156 -0.000000 0.152502 0.277716 -1.000000 0.302211 0.603111 -1.000000 0.519083 0.691395 -1.000000 0.299794 0.647601 -0.000000 0.384462 0.517522 -0.000000 0.221309 0.305192 -0.000000 0.367274 0.503765 -0.000000 0.337746 0.591077 -0.000000 0.145899 0.263625 -1.000000 0.414073 0.599591 -0.000000 0.152904 0.557253 -0.000000 0.345800 0.410097 -0.000000 0.283921 0.442342 -0.000000 0.420805 0.540704 -0.000000 0.382728 0.467405 -0.000000 0.332877 0.487902 -1.000000 0.483051 0.669946 -0.000000 0.311442 0.509110 -0.000000 0.304542 0.405824 -1.000000 0.476058 0.722197 -0.000000 0.087010 0.356679 -0.000000 0.286310 0.554956 -0.000000 0.262545 0.395812 -1.000000 0.456266 0.832514 -0.000000 0.162775 0.234763 -0.000000 0.237567 0.308066 -0.000000 0.304806 0.417123 -0.000000 0.187233 0.322587 -1.000000 0.509059 0.586988 -0.000000 0.337408 0.492913 -1.000000 0.637149 0.806485 -0.000000 0.290388 0.402845 -0.000000 0.409257 0.463213 -0.000000 0.205731 0.290107 -1.000000 0.703565 0.780184 -1.000000 0.762320 0.883820 -0.000000 0.430970 0.536012 -1.000000 0.453806 0.627823 -1.000000 0.421150 0.595644 -1.000000 0.778521 0.896804 -0.000000 0.263792 0.487992 -0.000000 0.314233 0.442666 -1.000000 0.340442 0.744408 -0.000000 0.265697 0.386746 -1.000000 0.588076 0.721545 -0.000000 0.476271 0.573887 -1.000000 0.537259 0.752284 -1.000000 0.540044 0.642390 -1.000000 0.789948 0.897157 -1.000000 0.499518 0.624615 -1.000000 0.674253 0.872026 -1.000000 0.439344 0.610095 -1.000000 0.407931 0.684587 -0.000000 0.227526 0.417340 -0.000000 0.245463 0.601820 -1.000000 0.647145 0.707758 -0.000000 0.292232 0.355189 -0.000000 0.394226 0.478338 -1.000000 0.729621 0.905942 -0.000000 0.326497 0.618210 -1.000000 0.424470 0.527758 -1.000000 0.398567 0.780571 -1.000000 0.644528 0.725458 -0.000000 0.263530 0.378992 -0.000000 0.215015 0.308514 -0.000000 0.317181 0.408162 -0.000000 0.124465 0.178991 -1.000000 0.467243 0.638149 -0.000000 0.281433 0.327047 -1.000000 0.779584 0.985479 -1.000000 0.539432 0.687287 -1.000000 0.698828 0.777052 -0.000000 0.396852 0.521239 -0.000000 0.361414 0.418624 -1.000000 0.443891 0.590845 -0.000000 0.278819 0.443288 -0.000000 0.163261 0.431671 -0.000000 0.279712 0.564609 -1.000000 0.600835 0.827839 -1.000000 0.534030 0.688263 -0.000000 0.198917 0.376790 -1.000000 0.510130 0.786455 -1.000000 0.450644 0.712191 -0.000000 0.088898 0.485186 -0.000000 0.209145 0.319709 -0.000000 0.260771 0.315615 -0.000000 0.256404 0.383797 -0.000000 0.454846 0.706750 -0.000000 0.446069 0.578452 -0.000000 0.282540 0.459406 -0.000000 0.254622 0.458421 -1.000000 0.386176 0.671750 +0.000000 0.063254 0.097981 +0.000000 0.195273 0.207024 +0.000000 0.080428 0.130979 +0.000000 0.161919 0.224062 +0.000000 0.159888 0.292871 +0.000000 0.130859 0.262355 +0.000000 0.055605 0.234676 +0.000000 0.154614 0.291912 +0.000000 0.100355 0.186234 +0.000000 0.271194 0.359921 +0.000000 0.195800 0.297624 +0.000000 0.306360 0.368946 +0.000000 0.394553 0.488798 +0.000000 0.129983 0.259065 +0.000000 0.072150 0.138995 +0.000000 0.287308 0.556645 +0.000000 0.208845 0.383221 +0.000000 0.219426 0.265090 +0.000000 0.112642 0.136059 +0.000000 0.296746 0.493424 +0.000000 0.139164 0.244939 +0.000000 0.212882 0.358598 +0.000000 0.249371 0.381788 +0.000000 0.235390 0.377699 +0.000000 0.354900 0.538839 +0.000000 0.118062 0.259427 +0.000000 0.137846 0.214422 +0.000000 0.270229 0.359664 +0.000000 0.332286 0.544358 +0.000000 0.238639 0.520331 +0.000000 0.226335 0.376857 +0.000000 0.286227 0.427360 +0.000000 0.363994 0.460808 +1.000000 0.439406 0.570028 +0.000000 0.213245 0.394100 +0.000000 0.265211 0.433544 +0.000000 0.334582 0.457471 +0.000000 0.210034 0.333600 +0.000000 0.265901 0.353179 +0.000000 0.121864 0.280419 +0.000000 0.338720 0.662083 +0.000000 0.385756 0.540913 +0.000000 0.066106 0.284537 +0.000000 0.140253 0.192532 +0.000000 0.224389 0.297352 +0.000000 0.274979 0.322848 +0.000000 0.338528 0.369949 +0.000000 0.293847 0.427852 +0.000000 0.434942 0.525146 +1.000000 0.535901 0.705668 +0.000000 0.391931 0.458663 +0.000000 0.255923 0.343710 +0.000000 0.380002 0.476756 +0.000000 0.279792 0.381307 +0.000000 0.266566 0.319867 +0.000000 0.355619 0.443840 +1.000000 0.399055 0.614496 +0.000000 0.178605 0.333045 +0.000000 0.434491 0.492606 +0.000000 0.225289 0.316254 +0.000000 0.301119 0.430593 +1.000000 0.610695 0.800837 +0.000000 0.307809 0.441826 +0.000000 0.349509 0.558426 +0.000000 0.386816 0.417430 +0.000000 0.401468 0.566613 +0.000000 0.180458 0.388247 +0.000000 0.205228 0.385539 +0.000000 0.206109 0.522599 +0.000000 0.258802 0.414362 +0.000000 0.170788 0.471904 +1.000000 0.328829 0.588173 +0.000000 0.125418 0.154364 +0.000000 0.141366 0.238340 +0.000000 0.211104 0.259811 +0.000000 0.219673 0.348892 +1.000000 0.538936 0.596301 +0.000000 0.244495 0.371420 +0.000000 0.166359 0.436095 +0.000000 0.233814 0.306213 +0.000000 0.214012 0.236293 +1.000000 0.367507 0.653899 +1.000000 0.536052 0.687675 +0.000000 0.200344 0.393470 +0.000000 0.074324 0.139967 +0.000000 0.253176 0.503452 +1.000000 0.712092 0.911009 +0.000000 0.148967 0.233552 +1.000000 0.420521 0.525129 +0.000000 0.281586 0.449643 +0.000000 0.415734 0.527373 +0.000000 0.180867 0.257021 +0.000000 0.287143 0.435619 +0.000000 0.245233 0.386528 +1.000000 0.542068 0.817527 +1.000000 0.407375 0.554181 +0.000000 0.307080 0.433866 +0.000000 0.412964 0.456685 +1.000000 0.393871 0.768310 +0.000000 0.466066 0.559906 +0.000000 0.124144 0.193973 +0.000000 0.322430 0.424669 +0.000000 0.151825 0.278742 +1.000000 0.301222 0.603838 +1.000000 0.522079 0.693242 +1.000000 0.298500 0.648683 +0.000000 0.383050 0.521281 +0.000000 0.219588 0.304680 +0.000000 0.366373 0.501866 +0.000000 0.336869 0.591129 +0.000000 0.148992 0.268347 +1.000000 0.414344 0.601728 +0.000000 0.152421 0.557340 +0.000000 0.346790 0.408564 +0.000000 0.283772 0.445531 +0.000000 0.422995 0.541149 +0.000000 0.383448 0.464324 +0.000000 0.331814 0.489881 +1.000000 0.482536 0.670951 +0.000000 0.310524 0.511148 +0.000000 0.306036 0.407844 +1.000000 0.472411 0.723238 +0.000000 0.086951 0.354961 +0.000000 0.287749 0.554533 +0.000000 0.261021 0.396569 +1.000000 0.456366 0.835763 +0.000000 0.161112 0.237505 +0.000000 0.235775 0.305934 +0.000000 0.300791 0.418951 +0.000000 0.189224 0.323355 +1.000000 0.510222 0.586318 +0.000000 0.336993 0.492896 +1.000000 0.634971 0.807615 +0.000000 0.289937 0.400814 +0.000000 0.408571 0.462840 +0.000000 0.207687 0.290693 +1.000000 0.702547 0.775726 +1.000000 0.763069 0.883828 +0.000000 0.430606 0.536159 +1.000000 0.458201 0.627050 +1.000000 0.421077 0.602258 +1.000000 0.783578 0.893522 +0.000000 0.265009 0.486779 +0.000000 0.313970 0.442856 +1.000000 0.341150 0.744139 +0.000000 0.266393 0.388470 +1.000000 0.588819 0.719345 +0.000000 0.478017 0.570976 +1.000000 0.535712 0.753140 +1.000000 0.540066 0.643340 +1.000000 0.788869 0.903525 +1.000000 0.500941 0.624359 +1.000000 0.672801 0.870738 +1.000000 0.440563 0.608410 +1.000000 0.408206 0.689143 +0.000000 0.226362 0.414947 +0.000000 0.244450 0.602926 +1.000000 0.651315 0.706431 +0.000000 0.294774 0.356835 +0.000000 0.393913 0.481877 +1.000000 0.728997 0.908197 +0.000000 0.326370 0.616926 +1.000000 0.426752 0.529047 +1.000000 0.399556 0.781778 +1.000000 0.650429 0.724167 +0.000000 0.265291 0.379296 +0.000000 0.214054 0.309486 +0.000000 0.315409 0.409288 +0.000000 0.123357 0.179197 +1.000000 0.465758 0.637765 +0.000000 0.283083 0.327480 +1.000000 0.780997 0.982180 +1.000000 0.539124 0.687386 +1.000000 0.697587 0.776066 +0.000000 0.395733 0.522453 +0.000000 0.362856 0.415778 +1.000000 0.443249 0.594379 +0.000000 0.280466 0.441391 +0.000000 0.162939 0.431504 +0.000000 0.281772 0.562431 +1.000000 0.601315 0.829984 +1.000000 0.536297 0.689820 +0.000000 0.198638 0.376431 +1.000000 0.508416 0.790084 +1.000000 0.450380 0.715270 +0.000000 0.091925 0.486855 +0.000000 0.210002 0.321692 +0.000000 0.259601 0.316226 +0.000000 0.257007 0.385543 +0.000000 0.454336 0.708163 +0.000000 0.447638 0.577024 +0.000000 0.282050 0.458769 +0.000000 0.253884 0.458618 +1.000000 0.386388 0.674689 diff --git a/test/train-sets/ref/cbadf_automl_readable.txt b/test/train-sets/ref/cbadf_automl_readable.txt index 0c90bb963b3..87a2578262d 100644 --- a/test/train-sets/ref/cbadf_automl_readable.txt +++ b/test/train-sets/ref/cbadf_automl_readable.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:-1 Max label:0 @@ -535,28 +535,11 @@ _automl_config_manager_index_queue[41].first = -12 _automl_config_manager_index_queue[41].second = 5 _automl_config_manager_index_queue[42].first = -12 _automl_config_manager_index_queue[42].second = 2 -_automl_config_manager_per_live_model_state_double.size() = 9 -_automl_config_manager_per_live_model_state_double[0] = 98.98610879850912 -_automl_config_manager_per_live_model_state_double[1] = 0.30612243711948395 -_automl_config_manager_per_live_model_state_double[2] = 0 -_automl_config_manager_per_live_model_state_double[3] = 96.53712930155325 -_automl_config_manager_per_live_model_state_double[4] = 0.30612243711948395 -_automl_config_manager_per_live_model_state_double[5] = 0 -_automl_config_manager_per_live_model_state_double[6] = 94.08814980459738 -_automl_config_manager_per_live_model_state_double[7] = 0.30612243711948395 -_automl_config_manager_per_live_model_state_double[8] = 0 -_automl_config_manager_per_live_model_state_uint64.size() = 6 -_automl_config_manager_per_live_model_state_uint64[0] = 3 -_automl_config_manager_per_live_model_state_uint64[1] = 36 -_automl_config_manager_per_live_model_state_uint64[2] = 3 -_automl_config_manager_per_live_model_state_uint64[3] = 36 -_automl_config_manager_per_live_model_state_uint64[4] = 3 -_automl_config_manager_per_live_model_state_uint64[5] = 36 -event_sum 0 -action_sum 0 +event_sum 3 +action_sum 36 :1 initial_t 0 -norm normalizer 0 +norm normalizer 98.9861 t 3 sum_loss 0 sum_loss_since_last_dump 0 @@ -568,7 +551,7 @@ weighted_labels 0 weighted_unlabeled_examples 0 example_number 3 total_features 29895 -total_weight 0 +total_weight 0.306122 sd::oec.weighted_labeled_examples 2 current_pass 1 l1_state 0 diff --git a/test/train-sets/ref/cbadf_automl_readable_predictonly.txt b/test/train-sets/ref/cbadf_automl_readable_predictonly.txt index 63cd1f1ca35..ee9fb12ff8a 100644 --- a/test/train-sets/ref/cbadf_automl_readable_predictonly.txt +++ b/test/train-sets/ref/cbadf_automl_readable_predictonly.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:-1 Max label:0 diff --git a/test/train-sets/ref/cbe_cover_e_dec_legacy.stderr b/test/train-sets/ref/cbe_cover_e_dec_legacy.stderr index 0d795b107b1..88293c01dff 100644 --- a/test/train-sets/ref/cbe_cover_e_dec_legacy.stderr +++ b/test/train-sets/ref/cbe_cover_e_dec_legacy.stderr @@ -15,11 +15,11 @@ loss last counter weight label predict feat 0.000000 0.000000 1 1.0 1:0:0.5 1:0.500000 3 0.323483 0.646967 2 2.0 2:1:0.5 1:0.500000 3 0.456686 0.589888 4 4.0 1:2:0.5 2:0.591752 3 -0.542755 0.628824 8 8.0 2:1:0.5 2:0.732739 3 +0.541718 0.626750 8 8.0 2:1:0.5 2:0.732739 3 finished run number of examples = 8 weighted example sum = 8.000000 weighted label sum = 0.000000 -average loss = 0.542755 +average loss = 0.541718 total feature number = 25 diff --git a/test/train-sets/ref/cbe_cover_e_fixed_legacy.stderr b/test/train-sets/ref/cbe_cover_e_fixed_legacy.stderr index a66659986d1..074e917f127 100644 --- a/test/train-sets/ref/cbe_cover_e_fixed_legacy.stderr +++ b/test/train-sets/ref/cbe_cover_e_fixed_legacy.stderr @@ -15,11 +15,11 @@ loss last counter weight label predict feat 0.000000 0.000000 1 1.0 1:0:0.5 1:0.750000 3 0.431311 0.862622 2 2.0 2:1:0.5 2:0.666667 3 0.392332 0.353353 4 4.0 1:2:0.5 2:0.750000 3 -0.511467 0.630603 8 8.0 2:1:0.5 2:0.750000 3 +0.510628 0.628924 8 8.0 2:1:0.5 2:0.750000 3 finished run number of examples = 8 weighted example sum = 8.000000 weighted label sum = 0.000000 -average loss = 0.511467 +average loss = 0.510628 total feature number = 25 diff --git a/test/train-sets/ref/cbzo_constant_invert_hash.txt b/test/train-sets/ref/cbzo_constant_invert_hash.txt index 8bb28832887..867258e5089 100644 --- a/test/train-sets/ref/cbzo_constant_invert_hash.txt +++ b/test/train-sets/ref/cbzo_constant_invert_hash.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:-1.13836 Max label:3.60884 diff --git a/test/train-sets/ref/cbzo_constant_readable_model.txt b/test/train-sets/ref/cbzo_constant_readable_model.txt index 6059e7b1f6b..e853c1977ad 100644 --- a/test/train-sets/ref/cbzo_constant_readable_model.txt +++ b/test/train-sets/ref/cbzo_constant_readable_model.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:-1.13836 Max label:3.60884 diff --git a/test/train-sets/ref/cbzo_linear_invert_hash.txt b/test/train-sets/ref/cbzo_linear_invert_hash.txt index d429c827326..d4f3076bc82 100644 --- a/test/train-sets/ref/cbzo_linear_invert_hash.txt +++ b/test/train-sets/ref/cbzo_linear_invert_hash.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:-3.35097 Max label:3.13689 diff --git a/test/train-sets/ref/cbzo_linear_readable_model.txt b/test/train-sets/ref/cbzo_linear_readable_model.txt index 574df27191b..117ef0ca263 100644 --- a/test/train-sets/ref/cbzo_linear_readable_model.txt +++ b/test/train-sets/ref/cbzo_linear_readable_model.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:-3.35097 Max label:3.13689 diff --git a/test/train-sets/ref/ccb_simple_invert_hash.txt b/test/train-sets/ref/ccb_simple_invert_hash.txt index 2635d3c8036..f07a86348dc 100644 --- a/test/train-sets/ref/ccb_simple_invert_hash.txt +++ b/test/train-sets/ref/ccb_simple_invert_hash.txt @@ -1,4 +1,4 @@ -Id +Id Min label:-1 Max label:0 bits:18 diff --git a/test/train-sets/ref/ccb_simple_pdrop_invert_hash.txt b/test/train-sets/ref/ccb_simple_pdrop_invert_hash.txt index 8d01cb166ed..33be3798fdc 100644 --- a/test/train-sets/ref/ccb_simple_pdrop_invert_hash.txt +++ b/test/train-sets/ref/ccb_simple_pdrop_invert_hash.txt @@ -1,4 +1,4 @@ -Id +Id Min label:-1 Max label:0 bits:18 diff --git a/test/train-sets/ref/coin.readable b/test/train-sets/ref/coin.readable index d6c6529eaa8..ace89cbea25 100644 --- a/test/train-sets/ref/coin.readable +++ b/test/train-sets/ref/coin.readable @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:0 Max label:2 diff --git a/test/train-sets/ref/coin_model_overflow.invert.txt b/test/train-sets/ref/coin_model_overflow.invert.txt index 677c3cceb75..28772db5436 100644 --- a/test/train-sets/ref/coin_model_overflow.invert.txt +++ b/test/train-sets/ref/coin_model_overflow.invert.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:0 Max label:1 diff --git a/test/train-sets/ref/dupeindex_self_cubic.txt b/test/train-sets/ref/dupeindex_self_cubic.txt index e9dffead440..12747215956 100644 --- a/test/train-sets/ref/dupeindex_self_cubic.txt +++ b/test/train-sets/ref/dupeindex_self_cubic.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:0 Max label:1 diff --git a/test/train-sets/ref/dupeindex_self_quadratic.txt b/test/train-sets/ref/dupeindex_self_quadratic.txt index 9874306d910..f64ecee9fc8 100644 --- a/test/train-sets/ref/dupeindex_self_quadratic.txt +++ b/test/train-sets/ref/dupeindex_self_quadratic.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:0 Max label:1 diff --git a/test/train-sets/ref/dupeindex_self_quartic.txt b/test/train-sets/ref/dupeindex_self_quartic.txt index de059b7691d..b57f28a5e64 100644 --- a/test/train-sets/ref/dupeindex_self_quartic.txt +++ b/test/train-sets/ref/dupeindex_self_quartic.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:0 Max label:1 diff --git a/test/train-sets/ref/ftrl.readable b/test/train-sets/ref/ftrl.readable index b19240c4158..d8426d5c423 100644 --- a/test/train-sets/ref/ftrl.readable +++ b/test/train-sets/ref/ftrl.readable @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:0 Max label:2 diff --git a/test/train-sets/ref/help.stdout b/test/train-sets/ref/help.stdout index 07679dd0c7c..16ffd877248 100644 --- a/test/train-sets/ref/help.stdout +++ b/test/train-sets/ref/help.stdout @@ -639,6 +639,11 @@ Weight Options: with quadratic interactions. (x86 Linux only) (type: bool, experimental) --two_pass_svd A more accurate svd that is much slower than the default (one pass svd) (type: bool, experimental) + --action_cache_slack arg If actions do not change between example calls then some calculations + for this algorithm are cached. If action cache size exceeds the + number of active actions plus action_cache_slack, then the cache + will be cleared. Setting this to -1 disables any caching (type: + uint, default: 50, keep, experimental) [Reduction] Explore Evaluation Options: --explore_eval Evaluate explore_eval adf policies (type: bool, keep, necessary) --multiplier arg Multiplier used to make all rejection sample probabilities <= diff --git a/test/train-sets/ref/houses.stdout b/test/train-sets/ref/houses.stdout index 7001c3a5586..e226a205887 100644 --- a/test/train-sets/ref/houses.stdout +++ b/test/train-sets/ref/houses.stdout @@ -1,30 +1,30 @@ 0 -HiddenBias:29015:1:0@0 + HiddenBias:29015:1:0@0 0 -HiddenBias:29015:1:0@0 + HiddenBias:29015:1:0@0 0 -price:197660:0.23:0@0 sqft:63562:0.25:0@0 age:68258:0.05:0@0 2006:4012:1:0@0 Constant:232120:1:0@0 + price:197660:0.23:0@0 sqft:63562:0.25:0@0 age:68258:0.05:0@0 2006:4012:1:0@0 Constant:232120:1:0@0 0 -OutputWeight:156908:1:0@0 + OutputWeight:156908:1:0@0 0 -OutputLayer0:156908:0:-0.302343@0.815926 OutputLayerConst:156909:1:0@0 + OutputLayer0:156908:0:-0.302343@0.815926 OutputLayerConst:156909:1:0@0 -0.316100 -HiddenBias:29015:1:-0.3161@0.99997 + HiddenBias:29015:1:-0.3161@0.99997 0 second_house -price:197660:0.18:0@0 sqft:63562:0.15:0@0 age:68258:0.35:0@0 1976:3952:1:0@0 Constant:232120:1:0@0 + price:197660:0.18:0@0 sqft:63562:0.15:0@0 age:68258:0.35:0@0 1976:3952:1:0@0 Constant:232120:1:0@0 -0.302343 -OutputWeight:156908:1:-0.302343@0.815926 + OutputWeight:156908:1:-0.302343@0.815926 0 -OutputLayer0:156908:0:-0.302343@0.815926 OutputLayerConst:156909:1:0@0 + OutputLayer0:156908:0:-0.302343@0.815926 OutputLayerConst:156909:1:0@0 -0.302343 -OutputWeight:156908:1:-0.302343@0.815926 + OutputWeight:156908:1:-0.302343@0.815926 -0.316100 -HiddenBias:29015:1:-0.3161@0.99997 --0.017576 third_house -price:197660:0.53:-0.335796@0.0236938 age:68258:0.87:-0.172693@0.089583 sqft:63562:0.32:-0.402875@0.016454 Constant:232120:1:-0.0604413@0.73129 1924:3848:1:0@0 + HiddenBias:29015:1:-0.3161@0.99997 +-0.016668 third_house + price:197660:0.53:-0.335207@0.0236938 age:68258:0.87:-0.17239@0.089583 sqft:63562:0.32:-0.402168@0.016454 Constant:232120:1:-0.0603353@0.73129 1924:3848:1:0@0 -0.302343 -OutputWeight:156908:1:-0.302343@0.815926 + OutputWeight:156908:1:-0.302343@0.815926 1 -OutputLayerConst:156909:1:0.506834@8 OutputLayer0:156908:-0.017558:-0.302343@0.815926 --0.290302 -OutputWeight:156908:1:-0.290302@0.817159 \ No newline at end of file + OutputLayerConst:156909:1:0.506834@8 OutputLayer0:156908:-0.0166478:-0.302343@0.815926 +-0.286349 + OutputWeight:156908:1:-0.286349@0.817034 diff --git a/test/train-sets/ref/ignore_feature.interactions b/test/train-sets/ref/ignore_feature.interactions index e5e0d608e6a..17e0de126bc 100644 --- a/test/train-sets/ref/ignore_feature.interactions +++ b/test/train-sets/ref/ignore_feature.interactions @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:-1 Max label:0 diff --git a/test/train-sets/ref/ignore_feature_default_ns.interactions b/test/train-sets/ref/ignore_feature_default_ns.interactions index 838134603b2..4fb16264052 100644 --- a/test/train-sets/ref/ignore_feature_default_ns.interactions +++ b/test/train-sets/ref/ignore_feature_default_ns.interactions @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:-1 Max label:0 diff --git a/test/train-sets/ref/ignore_multiple_features.interactions b/test/train-sets/ref/ignore_multiple_features.interactions index 2efdb7c41d5..01c12466913 100644 --- a/test/train-sets/ref/ignore_multiple_features.interactions +++ b/test/train-sets/ref/ignore_multiple_features.interactions @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:-1 Max label:0 diff --git a/test/train-sets/ref/inv_hash_load_model.invert.txt b/test/train-sets/ref/inv_hash_load_model.invert.txt index cebe3ef00ad..81f13c2ba2c 100644 --- a/test/train-sets/ref/inv_hash_load_model.invert.txt +++ b/test/train-sets/ref/inv_hash_load_model.invert.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:0 Max label:2 diff --git a/test/train-sets/ref/inv_hash_load_model.readable.txt b/test/train-sets/ref/inv_hash_load_model.readable.txt index ef0f06f4d92..1593e1f07e6 100644 --- a/test/train-sets/ref/inv_hash_load_model.readable.txt +++ b/test/train-sets/ref/inv_hash_load_model.readable.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:0 Max label:2 diff --git a/test/train-sets/ref/ksvm_train.rbf.predict b/test/train-sets/ref/ksvm_train.rbf.predict index 285a77d8b11..30021e4e73b 100644 --- a/test/train-sets/ref/ksvm_train.rbf.predict +++ b/test/train-sets/ref/ksvm_train.rbf.predict @@ -40,7 +40,7 @@ -0.298287 -0.322792 -0.359415 --0.337301 +-0.337300 -0.363496 -0.459323 -0.511590 @@ -61,190 +61,190 @@ -0.423798 -0.397756 -0.293423 --0.542385 +-0.542384 -0.388847 --0.410773 +-0.410774 -0.409145 -0.409086 -0.530262 --0.441563 +-0.441564 -0.334338 -0.306773 -0.376202 -0.437149 --0.254241 +-0.254242 -0.302737 --0.280581 +-0.280582 -0.468249 --0.445411 +-0.445410 -0.456937 --0.429883 --0.439088 --0.349915 +-0.429884 +-0.439089 +-0.349914 -0.410971 -0.501306 --0.514045 +-0.514044 -0.519062 --0.374990 +-0.374989 -0.329659 -0.569188 --0.409274 --0.434954 +-0.409275 +-0.434953 -0.133812 -0.319933 -0.256982 -0.498179 --0.386975 +-0.386976 -0.333555 -0.449723 --0.368091 --0.301244 +-0.368092 +-0.301243 -0.349987 --0.305834 +-0.305835 -0.337926 -0.358264 --0.314864 --0.375793 --0.466998 --0.418961 --0.394994 --0.284009 --0.277930 +-0.314863 +-0.375794 +-0.466999 +-0.418962 +-0.394993 +-0.284011 +-0.277929 -0.337629 -0.438446 --0.188560 +-0.188561 -0.385540 --0.420675 +-0.420676 -0.307711 -0.334184 --0.325543 --0.325641 +-0.325544 +-0.325642 -0.337942 -0.317093 -0.407631 --0.207016 --0.107704 +-0.207015 +-0.107705 -0.247503 --0.201117 +-0.201119 -0.397332 --0.337279 --0.222775 --0.257910 --0.227449 --0.322027 +-0.337280 +-0.222776 +-0.257909 +-0.227448 +-0.322030 -0.289908 --0.427382 +-0.427380 -0.267543 --0.201158 -0.164485 +-0.201160 +0.164487 -0.195240 -0.330030 --0.432331 --0.010992 +-0.432332 +-0.010991 0.344526 --0.247542 --0.082936 --0.198838 +-0.247541 +-0.082934 +-0.198839 0.217597 --0.416878 --0.223981 -0.018709 +-0.416876 +-0.223982 +0.018710 -0.145108 -0.087493 +0.087494 -0.131474 --0.378017 +-0.378019 -0.117230 -0.289694 --0.187535 +0.289692 +-0.187534 0.335303 --0.211335 +-0.211336 0.476274 -0.382407 --0.351695 +-0.351696 -0.057328 -0.216453 --0.194411 -0.285892 --0.256831 --0.292014 -0.216467 +-0.194413 +0.285891 +-0.256832 +-0.292013 +0.216466 0.093587 --0.234669 +-0.234670 -0.286890 --0.036496 +-0.036498 -0.316144 --0.151081 --0.550987 -0.197503 --0.016160 -0.313661 --0.027068 -0.066874 -0.022904 +-0.151082 +-0.550988 +0.197502 +-0.016162 +0.313660 +-0.027069 +0.066875 +0.022905 -0.262253 --0.227533 --0.277169 --0.064773 +-0.227532 +-0.277168 +-0.064775 -0.293002 --0.307918 --0.046608 -0.260579 --0.082755 --0.211580 --0.292219 --0.376719 +-0.307917 +-0.046607 +0.260577 +-0.082757 +-0.211583 +-0.292220 +-0.376718 -0.262074 --0.137004 --0.309522 --0.213425 --0.203285 --0.205892 --0.104864 +-0.137005 +-0.309521 +-0.213424 +-0.203284 +-0.205894 +-0.104865 -0.125947 --0.328827 --0.359252 --0.373672 -0.021654 --0.106385 --0.206097 --0.112396 --0.177456 --0.439561 +-0.328826 +-0.359251 +-0.373669 +0.021653 +-0.106386 +-0.206095 +-0.112397 +-0.177457 +-0.439562 -0.411683 --0.191273 -0.768048 +-0.191274 +0.768049 -0.454163 --0.279515 --0.329129 --0.255296 +-0.279516 +-0.329130 +-0.255297 -0.312587 --0.190189 --0.507048 --0.328184 --0.316488 +-0.190188 +-0.507047 +-0.328182 +-0.316486 -0.126934 -0.266179 -0.874625 -0.026328 --0.325729 --0.409636 +0.874626 +0.026330 +-0.325731 +-0.409635 -0.492323 --0.535633 --0.477274 --0.233937 --0.148593 --0.406306 --0.413641 +-0.535632 +-0.477271 +-0.233935 +-0.148594 +-0.406308 +-0.413642 -0.001290 -0.624692 --0.111970 -0.006651 --0.251943 --0.535996 -0.230161 --0.220745 --0.068769 +0.624695 +-0.111968 +0.006653 +-0.251939 +-0.535998 +0.230163 +-0.220746 +-0.068768 0.114005 --0.278020 --0.495117 +-0.278022 +-0.495115 0.143694 diff --git a/test/train-sets/ref/l1_l2_default_model.txt b/test/train-sets/ref/l1_l2_default_model.txt index 01a2171d545..cca44f2a02d 100644 --- a/test/train-sets/ref/l1_l2_default_model.txt +++ b/test/train-sets/ref/l1_l2_default_model.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:0 Max label:0 diff --git a/test/train-sets/ref/l1_l2_override_model.txt b/test/train-sets/ref/l1_l2_override_model.txt index d23ccefb270..49b6d52ade6 100644 --- a/test/train-sets/ref/l1_l2_override_model.txt +++ b/test/train-sets/ref/l1_l2_override_model.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:0 Max label:0 diff --git a/test/train-sets/ref/l1_override_l2_leave_model.txt b/test/train-sets/ref/l1_override_l2_leave_model.txt index bfbba893342..4f6d1080afb 100644 --- a/test/train-sets/ref/l1_override_l2_leave_model.txt +++ b/test/train-sets/ref/l1_override_l2_leave_model.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:0 Max label:0 diff --git a/test/train-sets/ref/l1l2_migrate.txt b/test/train-sets/ref/l1l2_migrate.txt index 2b5963731b4..6b02f243863 100644 --- a/test/train-sets/ref/l1l2_migrate.txt +++ b/test/train-sets/ref/l1l2_migrate.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:0 Max label:0 diff --git a/test/train-sets/ref/l1l2_migrate_override.txt b/test/train-sets/ref/l1l2_migrate_override.txt index 208a0c85420..235ef8b65f1 100644 --- a/test/train-sets/ref/l1l2_migrate_override.txt +++ b/test/train-sets/ref/l1l2_migrate_override.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:0 Max label:0 diff --git a/test/train-sets/ref/las_egreedy_ips.stderr b/test/train-sets/ref/las_egreedy_ips.stderr new file mode 100644 index 00000000000..c7e8858ec5a --- /dev/null +++ b/test/train-sets/ref/las_egreedy_ips.stderr @@ -0,0 +1,22 @@ +using no cache +Reading datafile = train-sets/las_100_actions.txt +num sources = 1 +Num weight bits = 18 +learning rate = 0.5 +initial_t = 0 +power_t = 0.5 +cb_type = ips +Enabled learners: gd, scorer-identity, csoaa_ldf-rank, cb_adf, cb_explore_adf_large_action_space, cb_explore_adf_greedy, cb_actions_mask, shared_feature_merger +Input label = CB +Output pred = ACTION_PROBS +average since example example current current current +loss last counter weight label predict features +0.000000 0.000000 1 1.0 47:-1:0.17 0:0.06 4761 +-0.00153 -0.00306 2 2.0 14:-1:0.96 47:0.95 4761 + +finished run +number of examples = 2 +weighted example sum = 2.000000 +weighted label sum = 0.000000 +average loss = -0.001535 +total feature number = 9522 diff --git a/test/train-sets/ref/lrqfa_nn_pred.stderr b/test/train-sets/ref/lrqfa_nn_pred.stderr index be67d2ed5fa..6a920b1d2fb 100644 --- a/test/train-sets/ref/lrqfa_nn_pred.stderr +++ b/test/train-sets/ref/lrqfa_nn_pred.stderr @@ -12,15 +12,15 @@ Input label = SIMPLE Output pred = SCALAR average since example example current current current loss last counter weight label predict features -0.126977 0.126977 1 1.0 1.0000 0.6556 6 -0.304709 0.482441 2 2.0 -1.0000 0.4242 5 -0.412701 0.520693 4 4.0 -1.0000 0.4818 6 +0.459158 0.459158 1 1.0 1.0000 0.5799 6 +0.245398 0.031637 2 2.0 -1.0000 0.3053 5 +0.273194 0.300991 4 4.0 -1.0000 0.3391 6 finished run number of examples = 4 weighted example sum = 4.000000 weighted label sum = 0.000000 -average loss = 0.412701 +average loss = 0.273194 best constant = 0.000000 best constant's loss = 1.000000 total feature number = 23 diff --git a/test/train-sets/ref/lrqfa_nn_pred_rev.stderr b/test/train-sets/ref/lrqfa_nn_pred_rev.stderr index 2d7476d421c..441b150279f 100644 --- a/test/train-sets/ref/lrqfa_nn_pred_rev.stderr +++ b/test/train-sets/ref/lrqfa_nn_pred_rev.stderr @@ -12,15 +12,15 @@ Input label = SIMPLE Output pred = SCALAR average since example example current current current loss last counter weight label predict features -0.859992 0.859992 1 1.0 -1.0000 0.4818 6 -0.520693 0.181394 2 2.0 1.0000 0.6397 6 -0.412701 0.304709 4 4.0 1.0000 0.6556 6 +0.110677 0.110677 1 1.0 -1.0000 0.3391 6 +0.300991 0.491306 2 2.0 1.0000 0.5742 6 +0.273194 0.245398 4 4.0 1.0000 0.5799 6 finished run number of examples = 4 weighted example sum = 4.000000 weighted label sum = 0.000000 -average loss = 0.412701 +average loss = 0.273194 best constant = 0.000000 best constant's loss = 1.000000 total feature number = 23 diff --git a/test/train-sets/ref/lrqfa_nn_train.stderr b/test/train-sets/ref/lrqfa_nn_train.stderr index 87345afd083..73f194101af 100644 --- a/test/train-sets/ref/lrqfa_nn_train.stderr +++ b/test/train-sets/ref/lrqfa_nn_train.stderr @@ -12,14 +12,14 @@ Output pred = SCALAR average since example example current current current loss last counter weight label predict features 0.566986 0.566986 1 1.0 1.0000 0.5672 6 -0.945758 1.324530 2 2.0 -1.0000 0.7341 5 -0.881015 0.816272 4 4.0 -1.0000 0.7297 6 +0.976705 1.386424 2 2.0 -1.0000 0.7500 5 +0.912181 0.847658 4 4.0 -1.0000 0.7110 6 finished run number of examples = 4 weighted example sum = 4.000000 weighted label sum = 0.000000 -average loss = 0.881015 +average loss = 0.912181 best constant = 0.000000 best constant's loss = 0.693147 total feature number = 23 diff --git a/test/train-sets/ref/marginal_invert_hash_readable_model.txt b/test/train-sets/ref/marginal_invert_hash_readable_model.txt index 4d372dcd6d3..d4fb69550bd 100644 --- a/test/train-sets/ref/marginal_invert_hash_readable_model.txt +++ b/test/train-sets/ref/marginal_invert_hash_readable_model.txt @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:0 Max label:1 diff --git a/test/train-sets/ref/nn-1-noquiet.stderr b/test/train-sets/ref/nn-1-noquiet.stderr index e5babce1154..80f60c6a0a1 100644 --- a/test/train-sets/ref/nn-1-noquiet.stderr +++ b/test/train-sets/ref/nn-1-noquiet.stderr @@ -12,18 +12,18 @@ average since example example current current cur loss last counter weight label predict features 1.000000 1.000000 1 1.0 1.0000 0.0000 51 0.583598 0.167197 2 2.0 0.0000 0.4089 104 -0.346844 0.110089 4 4.0 0.0000 0.3112 135 -0.273247 0.199650 8 8.0 0.0000 0.2907 146 -0.261045 0.248843 16 16.0 1.0000 0.2922 24 -0.251549 0.242054 32 32.0 0.0000 0.3430 32 -0.244875 0.238201 64 64.0 0.0000 0.3535 61 -0.243892 0.242908 128 128.0 1.0000 0.5382 106 +0.313688 0.043778 4 4.0 0.0000 0.1536 135 +0.289131 0.264573 8 8.0 0.0000 0.2468 146 +0.286736 0.284341 16 16.0 1.0000 0.3274 24 +0.281296 0.275857 32 32.0 0.0000 0.4074 32 +0.271335 0.261373 64 64.0 0.0000 0.3886 61 +0.260317 0.249299 128 128.0 1.0000 0.6949 106 finished run number of examples = 200 weighted example sum = 200.000000 weighted label sum = 91.000000 -average loss = 0.230748 +average loss = 0.220733 best constant = 0.455000 best constant's loss = 0.247975 total feature number = 15482 diff --git a/test/train-sets/ref/oaa_probabilities.stderr b/test/train-sets/ref/oaa_probabilities.stderr index 0be53e20da7..96efc040f54 100644 --- a/test/train-sets/ref/oaa_probabilities.stderr +++ b/test/train-sets/ref/oaa_probabilities.stderr @@ -13,8 +13,8 @@ average since example example current current cur loss last counter weight label predict features 0.000000 0.000000 1 1.0 1 1(25%) 51 0.500000 1.000000 2 2.0 2 1(31%) 104 -0.500000 0.500000 4 4.0 2 2(27%) 135 -0.375000 0.250000 8 8.0 2 2(55%) 146 +0.500000 0.500000 4 4.0 2 2(26%) 135 +0.375000 0.250000 8 8.0 2 2(54%) 146 0.500000 0.625000 16 16.0 1 2(38%) 24 0.500000 0.500000 32 32.0 2 2(35%) 32 0.578125 0.656250 64 64.0 2 2(62%) 61 @@ -25,5 +25,5 @@ number of examples = 200 weighted example sum = 200.000000 weighted label sum = 0.000000 average loss = 0.450000 -average multiclass log loss = 1.161186 +average multiclass log loss = 1.161130 total feature number = 15482 diff --git a/test/train-sets/ref/pistol.readable b/test/train-sets/ref/pistol.readable index c6027a344a0..d33f2e72629 100644 --- a/test/train-sets/ref/pistol.readable +++ b/test/train-sets/ref/pistol.readable @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:0 Max label:2 diff --git a/test/train-sets/ref/plt_predict.stderr b/test/train-sets/ref/plt_predict.stderr index d3dea9b4016..aa105fe6728 100644 --- a/test/train-sets/ref/plt_predict.stderr +++ b/test/train-sets/ref/plt_predict.stderr @@ -15,16 +15,16 @@ Input label = MULTILABEL Output pred = MULTILABELS average since example example current current current loss last counter weight label predict features -0.000000 0.000000 1 1.0 0,1 1 2 -0.000000 0.000000 2 2.0 1,2 2,1 2 -0.000000 0.000000 4 4.0 3,4 4,3 2 -0.000000 0.000000 8 8.0 8 8 2 +1.655385 1.655385 1 1.0 0,1 1 2 +2.160409 2.665433 2 2.0 1,2 2,1 2 +1.881014 1.601619 4 4.0 3,4 4,3 2 +1.582159 1.283305 8 8.0 8 8 2 finished run number of examples = 10 weighted example sum = 10.000000 weighted label sum = 0.000000 -average loss = 0.000000 +average loss = 1.375444 total feature number = 20 hamming loss = 0.200000 micro-precision = 1.000000 diff --git a/test/train-sets/ref/plt_predict_probabilities.stderr b/test/train-sets/ref/plt_predict_probabilities.stderr index 99042112722..b9b92409720 100644 --- a/test/train-sets/ref/plt_predict_probabilities.stderr +++ b/test/train-sets/ref/plt_predict_probabilities.stderr @@ -15,16 +15,16 @@ Input label = MULTILABEL Output pred = ACTION_PROBS average since example example current current current loss last counter weight label predict features -0.000000 0.000000 1 1.0 0,1 1 2 -0.000000 0.000000 2 2.0 1,2 2,1 2 -0.000000 0.000000 4 4.0 3,4 4,3 2 -0.000000 0.000000 8 8.0 8 8 2 +1.655385 1.655385 1 1.0 0,1 1 2 +2.160409 2.665433 2 2.0 1,2 2,1 2 +1.881014 1.601619 4 4.0 3,4 4,3 2 +1.582159 1.283305 8 8.0 8 8 2 finished run number of examples = 10 weighted example sum = 10.000000 weighted label sum = 0.000000 -average loss = 0.000000 +average loss = 1.375444 total feature number = 20 hamming loss = 0.200000 micro-precision = 1.000000 diff --git a/test/train-sets/ref/plt_sgd_predict.stderr b/test/train-sets/ref/plt_sgd_predict.stderr index d6451175cd0..6947e934701 100644 --- a/test/train-sets/ref/plt_sgd_predict.stderr +++ b/test/train-sets/ref/plt_sgd_predict.stderr @@ -15,16 +15,16 @@ Input label = MULTILABEL Output pred = MULTILABELS average since example example current current current loss last counter weight label predict features -0.000000 0.000000 1 1.0 0,1 0,1 2 -0.000000 0.000000 2 2.0 1,2 2,1,8 2 -0.000000 0.000000 4 4.0 3,4 8 2 -0.000000 0.000000 8 8.0 8 1,8 2 +0.249245 0.249245 1 1.0 0,1 0,1 2 +2.069833 3.890422 2 2.0 1,2 2,1,8 2 +3.383714 4.697595 4 4.0 3,4 8 2 +3.461616 3.539518 8 8.0 8 1,8 2 finished run number of examples = 10 weighted example sum = 10.000000 weighted label sum = 0.000000 -average loss = 0.000000 +average loss = 2.905217 total feature number = 20 hamming loss = 1.700000 micro-precision = 0.562500 diff --git a/test/train-sets/ref/plt_sgd_top1_predict.stderr b/test/train-sets/ref/plt_sgd_top1_predict.stderr index e3f9e7673fa..63af2af8111 100644 --- a/test/train-sets/ref/plt_sgd_top1_predict.stderr +++ b/test/train-sets/ref/plt_sgd_top1_predict.stderr @@ -15,16 +15,16 @@ Input label = MULTILABEL Output pred = MULTILABELS average since example example current current current loss last counter weight label predict features -0.000000 0.000000 1 1.0 0,1 1 2 -0.000000 0.000000 2 2.0 1,2 2 2 -0.000000 0.000000 4 4.0 3,4 8 2 -0.000000 0.000000 8 8.0 8 8 2 +0.249245 0.249245 1 1.0 0,1 1 2 +2.069833 3.890422 2 2.0 1,2 2 2 +3.383714 4.697595 4 4.0 3,4 8 2 +3.461616 3.539518 8 8.0 8 8 2 finished run number of examples = 10 weighted example sum = 10.000000 weighted label sum = 0.000000 -average loss = 0.000000 +average loss = 2.905217 total feature number = 20 p@1 = 0.600000 r@1 = 0.450000 diff --git a/test/train-sets/ref/plt_top1_predict.stderr b/test/train-sets/ref/plt_top1_predict.stderr index 88099c5566b..c15396706a4 100644 --- a/test/train-sets/ref/plt_top1_predict.stderr +++ b/test/train-sets/ref/plt_top1_predict.stderr @@ -15,16 +15,16 @@ Input label = MULTILABEL Output pred = MULTILABELS average since example example current current current loss last counter weight label predict features -0.000000 0.000000 1 1.0 0,1 1 2 -0.000000 0.000000 2 2.0 1,2 1 2 -0.000000 0.000000 4 4.0 3,4 3 2 -0.000000 0.000000 8 8.0 8 8 2 +1.655385 1.655385 1 1.0 0,1 1 2 +2.160409 2.665433 2 2.0 1,2 1 2 +1.881014 1.601619 4 4.0 3,4 3 2 +1.582159 1.283305 8 8.0 8 8 2 finished run number of examples = 10 weighted example sum = 10.000000 weighted label sum = 0.000000 -average loss = 0.000000 +average loss = 1.375444 total feature number = 20 p@1 = 1.000000 r@1 = 0.625000 diff --git a/test/train-sets/ref/plt_top1_predict_probabilities.stderr b/test/train-sets/ref/plt_top1_predict_probabilities.stderr index a8a7b8425b3..fbd5fc04b16 100644 --- a/test/train-sets/ref/plt_top1_predict_probabilities.stderr +++ b/test/train-sets/ref/plt_top1_predict_probabilities.stderr @@ -15,16 +15,16 @@ Input label = MULTILABEL Output pred = ACTION_PROBS average since example example current current current loss last counter weight label predict features -0.000000 0.000000 1 1.0 0,1 1 2 -0.000000 0.000000 2 2.0 1,2 1 2 -0.000000 0.000000 4 4.0 3,4 3 2 -0.000000 0.000000 8 8.0 8 8 2 +1.655385 1.655385 1 1.0 0,1 1 2 +2.160409 2.665433 2 2.0 1,2 1 2 +1.881014 1.601619 4 4.0 3,4 3 2 +1.582159 1.283305 8 8.0 8 8 2 finished run number of examples = 10 weighted example sum = 10.000000 weighted label sum = 0.000000 -average loss = 0.000000 +average loss = 1.375444 total feature number = 20 p@1 = 1.000000 r@1 = 0.625000 diff --git a/test/train-sets/ref/rcv1_multiclass_legacy.stderr b/test/train-sets/ref/rcv1_multiclass_legacy.stderr index 61216c59ca1..a8dd9a1d505 100644 --- a/test/train-sets/ref/rcv1_multiclass_legacy.stderr +++ b/test/train-sets/ref/rcv1_multiclass_legacy.stderr @@ -20,7 +20,7 @@ loss last counter weight label predict feat 0.265625 0.250000 64 64.0 1 1 54 0.296875 0.328125 128 128.0 2 2 67 0.277344 0.257812 256 256.0 1 1 86 -0.255859 0.234375 512 512.0 2 2 104 +0.253906 0.230469 512 512.0 2 2 104 finished run number of examples = 1000 diff --git a/test/train-sets/ref/rcv1_raw_cb_dm.stderr b/test/train-sets/ref/rcv1_raw_cb_dm.stderr index 9d830ba6131..12c4f542e22 100644 --- a/test/train-sets/ref/rcv1_raw_cb_dm.stderr +++ b/test/train-sets/ref/rcv1_raw_cb_dm.stderr @@ -16,16 +16,16 @@ loss last counter weight label predict feat 1.000000 0.000000 2 2.0 1:0:0.5 2 598 0.500000 0.000000 4 4.0 1:0:0.5 2 784 0.250000 0.000000 8 8.0 1:0:0.5 2 850 -0.620300 0.990600 16 16.0 1:1:0.5 2 118 -0.367739 0.115178 32 32.0 1:0:0.5 1 166 -0.355549 0.343359 64 64.0 2:1:0.5 1 340 -0.270033 0.184517 128 128.0 1:1:0.5 2 610 -0.214834 0.159635 256 256.0 2:0:0.5 2 712 -0.193233 0.171632 512 512.0 2:1:0.5 1 424 +0.620319 0.990638 16 16.0 1:1:0.5 2 118 +0.367899 0.115479 32 32.0 1:0:0.5 1 166 +0.355680 0.343460 64 64.0 2:1:0.5 1 340 +0.268485 0.181290 128 128.0 1:1:0.5 2 610 +0.208617 0.148750 256 256.0 2:0:0.5 2 712 +0.190153 0.171689 512 512.0 2:1:0.5 1 424 finished run number of examples = 1000 weighted example sum = 1000.000000 weighted label sum = 0.000000 -average loss = 0.140738 +average loss = 0.141290 total feature number = 440020 diff --git a/test/train-sets/ref/rcv1_raw_cb_ips.stderr b/test/train-sets/ref/rcv1_raw_cb_ips.stderr index 197991e8833..154c98f45ca 100644 --- a/test/train-sets/ref/rcv1_raw_cb_ips.stderr +++ b/test/train-sets/ref/rcv1_raw_cb_ips.stderr @@ -19,12 +19,12 @@ loss last counter weight label predict feat 0.312500 0.125000 32 32.0 1:0:0.5 1 166 0.343750 0.375000 64 64.0 2:1:0.5 1 340 0.312500 0.281250 128 128.0 1:1:0.5 1 610 -0.257812 0.203125 256 256.0 2:0:0.5 2 712 -0.218750 0.179688 512 512.0 2:1:0.5 1 424 +0.265625 0.218750 256 256.0 2:0:0.5 2 712 +0.222656 0.179688 512 512.0 2:1:0.5 1 424 finished run number of examples = 1000 weighted example sum = 1000.000000 weighted label sum = 0.000000 -average loss = 0.166000 +average loss = 0.168000 total feature number = 440020 diff --git a/test/train-sets/ref/rcv1_raw_new_cb_dr.stderr b/test/train-sets/ref/rcv1_raw_new_cb_dr.stderr index 9c90623edc4..72bd01a055c 100644 --- a/test/train-sets/ref/rcv1_raw_new_cb_dr.stderr +++ b/test/train-sets/ref/rcv1_raw_new_cb_dr.stderr @@ -16,15 +16,15 @@ loss last counter weight label predict feat 0.500000 0.000000 4 4.0 0:0:0.5 1:0 1568 0.250000 0.000000 8 8.0 0:0:0.5 1:0 1700 0.620199 0.990398 16 16.0 0:1:0.5 1:0.07 236 -0.326163 0.032127 32 32.0 0:0:0.5 0:0.07 332 -0.310652 0.295141 64 64.0 1:1:0.5 0:0.14 680 -0.249250 0.187847 128 128.0 0:1:0.5 1:0.32 1220 -0.214749 0.180249 256 256.0 1:0:0.5 1:0.6 1424 -0.189458 0.164166 512 512.0 1:1:0.5 0:0.29 848 +0.326201 0.032204 32 32.0 0:0:0.5 0:0.07 332 +0.310692 0.295182 64 64.0 1:1:0.5 0:0.14 680 +0.249281 0.187870 128 128.0 0:1:0.5 1:0.32 1220 +0.214765 0.180249 256 256.0 1:0:0.5 1:0.6 1424 +0.189467 0.164169 512 512.0 1:1:0.5 0:0.29 848 finished run number of examples = 1000 weighted example sum = 1000.000000 weighted label sum = 0.000000 -average loss = 0.132501 +average loss = 0.132504 total feature number = 880040 diff --git a/test/train-sets/ref/w_out_slot_ns.interactions b/test/train-sets/ref/w_out_slot_ns.interactions index 33b3bed749f..3ff01b3108e 100644 --- a/test/train-sets/ref/w_out_slot_ns.interactions +++ b/test/train-sets/ref/w_out_slot_ns.interactions @@ -1,4 +1,4 @@ -Version 9.7.0 +Version 9.8.0 Id Min label:-1 Max label:0 diff --git a/test/train-sets/ref/wiki1K.stderr b/test/train-sets/ref/wiki1K.stderr index fb069cbb027..a756fda6d2c 100644 --- a/test/train-sets/ref/wiki1K.stderr +++ b/test/train-sets/ref/wiki1K.stderr @@ -11,11 +11,11 @@ Output pred = NOPRED average since example example current current current loss last counter weight label predict features 10.40360 10.40360 128 128.0 none 0 732 -9.956976 9.510344 256 256.0 none 0 102 +9.956974 9.510342 256 256.0 none 0 102 finished run number of examples = 256 weighted example sum = 256.000000 weighted label sum = 0.000000 -average loss = 9.956976 +average loss = 9.956974 total feature number = 22158 diff --git a/test/train-sets/same_model_test.1.dat b/test/train-sets/same_model_test.1.dat index e60217589d1..fc949b57a46 100644 --- a/test/train-sets/same_model_test.1.dat +++ b/test/train-sets/same_model_test.1.dat @@ -1,2 +1,2 @@ -1.0 |a short |b good --1.0 |a median +1.0 |a short small |b good +-1.0 |a median |b mode diff --git a/utl/flatbuffer/vw_to_flat.cc b/utl/flatbuffer/vw_to_flat.cc index 965378de5b2..39f5903bcf9 100644 --- a/utl/flatbuffer/vw_to_flat.cc +++ b/utl/flatbuffer/vw_to_flat.cc @@ -418,7 +418,7 @@ void to_flat::convert_txt_to_flat(VW::workspace& all) break; } - uint64_t multiplier = (uint64_t)all.reduction_state.wpp << all.weights.stride_shift(); + uint64_t multiplier = (uint64_t)all.reduction_state.total_feature_width << all.weights.stride_shift(); if (multiplier != 1) { for (VW::features& fs : *ae) diff --git a/utl/version_number.py b/utl/version_number.py index 87133da738d..a89805c2278 100755 --- a/utl/version_number.py +++ b/utl/version_number.py @@ -27,6 +27,10 @@ def debug_print(msg): ) m = r.match(git_describe) +if m is None: + debug_print("Error: 'git describe' output did not match regex!") + sys.exit(1) + major = m.group("major") minor = m.group("minor") patch = m.group("patch") diff --git a/vcpkg.json b/vcpkg.json index 895d1be2e87..d2acf137cab 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,7 +1,7 @@ { "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg/master/scripts/vcpkg.schema.json", "name": "vowpal-wabbit", - "version": "9.7.0", + "version": "9.8.0", "dependencies": [ "boost-math", "eigen3", diff --git a/version.txt b/version.txt index a458a24cc21..834eb3fa85b 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -9.7.0 +9.8.0 diff --git a/vowpalwabbit/core/CMakeLists.txt b/vowpalwabbit/core/CMakeLists.txt index 1822ca3e3e0..ef0b59f3d91 100644 --- a/vowpalwabbit/core/CMakeLists.txt +++ b/vowpalwabbit/core/CMakeLists.txt @@ -446,6 +446,7 @@ vw_add_test_executable( tests/eigen_memory_tree_test.cc tests/epsilon_decay_test.cc tests/epsilon_test.cc + tests/example_ft_hash_test.cc tests/example_header_test.cc tests/example_test.cc tests/feature_group_test.cc diff --git a/vowpalwabbit/core/include/vw/core/automl_impl.h b/vowpalwabbit/core/include/vw/core/automl_impl.h index 1e6a1963adc..4a446668609 100644 --- a/vowpalwabbit/core/include/vw/core/automl_impl.h +++ b/vowpalwabbit/core/include/vw/core/automl_impl.h @@ -211,22 +211,13 @@ class interaction_config_manager dense_parameters& weights; double automl_significance_level; VW::io::logger* logger; - uint32_t& wpp; + uint32_t& feature_width; const bool _ccb_on; config_oracle_impl _config_oracle; bool reward_as_cost; double tol_x; bool is_brentq; - // TODO: delete all this, gd and cb_adf must respect ft_offset, see header import of automl.cc - std::vector per_live_model_state_double; - std::vector per_live_model_state_uint64; - double* _gd_normalized = nullptr; - double* _gd_total_weight = nullptr; - double* _sd_gravity = nullptr; - uint64_t* _cb_adf_event_sum = nullptr; - uint64_t* _cb_adf_action_sum = nullptr; - // Stores all namespaces currently seen std::map ns_counter; @@ -240,8 +231,8 @@ class interaction_config_manager interaction_config_manager(uint64_t global_lease, uint64_t max_live_configs, std::shared_ptr rand_state, uint64_t priority_challengers, const std::string& interaction_type, const std::string& oracle_type, dense_parameters& weights, priority_func calc_priority, - double automl_significance_level, VW::io::logger* logger, uint32_t& wpp, bool ccb_on, config_type conf_type, - std::string trace_prefix, bool reward_as_cost, double tol_x, bool is_brentq); + double automl_significance_level, VW::io::logger* logger, uint32_t& feature_width, bool ccb_on, + config_type conf_type, std::string trace_prefix, bool reward_as_cost, double tol_x, bool is_brentq); void do_learning(VW::LEARNER::learner& base, multi_ex& ec, uint64_t live_slot); void persist(metric_sink& metrics, bool verbose); @@ -266,7 +257,7 @@ class interaction_config_manager bool count_namespaces(const multi_ex& ecs, std::map& ns_counter); void apply_config(example* ec, interaction_vec_t* live_interactions); bool is_allowed_to_remove(const namespace_index ns); -void clear_non_champ_weights(dense_parameters& weights, uint32_t total, uint32_t& wpp); +void clear_non_champ_weights(dense_parameters& weights, uint32_t total, uint32_t& feature_width); bool worse(); // all possible states of automl diff --git a/vowpalwabbit/core/include/vw/core/cached_learner.h b/vowpalwabbit/core/include/vw/core/cached_learner.h index 2992fd04b01..361c9fb8a17 100644 --- a/vowpalwabbit/core/include/vw/core/cached_learner.h +++ b/vowpalwabbit/core/include/vw/core/cached_learner.h @@ -11,7 +11,7 @@ namespace VW class cached_learner : public setup_base_i { public: - std::shared_ptr setup_base_learner() override { return _cached; } + std::shared_ptr setup_base_learner(size_t) override { return _cached; } operator bool() const { return !(_cached == nullptr); } @@ -37,6 +37,8 @@ class cached_learner : public setup_base_i std::string get_setupfn_name(reduction_setup_fn) override { return ""; } + size_t get_feature_width_above() override { return 1; } + private: std::shared_ptr _cached; VW::config::options_i* _options_impl = nullptr; diff --git a/vowpalwabbit/core/include/vw/core/example.h b/vowpalwabbit/core/include/vw/core/example.h index 99f7e8bb0cd..c13a8975f3f 100644 --- a/vowpalwabbit/core/include/vw/core/example.h +++ b/vowpalwabbit/core/include/vw/core/example.h @@ -143,26 +143,8 @@ class example : public example_predict // core example datatype. class workspace; -class flat_example -{ -public: - polylabel l; - reduction_features ex_reduction_features; - - VW::v_array tag; // An identifier for the example. - - size_t example_counter; - uint64_t ft_offset; - float global_weight; - - size_t num_features; // precomputed, cause it's fast&easy. - float total_sum_feat_sq; // precomputed, cause it's kind of fast & easy. - features fs; // all the features -}; - -flat_example* flatten_example(VW::workspace& all, example* ec); -flat_example* flatten_sort_example(VW::workspace& all, example* ec); -void free_flatten_example(flat_example* fec); +// TODO: make workspace and example const +void flatten_features(VW::workspace& all, example& ec, features& fs); inline bool example_is_newline(const example& ec) { return ec.is_newline; } @@ -194,13 +176,6 @@ void truncate_example_namespace(VW::example& ec, VW::namespace_index ns, const f void append_example_namespaces_from_example(VW::example& target, const VW::example& source); void truncate_example_namespaces_from_example(VW::example& target, const VW::example& source); } // namespace details - -namespace model_utils -{ -size_t read_model_field(io_buf& io, flat_example& fe, VW::label_parser& lbl_parser); -size_t write_model_field(io_buf& io, const flat_example& fe, const std::string& upstream_name, bool text, - VW::label_parser& lbl_parser, uint64_t parse_mask); -} // namespace model_utils } // namespace VW // Deprecated compat definitions @@ -209,18 +184,6 @@ using polylabel VW_DEPRECATED("polylabel moved into VW namespace") = VW::polylab using polyprediction VW_DEPRECATED("polyprediction moved into VW namespace") = VW::polyprediction; using example VW_DEPRECATED("example moved into VW namespace") = VW::example; using multi_ex VW_DEPRECATED("multi_ex moved into VW namespace") = VW::multi_ex; -using flat_example VW_DEPRECATED("flat_example moved into VW namespace") = VW::flat_example; - -VW_DEPRECATED("flatten_example moved into VW namespace") -inline VW::flat_example* flatten_example(VW::workspace& all, VW::example* ec) { return VW::flatten_example(all, ec); } - -VW_DEPRECATED("flatten_sort_example moved into VW namespace") -inline VW::flat_example* flatten_sort_example(VW::workspace& all, VW::example* ec) -{ - return VW::flatten_sort_example(all, ec); -} -VW_DEPRECATED("free_flatten_example moved into VW namespace") -inline void free_flatten_example(VW::flat_example* fec) { return VW::free_flatten_example(fec); } VW_DEPRECATED("example_is_newline moved into VW namespace") inline bool example_is_newline(const VW::example& ec) { return VW::example_is_newline(ec); } diff --git a/vowpalwabbit/core/include/vw/core/example_predict.h b/vowpalwabbit/core/include/vw/core/example_predict.h index 2fa598fed8c..8294ebf7ea6 100644 --- a/vowpalwabbit/core/include/vw/core/example_predict.h +++ b/vowpalwabbit/core/include/vw/core/example_predict.h @@ -42,6 +42,10 @@ class example_predict example_predict(example_predict&& other) = default; example_predict& operator=(example_predict&& other) = default; + // this hashing function does not take into account the order of the features + // an example with the exact same namespaces / features-values but in a different order will have the same hash + uint64_t get_or_calculate_order_independent_feature_space_hash(); + /// If indices is modified this iterator is invalidated. iterator begin(); /// If indices is modified this iterator is invalidated. @@ -50,6 +54,8 @@ class example_predict VW::v_array indices; std::array feature_space; // Groups of feature values. uint64_t ft_offset = 0; // An offset for all feature values. + uint64_t feature_space_hash = 0; // A unique hash of the feature space and namespaces of the example. + bool is_set_feature_space_hash = false; // Interactions are specified by this struct's interactions vector of vectors of unsigned characters, where each // vector is an interaction and each char is a namespace. diff --git a/vowpalwabbit/core/include/vw/core/feature_group.h b/vowpalwabbit/core/include/vw/core/feature_group.h index f19e7ea4a80..569c54ca482 100644 --- a/vowpalwabbit/core/include/vw/core/feature_group.h +++ b/vowpalwabbit/core/include/vw/core/feature_group.h @@ -510,6 +510,10 @@ class features return all_extents_complete; } }; + +/// Both fs1 and fs2 must be sorted. +/// Most often used with VW::flatten_features +float features_dot_product(const features& fs1, const features& fs2); } // namespace VW using feature_value VW_DEPRECATED("Moved into VW namespace. Will be removed in VW 10.") = VW::feature_value; diff --git a/vowpalwabbit/core/include/vw/core/gen_cs_example.h b/vowpalwabbit/core/include/vw/core/gen_cs_example.h index 2b03225eceb..57eb0ad3391 100644 --- a/vowpalwabbit/core/include/vw/core/gen_cs_example.h +++ b/vowpalwabbit/core/include/vw/core/gen_cs_example.h @@ -12,6 +12,12 @@ namespace VW { namespace details { +class cb_to_cs_per_model_state +{ +public: + uint64_t action_sum = 0; + uint64_t event_sum = 0; +}; class cb_to_cs { @@ -28,18 +34,18 @@ class cb_to_cs VW::cb_class known_cost; }; -class cb_to_cs_adf +class cb_to_cs_adf_mtr { public: - VW::cb_type_t cb_type = VW::cb_type_t::DM; - - // for MTR - uint64_t action_sum = 0; - uint64_t event_sum = 0; + cb_to_cs_adf_mtr(size_t feature_width = 1) : per_model_state(feature_width) {} + std::vector per_model_state; uint32_t mtr_example = 0; VW::multi_ex mtr_ec_seq; // shared + the one example. +}; - // for DR +class cb_to_cs_adf_dr +{ +public: VW::cs_label pred_scores; VW::cb_class known_cost; VW::LEARNER::learner* scorer = nullptr; @@ -198,13 +204,13 @@ void gen_cs_example_ips( void gen_cs_example_dm(const VW::multi_ex& examples, VW::cs_label& cs_labels); -void gen_cs_example_mtr(cb_to_cs_adf& c, VW::multi_ex& ec_seq, VW::cs_label& cs_labels); +void gen_cs_example_mtr(cb_to_cs_adf_mtr& c, VW::multi_ex& ec_seq, VW::cs_label& cs_labels, uint64_t offset_index); void gen_cs_example_sm(VW::multi_ex& examples, uint32_t chosen_action, float sign_offset, const VW::action_scores& action_vals, VW::cs_label& cs_labels); template -void gen_cs_example_dr(cb_to_cs_adf& c, VW::multi_ex& examples, VW::cs_label& cs_labels, float clip_p = 0.f) +void gen_cs_example_dr(cb_to_cs_adf_dr& c, VW::multi_ex& examples, VW::cs_label& cs_labels, float clip_p = 0.f) { // size_t mysize = examples.size(); VW_DBG(*examples[0]) << "gen_cs_example_dr-adf:" << is_learn << std::endl; c.pred_scores.costs.clear(); @@ -239,26 +245,6 @@ void gen_cs_example_dr(cb_to_cs_adf& c, VW::multi_ex& examples, VW::cs_label& cs } } -template -void gen_cs_example(cb_to_cs_adf& c, VW::multi_ex& ec_seq, VW::cs_label& cs_labels, VW::io::logger& logger) -{ - VW_DBG(*ec_seq[0]) << "gen_cs_example:" << is_learn << std::endl; - switch (c.cb_type) - { - case VW::cb_type_t::IPS: - gen_cs_example_ips(ec_seq, cs_labels, logger); - break; - case VW::cb_type_t::DR: - gen_cs_example_dr(c, ec_seq, cs_labels); - break; - case VW::cb_type_t::MTR: - gen_cs_example_mtr(c, ec_seq, cs_labels); - break; - default: - THROW("Unknown cb_type specified for contextual bandit learning: " << VW::to_string(c.cb_type)); - } -} - void cs_prep_labels(VW::multi_ex& examples, std::vector& cb_labels, VW::cs_label& cs_labels, std::vector& prepped_cs_labels, uint64_t offset); diff --git a/vowpalwabbit/core/include/vw/core/global_data.h b/vowpalwabbit/core/include/vw/core/global_data.h index 3710e1187e6..c66792ca661 100644 --- a/vowpalwabbit/core/include/vw/core/global_data.h +++ b/vowpalwabbit/core/include/vw/core/global_data.h @@ -211,7 +211,7 @@ class reduction_state bool is_ccb_input_model = false; void* /*Search::search*/ searchstr; bool invariant_updates; // Should we use importance aware/safe updates, gd only - uint32_t wpp; + uint32_t total_feature_width; }; class runtime_config diff --git a/vowpalwabbit/core/include/vw/core/learner.h b/vowpalwabbit/core/include/vw/core/learner.h index 02a97cef1a9..d9ea5a8b92a 100644 --- a/vowpalwabbit/core/include/vw/core/learner.h +++ b/vowpalwabbit/core/include/vw/core/learner.h @@ -96,8 +96,8 @@ using add_subtract_with_all_func = std::function void debug_log_message(polymorphic_ex ex, const std::string& msg); public: - size_t weights; // this stores the number of "weight vectors" required by the learner. - size_t increment; + size_t feature_width = 1; // This stores the number of "weight vectors" required by the learner. + size_t feature_width_below = 1; // This stores the incremental product of feature widths for this learner and below + // in the stack // If true, learn will return a prediction. The framework should // not call predict before learn @@ -143,12 +144,22 @@ class learner final : public std::enable_shared_from_this void* get_internal_type_erased_data_pointer_test_use_only() { return _learner_data.get(); } + std::shared_ptr get_internal_type_erased_data_pointer_test_use_only_shared() { return _learner_data; } + + // This is very dangerous to use. + // This will not override the data object passed to functions, just the data object returned by + // get_internal_type_erased_data_pointer_test_use_only + void set_internal_type_erased_data_pointer_does_not_override_funcs(std::shared_ptr data) + { + _learner_data = std::move(data); + } + /// \brief Will update the model according to the labels and examples supplied. /// \param ec The ::example object or ::multi_ex to be operated on. This /// object **must** have a valid label set for every ::example in the field /// example::l that corresponds to the type this learner expects. - /// \param i This is the offset used for the weights in this call. If using - /// multiple regressors/learners you can increment this value for each call. + /// \param i This is the offset used for the feature_width in this call. If using + /// multiple regressors/learners you can feature_width_below this value for each call. /// \returns While some learner may fill the example::pred, this is not /// guaranteed and is undefined behavior if accessed. void learn(polymorphic_ex ec, size_t i = 0); @@ -157,8 +168,8 @@ class learner final : public std::enable_shared_from_this /// \param ec The ::example object or ::multi_ex to be operated on. This /// object **must** have a valid prediction allocated in the field /// example::pred that corresponds to this learner type. - /// \param i This is the offset used for the weights in this call. If using - /// multiple regressors/learners you can increment this value for each call. + /// \param i This is the offset used for the feature_width in this call. If using + /// multiple regressors/learners you can feature_width_below this value for each call. /// \returns The prediction calculated by this learner be set on /// example::pred. If the polymorphic_ex is ::multi_ex then the prediction is /// set on the 0th item in the list. @@ -540,7 +551,7 @@ class reduction_learner_builder // Default sensitivity calls the base learner's sensitivity recursively this->set_sensitivity(details::recur_sensitivity); - set_params_per_weight(1); + set_feature_width(1); this->set_learn_returns_prediction(false); // By default, will produce what the base learner expects @@ -589,9 +600,9 @@ class reduction_learner_builder this->learner_ptr->_sensitivity_f = [fn_ptr, data, base](example& ex) { return fn_ptr(*data, *base, ex); }; ) - LEARNER_BUILDER_DEFINE(set_params_per_weight(size_t params_per_weight), - this->learner_ptr->weights = params_per_weight; - this->learner_ptr->increment = this->learner_ptr->_base_learner->increment * this->learner_ptr->weights; + LEARNER_BUILDER_DEFINE(set_feature_width(size_t feature_width), + this->learner_ptr->feature_width = feature_width; + this->learner_ptr->feature_width_below = this->learner_ptr->_base_learner->feature_width_below * this->learner_ptr->feature_width; ) LEARNER_BUILDER_DEFINE(set_merge(void (*fn_ptr)(const std::vector&, const std::vector&, DataT&)), @@ -651,7 +662,7 @@ class reduction_no_data_learner_builder // Default sensitivity calls base learner's sensitivity recursively this->set_sensitivity(details::recur_sensitivity); - set_params_per_weight(1); + set_feature_width(1); // By default, will produce what the base learner expects super::set_input_label_type(base->get_input_label_type()); super::set_output_label_type(base->get_input_label_type()); @@ -693,9 +704,9 @@ class reduction_no_data_learner_builder this->learner_ptr->_sensitivity_f = [fn_ptr, base](example& ex) { return fn_ptr(*base, ex); }; ) - LEARNER_BUILDER_DEFINE(set_params_per_weight(size_t params_per_weight), - this->learner_ptr->weights = params_per_weight; - this->learner_ptr->increment = this->learner_ptr->_base_learner->increment * this->learner_ptr->weights; + LEARNER_BUILDER_DEFINE(set_feature_width(size_t feature_width), + this->learner_ptr->feature_width = feature_width; + this->learner_ptr->feature_width_below = this->learner_ptr->_base_learner->feature_width_below * this->learner_ptr->feature_width; ) // clang-format on @@ -720,8 +731,6 @@ class bottom_learner_builder : public common_learner_builderlearner_data.get(); @@ -758,11 +769,6 @@ class bottom_learner_builder : public common_learner_builderlearner_ptr->_sensitivity_f = [fn_ptr, data](example& ex) { return fn_ptr(*data, ex); }; ) - LEARNER_BUILDER_DEFINE(set_params_per_weight(size_t params_per_weight), - this->learner_ptr->weights = 1; - this->learner_ptr->increment = params_per_weight; - ) - LEARNER_BUILDER_DEFINE(set_merge_with_all(void (*fn_ptr)(const std::vector&, const std::vector&, const std::vector&, VW::workspace&, DataT&)), assert(fn_ptr != nullptr); @@ -856,4 +862,4 @@ bottom_learner_builder make_no_data_bottom_learner(void (*learn_ } } // namespace LEARNER -} // namespace VW \ No newline at end of file +} // namespace VW diff --git a/vowpalwabbit/core/include/vw/core/multi_model_utils.h b/vowpalwabbit/core/include/vw/core/multi_model_utils.h index 2778d48df7a..9aa14e9b861 100644 --- a/vowpalwabbit/core/include/vw/core/multi_model_utils.h +++ b/vowpalwabbit/core/include/vw/core/multi_model_utils.h @@ -14,11 +14,12 @@ namespace reductions namespace multi_model { /* -***** NOTE: overall_ppw_size must be of form 2^n for all functions ***** +***** NOTE: total_feature_width must be of form 2^n for all functions ***** These functions are used to maniputate the weights of a multi-model learner. As an example of what each variable is, consider a learner with the command line "--bag 4 --automl 2". In this scenario 'automl' is above 'bag' in the -stack, and would be considered to have the innermost ppw. Thus innermost_ppw_size = 2. The overall_ppw_size = 8 -here as it is the product of all ppw's in the stack (4 * 2). The actual weights here will look like this: +stack, and would be considered to have the innermost feature_widths. Thus innermost_feature_width_size = 2. The +total_feature_width = 8 here as it is the product of all feature_width's in the stack (4 * 2). The actual weights here +will look like this: 0 1 2 3 4 5 6 7 -- Indices as printed using --invert_hash 0 4 8 12 16 20 24 28 -- Actual weight indices included stride size of 4 @@ -26,44 +27,44 @@ bag0 bag0 bag1 bag1 bag2 bag2 bag3 bag3 -- Shows how weights are g aml0 aml1 aml0 aml1 aml0 aml1 aml0 aml1 -- Shows how weights are grouped by automl learner number */ -// This function is used to clear the weights of a specific offset in the innermost ppw. -inline void clear_innermost_offset( - dense_parameters& weights, const size_t offset, const size_t overall_ppw_size, const size_t innermost_ppw_size) +// This function is used to clear the weights of a specific offset in the innermost feature_width. +inline void clear_innermost_offset(dense_parameters& weights, const size_t offset, const size_t total_feature_width, + const size_t innermost_feature_width_size) { VW::weight* weights_arr = weights.first(); - const size_t overall_without_innermost_ppw_size = overall_ppw_size / innermost_ppw_size; - assert(offset < innermost_ppw_size); + const size_t overall_without_innermost_feature_width_size = total_feature_width / innermost_feature_width_size; + assert(offset < innermost_feature_width_size); - for (auto iterator_clear = weights.begin(); iterator_clear < weights.end(); iterator_clear += overall_ppw_size) + for (auto iterator_clear = weights.begin(); iterator_clear < weights.end(); iterator_clear += total_feature_width) { - for (size_t outer_offset = 0; outer_offset < overall_without_innermost_ppw_size; ++outer_offset) + for (size_t outer_offset = 0; outer_offset < overall_without_innermost_feature_width_size; ++outer_offset) { for (size_t stride_offset = 0; stride_offset < weights.stride(); ++stride_offset) { - weights_arr[iterator_clear.index() + outer_offset * innermost_ppw_size * weights.stride() + + weights_arr[iterator_clear.index() + outer_offset * innermost_feature_width_size * weights.stride() + offset * weights.stride() + stride_offset] = 0.0f; } } } } -// This function is used to move the weights of a specific offset in the innermost ppw to another offset. +// This function is used to move the weights of a specific offset in the innermost feature_width to another offset. inline void move_innermost_offsets(dense_parameters& weights, const size_t from, const size_t to, - const size_t overall_ppw_size, const size_t innermost_ppw_size, bool swap = false) + const size_t total_feature_width, const size_t innermost_feature_width_size, bool swap = false) { VW::weight* weights_arr = weights.first(); - const size_t overall_without_innermost_ppw_size = overall_ppw_size / innermost_ppw_size; - assert(from < innermost_ppw_size); - assert(to < innermost_ppw_size); + const size_t overall_without_innermost_feature_width_size = total_feature_width / innermost_feature_width_size; + assert(from < innermost_feature_width_size); + assert(to < innermost_feature_width_size); - for (auto iterator_move = weights.begin(); iterator_move < weights.end(); iterator_move += overall_ppw_size) + for (auto iterator_move = weights.begin(); iterator_move < weights.end(); iterator_move += total_feature_width) { - for (size_t outer_offset = 0; outer_offset < overall_without_innermost_ppw_size; ++outer_offset) + for (size_t outer_offset = 0; outer_offset < overall_without_innermost_feature_width_size; ++outer_offset) { for (size_t stride_offset = 0; stride_offset < weights.stride(); stride_offset++) { size_t outer_index = - iterator_move.index() + outer_offset * innermost_ppw_size * weights.stride() + stride_offset; + iterator_move.index() + outer_offset * innermost_feature_width_size * weights.stride() + stride_offset; if (weights_arr[outer_index + to * weights.stride()] != weights_arr[outer_index + from * weights.stride()]) { if (swap) @@ -81,9 +82,9 @@ inline void move_innermost_offsets(dense_parameters& weights, const size_t from, } } -/* This function is used to select one sub-offset within the innermost ppw and remove all others. For instance - if this called with "--bag 4 --automl 2" using offset = 1 (we will have innermost_ppw_size = 2 and overall_ppw_size - = 8) then this will remove all weights with automl = 0. The set of weights referenced above: +/* This function is used to select one sub-offset within the innermost feature_width and remove all others. For instance + if this called with "--bag 4 --automl 2" using offset = 1 (we will have innermost_feature_width_size = 2 and + total_feature_width = 8) then this will remove all weights with automl = 0. The set of weights referenced above: 0 1 2 3 4 5 6 7 0 4 8 12 16 20 24 28 @@ -97,23 +98,26 @@ inline void move_innermost_offsets(dense_parameters& weights, const size_t from, bag0 bag1 bag2 bag3 aml1 aml1 aml1 aml1 */ -inline void reduce_innermost_model_weights( - dense_parameters& weights, const size_t offset, const size_t overall_ppw_size, const size_t innermost_ppw_size) +inline void reduce_innermost_model_weights(dense_parameters& weights, const size_t offset, + const size_t total_feature_width, const size_t innermost_feature_width_size) { VW::weight* weights_arr = weights.first(); - const size_t overall_without_innermost_ppw_size = overall_ppw_size / innermost_ppw_size; - for (size_t inner_ppw = 0; inner_ppw < innermost_ppw_size; ++inner_ppw) + const size_t overall_without_innermost_feature_width_size = total_feature_width / innermost_feature_width_size; + for (size_t inner_feature_widths = 0; inner_feature_widths < innermost_feature_width_size; ++inner_feature_widths) { - if (inner_ppw != offset) { clear_innermost_offset(weights, inner_ppw, overall_ppw_size, innermost_ppw_size); } + if (inner_feature_widths != offset) + { + clear_innermost_offset(weights, inner_feature_widths, total_feature_width, innermost_feature_width_size); + } } - for (auto weights_it = weights.begin(); weights_it < weights.end(); weights_it += overall_ppw_size) + for (auto weights_it = weights.begin(); weights_it < weights.end(); weights_it += total_feature_width) { - uint32_t cb_ind = weights_it.index() / innermost_ppw_size; - for (size_t outer_offset = 0; outer_offset < overall_without_innermost_ppw_size; ++outer_offset) + uint32_t cb_ind = weights_it.index() / innermost_feature_width_size; + for (size_t outer_offset = 0; outer_offset < overall_without_innermost_feature_width_size; ++outer_offset) { for (size_t stride_offset = 0; stride_offset < weights.stride(); ++stride_offset) { - auto old_ind = weights_it.index() + outer_offset * innermost_ppw_size * weights.stride() + + auto old_ind = weights_it.index() + outer_offset * innermost_feature_width_size * weights.stride() + offset * weights.stride() + stride_offset; auto new_ind = cb_ind + outer_offset * weights.stride() + stride_offset; if (weights_arr[old_ind] != 0.0f) diff --git a/vowpalwabbit/core/include/vw/core/parse_example_json.h b/vowpalwabbit/core/include/vw/core/parse_example_json.h index 8155760e90a..a5003c9662c 100644 --- a/vowpalwabbit/core/include/vw/core/parse_example_json.h +++ b/vowpalwabbit/core/include/vw/core/parse_example_json.h @@ -15,7 +15,7 @@ void read_line_json_s(const VW::label_parser& lbl_parser, hash_func_t hash_func, uint64_t parse_mask, bool chain_hash, VW::label_parser_reuse_mem* reuse_mem, const VW::named_labels* ldict, VW::multi_ex& examples, char* line, size_t length, example_factory_t example_factory, VW::io::logger& logger, std::unordered_map>* ignore_features, - std::unordered_map* dedup_examples = nullptr) + const std::unordered_map* dedup_examples = nullptr) { VW::parsers::json::read_line_json(lbl_parser, hash_func, hash_seed, parse_mask, chain_hash, reuse_mem, ldict, examples, line, length, std::move(example_factory), logger, ignore_features, dedup_examples); @@ -24,7 +24,7 @@ void read_line_json_s(const VW::label_parser& lbl_parser, hash_func_t hash_func, template VW_DEPRECATED("read_line_json_s moved to VW::parsers::json::read_line_json") void read_line_json_s(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, - example_factory_t example_factory, std::unordered_map* dedup_examples = nullptr) + example_factory_t example_factory, const std::unordered_map* dedup_examples = nullptr) { VW::parsers::json::read_line_json(all, examples, line, length, std::move(example_factory), dedup_examples); } @@ -67,17 +67,17 @@ extern template void VW::read_line_json_s(const VW::label_parser& lbl_pars uint64_t hash_seed, uint64_t parse_mask, bool chain_hash, VW::label_parser_reuse_mem* reuse_mem, const VW::named_labels* ldict, VW::multi_ex& examples, char* line, size_t length, example_factory_t example_factory, VW::io::logger& logger, std::unordered_map>* ignore_features, - std::unordered_map* dedup_examples); + const std::unordered_map* dedup_examples); extern template void VW::read_line_json_s(const VW::label_parser& lbl_parser, hash_func_t hash_func, uint64_t hash_seed, uint64_t parse_mask, bool chain_hash, VW::label_parser_reuse_mem* reuse_mem, const VW::named_labels* ldict, VW::multi_ex& examples, char* line, size_t length, example_factory_t example_factory, VW::io::logger& logger, std::unordered_map>* ignore_features, - std::unordered_map* dedup_examples); + const std::unordered_map* dedup_examples); extern template void VW::read_line_json_s(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, - example_factory_t example_factory, std::unordered_map* dedup_examples); + example_factory_t example_factory, const std::unordered_map* dedup_examples); extern template void VW::read_line_json_s(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, - example_factory_t example_factory, std::unordered_map* dedup_examples); + example_factory_t example_factory, const std::unordered_map* dedup_examples); extern template bool VW::read_line_decision_service_json(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, bool copy_line, example_factory_t example_factory, diff --git a/vowpalwabbit/core/include/vw/core/parse_slates_example_json.h b/vowpalwabbit/core/include/vw/core/parse_slates_example_json.h index fcc3da39d4a..917d0bbb9f5 100644 --- a/vowpalwabbit/core/include/vw/core/parse_slates_example_json.h +++ b/vowpalwabbit/core/include/vw/core/parse_slates_example_json.h @@ -7,7 +7,7 @@ template VW_DEPRECATED("parse_slates_example_json moved to VW::parsers::json::details::parse_slates_example_json") void parse_slates_example_json(const VW::label_parser& lbl_parser, VW::hash_func_t hash_func, uint64_t hash_seed, uint64_t parse_mask, bool chain_hash, VW::multi_ex& examples, char* line, size_t length, - VW::example_factory_t example_factory, std::unordered_map* dedup_examples = nullptr) + VW::example_factory_t example_factory, const std::unordered_map* dedup_examples = nullptr) { VW::parsers::json::details::parse_slates_example_json(lbl_parser, hash_func, hash_seed, parse_mask, chain_hash, examples, line, length, std::move(example_factory), dedup_examples); @@ -16,7 +16,7 @@ void parse_slates_example_json(const VW::label_parser& lbl_parser, VW::hash_func template VW_DEPRECATED("parse_slates_example_json moved to VW::parsers::json::details::parse_slates_example_json") void parse_slates_example_json(const VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, - VW::example_factory_t example_factory, std::unordered_map* dedup_examples = nullptr) + VW::example_factory_t example_factory, const std::unordered_map* dedup_examples = nullptr) { VW::parsers::json::details::parse_slates_example_json( all, examples, line, length, std::move(example_factory), dedup_examples); @@ -26,7 +26,7 @@ template VW_DEPRECATED("parse_slates_example_dsjson moved to VW::parsers::json::details::parse_slates_example_dsjson") void parse_slates_example_dsjson(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, VW::example_factory_t example_factory, VW::parsers::json::decision_service_interaction* data, - std::unordered_map* dedup_examples = nullptr) + const std::unordered_map* dedup_examples = nullptr) { VW::parsers::json::details::parse_slates_example_dsjson( all, examples, line, length, std::move(example_factory), data, dedup_examples); @@ -35,19 +35,21 @@ void parse_slates_example_dsjson(VW::workspace& all, VW::multi_ex& examples, cha // Define extern template specializations so they don't get initialized when this file is included extern template void parse_slates_example_json(const VW::label_parser& lbl_parser, VW::hash_func_t hash_func, uint64_t hash_seed, uint64_t parse_mask, bool chain_hash, VW::multi_ex& examples, char* line, size_t length, - VW::example_factory_t example_factory, std::unordered_map* dedup_examples); + VW::example_factory_t example_factory, const std::unordered_map* dedup_examples); extern template void parse_slates_example_json(const VW::label_parser& lbl_parser, VW::hash_func_t hash_func, uint64_t hash_seed, uint64_t parse_mask, bool chain_hash, VW::multi_ex& examples, char* line, size_t length, - VW::example_factory_t example_factory, std::unordered_map* dedup_examples); + VW::example_factory_t example_factory, const std::unordered_map* dedup_examples); extern template void parse_slates_example_json(const VW::workspace& all, VW::multi_ex& examples, char* line, - size_t length, VW::example_factory_t example_factory, std::unordered_map* dedup_examples); + size_t length, VW::example_factory_t example_factory, + const std::unordered_map* dedup_examples); extern template void parse_slates_example_json(const VW::workspace& all, VW::multi_ex& examples, char* line, - size_t length, VW::example_factory_t example_factory, std::unordered_map* dedup_examples); + size_t length, VW::example_factory_t example_factory, + const std::unordered_map* dedup_examples); extern template void parse_slates_example_dsjson(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, VW::example_factory_t example_factory, VW::parsers::json::decision_service_interaction* data, - std::unordered_map* dedup_examples); + const std::unordered_map* dedup_examples); extern template void parse_slates_example_dsjson(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, VW::example_factory_t example_factory, VW::parsers::json::decision_service_interaction* data, - std::unordered_map* dedup_examples); + const std::unordered_map* dedup_examples); diff --git a/vowpalwabbit/core/include/vw/core/reduction_stack.h b/vowpalwabbit/core/include/vw/core/reduction_stack.h index d1c85f30c2f..2051ea37b15 100644 --- a/vowpalwabbit/core/include/vw/core/reduction_stack.h +++ b/vowpalwabbit/core/include/vw/core/reduction_stack.h @@ -21,7 +21,7 @@ class default_reduction_stack_setup : public setup_base_i // This function consumes all the reduction_stack until it's able to construct a learner // Same signature as the old setup_base(...) from parse_args.cc - std::shared_ptr setup_base_learner() override; + std::shared_ptr setup_base_learner(size_t increment = 1) override; VW::config::options_i* get_options() override { return _options_impl; } @@ -29,10 +29,13 @@ class default_reduction_stack_setup : public setup_base_i std::string get_setupfn_name(reduction_setup_fn setup) override; + size_t get_feature_width_above() override { return _feature_width_above; } + private: VW::config::options_i* _options_impl = nullptr; VW::workspace* _all_ptr = nullptr; std::shared_ptr _base; + size_t _feature_width_above = 1; protected: std::vector> _reduction_stack; diff --git a/vowpalwabbit/core/include/vw/core/reductions/cb/cb_adf.h b/vowpalwabbit/core/include/vw/core/reductions/cb/cb_adf.h index 860a3958fae..7c528153c08 100644 --- a/vowpalwabbit/core/include/vw/core/reductions/cb/cb_adf.h +++ b/vowpalwabbit/core/include/vw/core/reductions/cb/cb_adf.h @@ -22,36 +22,41 @@ std::shared_ptr cb_adf_setup(VW::setup_base_i& stack_build class cb_adf { public: - VW::details::cb_to_cs_adf gen_cs; - void learn(VW::LEARNER::learner& base, VW::multi_ex& ec_seq); void predict(VW::LEARNER::learner& base, VW::multi_ex& ec_seq); bool update_statistics(const VW::example& ec, const VW::multi_ex& ec_seq, VW::shared_data& sd) const; - cb_adf(VW::cb_type_t cb_type, bool rank_all, float clip_p, bool no_predict, VW::workspace* all) - : _no_predict(no_predict), _rank_all(rank_all), _clip_p(clip_p), _all(all) + cb_adf(VW::cb_type_t cb_type, bool rank_all, float clip_p, bool no_predict, size_t feature_width_above, + VW::workspace* all) + : _no_predict(no_predict) + , _rank_all(rank_all) + , _clip_p(clip_p) + , _gen_cs_mtr(feature_width_above) + , _cb_type(cb_type) + , _all(all) { - gen_cs.cb_type = cb_type; } - void set_scorer(VW::LEARNER::learner* scorer) { gen_cs.scorer = scorer; } + void set_scorer(VW::LEARNER::learner* scorer) { _gen_cs_dr.scorer = scorer; } bool get_rank_all() const { return _rank_all; } - const VW::details::cb_to_cs_adf& get_gen_cs() const { return gen_cs; } - VW::details::cb_to_cs_adf& get_gen_cs() { return gen_cs; } + const VW::details::cb_to_cs_adf_dr& get_gen_cs_dr() const { return _gen_cs_dr; } + const VW::details::cb_to_cs_adf_mtr& get_gen_cs_mtr() const { return _gen_cs_mtr; } + + VW::details::cb_to_cs_adf_dr& get_gen_cs_dr() { return _gen_cs_dr; } + VW::details::cb_to_cs_adf_mtr& get_gen_cs_mtr() { return _gen_cs_mtr; } const VW::version_struct* get_model_file_ver() const; bool learn_returns_prediction() const { - return ((gen_cs.cb_type == VW::cb_type_t::MTR) && !_no_predict) || gen_cs.cb_type == VW::cb_type_t::IPS || - gen_cs.cb_type == VW::cb_type_t::DR || gen_cs.cb_type == VW::cb_type_t::DM || - gen_cs.cb_type == VW::cb_type_t::SM; + return ((_cb_type == VW::cb_type_t::MTR) && !_no_predict) || _cb_type == VW::cb_type_t::IPS || + _cb_type == VW::cb_type_t::DR || _cb_type == VW::cb_type_t::DM || _cb_type == VW::cb_type_t::SM; } - VW::cb_class* known_cost() { return &gen_cs.known_cost; } - const VW::cb_class* known_cost() const { return &gen_cs.known_cost; } + VW::cb_class* known_cost() { return &_gen_cs_dr.known_cost; } + const VW::cb_class* known_cost() const { return &_gen_cs_dr.known_cost; } private: void learn_ips(VW::LEARNER::learner& base, VW::multi_ex& examples); @@ -61,7 +66,6 @@ class cb_adf template void learn_mtr(VW::LEARNER::learner& base, VW::multi_ex& examples); -private: std::vector _cb_labels; VW::cs_label _cs_labels; std::vector _prepped_cs_labels; @@ -73,9 +77,13 @@ class cb_adf VW::v_array _backup_weights; // temporary storage for sm; backup for weights in examples uint64_t _offset = 0; + uint64_t _offset_index = 0; const bool _no_predict; const bool _rank_all; const float _clip_p; + VW::details::cb_to_cs_adf_dr _gen_cs_dr; + VW::details::cb_to_cs_adf_mtr _gen_cs_mtr; + VW::cb_type_t _cb_type; VW::workspace* _all = nullptr; }; diff --git a/vowpalwabbit/core/include/vw/core/reductions/cb/cbify.h b/vowpalwabbit/core/include/vw/core/reductions/cb/cbify.h index 740dcea0eab..fa3e7a64a3d 100644 --- a/vowpalwabbit/core/include/vw/core/reductions/cb/cbify.h +++ b/vowpalwabbit/core/include/vw/core/reductions/cb/cbify.h @@ -19,10 +19,10 @@ class cbify_adf_data public: VW::multi_ex ecs; size_t num_actions = 0; - size_t increment = 0; + size_t feature_width_below = 0; uint64_t custom_index_mask = 0; - void init_adf_data(std::size_t num_actions, size_t increment, + void init_adf_data(std::size_t num_actions, size_t feature_width_below, std::vector>& interactions, std::vector>& extent_interactions); void copy_example_to_adf(parameters& weights, VW::example& ec); diff --git a/vowpalwabbit/core/include/vw/core/reductions/epsilon_decay.h b/vowpalwabbit/core/include/vw/core/reductions/epsilon_decay.h index 7937edcccbb..7569c1ca172 100644 --- a/vowpalwabbit/core/include/vw/core/reductions/epsilon_decay.h +++ b/vowpalwabbit/core/include/vw/core/reductions/epsilon_decay.h @@ -28,7 +28,7 @@ class epsilon_decay_data public: epsilon_decay_data(uint64_t model_count, uint64_t min_scope, double epsilon_decay_significance_level, double epsilon_decay_estimator_decay, dense_parameters& weights, std::string epsilon_decay_audit_str, - bool constant_epsilon, uint32_t& wpp, uint64_t _min_champ_examples, float initial_epsilon, + bool constant_epsilon, uint32_t& feature_width, uint64_t _min_champ_examples, float initial_epsilon, uint64_t shift_model_bounds, bool reward_as_cost, double tol_x, bool is_brentq, bool predict_only_model); void update_weights(float init_ep, VW::LEARNER::learner& base, VW::multi_ex& examples); void promote_model(int64_t model_ind, int64_t swap_dist); @@ -49,21 +49,12 @@ class epsilon_decay_data std::stringstream _audit_msg; uint64_t _global_counter = 1; bool _constant_epsilon; - uint32_t& _wpp; + uint32_t& _feature_width; uint64_t _min_champ_examples; float _initial_epsilon; uint64_t _shift_model_bounds; bool _reward_as_cost; bool _predict_only_model; - - // TODO: delete all this, gd and cb_adf must respect ft_offset, see header import of automl.cc - std::vector per_live_model_state_double; - std::vector per_live_model_state_uint64; - double* _gd_normalized = nullptr; - double* _gd_total_weight = nullptr; - double* _sd_gravity = nullptr; - uint64_t* _cb_adf_event_sum = nullptr; - uint64_t* _cb_adf_action_sum = nullptr; }; } // namespace epsilon_decay diff --git a/vowpalwabbit/core/include/vw/core/reductions/gd.h b/vowpalwabbit/core/include/vw/core/reductions/gd.h index 0fb43aef0be..462f05bdb77 100644 --- a/vowpalwabbit/core/include/vw/core/reductions/gd.h +++ b/vowpalwabbit/core/include/vw/core/reductions/gd.h @@ -26,7 +26,7 @@ std::shared_ptr gd_setup(VW::setup_base_i& stack_builder); namespace details { -class per_model_state +class gd_per_model_state { public: double normalized_sum_norm_x = 0.0; @@ -37,7 +37,9 @@ class per_model_state class gd { public: - std::vector per_model_states; + gd(size_t feature_width_above = 1) : gd_per_model_states(feature_width_above) {} + std::vector gd_per_model_states; + VW::reductions::details::gd_per_model_state* current_model_state = nullptr; size_t no_win_counter = 0; size_t early_stop_thres = 0; float initial_constant = 0.f; @@ -64,8 +66,9 @@ float finalize_prediction(VW::shared_data& sd, VW::io::logger& logger, float ret void print_features(VW::workspace& all, VW::example& ec); void print_audit_features(VW::workspace&, VW::example& ec); void save_load_regressor_gd(VW::workspace& all, VW::io_buf& model_file, bool read, bool text); -void save_load_online_state_gd(VW::workspace& all, VW::io_buf& model_file, bool read, bool text, double& total_weight, - double& normalized_sum_norm_x, VW::reductions::gd* g = nullptr, uint32_t ftrl_size = 0); +void save_load_online_state_gd(VW::workspace& all, VW::io_buf& model_file, bool read, bool text, + std::vector& pms, VW::reductions::gd* g = nullptr, + uint32_t ftrl_size = 0); template class multipredict_info @@ -194,6 +197,15 @@ inline float trunc_weight(const float w, const float gravity) } // namespace VW +namespace VW +{ +namespace model_utils +{ +size_t read_model_field(io_buf&, VW::reductions::details::gd_per_model_state&); +size_t write_model_field(io_buf&, const VW::reductions::details::gd_per_model_state&, const std::string&, bool); +} // namespace model_utils +} // namespace VW + namespace VW { template @@ -201,13 +213,13 @@ inline void generate_interactions(VW::workspace& all, VW::example_predict& ec, R { if (all.weights.sparse) { - generate_interactions(*ec.interactions, *ec.extent_interactions, + VW::generate_interactions(*ec.interactions, *ec.extent_interactions, all.fc.permutations, ec, dat, all.weights.sparse_weights, num_interacted_features, all.runtime_state.generate_interactions_object_cache_state); } else { - generate_interactions(*ec.interactions, *ec.extent_interactions, + VW::generate_interactions(*ec.interactions, *ec.extent_interactions, all.fc.permutations, ec, dat, all.weights.dense_weights, num_interacted_features, all.runtime_state.generate_interactions_object_cache_state); } @@ -219,13 +231,13 @@ inline void generate_interactions(VW::workspace& all, VW::example_predict& ec, R { if (all.weights.sparse) { - generate_interactions(all.fc.interactions, all.fc.extent_interactions, + VW::generate_interactions(all.fc.interactions, all.fc.extent_interactions, all.fc.permutations, ec, dat, all.weights.sparse_weights, num_interacted_features, all.runtime_state.generate_interactions_object_cache_state); } else { - generate_interactions(all.fc.interactions, all.fc.extent_interactions, + VW::generate_interactions(all.fc.interactions, all.fc.extent_interactions, all.fc.permutations, ec, dat, all.weights.dense_weights, num_interacted_features, all.runtime_state.generate_interactions_object_cache_state); } @@ -233,27 +245,6 @@ inline void generate_interactions(VW::workspace& all, VW::example_predict& ec, R } // namespace VW -namespace INTERACTIONS // NOLINT -{ -template -VW_DEPRECATED("Moved to VW namespace") -inline void generate_interactions(VW::workspace& all, VW::example_predict& ec, R& dat, size_t& num_interacted_features) -{ - // call version in VW namespace - VW::generate_interactions(all, ec, dat, num_interacted_features); -} - -// this code is for C++98/03 complience as I unable to pass null function-pointer as template argument in g++-4.6 -template -VW_DEPRECATED("Moved to VW namespace") -inline void generate_interactions(VW::workspace& all, VW::example_predict& ec, R& dat, size_t& num_interacted_features) -{ - // call version in VW namespace - VW::generate_interactions(all, ec, dat, num_interacted_features); -} - -} // namespace INTERACTIONS - namespace GD { diff --git a/vowpalwabbit/core/include/vw/core/reductions/search/search.h b/vowpalwabbit/core/include/vw/core/reductions/search/search.h index c9e73a1f0ee..b1a2afb6511 100644 --- a/vowpalwabbit/core/include/vw/core/reductions/search/search.h +++ b/vowpalwabbit/core/include/vw/core/reductions/search/search.h @@ -152,7 +152,7 @@ class search // of length equal to the total number of actions ("A"); otherwise // it should be of length allowed_actions_cnt. only valid // if ACTION_COSTS is specified as an option. - // learner_id the id for the underlying learner to use (via set_num_learners) + // learner_id the id for the underlying learner to use (via set_feature_width) action predict(VW::example& ec, ptag my_tag, const action* oracle_actions, size_t oracle_actions_cnt = 1, const ptag* condition_on = nullptr, const char* condition_on_names = nullptr // strlen(condition_on_names) should == |condition_on| @@ -197,7 +197,7 @@ class search std::stringstream& output(); // set the number of learners - void set_num_learners(size_t num_learners); + void set_feature_width(size_t feature_width); // get the action sequence from the test run (only run if test_only or -t or...) void get_test_action_sequence(std::vector&); diff --git a/vowpalwabbit/core/include/vw/core/setup_base.h b/vowpalwabbit/core/include/vw/core/setup_base.h index bf6afd65d26..fc82cc81bd9 100644 --- a/vowpalwabbit/core/include/vw/core/setup_base.h +++ b/vowpalwabbit/core/include/vw/core/setup_base.h @@ -22,7 +22,7 @@ class setup_base_i public: virtual void delayed_state_attach(VW::workspace&, VW::config::options_i&) = 0; - virtual std::shared_ptr setup_base_learner() = 0; + virtual std::shared_ptr setup_base_learner(size_t increment = 1) = 0; // this one we can share freely virtual VW::config::options_i* get_options() = 0; @@ -33,6 +33,8 @@ class setup_base_i virtual std::string get_setupfn_name(reduction_setup_fn setup) = 0; + virtual size_t get_feature_width_above() = 0; + virtual ~setup_base_i() = default; }; diff --git a/vowpalwabbit/core/src/example.cc b/vowpalwabbit/core/src/example.cc index 4269b5d6cf4..c49cde7b69f 100644 --- a/vowpalwabbit/core/src/example.cc +++ b/vowpalwabbit/core/src/example.cc @@ -48,6 +48,9 @@ float VW::example::get_total_sum_feat_sq() float collision_cleanup(VW::features& fs) { + // Input must be sorted. + assert(std::is_sorted(fs.indices.begin(), fs.indices.end())); + // This loops over the sequence of feature values and their indexes // when an index is repeated this combines them by adding their values. // This assumes that fs is sorted (which is the case in `flatten_sort_example`). @@ -105,46 +108,23 @@ void vec_ffs_store(full_features_and_source& p, float fx, uint64_t fi) } namespace VW { -flat_example* flatten_example(VW::workspace& all, example* ec) -{ - flat_example& fec = VW::details::calloc_or_throw(); - fec.l = ec->l; - fec.tag = ec->tag; - fec.ex_reduction_features = ec->ex_reduction_features; - fec.example_counter = ec->example_counter; - fec.ft_offset = ec->ft_offset; - fec.num_features = ec->num_features; +void flatten_features(VW::workspace& all, example& ec, features& fs) +{ + fs.clear(); full_features_and_source ffs; + ffs.fs = std::move(fs); ffs.stride_shift = all.weights.stride_shift(); if (all.weights.not_null()) - { // TODO:temporary fix. all.weights is not initialized at this point in some cases. + { + // TODO:temporary fix. all.weights is not initialized at this point in some cases. ffs.mask = all.weights.mask() >> all.weights.stride_shift(); } else { ffs.mask = static_cast(LONG_MAX) >> all.weights.stride_shift(); } - VW::foreach_feature(all, *ec, ffs); - - std::swap(fec.fs, ffs.fs); - - return &fec; -} - -flat_example* flatten_sort_example(VW::workspace& all, example* ec) -{ - flat_example* fec = flatten_example(all, ec); - fec->fs.sort(all.runtime_state.parse_mask); - fec->total_sum_feat_sq = collision_cleanup(fec->fs); - return fec; -} - -void free_flatten_example(flat_example* fec) -{ - // note: The label memory should be freed by by freeing the original example. - if (fec) - { - fec->fs.~features(); - free(fec); - } + VW::foreach_feature(all, ec, ffs); + ffs.fs.sort(all.runtime_state.parse_mask); + ffs.fs.sum_feat_sq = collision_cleanup(ffs.fs); + fs = std::move(ffs.fs); } void return_multiple_example(VW::workspace& all, VW::multi_ex& examples) @@ -213,42 +193,6 @@ void truncate_example_namespaces_from_example(VW::example& target, const VW::exa } } } // namespace details - -namespace model_utils -{ -size_t read_model_field(io_buf& io, flat_example& fe, VW::label_parser& lbl_parser) -{ - size_t bytes = 0; - lbl_parser.default_label(fe.l); - bytes += lbl_parser.read_cached_label(fe.l, fe.ex_reduction_features, io); - bytes += read_model_field(io, fe.tag); - bytes += read_model_field(io, fe.example_counter); - bytes += read_model_field(io, fe.ft_offset); - bytes += read_model_field(io, fe.global_weight); - bytes += read_model_field(io, fe.num_features); - bytes += read_model_field(io, fe.total_sum_feat_sq); - unsigned char index = 0; - bytes += ::VW::parsers::cache::details::read_cached_index(io, index); - bool sorted = true; - bytes += ::VW::parsers::cache::details::read_cached_features(io, fe.fs, sorted); - return bytes; -} -size_t write_model_field(io_buf& io, const flat_example& fe, const std::string& upstream_name, bool text, - VW::label_parser& lbl_parser, uint64_t parse_mask) -{ - size_t bytes = 0; - lbl_parser.cache_label(fe.l, fe.ex_reduction_features, io, upstream_name + "_label", text); - bytes += write_model_field(io, fe.tag, upstream_name + "_tag", text); - bytes += write_model_field(io, fe.example_counter, upstream_name + "_example_counter", text); - bytes += write_model_field(io, fe.ft_offset, upstream_name + "_ft_offset", text); - bytes += write_model_field(io, fe.global_weight, upstream_name + "_global_weight", text); - bytes += write_model_field(io, fe.num_features, upstream_name + "_num_features", text); - bytes += write_model_field(io, fe.total_sum_feat_sq, upstream_name + "_total_sum_feat_sq", text); - ::VW::parsers::cache::details::cache_index(io, 0); - ::VW::parsers::cache::details::cache_features(io, fe.fs, parse_mask); - return bytes; -} -} // namespace model_utils } // namespace VW namespace VW diff --git a/vowpalwabbit/core/src/example_predict.cc b/vowpalwabbit/core/src/example_predict.cc index d7239e70b8a..5cab683e236 100644 --- a/vowpalwabbit/core/src/example_predict.cc +++ b/vowpalwabbit/core/src/example_predict.cc @@ -26,3 +26,22 @@ bool VW::example_predict::iterator::operator!=(const iterator& rhs) const { retu VW::example_predict::iterator VW::example_predict::begin() { return {feature_space.data(), indices.begin()}; } VW::example_predict::iterator VW::example_predict::end() { return {feature_space.data(), indices.end()}; } + +uint64_t VW::example_predict::get_or_calculate_order_independent_feature_space_hash() +{ + if (!is_set_feature_space_hash) + { + is_set_feature_space_hash = true; + for (const auto ns : indices) + { + feature_space_hash += std::hash()(ns); + for (const auto& f : feature_space[ns]) + { + feature_space_hash += std::hash()(f.index()); + feature_space_hash += std::hash()(f.value()); + } + } + } + + return feature_space_hash; +} diff --git a/vowpalwabbit/core/src/feature_group.cc b/vowpalwabbit/core/src/feature_group.cc index 64f05a87390..ccb50e30484 100644 --- a/vowpalwabbit/core/src/feature_group.cc +++ b/vowpalwabbit/core/src/feature_group.cc @@ -297,3 +297,28 @@ void VW::features::end_ns_extent() } } } + +float VW::features_dot_product(const features& fs1, const features& fs2) +{ + assert(std::is_sorted(fs1.indices.begin(), fs1.indices.end())); + assert(std::is_sorted(fs2.indices.begin(), fs2.indices.end())); + + float dotprod = 0; + if (fs2.indices.empty()) { return 0.f; } + + for (size_t idx1 = 0, idx2 = 0; idx1 < fs1.size() && idx2 < fs2.size(); idx1++) + { + uint64_t ec1pos = fs1.indices[idx1]; + uint64_t ec2pos = fs2.indices[idx2]; + if (ec1pos < ec2pos) { continue; } + + while (ec1pos > ec2pos && ++idx2 < fs2.size()) { ec2pos = fs2.indices[idx2]; } + + if (ec1pos == ec2pos) + { + dotprod += fs1.values[idx1] * fs2.values[idx2]; + ++idx2; + } + } + return dotprod; +} \ No newline at end of file diff --git a/vowpalwabbit/core/src/gen_cs_example.cc b/vowpalwabbit/core/src/gen_cs_example.cc index 46d519ec832..7b2a0fef2ba 100644 --- a/vowpalwabbit/core/src/gen_cs_example.cc +++ b/vowpalwabbit/core/src/gen_cs_example.cc @@ -121,10 +121,11 @@ void VW::details::gen_cs_example_ips( } } -void VW::details::gen_cs_example_mtr(cb_to_cs_adf& c, VW::multi_ex& ec_seq, VW::cs_label& cs_labels) +void VW::details::gen_cs_example_mtr( + cb_to_cs_adf_mtr& c, VW::multi_ex& ec_seq, VW::cs_label& cs_labels, uint64_t offset_index) { - c.action_sum += ec_seq.size(); - c.event_sum++; + c.per_model_state[offset_index].action_sum += ec_seq.size(); + c.per_model_state[offset_index].event_sum++; c.mtr_ec_seq.clear(); cs_labels.costs.clear(); diff --git a/vowpalwabbit/core/src/learner.cc b/vowpalwabbit/core/src/learner.cc index b142f2ba9e6..89aca9bd0de 100644 --- a/vowpalwabbit/core/src/learner.cc +++ b/vowpalwabbit/core/src/learner.cc @@ -372,19 +372,19 @@ void learner::debug_log_message(polymorphic_ex ex, const std::string& msg) void learner::learn(polymorphic_ex ec, size_t i) { assert(is_multiline() == ec.is_multiline()); - details::increment_offset(ec, increment, i); + details::increment_offset(ec, feature_width_below, i); debug_log_message(ec, "learn"); _learn_f(ec); - details::decrement_offset(ec, increment, i); + details::decrement_offset(ec, feature_width_below, i); } void learner::predict(polymorphic_ex ec, size_t i) { assert(is_multiline() == ec.is_multiline()); - details::increment_offset(ec, increment, i); + details::increment_offset(ec, feature_width_below, i); debug_log_message(ec, "predict"); _predict_f(ec); - details::decrement_offset(ec, increment, i); + details::decrement_offset(ec, feature_width_below, i); } void learner::multipredict(polymorphic_ex ec, size_t lo, size_t count, polyprediction* pred, bool finalize_predictions) @@ -392,7 +392,7 @@ void learner::multipredict(polymorphic_ex ec, size_t lo, size_t count, polypredi assert(is_multiline() == ec.is_multiline()); if (_multipredict_f == nullptr) { - details::increment_offset(ec, increment, lo); + details::increment_offset(ec, feature_width_below, lo); debug_log_message(ec, "multipredict"); for (size_t c = 0; c < count; c++) { @@ -405,34 +405,34 @@ void learner::multipredict(polymorphic_ex ec, size_t lo, size_t count, polypredi else { pred[c].scalar = static_cast(ec).partial_prediction; } // pred[c].scalar = finalize_prediction ec.partial_prediction; // TODO: this breaks for complex labels because = // doesn't do deep copy! // note works if ec.partial_prediction, but only if finalize_prediction is run???? - details::increment_offset(ec, increment, 1); + details::increment_offset(ec, feature_width_below, 1); } - details::decrement_offset(ec, increment, lo + count); + details::decrement_offset(ec, feature_width_below, lo + count); } else { - details::increment_offset(ec, increment, lo); + details::increment_offset(ec, feature_width_below, lo); debug_log_message(ec, "multipredict"); - _multipredict_f(ec, count, increment, pred, finalize_predictions); - details::decrement_offset(ec, increment, lo); + _multipredict_f(ec, count, feature_width_below, pred, finalize_predictions); + details::decrement_offset(ec, feature_width_below, lo); } } void learner::update(polymorphic_ex ec, size_t i) { assert(is_multiline() == ec.is_multiline()); - details::increment_offset(ec, increment, i); + details::increment_offset(ec, feature_width_below, i); debug_log_message(ec, "update"); _update_f(ec); - details::decrement_offset(ec, increment, i); + details::decrement_offset(ec, feature_width_below, i); } float learner::sensitivity(example& ec, size_t i) { - details::increment_offset(ec, increment, i); + details::increment_offset(ec, feature_width_below, i); debug_log_message(ec, "sensitivity"); const float ret = _sensitivity_f(ec); - details::decrement_offset(ec, increment, i); + details::decrement_offset(ec, feature_width_below, i); return ret; } @@ -735,30 +735,30 @@ void VW::LEARNER::details::debug_decrement_depth(polymorphic_ex ex) } } -void VW::LEARNER::details::increment_offset(polymorphic_ex ex, const size_t increment, const size_t i) +void VW::LEARNER::details::increment_offset(polymorphic_ex ex, const size_t feature_width_below, const size_t i) { if (ex.is_multiline()) { - for (auto& ec : static_cast(ex)) { ec->ft_offset += static_cast(increment * i); } + for (auto& ec : static_cast(ex)) { ec->ft_offset += static_cast(feature_width_below * i); } } - else { static_cast(ex).ft_offset += static_cast(increment * i); } + else { static_cast(ex).ft_offset += static_cast(feature_width_below * i); } debug_increment_depth(ex); } -void VW::LEARNER::details::decrement_offset(polymorphic_ex ex, const size_t increment, const size_t i) +void VW::LEARNER::details::decrement_offset(polymorphic_ex ex, const size_t feature_width_below, const size_t i) { if (ex.is_multiline()) { for (auto ec : static_cast(ex)) { - assert(ec->ft_offset >= increment * i); - ec->ft_offset -= static_cast(increment * i); + assert(ec->ft_offset >= feature_width_below * i); + ec->ft_offset -= static_cast(feature_width_below * i); } } else { - assert(static_cast(ex).ft_offset >= increment * i); - static_cast(ex).ft_offset -= static_cast(increment * i); + assert(static_cast(ex).ft_offset >= feature_width_below * i); + static_cast(ex).ft_offset -= static_cast(feature_width_below * i); } debug_decrement_depth(ex); -} +} \ No newline at end of file diff --git a/vowpalwabbit/core/src/parse_args.cc b/vowpalwabbit/core/src/parse_args.cc index c7baff4e4b0..f2f152485f5 100644 --- a/vowpalwabbit/core/src/parse_args.cc +++ b/vowpalwabbit/core/src/parse_args.cc @@ -18,6 +18,7 @@ #include "vw/core/global_data.h" #include "vw/core/interactions.h" #include "vw/core/kskip_ngram_transformer.h" +#include "vw/core/label_parser.h" #include "vw/core/label_type.h" #include "vw/core/learner.h" #include "vw/core/loss_functions.h" @@ -1652,6 +1653,9 @@ void VW::details::instantiate_learner(VW::workspace& all, std::unique_ptrsetup_base_learner(); + // Setup label parser based on the stack which was just created. + all.parser_runtime.example_parser->lbl_parser = VW::get_label_parser(all.l->get_input_label_type()); + // explicit destroy of learner_builder state // avoids misuse of this interface: learner_builder.reset(); @@ -1666,11 +1670,10 @@ void VW::details::parse_sources(options_i& options, VW::workspace& all, VW::io_b auto parsed_source_options = parse_source(all, options); enable_sources(all, all.output_config.quiet, all.runtime_config.numpasses, parsed_source_options); - // force wpp to be a power of 2 to avoid 32-bit overflow - uint32_t i = 0; - const size_t params_per_problem = all.l->increment; - while (params_per_problem > (static_cast(1) << i)) { i++; } - all.reduction_state.wpp = (1 << i) >> all.weights.stride_shift(); + // force feature_width to be a power of 2 to avoid 32-bit overflow + uint32_t interleave_shifts = 0; + while (all.l->feature_width_below > (static_cast(1) << interleave_shifts)) { interleave_shifts++; } + all.reduction_state.total_feature_width = (1 << interleave_shifts) >> all.weights.stride_shift(); } void VW::details::print_enabled_learners(VW::workspace& all, std::vector& enabled_learners) diff --git a/vowpalwabbit/core/src/parse_example_json.cc b/vowpalwabbit/core/src/parse_example_json.cc index 4ae2479b909..488344d1dcb 100644 --- a/vowpalwabbit/core/src/parse_example_json.cc +++ b/vowpalwabbit/core/src/parse_example_json.cc @@ -9,17 +9,17 @@ template void VW::read_line_json_s(const VW::label_parser& lbl_parser, has uint64_t parse_mask, bool chain_hash, VW::label_parser_reuse_mem* reuse_mem, const VW::named_labels* ldict, VW::multi_ex& examples, char* line, size_t length, example_factory_t example_factory, VW::io::logger& logger, std::unordered_map>* ignore_features, - std::unordered_map* dedup_examples); + const std::unordered_map* dedup_examples); template void VW::read_line_json_s(const VW::label_parser& lbl_parser, hash_func_t hash_func, uint64_t hash_seed, uint64_t parse_mask, bool chain_hash, VW::label_parser_reuse_mem* reuse_mem, const VW::named_labels* ldict, VW::multi_ex& examples, char* line, size_t length, example_factory_t example_factory, VW::io::logger& logger, std::unordered_map>* ignore_features, - std::unordered_map* dedup_examples); + const std::unordered_map* dedup_examples); template void VW::read_line_json_s(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, - example_factory_t example_factory, std::unordered_map* dedup_examples); + example_factory_t example_factory, const std::unordered_map* dedup_examples); template void VW::read_line_json_s(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, - example_factory_t example_factory, std::unordered_map* dedup_examples); + example_factory_t example_factory, const std::unordered_map* dedup_examples); template bool VW::read_line_decision_service_json(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, bool copy_line, example_factory_t example_factory, diff --git a/vowpalwabbit/core/src/parse_slates_example_json.cc b/vowpalwabbit/core/src/parse_slates_example_json.cc index 296f52d8d25..91520d1266e 100644 --- a/vowpalwabbit/core/src/parse_slates_example_json.cc +++ b/vowpalwabbit/core/src/parse_slates_example_json.cc @@ -5,19 +5,21 @@ // Explicitly instantiate templates only in this source file template void parse_slates_example_json(const VW::label_parser& lbl_parser, VW::hash_func_t hash_func, uint64_t hash_seed, uint64_t parse_mask, bool chain_hash, VW::multi_ex& examples, char* line, size_t length, - VW::example_factory_t example_factory, std::unordered_map* dedup_examples); + VW::example_factory_t example_factory, const std::unordered_map* dedup_examples); template void parse_slates_example_json(const VW::label_parser& lbl_parser, VW::hash_func_t hash_func, uint64_t hash_seed, uint64_t parse_mask, bool chain_hash, VW::multi_ex& examples, char* line, size_t length, - VW::example_factory_t example_factory, std::unordered_map* dedup_examples); + VW::example_factory_t example_factory, const std::unordered_map* dedup_examples); template void parse_slates_example_json(const VW::workspace& all, VW::multi_ex& examples, char* line, - size_t length, VW::example_factory_t example_factory, std::unordered_map* dedup_examples); + size_t length, VW::example_factory_t example_factory, + const std::unordered_map* dedup_examples); template void parse_slates_example_json(const VW::workspace& all, VW::multi_ex& examples, char* line, - size_t length, VW::example_factory_t example_factory, std::unordered_map* dedup_examples); + size_t length, VW::example_factory_t example_factory, + const std::unordered_map* dedup_examples); template void parse_slates_example_dsjson(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, VW::example_factory_t example_factory, VW::parsers::json::decision_service_interaction* data, - std::unordered_map* dedup_examples); + const std::unordered_map* dedup_examples); template void parse_slates_example_dsjson(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, VW::example_factory_t example_factory, VW::parsers::json::decision_service_interaction* data, - std::unordered_map* dedup_examples); \ No newline at end of file + const std::unordered_map* dedup_examples); \ No newline at end of file diff --git a/vowpalwabbit/core/src/reduction_stack.cc b/vowpalwabbit/core/src/reduction_stack.cc index 8d1d0e46460..ba306d94306 100644 --- a/vowpalwabbit/core/src/reduction_stack.cc +++ b/vowpalwabbit/core/src/reduction_stack.cc @@ -265,7 +265,7 @@ std::string default_reduction_stack_setup::get_setupfn_name(reduction_setup_fn s // this function consumes all the _reduction_stack until it's able to construct a learner // same signature/code as the old setup_base(...) from parse_args.cc -std::shared_ptr default_reduction_stack_setup::setup_base_learner() +std::shared_ptr default_reduction_stack_setup::setup_base_learner(size_t feature_width) { if (!_reduction_stack.empty()) { @@ -274,6 +274,7 @@ std::shared_ptr default_reduction_stack_setup::setup_base_ std::string setup_func_name = std::get<0>(func_map); _reduction_stack.pop_back(); + _feature_width_above *= feature_width; // 'hacky' way of keeping track of the option group created by the setup_func about to be created _options_impl->tint(setup_func_name); std::shared_ptr result = setup_func(*this); @@ -285,6 +286,11 @@ std::shared_ptr default_reduction_stack_setup::setup_base_ if (result == nullptr) { return this->setup_base_learner(); } else { + if (result->get_base_learner() == nullptr) + { + result->feature_width = get_all_pointer()->weights.stride(); + result->feature_width_below = result->feature_width; + } _reduction_stack.clear(); return result; } diff --git a/vowpalwabbit/core/src/reductions/active_cover.cc b/vowpalwabbit/core/src/reductions/active_cover.cc index 915a3c28aa8..1453b19325c 100644 --- a/vowpalwabbit/core/src/reductions/active_cover.cc +++ b/vowpalwabbit/core/src/reductions/active_cover.cc @@ -248,7 +248,7 @@ std::shared_ptr VW::reductions::active_cover_setup(VW::set if (options.was_supplied("active")) THROW("--active_cover cannot be combined with --active"); - auto base = require_singleline(stack_builder.setup_base_learner()); + auto base = require_singleline(stack_builder.setup_base_learner(data->cover_size + 1)); data->lambda_n = new float[data->cover_size]; data->lambda_d = new float[data->cover_size]; @@ -262,7 +262,7 @@ std::shared_ptr VW::reductions::active_cover_setup(VW::set const auto saved_cover_size = data->cover_size; auto l = make_reduction_learner(std::move(data), base, predict_or_learn_active_cover, predict_or_learn_active_cover, stack_builder.get_setupfn_name(active_cover_setup)) - .set_params_per_weight(saved_cover_size + 1) + .set_feature_width(saved_cover_size + 1) .set_input_prediction_type(VW::prediction_type_t::SCALAR) .set_output_prediction_type(VW::prediction_type_t::SCALAR) .set_input_label_type(VW::label_type_t::SIMPLE) diff --git a/vowpalwabbit/core/src/reductions/audit_regressor.cc b/vowpalwabbit/core/src/reductions/audit_regressor.cc index 6c5fe03a92c..158483d06e7 100644 --- a/vowpalwabbit/core/src/reductions/audit_regressor.cc +++ b/vowpalwabbit/core/src/reductions/audit_regressor.cc @@ -38,7 +38,7 @@ class audit_regressor_data } VW::workspace* all; - size_t increment = 0; + size_t feature_width_below = 0; size_t cur_class = 0; size_t total_class_cnt = 0; std::vector ns_pre; @@ -172,7 +172,7 @@ void audit_regressor(audit_regressor_data& rd, VW::LEARNER::learner& base, VW::e rd.all->runtime_state.generate_interactions_object_cache_state); } - ec.ft_offset += rd.increment; + ec.ft_offset += rd.feature_width_below; ++rd.cur_class; } @@ -238,8 +238,8 @@ void init_driver(audit_regressor_data& dat) dat.all->sd->dump_interval = 1.; // regressor could initialize these if saved without --predict_only_model dat.all->sd->example_number = 0; - dat.increment = dat.all->l->increment / dat.all->l->weights; - dat.total_class_cnt = dat.all->l->weights; + dat.feature_width_below = dat.all->l->feature_width_below / dat.all->l->feature_width; + dat.total_class_cnt = dat.all->l->feature_width; if (dat.all->options->was_supplied("csoaa")) { @@ -247,7 +247,7 @@ void init_driver(audit_regressor_data& dat) if (n != dat.total_class_cnt) { dat.total_class_cnt = n; - dat.increment = dat.all->l->increment / n; + dat.feature_width_below = dat.all->l->feature_width_below / n; } } diff --git a/vowpalwabbit/core/src/reductions/automl.cc b/vowpalwabbit/core/src/reductions/automl.cc index f34a3f0b7c6..ef02ec29e6c 100644 --- a/vowpalwabbit/core/src/reductions/automl.cc +++ b/vowpalwabbit/core/src/reductions/automl.cc @@ -9,13 +9,13 @@ #include "vw/core/automl_impl.h" #include "vw/core/estimators/confidence_sequence_robust.h" #include "vw/core/multi_model_utils.h" +#include "vw/core/shared_data.h" +#include "vw/core/vw.h" // TODO: delete this three includes #include "vw/core/reductions/cb/cb_adf.h" #include "vw/core/reductions/gd.h" #include "vw/core/setup_base.h" -#include "vw/core/shared_data.h" -#include "vw/core/vw.h" #include #include @@ -79,17 +79,10 @@ void pre_save_load_automl(VW::workspace& all, automl& data) { options_i& options = *all.options; if (!data.should_save_predict_only_model) { return; } - // Clear non-champ weights first - - std::swap(*data.cm->_gd_normalized, data.cm->per_live_model_state_double[0]); - std::swap(*data.cm->_gd_total_weight, data.cm->per_live_model_state_double[1]); - std::swap(*data.cm->_sd_gravity, data.cm->per_live_model_state_double[2]); - std::swap(*data.cm->_cb_adf_event_sum, data.cm->per_live_model_state_uint64[0]); - std::swap(*data.cm->_cb_adf_action_sum, data.cm->per_live_model_state_uint64[1]); // Adjust champ weights to new single-model space VW::reductions::multi_model::reduce_innermost_model_weights( - data.cm->weights, 0, data.cm->wpp, data.cm->max_live_configs); + data.cm->weights, 0, data.cm->feature_width, data.cm->max_live_configs); for (auto& group : options.get_all_option_group_definitions()) { @@ -157,7 +150,7 @@ std::shared_ptr make_automl_with_impl(VW::setup_base_i& st else if (priority_type == "favor_popular_namespaces") { calc_priority = &calc_priority_favor_popular_namespaces; } else { THROW("Invalid priority function provided"); } - // Note that all.reduction_state.wpp will not be set correctly until after setup + // Note that all.reduction_state.total_feature_width will not be set correctly until after setup assert(oracle_type == "one_diff" || oracle_type == "rand" || oracle_type == "champdupe" || oracle_type == "one_diff_inclusion" || oracle_type == "qbase_cubic"); @@ -174,40 +167,28 @@ std::shared_ptr make_automl_with_impl(VW::setup_base_i& st auto cm = VW::make_unique(default_lease, max_live_configs, all.get_random_state(), static_cast(priority_challengers), interaction_type, oracle_type, all.weights.dense_weights, - calc_priority, automl_significance_level, &all.logger, all.reduction_state.wpp, ccb_on, conf_type, + calc_priority, automl_significance_level, &all.logger, all.reduction_state.total_feature_width, ccb_on, conf_type, trace_file_name_prefix, reward_as_cost, tol_x, is_brentq); auto data = VW::make_unique>( std::move(cm), &all.logger, predict_only_model, trace_file_name_prefix); data->debug_reverse_learning_order = reversed_learning_order; - data->cm->per_live_model_state_double = std::vector(max_live_configs * 3, 0.f); - data->cm->per_live_model_state_uint64 = std::vector(max_live_configs * 2, 0.f); - auto ppw = max_live_configs; + auto feature_width = max_live_configs; auto* persist_ptr = verbose_metrics ? persist : persist; - data->adf_learner = require_multiline(base_learner->get_learner_by_name_prefix("cb_adf")); - VW::reductions::gd& gd = *static_cast( - base_learner->get_learner_by_name_prefix("gd")->get_internal_type_erased_data_pointer_test_use_only()); - auto& adf_data = - *static_cast(data->adf_learner->get_internal_type_erased_data_pointer_test_use_only()); - data->cm->_gd_normalized = &(gd.per_model_states[0].normalized_sum_norm_x); - data->cm->_gd_total_weight = &(gd.per_model_states[0].total_weight); - data->cm->_cb_adf_event_sum = &(adf_data.gen_cs.event_sum); - data->cm->_cb_adf_action_sum = &(adf_data.gen_cs.action_sum); - data->cm->_sd_gravity = &(all.sd->gravity); - - auto l = make_reduction_learner(std::move(data), require_multiline(base_learner), - learn_automl, predict_automl, - stack_builder.get_setupfn_name(VW::reductions::automl_setup)) - .set_params_per_weight(ppw) // refactor pm - .set_input_prediction_type(VW::prediction_type_t::ACTION_SCORES) - .set_output_prediction_type(VW::prediction_type_t::ACTION_SCORES) - .set_input_label_type(VW::label_type_t::CB) - .set_output_label_type(VW::label_type_t::CB) - .set_save_load(save_load_automl) - .set_persist_metrics(persist_ptr) - .set_learn_returns_prediction(true) - .set_pre_save_load(pre_save_load_automl) - .build(); + + auto l = + make_reduction_learner(std::move(data), require_multiline(base_learner), learn_automl, + predict_automl, stack_builder.get_setupfn_name(VW::reductions::automl_setup)) + .set_feature_width(feature_width) + .set_input_prediction_type(VW::prediction_type_t::ACTION_SCORES) + .set_output_prediction_type(VW::prediction_type_t::ACTION_SCORES) + .set_input_label_type(VW::label_type_t::CB) + .set_output_label_type(VW::label_type_t::CB) + .set_save_load(save_load_automl) + .set_persist_metrics(persist_ptr) + .set_learn_returns_prediction(true) + .set_pre_save_load(pre_save_load_automl) + .build(); return l; } @@ -327,7 +308,7 @@ std::shared_ptr VW::reductions::automl_setup(VW::setup_bas // make sure we setup the rest of the stack with cleared interactions // to make sure there are not subtle bugs - auto learner = stack_builder.setup_base_learner(); + auto learner = stack_builder.setup_base_learner(max_live_configs); assert(all.fc.interactions.empty() == true); diff --git a/vowpalwabbit/core/src/reductions/baseline.cc b/vowpalwabbit/core/src/reductions/baseline.cc index 73a4b656544..8fc2d6c1333 100644 --- a/vowpalwabbit/core/src/reductions/baseline.cc +++ b/vowpalwabbit/core/src/reductions/baseline.cc @@ -56,7 +56,7 @@ void init_global(baseline_data& data) data.ec.indices.push_back(VW::details::CONSTANT_NAMESPACE); // different index from constant to avoid conflicts data.ec.feature_space[VW::details::CONSTANT_NAMESPACE].push_back(1, - ((VW::details::CONSTANT - 17) * data.all->reduction_state.wpp) << data.all->weights.stride_shift(), + ((VW::details::CONSTANT - 17) * data.all->reduction_state.total_feature_width) << data.all->weights.stride_shift(), VW::details::CONSTANT_NAMESPACE); data.ec.reset_total_sum_feat_sq(); data.ec.num_features++; diff --git a/vowpalwabbit/core/src/reductions/bfgs.cc b/vowpalwabbit/core/src/reductions/bfgs.cc index 3a25b3663da..c33fbe35b61 100644 --- a/vowpalwabbit/core/src/reductions/bfgs.cc +++ b/vowpalwabbit/core/src/reductions/bfgs.cc @@ -1202,7 +1202,6 @@ std::shared_ptr VW::reductions::bfgs_setup(VW::setup_base_ return make_bottom_learner( std::move(b), learn_ptr, predict_ptr, learner_name, VW::prediction_type_t::SCALAR, VW::label_type_t::SIMPLE) - .set_params_per_weight(all.weights.stride()) .set_save_load(save_load) .set_init_driver(init_driver) .set_end_pass(end_pass) diff --git a/vowpalwabbit/core/src/reductions/boosting.cc b/vowpalwabbit/core/src/reductions/boosting.cc index 19e91928f62..394b4df267b 100644 --- a/vowpalwabbit/core/src/reductions/boosting.cc +++ b/vowpalwabbit/core/src/reductions/boosting.cc @@ -391,7 +391,7 @@ std::shared_ptr VW::reductions::boosting_setup(VW::setup_b data->alpha = std::vector(data->N, 0); data->v = std::vector(data->N, 1); - size_t ws = data->N; + size_t feature_width = data->N; std::string name_addition; void (*learn_ptr)(boosting&, VW::LEARNER::learner&, VW::example&); void (*pred_ptr)(boosting&, VW::LEARNER::learner&, VW::example&); @@ -420,9 +420,9 @@ std::shared_ptr VW::reductions::boosting_setup(VW::setup_b } else { THROW("Unrecognized boosting algorithm: \'" << data->alg << "\'."); } - auto l = make_reduction_learner(std::move(data), require_singleline(stack_builder.setup_base_learner()), learn_ptr, - pred_ptr, stack_builder.get_setupfn_name(boosting_setup) + name_addition) - .set_params_per_weight(ws) + auto l = make_reduction_learner(std::move(data), require_singleline(stack_builder.setup_base_learner(feature_width)), + learn_ptr, pred_ptr, stack_builder.get_setupfn_name(boosting_setup) + name_addition) + .set_feature_width(feature_width) .set_input_prediction_type(VW::prediction_type_t::SCALAR) .set_output_prediction_type(VW::prediction_type_t::SCALAR) .set_input_label_type(VW::label_type_t::SIMPLE) diff --git a/vowpalwabbit/core/src/reductions/bs.cc b/vowpalwabbit/core/src/reductions/bs.cc index 34dd762d0c3..6013b4cfb66 100644 --- a/vowpalwabbit/core/src/reductions/bs.cc +++ b/vowpalwabbit/core/src/reductions/bs.cc @@ -235,7 +235,7 @@ std::shared_ptr VW::reductions::bs_setup(VW::setup_base_i& .help("Prediction type")); if (!options.add_parse_and_check_necessary(new_options)) { return nullptr; } - size_t ws = data->num_bootstrap_rounds; + size_t feature_width = data->num_bootstrap_rounds; if (options.was_supplied("bs_type")) { @@ -256,9 +256,9 @@ std::shared_ptr VW::reductions::bs_setup(VW::setup_base_i& data->all = &all; data->random_state = all.get_random_state(); - auto l = make_reduction_learner(std::move(data), require_singleline(stack_builder.setup_base_learner()), + auto l = make_reduction_learner(std::move(data), require_singleline(stack_builder.setup_base_learner(feature_width)), predict_or_learn, predict_or_learn, stack_builder.get_setupfn_name(bs_setup)) - .set_params_per_weight(ws) + .set_feature_width(feature_width) .set_learn_returns_prediction(true) .set_output_example_prediction(output_example_prediction_bs) .set_update_stats(VW::details::update_stats_simple_label) diff --git a/vowpalwabbit/core/src/reductions/cats_pdf.cc b/vowpalwabbit/core/src/reductions/cats_pdf.cc index f8ba8920121..e85459ec25f 100644 --- a/vowpalwabbit/core/src/reductions/cats_pdf.cc +++ b/vowpalwabbit/core/src/reductions/cats_pdf.cc @@ -171,8 +171,5 @@ std::shared_ptr VW::reductions::cats_pdf_setup(setup_base_ .set_print_update(print_update_cats_pdf) .set_update_stats(update_stats_cats_pdf) .build(); - - all.parser_runtime.example_parser->lbl_parser = cb_continuous::the_label_parser; - return l; } diff --git a/vowpalwabbit/core/src/reductions/cats_tree.cc b/vowpalwabbit/core/src/reductions/cats_tree.cc index 0d663a86e93..473d3bd6930 100644 --- a/vowpalwabbit/core/src/reductions/cats_tree.cc +++ b/vowpalwabbit/core/src/reductions/cats_tree.cc @@ -362,16 +362,15 @@ std::shared_ptr VW::reductions::cats_tree_setup(VW::setup_ tree->init(num_actions, bandwidth); tree->set_trace_message(all.output_runtime.trace_message, all.output_config.quiet); - auto base = stack_builder.setup_base_learner(); - int32_t params_per_weight = tree->learner_count(); + int32_t feature_width = tree->learner_count(); + auto base = stack_builder.setup_base_learner(feature_width); auto l = make_reduction_learner( std::move(tree), require_singleline(base), learn, predict, stack_builder.get_setupfn_name(cats_tree_setup)) - .set_params_per_weight(params_per_weight) + .set_feature_width(feature_width) .set_input_label_type(VW::label_type_t::CB) .set_output_label_type(VW::label_type_t::SIMPLE) .set_input_prediction_type(VW::prediction_type_t::SCALAR) .set_output_prediction_type(VW::prediction_type_t::MULTICLASS) .build(); - all.parser_runtime.example_parser->lbl_parser = VW::cb_label_parser_global; return l; } diff --git a/vowpalwabbit/core/src/reductions/cb/cb_adf.cc b/vowpalwabbit/core/src/reductions/cb/cb_adf.cc index b31f12e14bc..32cd8146de5 100644 --- a/vowpalwabbit/core/src/reductions/cb/cb_adf.cc +++ b/vowpalwabbit/core/src/reductions/cb/cb_adf.cc @@ -187,7 +187,7 @@ void VW::reductions::cb_adf::learn_sm(learner& base, VW::multi_ex& examples) void VW::reductions::cb_adf::learn_dr(learner& base, VW::multi_ex& examples) { - details::gen_cs_example_dr(gen_cs, examples, _cs_labels, _clip_p); + details::gen_cs_example_dr(_gen_cs_dr, examples, _cs_labels, _clip_p); details::cs_ldf_learn_or_predict(base, examples, _cb_labels, _cs_labels, _prepped_cs_labels, true, _offset); } @@ -210,21 +210,22 @@ void VW::reductions::cb_adf::learn_mtr(learner& base, VW::multi_ex& examples) // Train on _one_ action (which requires up to 3 examples). // We must go through the cost sensitive classifier layer to get // proper feature handling. - gen_cs_example_mtr(gen_cs, examples, _cs_labels); - uint32_t nf = static_cast(examples[gen_cs.mtr_example]->num_features); - float old_weight = examples[gen_cs.mtr_example]->weight; - const float clipped_p = std::max(examples[gen_cs.mtr_example]->l.cb.costs[0].probability, _clip_p); - examples[gen_cs.mtr_example]->weight *= - 1.f / clipped_p * (static_cast(gen_cs.event_sum) / static_cast(gen_cs.action_sum)); - - std::swap(gen_cs.mtr_ec_seq[0]->pred.a_s, _a_s_mtr_cs); + gen_cs_example_mtr(_gen_cs_mtr, examples, _cs_labels, _offset_index); + uint32_t nf = static_cast(examples[_gen_cs_mtr.mtr_example]->num_features); + float old_weight = examples[_gen_cs_mtr.mtr_example]->weight; + const float clipped_p = std::max(examples[_gen_cs_mtr.mtr_example]->l.cb.costs[0].probability, _clip_p); + examples[_gen_cs_mtr.mtr_example]->weight *= 1.f / clipped_p * + (static_cast(_gen_cs_mtr.per_model_state[_offset_index].event_sum) / + static_cast(_gen_cs_mtr.per_model_state[_offset_index].action_sum)); + + std::swap(_gen_cs_mtr.mtr_ec_seq[0]->pred.a_s, _a_s_mtr_cs); // TODO!!! cb_labels are not getting properly restored (empty costs are // dropped) details::cs_ldf_learn_or_predict( - base, gen_cs.mtr_ec_seq, _cb_labels, _cs_labels, _prepped_cs_labels, false, _offset); - examples[gen_cs.mtr_example]->num_features = nf; - examples[gen_cs.mtr_example]->weight = old_weight; - std::swap(gen_cs.mtr_ec_seq[0]->pred.a_s, _a_s_mtr_cs); + base, _gen_cs_mtr.mtr_ec_seq, _cb_labels, _cs_labels, _prepped_cs_labels, false, _offset); + examples[_gen_cs_mtr.mtr_example]->num_features = nf; + examples[_gen_cs_mtr.mtr_example]->weight = old_weight; + std::swap(_gen_cs_mtr.mtr_ec_seq[0]->pred.a_s, _a_s_mtr_cs); if (PREDICT) { // Return the saved prediction @@ -237,8 +238,9 @@ void VW::reductions::cb_adf::learn(learner& base, VW::multi_ex& ec_seq) if (VW::test_cb_adf_sequence(ec_seq) != nullptr) { _offset = ec_seq[0]->ft_offset; - gen_cs.known_cost = VW::get_observed_cost_or_default_cb_adf(ec_seq); // need to set for test case - switch (gen_cs.cb_type) + _offset_index = _offset / _all->weights.stride(); + _gen_cs_dr.known_cost = VW::get_observed_cost_or_default_cb_adf(ec_seq); // need to set for test case + switch (_cb_type) { case VW::cb_type_t::DR: learn_dr(base, ec_seq); @@ -264,7 +266,8 @@ void VW::reductions::cb_adf::learn(learner& base, VW::multi_ex& ec_seq) void VW::reductions::cb_adf::predict(learner& base, VW::multi_ex& ec_seq) { _offset = ec_seq[0]->ft_offset; - gen_cs.known_cost = VW::get_observed_cost_or_default_cb_adf(ec_seq); // need to set for test case + _offset_index = _offset / _all->weights.stride(); + _gen_cs_dr.known_cost = VW::get_observed_cost_or_default_cb_adf(ec_seq); // need to set for test case details::gen_cs_test_example(ec_seq, _cs_labels); // create test labels. details::cs_ldf_learn_or_predict(base, ec_seq, _cb_labels, _cs_labels, _prepped_cs_labels, false, _offset); } @@ -282,7 +285,10 @@ bool VW::reductions::cb_adf::update_statistics( float loss = 0.; bool labeled_example = true; - if (gen_cs.known_cost.probability > 0) { loss = get_cost_estimate(gen_cs.known_cost, gen_cs.pred_scores, action); } + if (_gen_cs_dr.known_cost.probability > 0) + { + loss = get_cost_estimate(_gen_cs_dr.known_cost, _gen_cs_dr.pred_scores, action); + } else { labeled_example = false; } bool holdout_example = labeled_example; @@ -325,7 +331,7 @@ void print_update_cb_adf(VW::workspace& all, VW::shared_data& /* sd */, const VW { if (ec_seq.empty()) { return; } - const bool labeled_example = data.gen_cs.known_cost.probability > 0; + const bool labeled_example = data.get_gen_cs_dr().known_cost.probability > 0; const auto& ec = *ec_seq.front(); if (labeled_example) { VW::details::print_update_cb(all, !labeled_example, ec, &ec_seq, true, data.known_cost()); } else { VW::details::print_update_cb(all, !labeled_example, ec, &ec_seq, true, nullptr); } @@ -340,13 +346,13 @@ void save_load(VW::reductions::cb_adf& c, VW::io_buf& model_file, bool read, boo } std::stringstream msg; - msg << "event_sum " << c.get_gen_cs().event_sum << "\n"; - VW::details::bin_text_read_write_fixed( - model_file, (char*)&c.get_gen_cs().event_sum, sizeof(c.get_gen_cs().event_sum), read, msg, text); + msg << "event_sum " << c.get_gen_cs_mtr().per_model_state[0].event_sum << "\n"; + VW::details::bin_text_read_write_fixed(model_file, (char*)&c.get_gen_cs_mtr().per_model_state[0].event_sum, + sizeof(c.get_gen_cs_mtr().per_model_state[0].event_sum), read, msg, text); - msg << "action_sum " << c.get_gen_cs().action_sum << "\n"; - VW::details::bin_text_read_write_fixed( - model_file, (char*)&c.get_gen_cs().action_sum, sizeof(c.get_gen_cs().action_sum), read, msg, text); + msg << "action_sum " << c.get_gen_cs_mtr().per_model_state[0].action_sum << "\n"; + VW::details::bin_text_read_write_fixed(model_file, (char*)&c.get_gen_cs_mtr().per_model_state[0].action_sum, + sizeof(c.get_gen_cs_mtr().per_model_state[0].action_sum), read, msg, text); } void cb_adf_merge(const std::vector& /* per_model_weights */, @@ -354,23 +360,38 @@ void cb_adf_merge(const std::vector& /* per_model_weights */, { for (const auto* source : sources) { - output_data.get_gen_cs().event_sum += source->get_gen_cs().event_sum; - output_data.get_gen_cs().action_sum += source->get_gen_cs().action_sum; + for (size_t i = 0; i < output_data.get_gen_cs_mtr().per_model_state.size(); i++) + { + output_data.get_gen_cs_mtr().per_model_state[i].event_sum += + source->get_gen_cs_mtr().per_model_state[i].event_sum; + output_data.get_gen_cs_mtr().per_model_state[i].action_sum += + source->get_gen_cs_mtr().per_model_state[i].action_sum; + } } } void cb_adf_add( const VW::reductions::cb_adf& data1, const VW::reductions::cb_adf& data2, VW::reductions::cb_adf& data_out) { - data_out.get_gen_cs().event_sum = data1.get_gen_cs().event_sum + data2.get_gen_cs().event_sum; - data_out.get_gen_cs().action_sum = data1.get_gen_cs().action_sum + data2.get_gen_cs().action_sum; + for (size_t i = 0; i < data1.get_gen_cs_mtr().per_model_state.size(); i++) + { + data_out.get_gen_cs_mtr().per_model_state[i].event_sum = + data1.get_gen_cs_mtr().per_model_state[i].event_sum + data2.get_gen_cs_mtr().per_model_state[i].event_sum; + data_out.get_gen_cs_mtr().per_model_state[i].action_sum = + data1.get_gen_cs_mtr().per_model_state[i].action_sum + data2.get_gen_cs_mtr().per_model_state[i].action_sum; + } } void cb_adf_subtract( const VW::reductions::cb_adf& data1, const VW::reductions::cb_adf& data2, VW::reductions::cb_adf& data_out) { - data_out.get_gen_cs().event_sum = data1.get_gen_cs().event_sum - data2.get_gen_cs().event_sum; - data_out.get_gen_cs().action_sum = data1.get_gen_cs().action_sum - data2.get_gen_cs().action_sum; + for (size_t i = 0; i < data1.get_gen_cs_mtr().per_model_state.size(); i++) + { + data_out.get_gen_cs_mtr().per_model_state[i].event_sum = + data1.get_gen_cs_mtr().per_model_state[i].event_sum - data2.get_gen_cs_mtr().per_model_state[i].event_sum; + data_out.get_gen_cs_mtr().per_model_state[i].action_sum = + data1.get_gen_cs_mtr().per_model_state[i].action_sum - data2.get_gen_cs_mtr().per_model_state[i].action_sum; + } } void learn(VW::reductions::cb_adf& c, learner& base, VW::multi_ex& ec_seq) { c.learn(base, ec_seq); } @@ -382,6 +403,7 @@ std::shared_ptr VW::reductions::cb_adf_setup(VW::setup_bas { options_i& options = *stack_builder.get_options(); VW::workspace& all = *stack_builder.get_all_pointer(); + size_t feature_width_above = stack_builder.get_feature_width_above(); bool cb_adf_option = false; std::string type_string = "mtr"; @@ -423,7 +445,7 @@ std::shared_ptr VW::reductions::cb_adf_setup(VW::setup_bas } // number of weight vectors needed - size_t problem_multiplier = 1; // default for IPS + size_t feature_width = 1; // default for IPS bool check_baseline_enabled = false; try @@ -439,7 +461,7 @@ std::shared_ptr VW::reductions::cb_adf_setup(VW::setup_bas if (cb_type == VW::cb_type_t::DR) { - problem_multiplier = 2; + feature_width = 2; // only use baseline when manually enabled for loss estimation check_baseline_enabled = true; } @@ -460,10 +482,9 @@ std::shared_ptr VW::reductions::cb_adf_setup(VW::setup_bas if (options.was_supplied("baseline") && check_baseline_enabled) { options.insert("check_enabled", ""); } - auto ld = VW::make_unique(cb_type, rank_all, clip_p, no_predict, &all); + auto ld = VW::make_unique(cb_type, rank_all, clip_p, no_predict, feature_width_above, &all); - auto base = require_multiline(stack_builder.setup_base_learner()); - all.parser_runtime.example_parser->lbl_parser = VW::cb_label_parser_global; + auto base = require_multiline(stack_builder.setup_base_learner(feature_width)); VW::reductions::cb_adf* bare = ld.get(); bool lrp = ld->learn_returns_prediction(); @@ -473,7 +494,7 @@ std::shared_ptr VW::reductions::cb_adf_setup(VW::setup_bas .set_input_prediction_type(VW::prediction_type_t::ACTION_SCORES) .set_output_prediction_type(VW::prediction_type_t::ACTION_SCORES) .set_learn_returns_prediction(lrp) - .set_params_per_weight(problem_multiplier) + .set_feature_width(feature_width) .set_save_load(::save_load) .set_merge(::cb_adf_merge) .set_add(::cb_adf_add) diff --git a/vowpalwabbit/core/src/reductions/cb/cb_algs.cc b/vowpalwabbit/core/src/reductions/cb/cb_algs.cc index 00afc0a2ba4..6bc95cdb13c 100644 --- a/vowpalwabbit/core/src/reductions/cb/cb_algs.cc +++ b/vowpalwabbit/core/src/reductions/cb/cb_algs.cc @@ -166,7 +166,7 @@ std::shared_ptr VW::reductions::cb_algs_setup(VW::setup_ba VW::details::cb_to_cs& c = data->cbcs; - size_t problem_multiplier = 2; // default for DR + size_t feature_width = 2; // default for DR c.cb_type = VW::cb_type_from_string(type_string); switch (c.cb_type) { @@ -174,10 +174,10 @@ std::shared_ptr VW::reductions::cb_algs_setup(VW::setup_ba break; case VW::cb_type_t::DM: if (eval) THROW("direct method can not be used for evaluation --- it is biased."); - problem_multiplier = 1; + feature_width = 1; break; case VW::cb_type_t::IPS: - problem_multiplier = 1; + feature_width = 1; break; case VW::cb_type_t::MTR: case VW::cb_type_t::SM: @@ -194,9 +194,7 @@ std::shared_ptr VW::reductions::cb_algs_setup(VW::setup_ba options.insert("csoaa", ss.str()); } - auto base = require_singleline(stack_builder.setup_base_learner()); - if (eval) { all.parser_runtime.example_parser->lbl_parser = VW::cb_eval_label_parser_global; } - else { all.parser_runtime.example_parser->lbl_parser = VW::cb_label_parser_global; } + auto base = require_singleline(stack_builder.setup_base_learner(feature_width)); c.scorer = VW::LEARNER::require_singleline(base->get_learner_by_name_prefix("scorer")); std::string name_addition = eval ? "-eval" : ""; @@ -216,7 +214,7 @@ std::shared_ptr VW::reductions::cb_algs_setup(VW::setup_ba .set_output_label_type(VW::label_type_t::CS) .set_input_prediction_type(VW::prediction_type_t::MULTICLASS) .set_output_prediction_type(VW::prediction_type_t::MULTICLASS) - .set_params_per_weight(problem_multiplier) + .set_feature_width(feature_width) .set_learn_returns_prediction(eval) .set_update_stats(update_stats_func) .set_output_example_prediction(output_example_prediction_func) diff --git a/vowpalwabbit/core/src/reductions/cb/cb_explore.cc b/vowpalwabbit/core/src/reductions/cb/cb_explore.cc index 81e0bcec113..ce4bfde3bad 100644 --- a/vowpalwabbit/core/src/reductions/cb/cb_explore.cc +++ b/vowpalwabbit/core/src/reductions/cb/cb_explore.cc @@ -374,12 +374,16 @@ std::shared_ptr VW::reductions::cb_explore_setup(VW::setup data->cbcs.cb_type = VW::cb_type_t::DR; data->model_file_version = all.runtime_state.model_file_ver; - auto base = require_singleline(stack_builder.setup_base_learner()); + size_t params_per_weight = 1; + if (options.was_supplied("cover")) { params_per_weight = data->cover_size + 1; } + else if (options.was_supplied("bag")) { params_per_weight = data->bag_size; } + + auto base = require_singleline(stack_builder.setup_base_learner(params_per_weight)); data->cbcs.scorer = VW::LEARNER::require_singleline(base->get_learner_by_name_prefix("scorer")); void (*learn_ptr)(cb_explore&, learner&, VW::example&); void (*predict_ptr)(cb_explore&, learner&, VW::example&); - size_t params_per_weight; + std::string name_addition; if (options.was_supplied("cover")) @@ -402,28 +406,24 @@ std::shared_ptr VW::reductions::cb_explore_setup(VW::setup data->preds.reserve(data->cover_size); learn_ptr = predict_or_learn_cover; predict_ptr = predict_or_learn_cover; - params_per_weight = data->cover_size + 1; name_addition = "-cover"; } else if (options.was_supplied("bag")) { learn_ptr = predict_or_learn_bag; predict_ptr = predict_or_learn_bag; - params_per_weight = data->bag_size; name_addition = "-bag"; } else if (options.was_supplied("first")) { learn_ptr = predict_or_learn_first; predict_ptr = predict_or_learn_first; - params_per_weight = 1; name_addition = "-first"; } else // greedy { learn_ptr = predict_or_learn_greedy; predict_ptr = predict_or_learn_greedy; - params_per_weight = 1; name_addition = "-greedy"; } auto l = make_reduction_learner( @@ -432,7 +432,7 @@ std::shared_ptr VW::reductions::cb_explore_setup(VW::setup .set_output_label_type(VW::label_type_t::CB) .set_input_prediction_type(VW::prediction_type_t::MULTICLASS) .set_output_prediction_type(VW::prediction_type_t::ACTION_PROBS) - .set_params_per_weight(params_per_weight) + .set_feature_width(params_per_weight) .set_update_stats(::update_stats_cb_explore) .set_output_example_prediction(::output_example_prediction_cb_explore) .set_print_update(::print_update_cb_explore) diff --git a/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_bag.cc b/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_bag.cc index 90f753b89f2..d13b60f2b8f 100644 --- a/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_bag.cc +++ b/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_bag.cc @@ -193,9 +193,9 @@ std::shared_ptr VW::reductions::cb_explore_adf_bag_setup(V // predict before training is called. if (!options.was_supplied("no_predict")) { options.insert("no_predict", ""); } - size_t problem_multiplier = VW::cast_to_smaller_type(bag_size); - auto base = require_multiline(stack_builder.setup_base_learner()); - all.parser_runtime.example_parser->lbl_parser = VW::cb_label_parser_global; + size_t feature_width = VW::cast_to_smaller_type(bag_size); + + auto base = require_multiline(stack_builder.setup_base_learner(feature_width)); using explore_type = cb_explore_adf_base; auto data = VW::make_unique(all.output_runtime.global_metrics.are_metrics_enabled(), epsilon, @@ -206,7 +206,7 @@ std::shared_ptr VW::reductions::cb_explore_adf_bag_setup(V .set_output_label_type(VW::label_type_t::CB) .set_input_prediction_type(VW::prediction_type_t::ACTION_SCORES) .set_output_prediction_type(VW::prediction_type_t::ACTION_PROBS) - .set_params_per_weight(problem_multiplier) + .set_feature_width(feature_width) .set_output_example_prediction(::output_example_prediction_bag) .set_update_stats(::update_stats_bag) .set_print_update(::print_update_bag) diff --git a/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_cover.cc b/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_cover.cc index 089e1523a64..cbcec777762 100644 --- a/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_cover.cc +++ b/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_cover.cc @@ -53,7 +53,8 @@ class cb_explore_adf_cover size_t _counter; VW::LEARNER::learner* _cs_ldf_learner; - VW::details::cb_to_cs_adf gen_cs; + VW::details::cb_to_cs_adf_dr _gen_cs_dr; + VW::cb_type_t _cb_type = VW::cb_type_t::DM; VW::version_struct _model_file_version; VW::io::logger _logger; @@ -79,29 +80,30 @@ cb_explore_adf_cover::cb_explore_adf_cover(size_t cover_size, float psi, bool no , _first_only(first_only) , _counter(0) , _cs_ldf_learner(cs_ldf_learner) + , _cb_type(cb_type) , _model_file_version(model_file_version) , _logger(std::move(logger)) { - gen_cs.cb_type = cb_type; - gen_cs.scorer = scorer; + _gen_cs_dr.scorer = scorer; } template void cb_explore_adf_cover::predict_or_learn_impl(VW::LEARNER::learner& base, VW::multi_ex& examples) { // Redundant with the call in cb_explore_adf_base, but encapsulation means we need to do this again here - gen_cs.known_cost = VW::get_observed_cost_or_default_cb_adf(examples); + _gen_cs_dr.known_cost = VW::get_observed_cost_or_default_cb_adf(examples); // Randomize over predictions from a base set of predictors // Use cost sensitive oracle to cover actions to form distribution. - const bool is_mtr = gen_cs.cb_type == VW::cb_type_t::MTR; if (is_learn) { - if (is_mtr) + VW_DBG(*examples[0]) << "gen_cs_example:" << is_learn << std::endl; + if (_cb_type == VW::cb_type_t::MTR) { // use DR estimates for non-ERM policies in MTR - VW::details::gen_cs_example_dr(gen_cs, examples, _cs_labels); + VW::details::gen_cs_example_dr(_gen_cs_dr, examples, _cs_labels); } - else { VW::details::gen_cs_example(gen_cs, examples, _cs_labels, _logger); } + else if (_cb_type == VW::cb_type_t::DR) { VW::details::gen_cs_example_dr(_gen_cs_dr, examples, _cs_labels); } + else { VW::details::gen_cs_example_ips(examples, _cs_labels, _logger); } if (base.learn_returns_prediction) { @@ -299,13 +301,12 @@ std::shared_ptr VW::reductions::cb_explore_adf_cover_setup } // Set explore_type - size_t problem_multiplier = cover_size + 1; + size_t feature_width = cover_size + 1; // Cover is using doubly robust without the cooperation of the base reduction - if (cb_type == VW::cb_type_t::MTR) { problem_multiplier *= 2; } + if (cb_type == VW::cb_type_t::MTR) { feature_width *= 2; } - auto base = VW::LEARNER::require_multiline(stack_builder.setup_base_learner()); - all.parser_runtime.example_parser->lbl_parser = VW::cb_label_parser_global; + auto base = VW::LEARNER::require_multiline(stack_builder.setup_base_learner(feature_width)); bool epsilon_decay; if (options.was_supplied("epsilon")) @@ -333,7 +334,7 @@ std::shared_ptr VW::reductions::cb_explore_adf_cover_setup .set_input_prediction_type(VW::prediction_type_t::ACTION_SCORES) .set_output_prediction_type(VW::prediction_type_t::ACTION_PROBS) .set_learn_returns_prediction(true) - .set_params_per_weight(problem_multiplier) + .set_feature_width(feature_width) .set_output_example_prediction(explore_type::output_example_prediction) .set_update_stats(explore_type::update_stats) .set_print_update(explore_type::print_update) diff --git a/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_first.cc b/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_first.cc index 7146513fcb9..2a1fcafd6a3 100644 --- a/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_first.cc +++ b/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_first.cc @@ -116,10 +116,9 @@ std::shared_ptr VW::reductions::cb_explore_adf_first_setup // Ensure serialization of cb_adf in all cases. if (!options.was_supplied("cb_adf")) { options.insert("cb_adf", ""); } - size_t problem_multiplier = 1; + size_t feature_width = 1; - auto base = require_multiline(stack_builder.setup_base_learner()); - all.parser_runtime.example_parser->lbl_parser = VW::cb_label_parser_global; + auto base = require_multiline(stack_builder.setup_base_learner(feature_width)); using explore_type = cb_explore_adf_base; auto data = VW::make_unique(all.output_runtime.global_metrics.are_metrics_enabled(), @@ -132,7 +131,7 @@ std::shared_ptr VW::reductions::cb_explore_adf_first_setup .set_output_label_type(VW::label_type_t::CB) .set_input_prediction_type(VW::prediction_type_t::ACTION_SCORES) .set_output_prediction_type(VW::prediction_type_t::ACTION_PROBS) - .set_params_per_weight(problem_multiplier) + .set_feature_width(feature_width) .set_output_example_prediction(explore_type::output_example_prediction) .set_update_stats(explore_type::update_stats) .set_print_update(explore_type::print_update) diff --git a/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_greedy.cc b/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_greedy.cc index 90b4909861c..9a0b7dd50c9 100644 --- a/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_greedy.cc +++ b/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_greedy.cc @@ -124,10 +124,9 @@ std::shared_ptr VW::reductions::cb_explore_adf_greedy_setu options.insert("no_predict", ""); } - size_t problem_multiplier = 1; + size_t feature_width = 1; - auto base = require_multiline(stack_builder.setup_base_learner()); - all.parser_runtime.example_parser->lbl_parser = VW::cb_label_parser_global; + auto base = require_multiline(stack_builder.setup_base_learner(feature_width)); using explore_type = cb_explore_adf_base; auto data = @@ -141,7 +140,7 @@ std::shared_ptr VW::reductions::cb_explore_adf_greedy_setu .set_output_label_type(VW::label_type_t::CB) .set_input_prediction_type(VW::prediction_type_t::ACTION_SCORES) .set_output_prediction_type(VW::prediction_type_t::ACTION_PROBS) - .set_params_per_weight(problem_multiplier) + .set_feature_width(feature_width) .set_output_example_prediction(explore_type::output_example_prediction) .set_update_stats(explore_type::update_stats) .set_print_update(explore_type::print_update) diff --git a/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_large_action_space.cc b/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_large_action_space.cc index 2a9ed8a6345..a2068227c3a 100644 --- a/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_large_action_space.cc +++ b/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_large_action_space.cc @@ -231,14 +231,14 @@ void generate_Z(const multi_ex& examples, Eigen::MatrixXf& Z, Eigen::MatrixXf& B template cb_explore_adf_large_action_space::cb_explore_adf_large_action_space(uint64_t d, float c, bool apply_shrink_factor, VW::workspace* all, uint64_t seed, size_t total_size, size_t thread_pool_size, - size_t block_size, bool use_explicit_simd, implementation_type impl_type) + size_t block_size, size_t action_cache_slack, bool use_explicit_simd, implementation_type impl_type) : _d(d) , _all(all) , _seed(seed) , _impl_type(impl_type) , spanner_state(c, d) , shrink_fact_config(apply_shrink_factor) - , impl(all, d, _seed, total_size, thread_pool_size, block_size, use_explicit_simd) + , impl(all, d, _seed, total_size, thread_pool_size, block_size, action_cache_slack, use_explicit_simd) { } @@ -287,14 +287,13 @@ void learn(cb_explore_adf_large_action_space& data, VW::LEARNER::learner& template std::shared_ptr make_las_with_impl(VW::setup_base_i& stack_builder, std::shared_ptr base, implementation_type& impl_type, VW::workspace& all, uint64_t d, float c, - bool apply_shrink_factor, size_t thread_pool_size, size_t block_size, bool use_explicit_simd) + bool apply_shrink_factor, size_t thread_pool_size, size_t block_size, size_t action_cache_slack, + bool use_explicit_simd) { - size_t problem_multiplier = 1; - float seed = (all.get_random_state()->get_random() + 1) * 10.f; auto data = VW::make_unique>(d, c, apply_shrink_factor, &all, seed, - 1 << all.iwc.num_bits, thread_pool_size, block_size, use_explicit_simd, impl_type); + 1 << all.iwc.num_bits, thread_pool_size, block_size, action_cache_slack, use_explicit_simd, impl_type); auto l = make_reduction_learner(std::move(data), base, learn, predict, stack_builder.get_setupfn_name(VW::reductions::cb_explore_adf_large_action_space_setup)) @@ -302,7 +301,6 @@ std::shared_ptr make_las_with_impl(VW::setup_base_i& stack .set_output_label_type(VW::label_type_t::CB) .set_input_prediction_type(VW::prediction_type_t::ACTION_SCORES) .set_output_prediction_type(VW::prediction_type_t::ACTION_SCORES) - .set_params_per_weight(problem_multiplier) .set_persist_metrics(persist_metrics) .set_learn_returns_prediction(false) .build(); @@ -326,6 +324,7 @@ std::shared_ptr VW::reductions::cb_explore_adf_large_actio // leave some resources available in the case of few core's (for parser) uint64_t thread_pool_size = (std::thread::hardware_concurrency() - 1) / 2; uint64_t block_size = 0; + uint64_t action_cache_slack = 50; config::option_group_definition new_options( "[Reduction] Experimental: Contextual Bandit Exploration with ADF with large action space filtering"); @@ -365,25 +364,21 @@ std::shared_ptr VW::reductions::cb_explore_adf_large_actio "(x86 Linux only)")) .add(make_option("two_pass_svd", use_two_pass_svd_impl) .experimental() - .help("A more accurate svd that is much slower than the default (one pass svd)")); + .help("A more accurate svd that is much slower than the default (one pass svd)")) + .add(make_option("action_cache_slack", action_cache_slack) + .keep() + .allow_override() + .default_value(50) + .help("If actions do not change between example calls then some calculations for this algorithm are " + "cached. If action cache size exceeds the number of active actions plus action_cache_slack, then " + "the cache will be cleared. Setting this to -1 disables any caching") + .experimental()); auto enabled = options.add_parse_and_check_necessary(new_options) && large_action_space; if (!enabled) { return nullptr; } if (options.was_supplied("squarecb")) { apply_shrink_factor = true; } - if (options.was_supplied("cb_type")) - { - auto cb_type = options.get_typed_option("cb_type").value(); - if (cb_type != "mtr") - { - all.logger.err_warn( - "Only cb_type 'mtr' is currently supported with large action spaces, resetting to mtr. Input was: '{}'", - cb_type); - options.replace("cb_type", "mtr"); - } - } - if (use_simd_in_one_pass_svd_impl && (options.was_supplied("cubic") || options.was_supplied("interactions") || options.was_supplied("experimental_full_name_interactions"))) @@ -394,19 +389,18 @@ std::shared_ptr VW::reductions::cb_explore_adf_large_actio } auto base = require_multiline(stack_builder.setup_base_learner()); - all.parser_runtime.example_parser->lbl_parser = VW::cb_label_parser_global; if (use_two_pass_svd_impl) { auto impl_type = implementation_type::two_pass_svd; return make_las_with_impl(stack_builder, base, impl_type, all, d, c, - apply_shrink_factor, thread_pool_size, block_size, + apply_shrink_factor, thread_pool_size, block_size, action_cache_slack, /*use_explicit_simd=*/false); } else { auto impl_type = implementation_type::one_pass_svd; return make_las_with_impl(stack_builder, base, impl_type, all, d, c, - apply_shrink_factor, thread_pool_size, block_size, use_simd_in_one_pass_svd_impl); + apply_shrink_factor, thread_pool_size, block_size, action_cache_slack, use_simd_in_one_pass_svd_impl); } } diff --git a/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_regcb.cc b/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_regcb.cc index fba872e2ad2..ddb3f3c927d 100644 --- a/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_regcb.cc +++ b/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_regcb.cc @@ -297,10 +297,9 @@ std::shared_ptr VW::reductions::cb_explore_adf_regcb_setup if (!options.was_supplied("cb_adf")) { options.insert("cb_adf", ""); } // Set explore_type - size_t problem_multiplier = 1; + size_t feature_width = 1; - auto base = require_multiline(stack_builder.setup_base_learner()); - all.parser_runtime.example_parser->lbl_parser = VW::cb_label_parser_global; + auto base = require_multiline(stack_builder.setup_base_learner(feature_width)); using explore_type = cb_explore_adf_base; auto data = VW::make_unique(all.output_runtime.global_metrics.are_metrics_enabled(), regcbopt, c0, @@ -311,7 +310,7 @@ std::shared_ptr VW::reductions::cb_explore_adf_regcb_setup .set_output_label_type(VW::label_type_t::CB) .set_input_prediction_type(VW::prediction_type_t::ACTION_SCORES) .set_output_prediction_type(VW::prediction_type_t::ACTION_PROBS) - .set_params_per_weight(problem_multiplier) + .set_feature_width(feature_width) .set_output_example_prediction(explore_type::output_example_prediction) .set_update_stats(explore_type::update_stats) .set_print_update(explore_type::print_update) diff --git a/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_rnd.cc b/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_rnd.cc index 2b0c240101c..143a0760033 100644 --- a/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_rnd.cc +++ b/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_rnd.cc @@ -41,13 +41,13 @@ namespace class cb_explore_adf_rnd { public: - cb_explore_adf_rnd( - float _epsilon, float _alpha, float _invlambda, uint32_t _numrnd, size_t _increment, VW::workspace* _all) + cb_explore_adf_rnd(float _epsilon, float _alpha, float _invlambda, uint32_t _numrnd, size_t _feature_width_below, + VW::workspace* _all) : _epsilon(_epsilon) , _alpha(_alpha) , _sqrtinvlambda(std::sqrt(_invlambda)) , _numrnd(_numrnd) - , _increment(_increment) + , _feature_width_below(_feature_width_below) , _all(_all) { } @@ -63,7 +63,7 @@ class cb_explore_adf_rnd float _sqrtinvlambda; uint32_t _numrnd; - size_t _increment; + size_t _feature_width_below; VW::workspace* _all; std::vector _bonuses; @@ -175,9 +175,9 @@ void cb_explore_adf_rnd::get_initial_predictions(VW::multi_ex& examples, uint32_ { auto* ec = examples[i]; - VW::LEARNER::details::increment_offset(*ec, _increment, id); + VW::LEARNER::details::increment_offset(*ec, _feature_width_below, id); _initials.push_back(get_initial_prediction(ec)); - VW::LEARNER::details::decrement_offset(*ec, _increment, id); + VW::LEARNER::details::decrement_offset(*ec, _feature_width_below, id); } } @@ -298,14 +298,13 @@ std::shared_ptr VW::reductions::cb_explore_adf_rnd_setup(V // Ensure serialization of cb_adf in all cases. if (!options.was_supplied("cb_adf")) { options.insert("cb_adf", ""); } - size_t problem_multiplier = 1 + numrnd; + size_t feature_width = 1 + numrnd; - auto base = require_multiline(stack_builder.setup_base_learner()); - all.parser_runtime.example_parser->lbl_parser = VW::cb_label_parser_global; + auto base = require_multiline(stack_builder.setup_base_learner(feature_width)); using explore_type = cb_explore_adf_base; - auto data = VW::make_unique(all.output_runtime.global_metrics.are_metrics_enabled(), epsilon, alpha, - invlambda, numrnd, base->increment * problem_multiplier, &all); + auto data = VW::make_unique(all.output_runtime.global_metrics.are_metrics_enabled(), epsilon, alpha, invlambda, numrnd, + base->feature_width_below * feature_width, &all); if (epsilon < 0.0 || epsilon > 1.0) { THROW("The value of epsilon must be in [0,1]"); } auto l = make_reduction_learner(std::move(data), base, explore_type::learn, explore_type::predict, @@ -314,7 +313,7 @@ std::shared_ptr VW::reductions::cb_explore_adf_rnd_setup(V .set_output_label_type(VW::label_type_t::CB) .set_input_prediction_type(VW::prediction_type_t::ACTION_SCORES) .set_output_prediction_type(VW::prediction_type_t::ACTION_PROBS) - .set_params_per_weight(problem_multiplier) + .set_feature_width(feature_width) .set_output_example_prediction(explore_type::output_example_prediction) .set_update_stats(explore_type::update_stats) .set_print_update(explore_type::print_update) diff --git a/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_softmax.cc b/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_softmax.cc index 18bdaeeace5..de6ff0843c2 100644 --- a/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_softmax.cc +++ b/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_softmax.cc @@ -84,10 +84,9 @@ std::shared_ptr VW::reductions::cb_explore_adf_softmax_set if (!options.was_supplied("cb_adf")) { options.insert("cb_adf", ""); } // Set explore_type - size_t problem_multiplier = 1; + size_t feature_width = 1; - auto base = require_multiline(stack_builder.setup_base_learner()); - all.parser_runtime.example_parser->lbl_parser = VW::cb_label_parser_global; + auto base = require_multiline(stack_builder.setup_base_learner(feature_width)); using explore_type = cb_explore_adf_base; auto data = VW::make_unique(all.output_runtime.global_metrics.are_metrics_enabled(), epsilon, lambda); @@ -99,7 +98,7 @@ std::shared_ptr VW::reductions::cb_explore_adf_softmax_set .set_output_label_type(VW::label_type_t::CB) .set_input_prediction_type(VW::prediction_type_t::ACTION_SCORES) .set_output_prediction_type(VW::prediction_type_t::ACTION_PROBS) - .set_params_per_weight(problem_multiplier) + .set_feature_width(feature_width) .set_output_example_prediction(explore_type::output_example_prediction) .set_update_stats(explore_type::update_stats) .set_print_update(explore_type::print_update) diff --git a/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_squarecb.cc b/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_squarecb.cc index fa13545e73b..7eba27a5bf6 100644 --- a/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_squarecb.cc +++ b/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_squarecb.cc @@ -388,10 +388,9 @@ std::shared_ptr VW::reductions::cb_explore_adf_squarecb_se if (options.was_supplied("large_action_space")) { store_gamma_in_reduction_features = true; } // Set explore_type - size_t problem_multiplier = 1; + size_t feature_width = 1; - auto base = require_multiline(stack_builder.setup_base_learner()); - all.parser_runtime.example_parser->lbl_parser = VW::cb_label_parser_global; + auto base = require_multiline(stack_builder.setup_base_learner(feature_width)); if (epsilon < 0.0 || epsilon > 1.0) { THROW("The value of epsilon must be in [0,1]"); } @@ -405,7 +404,7 @@ std::shared_ptr VW::reductions::cb_explore_adf_squarecb_se .set_output_label_type(VW::label_type_t::CB) .set_input_prediction_type(VW::prediction_type_t::ACTION_SCORES) .set_output_prediction_type(VW::prediction_type_t::ACTION_PROBS) - .set_params_per_weight(problem_multiplier) + .set_feature_width(feature_width) .set_output_example_prediction(explore_type::output_example_prediction) .set_update_stats(explore_type::update_stats) .set_print_update(explore_type::print_update) diff --git a/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_synthcover.cc b/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_synthcover.cc index 9ddb0f17064..a9757248256 100644 --- a/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_synthcover.cc +++ b/vowpalwabbit/core/src/reductions/cb/cb_explore_adf_synthcover.cc @@ -191,9 +191,8 @@ std::shared_ptr VW::reductions::cb_explore_adf_synthcover_ *(all.output_runtime.trace_message) << "synthcoverpsi = " << psi << std::endl; } - size_t problem_multiplier = 1; - auto base = require_multiline(stack_builder.setup_base_learner()); - all.parser_runtime.example_parser->lbl_parser = VW::cb_label_parser_global; + size_t feature_width = 1; + auto base = require_multiline(stack_builder.setup_base_learner(feature_width)); using explore_type = cb_explore_adf_base; auto data = VW::make_unique(all.output_runtime.global_metrics.are_metrics_enabled(), epsilon, psi, @@ -204,7 +203,7 @@ std::shared_ptr VW::reductions::cb_explore_adf_synthcover_ .set_output_label_type(VW::label_type_t::CB) .set_input_prediction_type(VW::prediction_type_t::ACTION_SCORES) .set_output_prediction_type(VW::prediction_type_t::ACTION_PROBS) - .set_params_per_weight(problem_multiplier) + .set_feature_width(feature_width) .set_output_example_prediction(explore_type::output_example_prediction) .set_update_stats(explore_type::update_stats) .set_print_update(explore_type::print_update) diff --git a/vowpalwabbit/core/src/reductions/cb/cb_to_cb_adf.cc b/vowpalwabbit/core/src/reductions/cb/cb_to_cb_adf.cc index b5d6a1c03c5..857e61b30b0 100644 --- a/vowpalwabbit/core/src/reductions/cb/cb_to_cb_adf.cc +++ b/vowpalwabbit/core/src/reductions/cb/cb_to_cb_adf.cc @@ -217,7 +217,7 @@ std::shared_ptr VW::reductions::cb_to_cb_adf_setup(VW::set if (num_actions <= 0) { THROW("cb num actions must be positive"); } - data->adf_data.init_adf_data(num_actions, base->increment, all.fc.interactions, all.fc.extent_interactions); + data->adf_data.init_adf_data(num_actions, base->feature_width_below, all.fc.interactions, all.fc.extent_interactions); // see csoaa.cc ~ line 894 / setup for csldf_setup all.parser_runtime.example_parser->emptylines_separate_examples = false; diff --git a/vowpalwabbit/core/src/reductions/cb/cbify.cc b/vowpalwabbit/core/src/reductions/cb/cbify.cc index db87dc91713..932ef92a88e 100644 --- a/vowpalwabbit/core/src/reductions/cb/cbify.cc +++ b/vowpalwabbit/core/src/reductions/cb/cbify.cc @@ -33,12 +33,12 @@ namespace VW { namespace reductions { -void cbify_adf_data::init_adf_data(std::size_t num_actions_, std::size_t increment_, +void cbify_adf_data::init_adf_data(std::size_t num_actions_, std::size_t feature_width_below_, std::vector>& interactions, std::vector>& extent_interactions) { this->num_actions = num_actions_; - this->increment = increment_; + this->feature_width_below = feature_width_below_; ecs.resize(num_actions_); for (size_t a = 0; a < num_actions_; ++a) @@ -51,7 +51,7 @@ void cbify_adf_data::init_adf_data(std::size_t num_actions_, std::size_t increme } // cache mask for copy routine - uint64_t total = num_actions_ * increment_; + uint64_t total = num_actions_ * feature_width_below_; uint64_t power_2 = 0; while (total > 0) @@ -89,7 +89,7 @@ void cbify_adf_data::copy_example_to_adf(parameters& weights, VW::example& ec) { auto rawidx = idx; rawidx -= rawidx & custom_index_mask; - rawidx += a * increment; + rawidx += a * feature_width_below; idx = rawidx & mask; } } @@ -778,7 +778,7 @@ std::shared_ptr VW::reductions::cbify_setup(VW::setup_base if (data->use_adf) { - data->adf_data.init_adf_data(num_actions, base->increment, all.fc.interactions, all.fc.extent_interactions); + data->adf_data.init_adf_data(num_actions, base->feature_width_below, all.fc.interactions, all.fc.extent_interactions); } if (use_cs) @@ -790,7 +790,6 @@ std::shared_ptr VW::reductions::cbify_setup(VW::setup_base output_example_prediction_func = VW::details::output_example_prediction_cs_label; print_update_func = VW::details::print_update_cs_label; name_addition = "-adf-cs"; - all.parser_runtime.example_parser->lbl_parser = VW::cs_label_parser_global; } else { @@ -801,7 +800,6 @@ std::shared_ptr VW::reductions::cbify_setup(VW::setup_base output_example_prediction_func = VW::details::output_example_prediction_multiclass_label; print_update_func = VW::details::print_update_multiclass_label; name_addition = "-adf"; - all.parser_runtime.example_parser->lbl_parser = VW::multiclass_label_parser_global; } l = make_reduction_learner( std::move(data), base, learn_ptr, predict_ptr, stack_builder.get_setupfn_name(cbify_setup) + name_addition) @@ -823,7 +821,6 @@ std::shared_ptr VW::reductions::cbify_setup(VW::setup_base { in_label_type = VW::label_type_t::SIMPLE; out_pred_type = VW::prediction_type_t::SCALAR; - all.parser_runtime.example_parser->lbl_parser = simple_label_parser_global; if (use_discrete) { out_label_type = VW::label_type_t::CB; @@ -863,7 +860,6 @@ std::shared_ptr VW::reductions::cbify_setup(VW::setup_base output_example_prediction_func = VW::details::output_example_prediction_cs_label; print_update_func = VW::details::print_update_cs_label; name_addition = "-cs"; - all.parser_runtime.example_parser->lbl_parser = VW::cs_label_parser_global; } else { @@ -877,7 +873,6 @@ std::shared_ptr VW::reductions::cbify_setup(VW::setup_base output_example_prediction_func = VW::details::output_example_prediction_multiclass_label; print_update_func = VW::details::print_update_multiclass_label; name_addition = ""; - all.parser_runtime.example_parser->lbl_parser = VW::multiclass_label_parser_global; } auto builder = make_reduction_learner( std::move(data), base, learn_ptr, predict_ptr, stack_builder.get_setupfn_name(cbify_setup) + name_addition) @@ -944,7 +939,6 @@ std::shared_ptr VW::reductions::cbifyldf_setup(VW::setup_b .set_print_update(print_update_cbify_ldf) .set_update_stats(update_stats_cbify_ldf) .build(); - all.parser_runtime.example_parser->lbl_parser = VW::cs_label_parser_global; return l; } diff --git a/vowpalwabbit/core/src/reductions/cb/details/large_action/one_pass_svd_impl.cc b/vowpalwabbit/core/src/reductions/cb/details/large_action/one_pass_svd_impl.cc index 7a060a0556f..e766f10bcf9 100644 --- a/vowpalwabbit/core/src/reductions/cb/details/large_action/one_pass_svd_impl.cc +++ b/vowpalwabbit/core/src/reductions/cb/details/large_action/one_pass_svd_impl.cc @@ -52,7 +52,36 @@ void one_pass_svd_impl::generate_AOmega(const multi_ex& examples, const std::vec // matrix const uint64_t sampling_slack = 10; auto p = std::min(num_actions, static_cast(_d + sampling_slack)); + + auto& red_features = + examples[0]->ex_reduction_features.template get(); + auto* shared_example = red_features.shared_example; + + if (static_cast(p) != AOmega.cols() || + cached_example_hashes.size() > (num_actions + _action_cache_slack)) + { + // we need to recompute the cache if the column size has changed + // or if we have been caching too many actions (rough heuristic to avoid caching too many actions) + cached_example_hashes.clear(); + } + + // example hash has been generated before shared feature merger, so each one should be a represenative of what the + // hash is without the shared features which will be removed anyway before calculating AOmega + for (size_t i = 0; i < examples.size(); ++i) + { + if (shared_example != nullptr) + { + VW::details::truncate_example_namespaces_from_example(*examples[i], *shared_example); + } + if (cached_example_hashes.find(examples[i]->get_or_calculate_order_independent_feature_space_hash()) == + cached_example_hashes.end()) + { + cached_example_hashes.clear(); + } + } + const float scaling_factor = 1.f / std::sqrt(p); + // resize is a no-op if size does not change AOmega.resize(num_actions, p); auto compute_dot_prod = compute_dot_prod_scalar; @@ -70,26 +99,28 @@ void one_pass_svd_impl::generate_AOmega(const multi_ex& examples, const std::vec } #endif - auto calculate_aomega_row = [compute_dot_prod](uint64_t row_index_begin, uint64_t row_index_end, uint64_t p, - VW::workspace* _all, uint64_t _seed, const multi_ex& examples, - Eigen::MatrixXf& AOmega, const std::vector& shrink_factors, - float scaling_factor) -> void + auto calculate_aomega_row = + [compute_dot_prod](uint64_t row_index_begin, uint64_t row_index_end, uint64_t p, VW::workspace* _all, + uint64_t _seed, const multi_ex& examples, Eigen::MatrixXf& AOmega, const std::vector& shrink_factors, + float scaling_factor, std::unordered_map& cached_example_hashes) -> void { for (auto row_index = row_index_begin; row_index < row_index_end; ++row_index) { VW::example* ex = examples[row_index]; - - auto& red_features = ex->ex_reduction_features.template get(); - auto* shared_example = red_features.shared_example; - if (shared_example != nullptr) { VW::details::truncate_example_namespaces_from_example(*ex, *shared_example); } - - for (uint64_t col = 0; col < p; ++col) + if (cached_example_hashes.find(ex->get_or_calculate_order_independent_feature_space_hash()) == + cached_example_hashes.end()) { - float final_dot_prod = compute_dot_prod(col, _all, _seed, ex); - AOmega(row_index, col) = final_dot_prod * shrink_factors[row_index] * scaling_factor; + for (uint64_t col = 0; col < p; ++col) + { + float final_dot_prod = compute_dot_prod(col, _all, _seed, ex); + AOmega(row_index, col) = final_dot_prod * shrink_factors[row_index] * scaling_factor; + } + } + else + { + AOmega.row(row_index) = cached_example_hashes[ex->get_or_calculate_order_independent_feature_space_hash()] * + shrink_factors[row_index]; } - - if (shared_example != nullptr) { VW::details::append_example_namespaces_from_example(*ex, *shared_example); } } }; @@ -105,14 +136,26 @@ void one_pass_svd_impl::generate_AOmega(const multi_ex& examples, const std::vec size_t row_index_end = row_index_begin + _block_size; if ((row_index_end + _block_size) > examples.size()) { row_index_end = examples.size(); } - _futures.emplace_back(_thread_pool.submit(calculate_aomega_row, row_index_begin, row_index_end, p, _all, _seed, - std::cref(examples), std::ref(AOmega), std::cref(shrink_factors), scaling_factor)); + _futures.emplace_back( + _thread_pool.submit(calculate_aomega_row, row_index_begin, row_index_end, p, _all, _seed, std::cref(examples), + std::ref(AOmega), std::cref(shrink_factors), scaling_factor, std::ref(cached_example_hashes))); row_index_begin = row_index_end; } for (auto& ft : _futures) { ft.get(); } _futures.clear(); + + for (size_t i = 0; i < examples.size(); i++) + { + auto ex = examples[i]; + if (cached_example_hashes.find(ex->feature_space_hash) == cached_example_hashes.end() && + ex->is_set_feature_space_hash) + { + cached_example_hashes.emplace(ex->feature_space_hash, AOmega.row(i) / shrink_factors[i]); + } + if (shared_example != nullptr) { VW::details::append_example_namespaces_from_example(*ex, *shared_example); } + } } void one_pass_svd_impl::_test_only_set_rank(uint64_t rank) { _d = rank; } @@ -129,8 +172,13 @@ void one_pass_svd_impl::run(const multi_ex& examples, const std::vector& } one_pass_svd_impl::one_pass_svd_impl(VW::workspace* all, uint64_t d, uint64_t seed, size_t, size_t thread_pool_size, - size_t block_size, bool use_explicit_simd) - : _all(all), _d(d), _seed(seed), _thread_pool(thread_pool_size), _block_size(block_size) + size_t block_size, size_t action_cache_slack, bool use_explicit_simd) + : _all(all) + , _d(d) + , _seed(seed) + , _thread_pool(thread_pool_size) + , _block_size(block_size) + , _action_cache_slack(action_cache_slack) { #ifdef BUILD_LAS_WITH_SIMD _use_simd = simd_type::NO_SIMD; diff --git a/vowpalwabbit/core/src/reductions/cb/details/large_action/two_pass_svd_impl.cc b/vowpalwabbit/core/src/reductions/cb/details/large_action/two_pass_svd_impl.cc index 5e276414bf2..b425e748af2 100644 --- a/vowpalwabbit/core/src/reductions/cb/details/large_action/two_pass_svd_impl.cc +++ b/vowpalwabbit/core/src/reductions/cb/details/large_action/two_pass_svd_impl.cc @@ -209,7 +209,8 @@ void two_pass_svd_impl::run(const multi_ex& examples, const std::vector& if (_set_testing_components) { _V = Y * svd.matrixV(); } } -two_pass_svd_impl::two_pass_svd_impl(VW::workspace* all, uint64_t d, uint64_t seed, size_t, size_t, size_t, bool) +two_pass_svd_impl::two_pass_svd_impl( + VW::workspace* all, uint64_t d, uint64_t seed, size_t, size_t, size_t, size_t, bool) : _all(all), _d(d), _seed(seed) { } diff --git a/vowpalwabbit/core/src/reductions/cb/details/large_action_space.h b/vowpalwabbit/core/src/reductions/cb/details/large_action_space.h index 7b315e94d78..936aa5784d5 100644 --- a/vowpalwabbit/core/src/reductions/cb/details/large_action_space.h +++ b/vowpalwabbit/core/src/reductions/cb/details/large_action_space.h @@ -41,7 +41,7 @@ class two_pass_svd_impl Eigen::MatrixXf Z; two_pass_svd_impl(VW::workspace* all, uint64_t d, uint64_t seed, size_t total_size, size_t thread_pool_size, - size_t block_size, bool use_explicit_simd); + size_t block_size, size_t action_cache_slack, bool use_explicit_simd); void run(const multi_ex& examples, const std::vector& shrink_factors, Eigen::MatrixXf& U, Eigen::VectorXf& S, Eigen::MatrixXf& _V); bool generate_Y(const multi_ex& examples, const std::vector& shrink_factors); @@ -62,9 +62,10 @@ class one_pass_svd_impl { public: Eigen::MatrixXf AOmega; + std::unordered_map cached_example_hashes; one_pass_svd_impl(VW::workspace* all, uint64_t d, uint64_t seed, size_t total_size, size_t thread_pool_size, - size_t block_size, bool use_explicit_simd); + size_t block_size, size_t action_cache_slack, bool use_explicit_simd); void run(const multi_ex& examples, const std::vector& shrink_factors, Eigen::MatrixXf& U, Eigen::VectorXf& S, Eigen::MatrixXf& _V); void generate_AOmega(const multi_ex& examples, const std::vector& shrink_factors); @@ -82,6 +83,7 @@ class one_pass_svd_impl uint64_t _seed; thread_pool _thread_pool; size_t _block_size; + size_t _action_cache_slack; #ifdef BUILD_LAS_WITH_SIMD enum class simd_type { @@ -154,7 +156,7 @@ class cb_explore_adf_large_action_space Eigen::MatrixXf _V; cb_explore_adf_large_action_space(uint64_t d, float c, bool apply_shrink_factor, VW::workspace* all, uint64_t seed, - size_t total_size, size_t thread_pool_size, size_t block_size, bool use_explicit_simd, + size_t total_size, size_t thread_pool_size, size_t block_size, size_t action_cache_slack, bool use_explicit_simd, implementation_type impl_type); ~cb_explore_adf_large_action_space() = default; diff --git a/vowpalwabbit/core/src/reductions/cb/warm_cb.cc b/vowpalwabbit/core/src/reductions/cb/warm_cb.cc index 7dc58f7d092..035e11c189a 100644 --- a/vowpalwabbit/core/src/reductions/cb/warm_cb.cc +++ b/vowpalwabbit/core/src/reductions/cb/warm_cb.cc @@ -592,7 +592,8 @@ std::shared_ptr VW::reductions::warm_cb_setup(VW::setup_ba options.insert("lr_multiplier", ss.str()); } - auto base = require_multiline(stack_builder.setup_base_learner()); + size_t feature_width = data->choices_lambda; + auto base = require_multiline(stack_builder.setup_base_learner(feature_width)); // Note: the current version of warm start CB can only support epsilon-greedy exploration // We need to wait for the epsilon value to be passed from the base // cb_explore learner, if there is one @@ -604,7 +605,6 @@ std::shared_ptr VW::reductions::warm_cb_setup(VW::setup_ba } void (*learn_pred_ptr)(warm_cb&, learner&, VW::example&); - size_t ws = data->choices_lambda; std::string name_addition; VW::label_type_t label_type; VW::learner_update_stats_func* update_stats_func = nullptr; @@ -620,7 +620,6 @@ std::shared_ptr VW::reductions::warm_cb_setup(VW::setup_ba update_stats_func = VW::details::update_stats_cs_label; output_example_prediction_func = VW::details::output_example_prediction_cs_label; print_update_func = VW::details::print_update_cs_label; - all.parser_runtime.example_parser->lbl_parser = VW::cs_label_parser_global; label_type = VW::label_type_t::CS; } else @@ -630,7 +629,6 @@ std::shared_ptr VW::reductions::warm_cb_setup(VW::setup_ba update_stats_func = VW::details::update_stats_multiclass_label; output_example_prediction_func = VW::details::output_example_prediction_multiclass_label; print_update_func = VW::details::print_update_multiclass_label; - all.parser_runtime.example_parser->lbl_parser = VW::multiclass_label_parser_global; label_type = VW::label_type_t::MULTICLASS; } @@ -640,7 +638,7 @@ std::shared_ptr VW::reductions::warm_cb_setup(VW::setup_ba .set_output_label_type(VW::label_type_t::CB) .set_input_prediction_type(VW::prediction_type_t::ACTION_PROBS) .set_output_prediction_type(VW::prediction_type_t::MULTICLASS) - .set_params_per_weight(ws) + .set_feature_width(feature_width) .set_learn_returns_prediction(true) .set_update_stats(update_stats_func) .set_output_example_prediction(output_example_prediction_func) diff --git a/vowpalwabbit/core/src/reductions/cbzo.cc b/vowpalwabbit/core/src/reductions/cbzo.cc index 18e23f39b69..3512167aaed 100644 --- a/vowpalwabbit/core/src/reductions/cbzo.cc +++ b/vowpalwabbit/core/src/reductions/cbzo.cc @@ -46,22 +46,22 @@ class linear_update_data }; // uint64_t index variant of VW::get_weight -inline float get_weight(VW::workspace& all, uint64_t index, uint32_t offset) +inline float get_weight(VW::workspace& all, uint64_t index) { - return (&all.weights[(index) << all.weights.stride_shift()])[offset]; + return all.weights[(index) << all.weights.stride_shift()]; } // uint64_t index variant of VW::set_weight -inline void set_weight(VW::workspace& all, uint64_t index, uint32_t offset, float value) +inline void set_weight(VW::workspace& all, uint64_t index, float value) { - (&all.weights[(index) << all.weights.stride_shift()])[offset] = value; + all.weights[(index) << all.weights.stride_shift()] = value; } float l1_grad(VW::workspace& all, uint64_t fi) { if (all.lc.no_bias && fi == VW::details::CONSTANT) { return 0.0f; } - float fw = get_weight(all, fi, 0); + float fw = get_weight(all, fi); return fw >= 0.0f ? all.lc.l1_lambda : -all.lc.l1_lambda; } @@ -69,7 +69,7 @@ float l2_grad(VW::workspace& all, uint64_t fi) { if (all.lc.no_bias && fi == VW::details::CONSTANT) { return 0.0f; } - float fw = get_weight(all, fi, 0); + float fw = get_weight(all, fi); return all.lc.l2_lambda * fw; } @@ -77,7 +77,7 @@ inline void accumulate_dotprod(float& dotprod, float x, float& fw) { dotprod += inline float constant_inference(VW::workspace& all) { - float wt = get_weight(all, VW::details::CONSTANT, 0); + float wt = get_weight(all, VW::details::CONSTANT); return wt; } @@ -102,7 +102,7 @@ float inference(VW::workspace& all, VW::example& ec) template void constant_update(cbzo& data, VW::example& ec) { - float fw = get_weight(*data.all, VW::details::CONSTANT, 0); + float fw = get_weight(*data.all, VW::details::CONSTANT); if (feature_mask_off || fw != 0.0f) { float action_centroid = inference(*data.all, ec); @@ -110,19 +110,19 @@ void constant_update(cbzo& data, VW::example& ec) float update = -data.all->uc.eta * (grad + l1_grad(*data.all, VW::details::CONSTANT) + l2_grad(*data.all, VW::details::CONSTANT)); - set_weight(*data.all, VW::details::CONSTANT, 0, fw + update); + set_weight(*data.all, VW::details::CONSTANT, fw + update); } } template void linear_per_feature_update(linear_update_data& upd_data, float x, uint64_t fi) { - float fw = get_weight(*upd_data.all, fi, 0); + float fw = get_weight(*upd_data.all, fi); if (feature_mask_off || fw != 0.0f) { float update = upd_data.mult * (upd_data.part_grad * x + (l1_grad(*upd_data.all, fi) + l2_grad(*upd_data.all, fi))); - set_weight(*upd_data.all, fi, 0, fw + update); + set_weight(*upd_data.all, fi, fw + update); } } @@ -236,10 +236,7 @@ void save_load(cbzo& data, VW::io_buf& model_file, bool read, bool text) if (read) { VW::details::initialize_regressor(all); - if (data.all->fc.initial_constant != 0.0f) - { - set_weight(all, VW::details::CONSTANT, 0, data.all->fc.initial_constant); - } + if (data.all->fc.initial_constant != 0.0f) { set_weight(all, VW::details::CONSTANT, data.all->fc.initial_constant); } } if (model_file.num_files() > 0) { save_load_regressor(all, model_file, read, text); } } @@ -368,14 +365,12 @@ std::shared_ptr VW::reductions::cbzo_setup(VW::setup_base_ } } - all.parser_runtime.example_parser->lbl_parser = cb_continuous::the_label_parser; data->all = &all; data->min_prediction_supplied = options.was_supplied("min_prediction"); data->max_prediction_supplied = options.was_supplied("max_prediction"); auto l = make_bottom_learner(std::move(data), get_learn(all, policy, feature_mask_off), get_predict(all, policy), stack_builder.get_setupfn_name(cbzo_setup), prediction_type_t::PDF, label_type_t::CONTINUOUS) - .set_params_per_weight(0) .set_save_load(save_load) .set_output_example_prediction(output_example_prediction_cbzo) .set_print_update(print_update_cbzo) diff --git a/vowpalwabbit/core/src/reductions/conditional_contextual_bandit.cc b/vowpalwabbit/core/src/reductions/conditional_contextual_bandit.cc index 1aaad5e8f34..9c5b82321b5 100644 --- a/vowpalwabbit/core/src/reductions/conditional_contextual_bandit.cc +++ b/vowpalwabbit/core/src/reductions/conditional_contextual_bandit.cc @@ -249,7 +249,7 @@ void inject_slot_id(ccb_data& data, VW::example* shared, size_t id) index = VW::hash_feature(*data.all, current_index_str, data.id_namespace_hash); // To maintain indices consistent with what the parser does we must scale. - index *= static_cast(data.all->reduction_state.wpp) << data.base_learner_stride_shift; + index *= static_cast(data.all->reduction_state.total_feature_width) << data.base_learner_stride_shift; data.slot_id_hashes[id] = index; } else { index = data.slot_id_hashes[id]; } @@ -688,7 +688,6 @@ std::shared_ptr VW::reductions::ccb_explore_adf_setup(VW:: } auto base = require_multiline(stack_builder.setup_base_learner()); - all.parser_runtime.example_parser->lbl_parser = VW::ccb_label_parser_global; // Stash the base learners stride_shift so we can properly add a feature // later. diff --git a/vowpalwabbit/core/src/reductions/count_label.cc b/vowpalwabbit/core/src/reductions/count_label.cc index 046aaaeb1f8..fc76e84a342 100644 --- a/vowpalwabbit/core/src/reductions/count_label.cc +++ b/vowpalwabbit/core/src/reductions/count_label.cc @@ -62,7 +62,7 @@ std::shared_ptr VW::reductions::count_label_setup(VW::setu if (base == nullptr) { return nullptr; } auto* all = stack_builder.get_all_pointer(); - auto base_label_type = all->parser_runtime.example_parser->lbl_parser.label_type; + auto base_label_type = base->get_input_label_type(); if (dont_output_best_constant) { if (base_label_type != label_type_t::SIMPLE) diff --git a/vowpalwabbit/core/src/reductions/cs_active.cc b/vowpalwabbit/core/src/reductions/cs_active.cc index 36d6d400f97..98be14836ea 100644 --- a/vowpalwabbit/core/src/reductions/cs_active.cc +++ b/vowpalwabbit/core/src/reductions/cs_active.cc @@ -463,10 +463,10 @@ std::shared_ptr VW::reductions::cs_active_setup(VW::setup_ name_addition = ""; } - size_t ws = data->num_classes; - auto l = make_reduction_learner(std::move(data), require_singleline(stack_builder.setup_base_learner()), learn_ptr, - predict_ptr, stack_builder.get_setupfn_name(cs_active_setup) + name_addition) - .set_params_per_weight(ws) + size_t feature_width = data->num_classes; + auto l = make_reduction_learner(std::move(data), require_singleline(stack_builder.setup_base_learner(feature_width)), + learn_ptr, predict_ptr, stack_builder.get_setupfn_name(cs_active_setup) + name_addition) + .set_feature_width(feature_width) .set_learn_returns_prediction(true) .set_input_prediction_type(VW::prediction_type_t::SCALAR) .set_output_prediction_type(VW::prediction_type_t::ACTIVE_MULTICLASS) @@ -476,9 +476,5 @@ std::shared_ptr VW::reductions::cs_active_setup(VW::setup_ .set_print_update(print_update_cs_active) .set_update_stats(update_stats_cs_active) .build(); - - // Label parser set to cost sensitive label parser - all.parser_runtime.example_parser->lbl_parser = VW::cs_label_parser_global; - return l; } diff --git a/vowpalwabbit/core/src/reductions/csoaa.cc b/vowpalwabbit/core/src/reductions/csoaa.cc index e1cee7c4c94..00a9caf5168 100644 --- a/vowpalwabbit/core/src/reductions/csoaa.cc +++ b/vowpalwabbit/core/src/reductions/csoaa.cc @@ -200,12 +200,12 @@ std::shared_ptr VW::reductions::csoaa_setup(VW::setup_base c->search = options.was_supplied("search"); c->pred = VW::details::calloc_or_throw(c->num_classes); - size_t ws = c->num_classes; - auto l = make_reduction_learner(std::move(c), require_singleline(stack_builder.setup_base_learner()), + size_t feature_width = c->num_classes; + auto l = make_reduction_learner(std::move(c), require_singleline(stack_builder.setup_base_learner(feature_width)), predict_or_learn, predict_or_learn, stack_builder.get_setupfn_name(csoaa_setup)) .set_learn_returns_prediction( true) /* csoaa.learn calls gd.learn. nothing to be gained by calling csoaa.predict first */ - .set_params_per_weight(ws) + .set_feature_width(feature_width) .set_input_prediction_type(VW::prediction_type_t::SCALAR) .set_output_prediction_type(VW::prediction_type_t::MULTICLASS) .set_input_label_type(VW::label_type_t::CS) @@ -214,8 +214,5 @@ std::shared_ptr VW::reductions::csoaa_setup(VW::setup_base .set_output_example_prediction(VW::details::output_example_prediction_cs_label) .set_print_update(VW::details::print_update_cs_label) .build(); - - all.parser_runtime.example_parser->lbl_parser = VW::cs_label_parser_global; - return l; } diff --git a/vowpalwabbit/core/src/reductions/csoaa_ldf.cc b/vowpalwabbit/core/src/reductions/csoaa_ldf.cc index 4725eb25478..b729b52aaa8 100644 --- a/vowpalwabbit/core/src/reductions/csoaa_ldf.cc +++ b/vowpalwabbit/core/src/reductions/csoaa_ldf.cc @@ -815,8 +815,5 @@ std::shared_ptr VW::reductions::csldf_setup(VW::setup_base .set_output_example_prediction(output_example_prediction_func) .set_print_update(print_update_func) .build(); - - all.parser_runtime.example_parser->lbl_parser = VW::cs_label_parser_global; - return l; } diff --git a/vowpalwabbit/core/src/reductions/details/automl/automl_impl.cc b/vowpalwabbit/core/src/reductions/details/automl/automl_impl.cc index 9103df2346b..969aac06bd4 100644 --- a/vowpalwabbit/core/src/reductions/details/automl/automl_impl.cc +++ b/vowpalwabbit/core/src/reductions/details/automl/automl_impl.cc @@ -64,15 +64,15 @@ template interaction_config_manager::interaction_config_manager(uint64_t default_lease, uint64_t max_live_configs, std::shared_ptr rand_state, uint64_t priority_challengers, const std::string& interaction_type, const std::string& oracle_type, dense_parameters& weights, - priority_func calc_priority, double automl_significance_level, VW::io::logger* logger, uint32_t& wpp, bool ccb_on, - config_type conf_type, std::string trace_prefix, bool reward_as_cost, double tol_x, bool is_brentq) + priority_func calc_priority, double automl_significance_level, VW::io::logger* logger, uint32_t& feature_width, + bool ccb_on, config_type conf_type, std::string trace_prefix, bool reward_as_cost, double tol_x, bool is_brentq) : default_lease(default_lease) , max_live_configs(max_live_configs) , priority_challengers(priority_challengers) , weights(weights) , automl_significance_level(automl_significance_level) , logger(logger) - , wpp(wpp) + , feature_width(feature_width) , _ccb_on(ccb_on) , _config_oracle(config_oracle_impl( default_lease, std::move(calc_priority), interaction_type, oracle_type, rand_state, conf_type)) @@ -193,7 +193,8 @@ void interaction_config_manager::schedule() } // copy the weights of the champ to the new slot - VW::reductions::multi_model::move_innermost_offsets(weights, current_champ, live_slot, wpp, max_live_configs); + VW::reductions::multi_model::move_innermost_offsets( + weights, current_champ, live_slot, feature_width, max_live_configs); // Regenerate interactions each time an exclusion is swapped in ns_based_config::apply_config_to_interactions(_ccb_on, ns_counter, _config_oracle._interaction_type, _config_oracle.configs[estimators[live_slot].first.config_index], @@ -268,11 +269,11 @@ void interaction_config_manager::check_for_n // this is a swap, see last bool argument in move_innermost_offsets VW::reductions::multi_model::move_innermost_offsets( - weights, winning_challenger_slot, old_champ_slot, wpp, max_live_configs, true); + weights, winning_challenger_slot, old_champ_slot, feature_width, max_live_configs, true); if (winning_challenger_slot != 1) { VW::reductions::multi_model::move_innermost_offsets( - weights, winning_challenger_slot, 1, wpp, max_live_configs, false); + weights, winning_challenger_slot, 1, feature_width, max_live_configs, false); } apply_new_champ(_config_oracle, winning_challenger_slot, estimators, priority_challengers, ns_counter); @@ -334,19 +335,9 @@ void interaction_config_manager::do_learning { assert(live_slot < max_live_configs); // TODO: what to do if that slot is switched with a new config? - std::swap(*_gd_normalized, per_live_model_state_double[live_slot * 3]); - std::swap(*_gd_total_weight, per_live_model_state_double[live_slot * 3 + 1]); - std::swap(*_sd_gravity, per_live_model_state_double[live_slot * 3 + 2]); - std::swap(*_cb_adf_event_sum, per_live_model_state_uint64[live_slot * 2]); - std::swap(*_cb_adf_action_sum, per_live_model_state_uint64[live_slot * 2 + 1]); for (example* ex : ec) { apply_config(ex, &estimators[live_slot].first.live_interactions); } if (!base.learn_returns_prediction) { base.predict(ec, live_slot); } base.learn(ec, live_slot); - std::swap(*_gd_normalized, per_live_model_state_double[live_slot * 3]); - std::swap(*_gd_total_weight, per_live_model_state_double[live_slot * 3 + 1]); - std::swap(*_sd_gravity, per_live_model_state_double[live_slot * 3 + 2]); - std::swap(*_cb_adf_event_sum, per_live_model_state_uint64[live_slot * 2]); - std::swap(*_cb_adf_action_sum, per_live_model_state_uint64[live_slot * 2 + 1]); } template diff --git a/vowpalwabbit/core/src/reductions/details/automl/automl_iomodel.cc b/vowpalwabbit/core/src/reductions/details/automl/automl_iomodel.cc index 791ac48bfe3..0a2f4610dac 100644 --- a/vowpalwabbit/core/src/reductions/details/automl/automl_iomodel.cc +++ b/vowpalwabbit/core/src/reductions/details/automl/automl_iomodel.cc @@ -124,8 +124,6 @@ size_t read_model_field( { cm.estimators.clear(); cm._config_oracle.configs.clear(); - cm.per_live_model_state_double.clear(); - cm.per_live_model_state_uint64.clear(); size_t bytes = 0; uint64_t current_champ = 0; bytes += read_model_field(io, cm.total_learn_count); @@ -135,8 +133,6 @@ size_t read_model_field( bytes += read_model_field(io, cm._config_oracle.configs); bytes += read_model_field(io, cm.estimators); bytes += read_model_field(io, cm._config_oracle.index_queue); - bytes += read_model_field(io, cm.per_live_model_state_double); - bytes += read_model_field(io, cm.per_live_model_state_uint64); for (uint64_t live_slot = 0; live_slot < cm.estimators.size(); ++live_slot) { auto& exclusions = cm._config_oracle.configs[cm.estimators[live_slot].first.config_index]; @@ -161,8 +157,6 @@ size_t write_model_field(io_buf& io, bytes += write_model_field(io, cm._config_oracle.configs, upstream_name + "_configs", text); bytes += write_model_field(io, cm.estimators, upstream_name + "_estimators", text); bytes += write_model_field(io, cm._config_oracle.index_queue, upstream_name + "_index_queue", text); - bytes += write_model_field(io, cm.per_live_model_state_double, upstream_name + "_per_live_model_state_double", text); - bytes += write_model_field(io, cm.per_live_model_state_uint64, upstream_name + "_per_live_model_state_uint64", text); return bytes; } diff --git a/vowpalwabbit/core/src/reductions/details/automl/automl_oracle.cc b/vowpalwabbit/core/src/reductions/details/automl/automl_oracle.cc index 6f6efd4e112..9072b8f3cad 100644 --- a/vowpalwabbit/core/src/reductions/details/automl/automl_oracle.cc +++ b/vowpalwabbit/core/src/reductions/details/automl/automl_oracle.cc @@ -3,7 +3,7 @@ // license as described in the file LICENSE. #include "vw/core/automl_impl.h" -#include "vw/core/reductions/conditional_contextual_bandit.h" +#include "vw/core/interactions.h" namespace VW { @@ -180,11 +180,34 @@ void ns_based_config::apply_config_to_interactions(const bool ccb_on, } } + // expanded version/equivalent of ccb::insert_ccb_interactions(interactions, ...); + // we cannot use that function since it relies on WILDCARD_NAMESPACE and on generate_interactions reduction + // which we don't use in automl stack + // + // CCB adds the following interactions: + // 1. Every existing interaction + VW::details::CCB_ID_NAMESPACE + // 2. wildcard_namespace + VW::details::CCB_ID_NAMESPACE if (ccb_on) { - std::vector> empty; - ccb::insert_ccb_interactions(interactions, empty); + auto total = interactions.size(); + auto reserve_size = 2 * total + ns_counter.size(); + interactions.reserve(reserve_size); + for (size_t i = 0; i < total; ++i) + { + auto copy = interactions[i]; + copy.push_back(VW::details::CCB_ID_NAMESPACE); + interactions.emplace_back(std::move(copy)); + } + + for (auto it = ns_counter.begin(); it != ns_counter.end(); ++it) + { + interactions.emplace_back(std::vector{(*it).first, VW::details::CCB_ID_NAMESPACE}); + } + + assert(interactions.size() == reserve_size); } + + std::sort(interactions.begin(), interactions.end(), VW::details::sort_interactions_comparator); } // Helper function to insert new configs from oracle into map of configs as well as index_queue. diff --git a/vowpalwabbit/core/src/reductions/details/automl/automl_util.cc b/vowpalwabbit/core/src/reductions/details/automl/automl_util.cc index 31ea9b823cc..0615d7e17b8 100644 --- a/vowpalwabbit/core/src/reductions/details/automl/automl_util.cc +++ b/vowpalwabbit/core/src/reductions/details/automl/automl_util.cc @@ -38,7 +38,9 @@ bool count_namespaces(const multi_ex& ecs, std::map& for (const auto& ns : ex->indices) { if (!VW::is_interaction_ns(ns)) { continue; } - if (!is_allowed_to_remove(ns)) { continue; } + // CCB_SLOT_NAMESPACE should be accounted for since generate_interactions treats it as a normal namespace + // it should still not be removed from oracle so we have to keep track of it here + if (ns != VW::details::CCB_SLOT_NAMESPACE && !is_allowed_to_remove(ns)) { continue; } ns_counter[ns]++; if (ns_counter[ns] == 1) { new_ns_seen = true; } } diff --git a/vowpalwabbit/core/src/reductions/ect.cc b/vowpalwabbit/core/src/reductions/ect.cc index 60e0cb3e9bb..9dcfb61c3e7 100644 --- a/vowpalwabbit/core/src/reductions/ect.cc +++ b/vowpalwabbit/core/src/reductions/ect.cc @@ -318,9 +318,9 @@ std::shared_ptr VW::reductions::ect_setup(VW::setup_base_i if (!options.add_parse_and_check_necessary(new_options)) { return nullptr; } - size_t wpp = create_circuit(*data.get(), data->k, data->errors + 1); + size_t feature_width = create_circuit(*data.get(), data->k, data->errors + 1); - auto base = stack_builder.setup_base_learner(); + auto base = stack_builder.setup_base_learner(feature_width); if (link == "logistic") { data->class_boundary = 0.5; // as --link=logistic maps predictions in [0;1] @@ -328,7 +328,7 @@ std::shared_ptr VW::reductions::ect_setup(VW::setup_base_i auto l = make_reduction_learner( std::move(data), require_singleline(base), learn, predict, stack_builder.get_setupfn_name(ect_setup)) - .set_params_per_weight(wpp) + .set_feature_width(feature_width) .set_input_label_type(VW::label_type_t::MULTICLASS) .set_output_label_type(VW::label_type_t::SIMPLE) .set_input_prediction_type(VW::prediction_type_t::SCALAR) @@ -337,8 +337,5 @@ std::shared_ptr VW::reductions::ect_setup(VW::setup_base_i .set_output_example_prediction(VW::details::output_example_prediction_multiclass_label) .set_print_update(VW::details::print_update_multiclass_label) .build(); - - all.parser_runtime.example_parser->lbl_parser = VW::multiclass_label_parser_global; - return l; } diff --git a/vowpalwabbit/core/src/reductions/eigen_memory_tree.cc b/vowpalwabbit/core/src/reductions/eigen_memory_tree.cc index 643dc781009..f62dd863ac7 100644 --- a/vowpalwabbit/core/src/reductions/eigen_memory_tree.cc +++ b/vowpalwabbit/core/src/reductions/eigen_memory_tree.cc @@ -11,6 +11,7 @@ #include "vw/config/options.h" #include "vw/core/array_parameters.h" #include "vw/core/example.h" +#include "vw/core/feature_group.h" #include "vw/core/learner.h" #include "vw/core/memory.h" #include "vw/core/model_utils.h" @@ -75,14 +76,14 @@ emt_example::emt_example(VW::workspace& all, VW::example* ex) std::vector> base_interactions; ex->interactions = &base_interactions; - auto* ex1 = VW::flatten_sort_example(all, ex); - for (auto& f : ex1->fs) { base.emplace_back(f.index(), f.value()); } - VW::free_flatten_example(ex1); + VW::features fs; + VW::flatten_features(all, *ex, fs); + for (auto& f : fs) { base.emplace_back(f.index(), f.value()); } + fs.clear(); ex->interactions = full_interactions; - auto* ex2 = VW::flatten_sort_example(all, ex); - for (auto& f : ex2->fs) { full.emplace_back(f.index(), f.value()); } - VW::free_flatten_example(ex2); + VW::flatten_features(all, *ex, fs); + for (auto& f : fs) { full.emplace_back(f.index(), f.value()); } } emt_lru::emt_lru(uint64_t max_size) : max_size(max_size) {} @@ -426,7 +427,7 @@ void scorer_example(emt_tree& b, const emt_example& ex1, const emt_example& ex2) // a model weight w[i] then we may also store information about our confidence in // w[i] at w[i+1] and information about the scale of feature f[i] at w[i+2] and so on. // This variable indicates how many such meta-data places we need to save in between actual weights. - uint64_t floats_per_feature_index = static_cast(b.all->reduction_state.wpp) + uint64_t floats_per_feature_index = static_cast(b.all->reduction_state.total_feature_width) << b.all->weights.stride_shift(); // In both of the example_types above we construct our scorer_example from flat_examples. The VW routine @@ -793,9 +794,6 @@ std::shared_ptr VW::reductions::eigen_memory_tree_setup(VW auto t = VW::make_unique(&all, all.get_random_state(), leaf_split, emt_scorer_type_from_string(scorer_type), emt_router_type_from_string(router_type), tree_bound); - // multi-class classification - all.parser_runtime.example_parser->lbl_parser = VW::multiclass_label_parser_global; - auto l = make_reduction_learner(std::move(t), require_singleline(stack_builder.setup_base_learner()), emt_learn, emt_predict, diff --git a/vowpalwabbit/core/src/reductions/epsilon_decay.cc b/vowpalwabbit/core/src/reductions/epsilon_decay.cc index bc525303696..1bf9b28dc37 100644 --- a/vowpalwabbit/core/src/reductions/epsilon_decay.cc +++ b/vowpalwabbit/core/src/reductions/epsilon_decay.cc @@ -18,7 +18,7 @@ // TODO: delete this three includes #include "vw/core/reductions/cb/cb_adf.h" #include "vw/core/reductions/gd.h" -#include "vw/core/shared_data.h" +#include "vw/core/setup_base.h" #include @@ -38,7 +38,7 @@ float decayed_epsilon(float init_ep, uint64_t update_count) epsilon_decay_data::epsilon_decay_data(uint64_t model_count, uint64_t min_scope, double epsilon_decay_significance_level, double epsilon_decay_estimator_decay, dense_parameters& weights, - std::string epsilon_decay_audit_str, bool constant_epsilon, uint32_t& wpp, uint64_t min_champ_examples, + std::string epsilon_decay_audit_str, bool constant_epsilon, uint32_t& feature_width, uint64_t min_champ_examples, float initial_epsilon, uint64_t shift_model_bounds, bool reward_as_cost, double tol_x, bool is_brentq, bool predict_only_model) : _model_count(model_count) @@ -48,7 +48,7 @@ epsilon_decay_data::epsilon_decay_data(uint64_t model_count, uint64_t min_scope, , _weights(weights) , _epsilon_decay_audit_str(std::move(epsilon_decay_audit_str)) , _constant_epsilon(constant_epsilon) - , _wpp(wpp) + , _feature_width(feature_width) , _min_champ_examples(min_champ_examples) , _initial_epsilon(initial_epsilon) , _shift_model_bounds(shift_model_bounds) @@ -88,6 +88,9 @@ void epsilon_decay_data::update_weights(float init_ep, VW::LEARNER::learner& bas ++_global_counter; } auto& ep_fts = examples[0]->ex_reduction_features.template get(); + + VW::action_scores champ_a_s; + // Process each model, then update the upper/lower bounds for each model for (int64_t model_ind = model_count - 1; model_ind >= 0; --model_ind) { @@ -96,18 +99,9 @@ void epsilon_decay_data::update_weights(float init_ep, VW::LEARNER::learner& bas ep_fts.epsilon = VW::reductions::epsilon_decay::decayed_epsilon( init_ep, conf_seq_estimators[model_ind][model_ind].update_count); } - std::swap(*_gd_normalized, per_live_model_state_double[_weight_indices[model_ind] * 3]); - std::swap(*_gd_total_weight, per_live_model_state_double[_weight_indices[model_ind] * 3 + 1]); - std::swap(*_sd_gravity, per_live_model_state_double[_weight_indices[model_ind] * 3 + 2]); - std::swap(*_cb_adf_event_sum, per_live_model_state_uint64[_weight_indices[model_ind] * 2]); - std::swap(*_cb_adf_action_sum, per_live_model_state_uint64[_weight_indices[model_ind] * 2 + 1]); if (!base.learn_returns_prediction) { base.predict(examples, _weight_indices[model_ind]); } base.learn(examples, _weight_indices[model_ind]); - std::swap(*_gd_normalized, per_live_model_state_double[_weight_indices[model_ind] * 3]); - std::swap(*_gd_total_weight, per_live_model_state_double[_weight_indices[model_ind] * 3 + 1]); - std::swap(*_sd_gravity, per_live_model_state_double[_weight_indices[model_ind] * 3 + 2]); - std::swap(*_cb_adf_event_sum, per_live_model_state_uint64[_weight_indices[model_ind] * 2]); - std::swap(*_cb_adf_action_sum, per_live_model_state_uint64[_weight_indices[model_ind] * 2 + 1]); + for (const auto& a_s : examples[0]->pred.a_s) { if (a_s.action == labelled_action) @@ -129,7 +123,9 @@ void epsilon_decay_data::update_weights(float init_ep, VW::LEARNER::learner& bas break; } } + if (model_ind == model_count - 1) { champ_a_s = examples[0]->pred.a_s; } } + examples[0]->pred.a_s = champ_a_s; } } @@ -175,7 +171,7 @@ void epsilon_decay_data::clear_weights_and_estimators(int64_t swap_dist, int64_t } for (int64_t ind = 0; ind < swap_dist; ++ind) { - VW::reductions::multi_model::clear_innermost_offset(_weights, _weight_indices[ind], _wpp, _model_count); + VW::reductions::multi_model::clear_innermost_offset(_weights, _weight_indices[ind], _feature_width, _model_count); } } @@ -300,17 +296,10 @@ void pre_save_load_epsilon_decay(VW::workspace& all, VW::reductions::epsilon_dec { options_i& options = *all.options; if (!data._predict_only_model) { return; } - // Clear non-champ weights first - - std::swap(*data._gd_normalized, data.per_live_model_state_double[0]); - std::swap(*data._gd_total_weight, data.per_live_model_state_double[1]); - std::swap(*data._sd_gravity, data.per_live_model_state_double[2]); - std::swap(*data._cb_adf_event_sum, data.per_live_model_state_uint64[0]); - std::swap(*data._cb_adf_action_sum, data.per_live_model_state_uint64[1]); // Adjust champ weights to new single-model space VW::reductions::multi_model::reduce_innermost_model_weights( - data._weights, data._weight_indices[data.conf_seq_estimators.size() - 1], data._wpp, data._model_count); + data._weights, data._weight_indices[data.conf_seq_estimators.size() - 1], data._feature_width, data._model_count); for (auto& group : options.get_all_option_group_definitions()) { @@ -320,7 +309,7 @@ void pre_save_load_epsilon_decay(VW::workspace& all, VW::reductions::epsilon_dec } } - all.iwc.num_bits = all.iwc.num_bits - static_cast(std::log2(data._wpp)); + all.iwc.num_bits = all.iwc.num_bits - static_cast(std::log2(data._feature_width)); options.get_typed_option("bit_precision").value(all.iwc.num_bits); } @@ -429,24 +418,12 @@ std::shared_ptr VW::reductions::epsilon_decay_setup(VW::se auto data = VW::make_unique(model_count, min_scope, epsilon_decay_significance_level, epsilon_decay_estimator_decay, all.weights.dense_weights, - epsilon_decay_audit_str, constant_epsilon, all.reduction_state.wpp, min_champ_examples, initial_epsilon, + epsilon_decay_audit_str, constant_epsilon, all.reduction_state.total_feature_width, min_champ_examples, initial_epsilon, shift_model_bounds, reward_as_cost, tol_x, is_brentq, predict_only_model); // make sure we setup the rest of the stack with cleared interactions // to make sure there are not subtle bugs - auto base = stack_builder.setup_base_learner(); - - VW::reductions::gd& gd = *static_cast( - base->get_learner_by_name_prefix("gd")->get_internal_type_erased_data_pointer_test_use_only()); - auto& adf_data = *static_cast(require_multiline(base->get_learner_by_name_prefix("cb_adf")) - ->get_internal_type_erased_data_pointer_test_use_only()); - data->per_live_model_state_double = std::vector(model_count * 3, 0.f); - data->per_live_model_state_uint64 = std::vector(model_count * 2, 0.f); - data->_gd_normalized = &(gd.per_model_states[0].normalized_sum_norm_x); - data->_gd_total_weight = &(gd.per_model_states[0].total_weight); - data->_cb_adf_event_sum = &(adf_data.gen_cs.event_sum); - data->_cb_adf_action_sum = &(adf_data.gen_cs.action_sum); - data->_sd_gravity = &(all.sd->gravity); + auto base = stack_builder.setup_base_learner(model_count); if (base->is_multiline()) { @@ -456,7 +433,7 @@ std::shared_ptr VW::reductions::epsilon_decay_setup(VW::se .set_output_label_type(VW::label_type_t::CB) .set_input_prediction_type(VW::prediction_type_t::ACTION_PROBS) .set_output_prediction_type(VW::prediction_type_t::ACTION_PROBS) - .set_params_per_weight(model_count) + .set_feature_width(model_count) .set_save_load(save_load_epsilon_decay) .set_finish(::finish) .set_pre_save_load(pre_save_load_epsilon_decay) diff --git a/vowpalwabbit/core/src/reductions/explore_eval.cc b/vowpalwabbit/core/src/reductions/explore_eval.cc index 07cc0fec721..c2751bc25b1 100644 --- a/vowpalwabbit/core/src/reductions/explore_eval.cc +++ b/vowpalwabbit/core/src/reductions/explore_eval.cc @@ -319,7 +319,6 @@ std::shared_ptr VW::reductions::explore_eval_setup(VW::set if (!options.was_supplied("cb_explore_adf")) { options.insert("cb_explore_adf", ""); } auto base = require_multiline(stack_builder.setup_base_learner()); - all.parser_runtime.example_parser->lbl_parser = VW::cb_label_parser_global; auto l = make_reduction_learner(std::move(data), base, do_actual_learning, do_actual_learning, stack_builder.get_setupfn_name(explore_eval_setup)) diff --git a/vowpalwabbit/core/src/reductions/freegrad.cc b/vowpalwabbit/core/src/reductions/freegrad.cc index ae9a5020b08..b5838997d0d 100644 --- a/vowpalwabbit/core/src/reductions/freegrad.cc +++ b/vowpalwabbit/core/src/reductions/freegrad.cc @@ -61,8 +61,7 @@ class freegrad size_t no_win_counter; size_t early_stop_thres; uint32_t freegrad_size; - double total_weight = 0.0; - double normalized_sum_norm_x = 0.0; + std::vector gd_per_model_states; }; template @@ -104,7 +103,7 @@ void freegrad_predict(freegrad& fg, VW::example& ec) fg.update_data.predict = 0.; fg.update_data.squared_norm_prediction = 0.; size_t num_features_from_interactions = 0; - fg.total_weight += ec.weight; + fg.gd_per_model_states[0].total_weight += ec.weight; float norm_w_pred; float projection_radius; @@ -299,7 +298,7 @@ void save_load(freegrad& fg, VW::io_buf& model_file, bool read, bool text) if (resume) { VW::details::save_load_online_state_gd( - *all, model_file, read, text, fg.total_weight, fg.normalized_sum_norm_x, nullptr, fg.freegrad_size); + *all, model_file, read, text, fg.gd_per_model_states, nullptr, fg.freegrad_size); } else { VW::details::save_load_regressor_gd(*all, model_file, read, text); } } @@ -367,8 +366,10 @@ std::shared_ptr VW::reductions::freegrad_setup(VW::setup_b fg_ptr->project = project; fg_ptr->adaptiveradius = adaptiveradius; fg_ptr->no_win_counter = 0; - fg_ptr->total_weight = 0; - fg_ptr->normalized_sum_norm_x = 0; + auto single_model_state = VW::reductions::details::gd_per_model_state(); + single_model_state.normalized_sum_norm_x = 0; + single_model_state.total_weight = 0.; + fg_ptr->gd_per_model_states.emplace_back(single_model_state); fg_ptr->epsilon = fepsilon; fg_ptr->lipschitz_const = flipschitz_const; @@ -389,21 +390,17 @@ std::shared_ptr VW::reductions::freegrad_setup(VW::setup_b fg_ptr->early_stop_thres = options.get_typed_option("early_terminate").value(); } - auto predict_ptr = - (fg_ptr->all->output_config.audit || fg_ptr->all->output_config.hash_inv) ? predict : predict; - auto learn_ptr = (fg_ptr->all->output_config.audit || fg_ptr->all->output_config.hash_inv) ? learn_freegrad - : learn_freegrad; - auto l = - VW::LEARNER::make_bottom_learner(std::move(fg_ptr), learn_ptr, predict_ptr, - stack_builder.get_setupfn_name(freegrad_setup), VW::prediction_type_t::SCALAR, VW::label_type_t::SIMPLE) - .set_learn_returns_prediction(true) - .set_params_per_weight(VW::details::UINT64_ONE << stack_builder.get_all_pointer()->weights.stride_shift()) - .set_save_load(save_load) - .set_end_pass(end_pass) - .set_output_example_prediction(VW::details::output_example_prediction_simple_label) - .set_update_stats(VW::details::update_stats_simple_label) - .set_print_update(VW::details::print_update_simple_label) - .build(); + auto predict_ptr = (fg_ptr->all->output_config.audit || fg_ptr->all->output_config.hash_inv) ? predict : predict; + auto learn_ptr = (fg_ptr->all->output_config.audit || fg_ptr->all->output_config.hash_inv) ? learn_freegrad : learn_freegrad; + auto l = VW::LEARNER::make_bottom_learner(std::move(fg_ptr), learn_ptr, predict_ptr, + stack_builder.get_setupfn_name(freegrad_setup), VW::prediction_type_t::SCALAR, VW::label_type_t::SIMPLE) + .set_learn_returns_prediction(true) + .set_save_load(save_load) + .set_end_pass(end_pass) + .set_output_example_prediction(VW::details::output_example_prediction_simple_label) + .set_update_stats(VW::details::update_stats_simple_label) + .set_print_update(VW::details::print_update_simple_label) + .build(); return l; } \ No newline at end of file diff --git a/vowpalwabbit/core/src/reductions/ftrl.cc b/vowpalwabbit/core/src/reductions/ftrl.cc index b9787689e9f..7cfaea0d27a 100644 --- a/vowpalwabbit/core/src/reductions/ftrl.cc +++ b/vowpalwabbit/core/src/reductions/ftrl.cc @@ -57,8 +57,7 @@ class ftrl size_t no_win_counter = 0; size_t early_stop_thres = 0; uint32_t ftrl_size = 0; - double total_weight = 0.0; - double normalized_sum_norm_x = 0.0; + std::vector gd_per_model_states; }; class uncertainty @@ -258,9 +257,10 @@ void coin_betting_predict(ftrl& b, VW::example& ec) VW::foreach_feature(*b.all, ec, b.data, num_features_from_interactions); ec.num_features_from_interactions = num_features_from_interactions; - b.normalized_sum_norm_x += (static_cast(ec.weight)) * b.data.normalized_squared_norm_x; - b.total_weight += ec.weight; - b.data.average_squared_norm_x = (static_cast((b.normalized_sum_norm_x + 1e-6) / b.total_weight)); + b.gd_per_model_states[0].normalized_sum_norm_x += (static_cast(ec.weight)) * b.data.normalized_squared_norm_x; + b.gd_per_model_states[0].total_weight += ec.weight; + b.data.average_squared_norm_x = (static_cast( + (b.gd_per_model_states[0].normalized_sum_norm_x + 1e-6) / b.gd_per_model_states[0].total_weight)); ec.partial_prediction = b.data.predict / b.data.average_squared_norm_x; @@ -343,8 +343,7 @@ void save_load(ftrl& b, VW::io_buf& model_file, bool read, bool text) if (resume) { - VW::details::save_load_online_state_gd( - *all, model_file, read, text, b.total_weight, b.normalized_sum_norm_x, nullptr, b.ftrl_size); + VW::details::save_load_online_state_gd(*all, model_file, read, text, b.gd_per_model_states, nullptr, b.ftrl_size); } else { VW::details::save_load_regressor_gd(*all, model_file, read, text); } } @@ -417,8 +416,10 @@ std::shared_ptr VW::reductions::ftrl_setup(VW::setup_base_ b->all = &all; b->no_win_counter = 0; - b->normalized_sum_norm_x = 0; - b->total_weight = 0; + auto single_model_state = VW::reductions::details::gd_per_model_state(); + single_model_state.normalized_sum_norm_x = 0; + single_model_state.total_weight = 0.; + b->gd_per_model_states.emplace_back(single_model_state); std::string algorithm_name; void (*learn_ptr)(ftrl&, VW::example&) = nullptr; @@ -484,7 +485,6 @@ std::shared_ptr VW::reductions::ftrl_setup(VW::setup_base_ stack_builder.get_setupfn_name(ftrl_setup) + "-" + algorithm_name + name_addition, VW::prediction_type_t::SCALAR, VW::label_type_t::SIMPLE) .set_learn_returns_prediction(learn_returns_prediction) - .set_params_per_weight(VW::details::UINT64_ONE << all.weights.stride_shift()) .set_sensitivity(sensitivity) .set_multipredict(multipredict_ptr) .set_save_load(save_load) diff --git a/vowpalwabbit/core/src/reductions/gd.cc b/vowpalwabbit/core/src/reductions/gd.cc index 58835cf6f6a..d9e255e2a9f 100644 --- a/vowpalwabbit/core/src/reductions/gd.cc +++ b/vowpalwabbit/core/src/reductions/gd.cc @@ -2,6 +2,8 @@ // individual contributors. All rights reserved. Released under a BSD (revised) // license as described in the file LICENSE. +#include "vw/core/reductions/gd.h" + #include "vw/core/array_parameters.h" #include "vw/core/array_parameters_dense.h" #include "vw/core/crossplat_compat.h" @@ -30,8 +32,8 @@ #include "vw/core/accumulate.h" #include "vw/core/debug_log.h" #include "vw/core/label_parser.h" +#include "vw/core/model_utils.h" #include "vw/core/parse_regressor.h" -#include "vw/core/reductions/gd.h" #include "vw/core/shared_data.h" #include "vw/core/vw.h" #include "vw/core/vw_versions.h" @@ -273,15 +275,15 @@ void merge(const std::vector& per_model_weighting, const std::vectorper_model_states[i].normalized_sum_norm_x; + output_data.gd_per_model_states[i].normalized_sum_norm_x += + source_data_obj->gd_per_model_states[i].normalized_sum_norm_x; // total_weight is additive - output_data.per_model_states[i].total_weight += source_data_obj->per_model_states[i].total_weight; + output_data.gd_per_model_states[i].total_weight += source_data_obj->gd_per_model_states[i].total_weight; } } } @@ -296,14 +298,14 @@ void add(const VW::workspace& ws1, const VW::reductions::gd& data1, const VW::wo } else { add_weights(ws_out.weights.dense_weights, ws1.weights.dense_weights, ws2.weights.dense_weights, length); } - for (size_t i = 0; i < data_out.per_model_states.size(); i++) + for (size_t i = 0; i < data_out.gd_per_model_states.size(); i++) { // normalized_sum_norm_x is additive - data_out.per_model_states[i].normalized_sum_norm_x = - data1.per_model_states[i].normalized_sum_norm_x + data2.per_model_states[i].normalized_sum_norm_x; + data_out.gd_per_model_states[i].normalized_sum_norm_x = + data1.gd_per_model_states[i].normalized_sum_norm_x + data2.gd_per_model_states[i].normalized_sum_norm_x; // total_weight is additive - data_out.per_model_states[i].total_weight = - data1.per_model_states[i].total_weight + data2.per_model_states[i].total_weight; + data_out.gd_per_model_states[i].total_weight = + data1.gd_per_model_states[i].total_weight + data2.gd_per_model_states[i].total_weight; } } @@ -317,14 +319,14 @@ void subtract(const VW::workspace& ws1, const VW::reductions::gd& data1, const V } else { subtract_weights(ws_out.weights.dense_weights, ws1.weights.dense_weights, ws2.weights.dense_weights, length); } - for (size_t i = 0; i < data_out.per_model_states.size(); i++) + for (size_t i = 0; i < data_out.gd_per_model_states.size(); i++) { // normalized_sum_norm_x is additive - data_out.per_model_states[i].normalized_sum_norm_x = - data1.per_model_states[i].normalized_sum_norm_x - data2.per_model_states[i].normalized_sum_norm_x; + data_out.gd_per_model_states[i].normalized_sum_norm_x = + data1.gd_per_model_states[i].normalized_sum_norm_x - data2.gd_per_model_states[i].normalized_sum_norm_x; // total_weight is additive - data_out.per_model_states[i].total_weight = - data1.per_model_states[i].total_weight - data2.per_model_states[i].total_weight; + data_out.gd_per_model_states[i].total_weight = + data1.gd_per_model_states[i].total_weight - data2.gd_per_model_states[i].total_weight; } } @@ -723,16 +725,16 @@ float get_pred_per_update(VW::reductions::gd& g, VW::example& ec) { if (!stateless) { - g.per_model_states[0].normalized_sum_norm_x += (static_cast(ec.weight)) * nd.norm_x; - g.per_model_states[0].total_weight += ec.weight; + g.current_model_state->normalized_sum_norm_x += (static_cast(ec.weight)) * nd.norm_x; + g.current_model_state->total_weight += ec.weight; g.update_multiplier = - average_update(static_cast(g.per_model_states[0].total_weight), - static_cast(g.per_model_states[0].normalized_sum_norm_x), g.neg_norm_power); + average_update(static_cast(g.current_model_state->total_weight), + static_cast(g.current_model_state->normalized_sum_norm_x), g.neg_norm_power); } else { - float nsnx = (static_cast(g.per_model_states[0].normalized_sum_norm_x)) + ec.weight * nd.norm_x; - float tw = static_cast(g.per_model_states[0].total_weight) + ec.weight; + float nsnx = (static_cast(g.current_model_state->normalized_sum_norm_x)) + ec.weight * nd.norm_x; + float tw = static_cast(g.current_model_state->total_weight) + ec.weight; g.update_multiplier = average_update(tw, nsnx, g.neg_norm_power); } nd.pred_per_update *= g.update_multiplier; @@ -772,8 +774,13 @@ float get_scale(VW::reductions::gd& g, VW::example& /* ec */, float weight) template float sensitivity(VW::reductions::gd& g, VW::example& ec) { + if (g.current_model_state == nullptr) + { + g.current_model_state = &(g.gd_per_model_states[ec.ft_offset / g.all->weights.stride()]); + } return get_scale(g, ec, 1.) * sensitivity(g, ec); + g.current_model_state = nullptr; } template void update(VW::reductions::gd& g, VW::example& ec) { + if (g.current_model_state == nullptr) + { + g.current_model_state = &(g.gd_per_model_states[ec.ft_offset / g.all->weights.stride()]); + } // invariant: not a test label, importance weight > 0 float update; if ((update = compute_update( @@ -832,6 +843,7 @@ void update(VW::reductions::gd& g, VW::example& ec) { // updating weights now to avoid numerical instability sync_weights(*g.all); } + g.current_model_state = nullptr; } template 0.); g.predict(g, ec); + g.current_model_state = &(g.gd_per_model_states[ec.ft_offset / g.all->weights.stride()]); update(g, ec); + assert(g.current_model_state == nullptr); // update clears this pointer + // this state should only matter on learn and not predict + // assert(g.current_model_state == nullptr); + // Guard example state restore against throws + auto restore_guard = VW::scope_exit([&g] { g.current_model_state = nullptr; }); } size_t write_index(VW::io_buf& model_file, std::stringstream& msg, bool text, uint32_t num_bits, uint64_t i) @@ -1080,7 +1098,7 @@ void save_load_online_state_weights(VW::workspace& all, VW::io_buf& model_file, } // namespace void VW::details::save_load_online_state_gd(VW::workspace& all, VW::io_buf& model_file, bool read, bool text, - double& total_weight, double& normalized_sum_norm_x, VW::reductions::gd* g, uint32_t ftrl_size) + std::vector& pms, VW::reductions::gd* g, uint32_t ftrl_size) { std::stringstream msg; @@ -1088,9 +1106,10 @@ void VW::details::save_load_online_state_gd(VW::workspace& all, VW::io_buf& mode VW::details::bin_text_read_write_fixed( model_file, reinterpret_cast(&all.uc.initial_t), sizeof(all.uc.initial_t), read, msg, text); - msg << "norm normalizer " << normalized_sum_norm_x << "\n"; - VW::details::bin_text_read_write_fixed( - model_file, reinterpret_cast(&normalized_sum_norm_x), sizeof(normalized_sum_norm_x), read, msg, text); + assert(pms.size() >= 1); + msg << "norm normalizer " << pms[0].normalized_sum_norm_x << "\n"; + VW::details::bin_text_read_write_fixed(model_file, reinterpret_cast(&pms[0].normalized_sum_norm_x), + sizeof(pms[0].normalized_sum_norm_x), read, msg, text); msg << "t " << all.sd->t << "\n"; VW::details::bin_text_read_write_fixed( @@ -1143,12 +1162,13 @@ void VW::details::save_load_online_state_gd(VW::workspace& all, VW::io_buf& mode if (!read || all.runtime_state.model_file_ver >= VW::version_definitions::VERSION_SAVE_RESUME_FIX) { + assert(pms.size() >= 1); // restore some data to allow save_resume work more accurate // fix average loss - msg << "total_weight " << total_weight << "\n"; + msg << "total_weight " << pms[0].total_weight << "\n"; VW::details::bin_text_read_write_fixed( - model_file, reinterpret_cast(&total_weight), sizeof(total_weight), read, msg, text); + model_file, reinterpret_cast(&pms[0].total_weight), sizeof(pms[0].total_weight), read, msg, text); // fix "loss since last" for first printed out example details msg << "sd::oec.weighted_labeled_examples " << all.sd->old_weighted_labeled_examples << "\n"; @@ -1272,8 +1292,7 @@ void save_load(VW::reductions::gd& g, VW::io_buf& model_file, bool read, bool te "save_resume functionality is known to have inaccuracy in model files version less than '{}'", VW::version_definitions::VERSION_SAVE_RESUME_FIX.to_string()); } - VW::details::save_load_online_state_gd(all, model_file, read, text, g.per_model_states[0].total_weight, - g.per_model_states[0].normalized_sum_norm_x, &g); + VW::details::save_load_online_state_gd(all, model_file, read, text, g.gd_per_model_states, &g); } else { @@ -1367,18 +1386,41 @@ uint64_t ceil_log_2(uint64_t v) } // namespace +namespace VW +{ +namespace model_utils +{ +size_t read_model_field(io_buf& model, VW::reductions::details::gd_per_model_state& pms) +{ + size_t bytes = 0; + bytes += read_model_field(model, pms.normalized_sum_norm_x); + bytes += read_model_field(model, pms.total_weight); + return bytes; +} + +size_t write_model_field( + io_buf& model, const VW::reductions::details::gd_per_model_state& pms, const std::string& name, bool text) +{ + size_t bytes = 0; + bytes += write_model_field(model, pms.normalized_sum_norm_x, name + "_normalized_sum_norm_x", text); + bytes += write_model_field(model, pms.total_weight, name + "_total_weight", text); + return bytes; +} +} // namespace model_utils +} // namespace VW + std::shared_ptr VW::reductions::gd_setup(VW::setup_base_i& stack_builder) { options_i& options = *stack_builder.get_options(); VW::workspace& all = *stack_builder.get_all_pointer(); - - auto g = VW::make_unique(); + size_t feature_width_above = stack_builder.get_feature_width_above(); bool sgd = false; bool adaptive = false; bool adax = false; bool invariant = false; bool normalized = false; + float sparse_l2 = 0.f; all.sd->gravity = L1_STATE_DEFAULT; all.sd->contraction = L2_STATE_DEFAULT; @@ -1392,7 +1434,7 @@ std::shared_ptr VW::reductions::gd_setup(VW::setup_base_i& .add(make_option("adax", adax).help("Use adaptive learning rates with x^2 instead of g^2x^2")) .add(make_option("invariant", invariant).help("Use safe/importance aware updates").keep(all.om.save_resume)) .add(make_option("normalized", normalized).help("Use per feature normalized updates").keep(all.om.save_resume)) - .add(make_option("sparse_l2", g->sparse_l2) + .add(make_option("sparse_l2", sparse_l2) .default_value(0.f) .help("Degree of l2 regularization applied to activated sparse parameters")) .add(make_option("l1_state", local_gravity) @@ -1407,23 +1449,21 @@ std::shared_ptr VW::reductions::gd_setup(VW::setup_base_i& if (options.was_supplied("l1_state")) { all.sd->gravity = local_gravity; } if (options.was_supplied("l2_state")) { all.sd->contraction = local_contraction; } + all.weights.adaptive = true; + all.weights.normalized = true; + auto g = VW::make_unique(feature_width_above); g->all = &all; - auto single_model_state = details::per_model_state(); - single_model_state.normalized_sum_norm_x = 0; - single_model_state.total_weight = 0.; - g->per_model_states.emplace_back(single_model_state); g->no_win_counter = 0; - all.weights.adaptive = true; - all.weights.normalized = true; g->neg_norm_power = (all.weights.adaptive ? (all.uc.power_t - 1.f) : -1.f); g->neg_power_t = -all.uc.power_t; + g->sparse_l2 = sparse_l2; if (all.uc.initial_t > 0) // for the normalized update: if initial_t is bigger than 1 we interpret this as if we had // seen (all.uc.initial_t) previous fake datapoints all with norm 1 { - g->per_model_states[0].normalized_sum_norm_x = all.uc.initial_t; - g->per_model_states[0].total_weight = all.uc.initial_t; + g->gd_per_model_states[0].normalized_sum_norm_x = all.uc.initial_t; + g->gd_per_model_states[0].total_weight = all.uc.initial_t; } bool feature_mask_off = true; @@ -1513,7 +1553,6 @@ std::shared_ptr VW::reductions::gd_setup(VW::setup_base_i& auto l = make_bottom_learner(std::move(g), g->learn, bare->predict, stack_builder.get_setupfn_name(gd_setup), VW::prediction_type_t::SCALAR, VW::label_type_t::SIMPLE) .set_learn_returns_prediction(true) - .set_params_per_weight(VW::details::UINT64_ONE << all.weights.stride_shift()) .set_sensitivity(bare->sensitivity) .set_multipredict(bare->multipredict) .set_update(bare->update) diff --git a/vowpalwabbit/core/src/reductions/gd_mf.cc b/vowpalwabbit/core/src/reductions/gd_mf.cc index f5bac506215..cad95d34f0e 100644 --- a/vowpalwabbit/core/src/reductions/gd_mf.cc +++ b/vowpalwabbit/core/src/reductions/gd_mf.cc @@ -393,7 +393,6 @@ std::shared_ptr VW::reductions::gd_mf_setup(VW::setup_base auto l = make_bottom_learner(std::move(data), learn, predict, stack_builder.get_setupfn_name(gd_mf_setup), VW::prediction_type_t::SCALAR, VW::label_type_t::SIMPLE) - .set_params_per_weight(VW::details::UINT64_ONE << all.weights.stride_shift()) .set_learn_returns_prediction(true) .set_save_load(save_load) .set_end_pass(end_pass) diff --git a/vowpalwabbit/core/src/reductions/generate_interactions.cc b/vowpalwabbit/core/src/reductions/generate_interactions.cc index 4c94c0e37a4..794ea065b90 100644 --- a/vowpalwabbit/core/src/reductions/generate_interactions.cc +++ b/vowpalwabbit/core/src/reductions/generate_interactions.cc @@ -206,6 +206,8 @@ std::shared_ptr VW::reductions::generate_interactions_setu } } + if (options.was_supplied("automl")) { return nullptr; } + // If there are no wildcards, then no expansion is required. // ccb_explore_adf adds a wildcard post setup and so this reduction must be turned on. if (!(interactions_spec_contains_wildcards || interactions_spec_contains_extent_wildcards || diff --git a/vowpalwabbit/core/src/reductions/interaction_ground.cc b/vowpalwabbit/core/src/reductions/interaction_ground.cc index dcebc5b732d..924e80b8d5a 100644 --- a/vowpalwabbit/core/src/reductions/interaction_ground.cc +++ b/vowpalwabbit/core/src/reductions/interaction_ground.cc @@ -92,16 +92,16 @@ std::shared_ptr VW::reductions::interaction_ground_setup(V if (!options.add_parse_and_check_necessary(new_options)) { return nullptr; } // number of weight vectors needed - size_t problem_multiplier = 2; // One for reward and one for loss + size_t feature_width = 2; // One for reward and one for loss auto ld = VW::make_unique(); // Ensure cb_adf so we are reducing to something useful. if (!options.was_supplied("cb_adf")) { options.insert("cb_adf", ""); } - auto base = require_multiline(stack_builder.setup_base_learner()); + auto base = require_multiline(stack_builder.setup_base_learner(feature_width)); auto l = make_reduction_learner( std::move(ld), base, learn, predict, stack_builder.get_setupfn_name(interaction_ground_setup)) - .set_params_per_weight(problem_multiplier) + .set_feature_width(feature_width) .set_input_label_type(label_type_t::CB) .set_output_label_type(label_type_t::CB) .set_output_prediction_type(prediction_type_t::ACTION_SCORES) diff --git a/vowpalwabbit/core/src/reductions/kernel_svm.cc b/vowpalwabbit/core/src/reductions/kernel_svm.cc index aeb72b9d844..7dce646b6b4 100644 --- a/vowpalwabbit/core/src/reductions/kernel_svm.cc +++ b/vowpalwabbit/core/src/reductions/kernel_svm.cc @@ -9,6 +9,7 @@ #include "vw/core/accumulate.h" #include "vw/core/constant.h" #include "vw/core/example.h" +#include "vw/core/feature_group.h" #include "vw/core/learner.h" #include "vw/core/loss_functions.h" #include "vw/core/memory.h" @@ -43,6 +44,75 @@ using std::endl; namespace { + +class flat_example +{ +public: + VW::polylabel l; + VW::reduction_features ex_reduction_features; + + VW::v_array tag; // An identifier for the example. + + size_t example_counter; + uint64_t ft_offset; + float global_weight; + + size_t num_features; // precomputed, cause it's fast&easy. + float total_sum_feat_sq; // precomputed, cause it's kind of fast & easy. + VW::features fs; // all the features +}; + +flat_example* flatten_sort_example(VW::workspace& all, VW::example* ec) +{ + auto& fec = VW::details::calloc_or_throw(); + fec.l = ec->l; + fec.tag = ec->tag; + fec.ex_reduction_features = ec->ex_reduction_features; + fec.example_counter = ec->example_counter; + fec.ft_offset = ec->ft_offset; + fec.num_features = ec->num_features; + + flatten_features(all, *ec, fec.fs); + fec.total_sum_feat_sq = fec.fs.sum_feat_sq; + + return &fec; +} + +// TODO: do not depend on unstable cache format for model format of KSVM. +size_t read_model_field_flat_example(VW::io_buf& io, flat_example& fe, VW::label_parser& lbl_parser) +{ + size_t bytes = 0; + lbl_parser.default_label(fe.l); + bytes += lbl_parser.read_cached_label(fe.l, fe.ex_reduction_features, io); + bytes += VW::model_utils::read_model_field(io, fe.tag); + bytes += VW::model_utils::read_model_field(io, fe.example_counter); + bytes += VW::model_utils::read_model_field(io, fe.ft_offset); + bytes += VW::model_utils::read_model_field(io, fe.global_weight); + bytes += VW::model_utils::read_model_field(io, fe.num_features); + bytes += VW::model_utils::read_model_field(io, fe.total_sum_feat_sq); + unsigned char index = 0; + bytes += ::VW::parsers::cache::details::read_cached_index(io, index); + bool sorted = true; + bytes += ::VW::parsers::cache::details::read_cached_features(io, fe.fs, sorted); + return bytes; +} + +size_t write_model_field_flat_example(VW::io_buf& io, const flat_example& fe, const std::string& upstream_name, + bool text, VW::label_parser& lbl_parser, uint64_t parse_mask) +{ + size_t bytes = 0; + lbl_parser.cache_label(fe.l, fe.ex_reduction_features, io, upstream_name + "_label", text); + bytes += VW::model_utils::write_model_field(io, fe.tag, upstream_name + "_tag", text); + bytes += VW::model_utils::write_model_field(io, fe.example_counter, upstream_name + "_example_counter", text); + bytes += VW::model_utils::write_model_field(io, fe.ft_offset, upstream_name + "_ft_offset", text); + bytes += VW::model_utils::write_model_field(io, fe.global_weight, upstream_name + "_global_weight", text); + bytes += VW::model_utils::write_model_field(io, fe.num_features, upstream_name + "_num_features", text); + bytes += VW::model_utils::write_model_field(io, fe.total_sum_feat_sq, upstream_name + "_total_sum_feat_sq", text); + ::VW::parsers::cache::details::cache_index(io, 0); + ::VW::parsers::cache::details::cache_features(io, fe.fs, parse_mask); + return bytes; +} + class svm_params; static size_t num_kernel_evals = 0; @@ -52,10 +122,10 @@ class svm_example { public: VW::v_array krow; - VW::flat_example ex; + flat_example ex; ~svm_example(); - void init_svm_example(VW::flat_example* fec); + void init_svm_example(flat_example* fec); int compute_kernels(svm_params& params); int clear_kernels(); }; @@ -132,7 +202,7 @@ class svm_params } }; -void svm_example::init_svm_example(VW::flat_example* fec) +void svm_example::init_svm_example(flat_example* fec) { ex = std::move(*fec); free(fec); @@ -146,7 +216,7 @@ svm_example::~svm_example() // free_flatten_example(fec); // free contents of flat example and frees fec. } -float kernel_function(const VW::flat_example* fec1, const VW::flat_example* fec2, void* params, size_t kernel_type); +float kernel_function(const flat_example* fec1, const flat_example* fec2, void* params, size_t kernel_type); int svm_example::compute_kernels(svm_params& params) { @@ -252,15 +322,15 @@ void save_load_svm_model(svm_params& params, VW::io_buf& model_file, bool read, { if (read) { - auto fec = VW::make_unique(); + auto fec = VW::make_unique(); auto* tmp = &VW::details::calloc_or_throw(); - VW::model_utils::read_model_field(model_file, *fec, params.all->parser_runtime.example_parser->lbl_parser); + read_model_field_flat_example(model_file, *fec, params.all->parser_runtime.example_parser->lbl_parser); tmp->ex = *fec; model->support_vec.push_back(tmp); } else { - VW::model_utils::write_model_field(model_file, model->support_vec[i]->ex, "_flat_example", false, + write_model_field_flat_example(model_file, model->support_vec[i]->ex, "_flat_example", false, params.all->parser_runtime.example_parser->lbl_parser, params.all->runtime_state.parse_mask); } } @@ -289,45 +359,19 @@ void save_load(svm_params& params, VW::io_buf& model_file, bool read, bool text) save_load_svm_model(params, model_file, read, text); } -float linear_kernel(const VW::flat_example* fec1, const VW::flat_example* fec2) -{ - float dotprod = 0; - - auto& fs_1 = const_cast(fec1->fs); - auto& fs_2 = const_cast(fec2->fs); - if (fs_2.indices.size() == 0) { return 0.f; } - - for (size_t idx1 = 0, idx2 = 0; idx1 < fs_1.size() && idx2 < fs_2.size(); idx1++) - { - uint64_t ec1pos = fs_1.indices[idx1]; - uint64_t ec2pos = fs_2.indices[idx2]; - // params.all->opts_n_args.trace_message<x<<" "<x<< endl; - if (ec1pos < ec2pos) { continue; } - - while (ec1pos > ec2pos && ++idx2 < fs_2.size()) { ec2pos = fs_2.indices[idx2]; } - - if (ec1pos == ec2pos) - { - dotprod += fs_1.values[idx1] * fs_2.values[idx2]; - ++idx2; - } - } - return dotprod; -} - -float poly_kernel(const VW::flat_example* fec1, const VW::flat_example* fec2, int power) +float poly_kernel(const flat_example* fec1, const flat_example* fec2, int power) { - float dotprod = linear_kernel(fec1, fec2); + float dotprod = VW::features_dot_product(fec1->fs, fec2->fs); return static_cast(std::pow(1 + dotprod, power)); } -float rbf_kernel(const VW::flat_example* fec1, const VW::flat_example* fec2, float bandwidth) +float rbf_kernel(const flat_example* fec1, const flat_example* fec2, float bandwidth) { - float dotprod = linear_kernel(fec1, fec2); + float dotprod = VW::features_dot_product(fec1->fs, fec2->fs); return expf(-(fec1->total_sum_feat_sq + fec2->total_sum_feat_sq - 2 * dotprod) * bandwidth); } -float kernel_function(const VW::flat_example* fec1, const VW::flat_example* fec2, void* params, size_t kernel_type) +float kernel_function(const flat_example* fec1, const flat_example* fec2, void* params, size_t kernel_type) { switch (kernel_type) { @@ -336,7 +380,7 @@ float kernel_function(const VW::flat_example* fec1, const VW::flat_example* fec2 case SVM_KER_POLY: return poly_kernel(fec1, fec2, *(static_cast(params))); case SVM_KER_LIN: - return linear_kernel(fec1, fec2); + return VW::features_dot_product(fec1->fs, fec2->fs); } return 0; } @@ -365,7 +409,7 @@ void predict(svm_params& params, svm_example** ec_arr, float* scores, size_t n) void predict(svm_params& params, VW::example& ec) { - VW::flat_example* fec = VW::flatten_sort_example(*(params.all), &ec); + flat_example* fec = flatten_sort_example(*(params.all), &ec); if (fec) { svm_example* sec = &VW::details::calloc_or_throw(); @@ -502,15 +546,14 @@ void sync_queries(VW::workspace& all, svm_params& params, bool* train_pool) VW::io_buf* b = new VW::io_buf(); char* queries; - VW::flat_example* fec = nullptr; + flat_example* fec = nullptr; for (size_t i = 0; i < params.pool_pos; i++) { if (!train_pool[i]) { continue; } fec = &(params.pool[i]->ex); - VW::model_utils::write_model_field( - *b, *fec, "_flat_example", false, all.parser_runtime.example_parser->lbl_parser, all.runtime_state.parse_mask); + write_model_field_flat_example(*b, *fec, "_flat_example", false, all.parser_runtime.example_parser->lbl_parser, all.runtime_state.parse_mask); delete params.pool[i]; } @@ -539,7 +582,7 @@ void sync_queries(VW::workspace& all, svm_params& params, bool* train_pool) for (size_t i = 0; i < params.pool_size; i++) { - if (!VW::model_utils::read_model_field(*b, *fec, all.parser_runtime.example_parser->lbl_parser)) + if (!read_model_field_flat_example(*b, *fec, all.parser_runtime.example_parser->lbl_parser)) { params.pool[i] = &VW::details::calloc_or_throw(); params.pool[i]->init_svm_example(fec); @@ -672,7 +715,7 @@ void train(svm_params& params) void learn(svm_params& params, VW::example& ec) { - VW::flat_example* fec = VW::flatten_sort_example(*(params.all), &ec); + flat_example* fec = flatten_sort_example(*(params.all), &ec); if (fec) { svm_example* sec = &VW::details::calloc_or_throw(); diff --git a/vowpalwabbit/core/src/reductions/lda_core.cc b/vowpalwabbit/core/src/reductions/lda_core.cc index 6870872624f..aa9d9a40dd5 100644 --- a/vowpalwabbit/core/src/reductions/lda_core.cc +++ b/vowpalwabbit/core/src/reductions/lda_core.cc @@ -1369,8 +1369,6 @@ std::shared_ptr VW::reductions::lda_setup(VW::setup_base_i ld->decay_levels.push_back(0.f); - all.parser_runtime.example_parser->lbl_parser = VW::no_label_parser_global; - // If minibatch is > 1, then the predict function does not actually produce predictions. const auto pred_type = ld->minibatch > 1 ? VW::prediction_type_t::NOPRED : VW::prediction_type_t::SCALARS; @@ -1378,7 +1376,6 @@ std::shared_ptr VW::reductions::lda_setup(VW::setup_base_i auto l = make_bottom_learner(std::move(ld), ld->compute_coherence_metrics ? learn_with_metrics : learn, ld->compute_coherence_metrics ? predict_with_metrics : predict, stack_builder.get_setupfn_name(lda_setup), pred_type, VW::label_type_t::NOLABEL) - .set_params_per_weight(VW::details::UINT64_ONE << all.weights.stride_shift()) .set_learn_returns_prediction(true) .set_save_load(save_load) .set_end_examples(end_examples) diff --git a/vowpalwabbit/core/src/reductions/log_multi.cc b/vowpalwabbit/core/src/reductions/log_multi.cc index de0d3473089..596c9805186 100644 --- a/vowpalwabbit/core/src/reductions/log_multi.cc +++ b/vowpalwabbit/core/src/reductions/log_multi.cc @@ -449,10 +449,10 @@ std::shared_ptr VW::reductions::log_multi_setup(VW::setup_ data->max_predictors = data->k - 1; init_tree(*data.get()); - size_t ws = data->max_predictors; - auto l = make_reduction_learner(std::move(data), require_singleline(stack_builder.setup_base_learner()), learn, - predict, stack_builder.get_setupfn_name(log_multi_setup)) - .set_params_per_weight(ws) + size_t feature_width = data->max_predictors; + auto l = make_reduction_learner(std::move(data), require_singleline(stack_builder.setup_base_learner(feature_width)), + learn, predict, stack_builder.get_setupfn_name(log_multi_setup)) + .set_feature_width(feature_width) .set_update_stats(VW::details::update_stats_multiclass_label) .set_output_example_prediction(VW::details::output_example_prediction_multiclass_label) .set_print_update(VW::details::print_update_multiclass_label) @@ -462,8 +462,5 @@ std::shared_ptr VW::reductions::log_multi_setup(VW::setup_ .set_input_label_type(VW::label_type_t::MULTICLASS) .set_output_label_type(VW::label_type_t::SIMPLE) .build(); - - all.parser_runtime.example_parser->lbl_parser = VW::multiclass_label_parser_global; - return l; } diff --git a/vowpalwabbit/core/src/reductions/lrq.cc b/vowpalwabbit/core/src/reductions/lrq.cc index 8bca2ab6e15..3227d53b30f 100644 --- a/vowpalwabbit/core/src/reductions/lrq.cc +++ b/vowpalwabbit/core/src/reductions/lrq.cc @@ -224,12 +224,12 @@ std::shared_ptr VW::reductions::lrq_setup(VW::setup_base_i if (!all.output_config.quiet) { *(all.output_runtime.trace_message) << std::endl; } - all.reduction_state.wpp = all.reduction_state.wpp * static_cast(1 + maxk); - auto base = stack_builder.setup_base_learner(); + all.reduction_state.total_feature_width = all.reduction_state.total_feature_width * static_cast(1 + maxk); + auto base = stack_builder.setup_base_learner(1 + maxk); auto l = make_reduction_learner(std::move(lrq), require_singleline(base), predict_or_learn, predict_or_learn, stack_builder.get_setupfn_name(lrq_setup)) - .set_params_per_weight(1 + maxk) + .set_feature_width(1 + maxk) .set_learn_returns_prediction(base->learn_returns_prediction) .set_end_pass(reset_seed) .build(); diff --git a/vowpalwabbit/core/src/reductions/lrqfa.cc b/vowpalwabbit/core/src/reductions/lrqfa.cc index 488c9025505..4067242d2c8 100644 --- a/vowpalwabbit/core/src/reductions/lrqfa.cc +++ b/vowpalwabbit/core/src/reductions/lrqfa.cc @@ -164,13 +164,13 @@ std::shared_ptr VW::reductions::lrqfa_setup(VW::setup_base int fd_id = 0; for (char i : lrq->field_name) { lrq->field_id[static_cast(i)] = fd_id++; } - all.reduction_state.wpp = all.reduction_state.wpp * static_cast(1 + lrq->k); - auto base = stack_builder.setup_base_learner(); - size_t ws = 1 + lrq->field_name.size() * lrq->k; + all.reduction_state.total_feature_width = all.reduction_state.total_feature_width * static_cast(1 + lrq->k); + size_t feature_width = 1 + lrq->field_name.size() * lrq->k; + auto base = stack_builder.setup_base_learner(feature_width); auto l = make_reduction_learner(std::move(lrq), require_singleline(base), predict_or_learn, predict_or_learn, stack_builder.get_setupfn_name(lrqfa_setup)) - .set_params_per_weight(ws) + .set_feature_width(feature_width) .set_learn_returns_prediction(base->learn_returns_prediction) .build(); diff --git a/vowpalwabbit/core/src/reductions/memory_tree.cc b/vowpalwabbit/core/src/reductions/memory_tree.cc index 8865882764d..86d75b9f403 100644 --- a/vowpalwabbit/core/src/reductions/memory_tree.cc +++ b/vowpalwabbit/core/src/reductions/memory_tree.cc @@ -8,6 +8,7 @@ #include "vw/common/random.h" #include "vw/config/options.h" #include "vw/core/example.h" +#include "vw/core/feature_group.h" #include "vw/core/learner.h" #include "vw/core/multiclass.h" #include "vw/core/multilabel.h" @@ -226,39 +227,14 @@ class memory_tree } }; -float linear_kernel(const VW::flat_example* fec1, const VW::flat_example* fec2) -{ - float dotprod = 0; - - auto& fs_1 = const_cast(fec1->fs); - auto& fs_2 = const_cast(fec2->fs); - if (fs_2.indices.size() == 0) { return 0.f; } - - for (size_t idx1 = 0, idx2 = 0; idx1 < fs_1.size() && idx2 < fs_2.size(); idx1++) - { - uint64_t ec1pos = fs_1.indices[idx1]; - uint64_t ec2pos = fs_2.indices[idx2]; - if (ec1pos < ec2pos) { continue; } - - while (ec1pos > ec2pos && ++idx2 < fs_2.size()) { ec2pos = fs_2.indices[idx2]; } - - if (ec1pos == ec2pos) - { - dotprod += fs_1.values[idx1] * fs_2.values[idx2]; - ++idx2; - } - } - return dotprod; -} - float normalized_linear_prod(memory_tree& b, VW::example* ec1, VW::example* ec2) { - VW::flat_example* fec1 = VW::flatten_sort_example(*b.all, ec1); - VW::flat_example* fec2 = VW::flatten_sort_example(*b.all, ec2); - float norm_sqrt = std::pow(fec1->total_sum_feat_sq * fec2->total_sum_feat_sq, 0.5f); - float linear_prod = linear_kernel(fec1, fec2); - VW::free_flatten_example(fec1); - VW::free_flatten_example(fec2); + VW::features fs1; + VW::features fs2; + flatten_features(*b.all, *ec1, fs1); + flatten_features(*b.all, *ec2, fs2); + float norm_sqrt = std::pow(fs1.sum_feat_sq * fs2.sum_feat_sq, 0.5f); + float linear_prod = VW::features_dot_product(fs1, fs2); return linear_prod / norm_sqrt; } @@ -1270,7 +1246,7 @@ std::shared_ptr VW::reductions::memory_tree_setup(VW::setu << "online =" << tree->online << " " << std::endl; } - size_t num_learners; + size_t feature_width; VW::prediction_type_t pred_type; VW::label_type_t label_type; bool oas = tree->oas; @@ -1278,22 +1254,20 @@ std::shared_ptr VW::reductions::memory_tree_setup(VW::setu // multi-class classification if (!oas) { - num_learners = tree->max_nodes + 1; - all.parser_runtime.example_parser->lbl_parser = VW::multiclass_label_parser_global; + feature_width = tree->max_nodes + 1; pred_type = VW::prediction_type_t::MULTICLASS; label_type = VW::label_type_t::MULTICLASS; } // multi-label classification else { - num_learners = tree->max_nodes + 1 + tree->max_num_labels; - all.parser_runtime.example_parser->lbl_parser = VW::multilabel_label_parser_global; + feature_width = tree->max_nodes + 1 + tree->max_num_labels; pred_type = VW::prediction_type_t::MULTILABELS; label_type = VW::label_type_t::MULTILABEL; } - auto l = make_reduction_learner(std::move(tree), require_singleline(stack_builder.setup_base_learner()), learn, - predict, stack_builder.get_setupfn_name(memory_tree_setup)) - .set_params_per_weight(num_learners) + auto l = make_reduction_learner(std::move(tree), require_singleline(stack_builder.setup_base_learner(feature_width)), + learn, predict, stack_builder.get_setupfn_name(memory_tree_setup)) + .set_feature_width(feature_width) .set_end_pass(end_pass) .set_save_load(save_load_memory_tree) .set_input_label_type(label_type) diff --git a/vowpalwabbit/core/src/reductions/mf.cc b/vowpalwabbit/core/src/reductions/mf.cc index 1ac423a38ee..64ceb042a1b 100644 --- a/vowpalwabbit/core/src/reductions/mf.cc +++ b/vowpalwabbit/core/src/reductions/mf.cc @@ -21,8 +21,6 @@ class mf public: size_t rank = 0; - uint32_t increment = 0; - // array to cache w*x, (l^k * x_l) and (r^k * x_r) // [ w*(1,x_l,x_r) , l^1*x_l, r^1*x_r, l^2*x_l, r^2*x_2, ... ] VW::v_array sub_predictions; @@ -208,11 +206,11 @@ std::shared_ptr VW::reductions::mf_setup(VW::setup_base_i& all.iwc.random_positive_weights = true; - size_t ws = 2 * data->rank + 1; + size_t feature_width = 2 * data->rank + 1; - auto l = make_reduction_learner(std::move(data), require_singleline(stack_builder.setup_base_learner()), learn, - predict, stack_builder.get_setupfn_name(mf_setup)) - .set_params_per_weight(ws) + auto l = make_reduction_learner(std::move(data), require_singleline(stack_builder.setup_base_learner(feature_width)), + learn, predict, stack_builder.get_setupfn_name(mf_setup)) + .set_feature_width(feature_width) .set_output_prediction_type(VW::prediction_type_t::SCALAR) .build(); diff --git a/vowpalwabbit/core/src/reductions/multilabel_oaa.cc b/vowpalwabbit/core/src/reductions/multilabel_oaa.cc index 3403ac57cf7..6b8ae67b05f 100644 --- a/vowpalwabbit/core/src/reductions/multilabel_oaa.cc +++ b/vowpalwabbit/core/src/reductions/multilabel_oaa.cc @@ -134,7 +134,7 @@ std::shared_ptr VW::reductions::multilabel_oaa_setup(VW::s data->k = VW::cast_to_smaller_type(k); std::string name_addition; VW::prediction_type_t pred_type; - size_t ws = data->k; + size_t feature_width = data->k; if (data->probabilities) { @@ -161,10 +161,10 @@ std::shared_ptr VW::reductions::multilabel_oaa_setup(VW::s pred_type = VW::prediction_type_t::MULTILABELS; } - auto l = make_reduction_learner(std::move(data), require_singleline(stack_builder.setup_base_learner()), + auto l = make_reduction_learner(std::move(data), require_singleline(stack_builder.setup_base_learner(feature_width)), predict_or_learn, predict_or_learn, stack_builder.get_setupfn_name(multilabel_oaa_setup) + name_addition) - .set_params_per_weight(ws) + .set_feature_width(feature_width) .set_learn_returns_prediction(true) .set_input_label_type(VW::label_type_t::MULTILABEL) .set_output_label_type(VW::label_type_t::SIMPLE) @@ -175,7 +175,5 @@ std::shared_ptr VW::reductions::multilabel_oaa_setup(VW::s .set_print_update(print_update_multilabel_oaa) .build(); - all.parser_runtime.example_parser->lbl_parser = VW::multilabel_label_parser_global; - return l; } diff --git a/vowpalwabbit/core/src/reductions/mwt.cc b/vowpalwabbit/core/src/reductions/mwt.cc index d832deb184b..c5bf74f34ee 100644 --- a/vowpalwabbit/core/src/reductions/mwt.cc +++ b/vowpalwabbit/core/src/reductions/mwt.cc @@ -331,7 +331,5 @@ std::shared_ptr VW::reductions::mwt_setup(VW::setup_base_i .set_update_stats(::update_stats_mwt) .set_print_update(::print_update_mwt) .build(); - - all.parser_runtime.example_parser->lbl_parser = VW::cb_label_parser_global; return l; } diff --git a/vowpalwabbit/core/src/reductions/nn.cc b/vowpalwabbit/core/src/reductions/nn.cc index 666399663bb..55dc166d82a 100644 --- a/vowpalwabbit/core/src/reductions/nn.cc +++ b/vowpalwabbit/core/src/reductions/nn.cc @@ -41,7 +41,7 @@ class nn VW::example hiddenbias; VW::example outputweight; float prediction = 0.f; - size_t increment = 0; + size_t feature_width_below = 0; bool dropout = false; uint64_t xsubi = 0; uint64_t save_xsubi = 0; @@ -105,7 +105,7 @@ void finish_setup(nn& n, VW::workspace& all) ss << "OutputLayer" << i; fs.space_names.emplace_back("", ss.str()); } - nn_index += static_cast(n.increment); + nn_index += static_cast(n.feature_width_below); } n.output_layer.num_features += n.k; @@ -486,15 +486,14 @@ std::shared_ptr VW::reductions::nn_setup(VW::setup_base_i& n->hidden_units_pred = VW::details::calloc_or_throw(n->k); n->hiddenbias_pred = VW::details::calloc_or_throw(n->k); - auto base = require_singleline(stack_builder.setup_base_learner()); - n->increment = base->increment; // Indexing of output layer is odd. + size_t feature_width = n->k + 1; + auto base = require_singleline(stack_builder.setup_base_learner(feature_width)); + n->feature_width_below = base->feature_width_below; // Indexing of output layer is odd. nn& nv = *n.get(); - size_t ws = n->k + 1; - auto builder = make_reduction_learner(std::move(n), base, predict_or_learn_multi, predict_or_learn_multi, stack_builder.get_setupfn_name(nn_setup)) - .set_params_per_weight(ws) + .set_feature_width(feature_width) .set_learn_returns_prediction(true) .set_input_prediction_type(VW::prediction_type_t::SCALAR) .set_output_prediction_type(VW::prediction_type_t::SCALAR) diff --git a/vowpalwabbit/core/src/reductions/oaa.cc b/vowpalwabbit/core/src/reductions/oaa.cc index ea84f9a3a9b..fb3fbc861c6 100644 --- a/vowpalwabbit/core/src/reductions/oaa.cc +++ b/vowpalwabbit/core/src/reductions/oaa.cc @@ -382,7 +382,7 @@ std::shared_ptr VW::reductions::oaa_setup(VW::setup_base_i oaa* data_ptr = data.get(); uint64_t k_value = data->k; - auto base = require_singleline(stack_builder.setup_base_learner()); + auto base = require_singleline(stack_builder.setup_base_learner(k_value)); void (*learn_ptr)(oaa&, VW::LEARNER::learner&, VW::example&) = nullptr; void (*pred_ptr)(oaa&, VW::LEARNER::learner&, VW::example&) = nullptr; std::string name_addition; @@ -440,8 +440,6 @@ std::shared_ptr VW::reductions::oaa_setup(VW::setup_base_i } } - all.parser_runtime.example_parser->lbl_parser = VW::multiclass_label_parser_global; - if (data_ptr->num_subsample > 0) { learn_ptr = learn_randomized; @@ -462,7 +460,7 @@ std::shared_ptr VW::reductions::oaa_setup(VW::setup_base_i auto l = make_reduction_learner( std::move(data), base, learn_ptr, pred_ptr, stack_builder.get_setupfn_name(oaa_setup) + name_addition) - .set_params_per_weight(k_value) + .set_feature_width(k_value) .set_input_label_type(VW::label_type_t::MULTICLASS) .set_output_label_type(VW::label_type_t::SIMPLE) .set_input_prediction_type(VW::prediction_type_t::SCALAR) diff --git a/vowpalwabbit/core/src/reductions/offset_tree.cc b/vowpalwabbit/core/src/reductions/offset_tree.cc index 2d7d8c27cdf..4b83a072615 100644 --- a/vowpalwabbit/core/src/reductions/offset_tree.cc +++ b/vowpalwabbit/core/src/reductions/offset_tree.cc @@ -279,12 +279,12 @@ std::shared_ptr VW::reductions::offset_tree_setup(VW::setu auto otree = VW::make_unique(num_actions); otree->init(); - auto base = stack_builder.setup_base_learner(); - size_t ws = otree->learner_count(); + size_t feature_width = otree->learner_count(); + auto base = stack_builder.setup_base_learner(feature_width); auto l = make_reduction_learner( std::move(otree), require_singleline(base), learn, predict, stack_builder.get_setupfn_name(offset_tree_setup)) - .set_params_per_weight(ws) + .set_feature_width(feature_width) .set_input_prediction_type(prediction_type_t::ACTION_PROBS) .set_output_prediction_type(prediction_type_t::ACTION_PROBS) .set_input_label_type(label_type_t::CB) diff --git a/vowpalwabbit/core/src/reductions/oja_newton.cc b/vowpalwabbit/core/src/reductions/oja_newton.cc index ba0a3da31f4..3e84c8ad9fb 100644 --- a/vowpalwabbit/core/src/reductions/oja_newton.cc +++ b/vowpalwabbit/core/src/reductions/oja_newton.cc @@ -482,12 +482,8 @@ void save_load(OjaNewton& oja_newton_ptr, VW::io_buf& model_file, bool read, boo VW::details::bin_text_read_write_fixed( model_file, reinterpret_cast(&resume), sizeof(resume), read, msg, text); - double temp = 0.; - double temp_normalized_sum_norm_x = 0.; - if (resume) - { - VW::details::save_load_online_state_gd(all, model_file, read, text, temp, temp_normalized_sum_norm_x); - } + std::vector temp_pms = {VW::reductions::details::gd_per_model_state()}; + if (resume) { VW::details::save_load_online_state_gd(all, model_file, read, text, temp_pms); } else { VW::details::save_load_regressor_gd(all, model_file, read, text); } } } @@ -569,7 +565,6 @@ std::shared_ptr VW::reductions::oja_newton_setup(VW::setup auto l = make_bottom_learner(std::move(oja_newton_ptr), learn, predict, stack_builder.get_setupfn_name(oja_newton_setup), VW::prediction_type_t::SCALAR, VW::label_type_t::SIMPLE) - .set_params_per_weight(all.weights.stride()) .set_save_load(save_load) .set_output_example_prediction(VW::details::output_example_prediction_simple_label) .set_update_stats(VW::details::update_stats_simple_label) diff --git a/vowpalwabbit/core/src/reductions/plt.cc b/vowpalwabbit/core/src/reductions/plt.cc index 4517119dbbc..daa2f367da2 100644 --- a/vowpalwabbit/core/src/reductions/plt.cc +++ b/vowpalwabbit/core/src/reductions/plt.cc @@ -90,15 +90,8 @@ inline float learn_node(plt& p, uint32_t n, learner& base, VW::example& ec) return ec.loss; } -void learn(plt& p, learner& base, VW::example& ec) +void get_nodes_to_update(plt& p, VW::multilabel_label& multilabels) { - auto multilabels = std::move(ec.l.multilabels); - VW::polyprediction pred = std::move(ec.pred); - - double t = p.all->sd->t; - double weighted_holdout_examples = p.all->sd->weighted_holdout_examples; - p.all->sd->weighted_holdout_examples = 0; - p.positive_nodes.clear(); p.negative_nodes.clear(); @@ -139,7 +132,16 @@ void learn(plt& p, learner& base, VW::example& ec) } } else { p.negative_nodes.insert(0); } +} + +void learn(plt& p, learner& base, VW::example& ec) +{ + auto multilabels = std::move(ec.l.multilabels); + VW::polyprediction pred = std::move(ec.pred); + get_nodes_to_update(p, multilabels); + + double t = p.all->sd->t; float loss = 0; ec.l.simple = {1.f}; ec.ex_reduction_features.template get().reset_to_default(); @@ -149,8 +151,6 @@ void learn(plt& p, learner& base, VW::example& ec) for (auto& n : p.negative_nodes) { loss += learn_node(p, n, base, ec); } p.all->sd->t = t; - p.all->sd->weighted_holdout_examples = weighted_holdout_examples; - ec.loss = loss; ec.pred = std::move(pred); ec.l.multilabels = std::move(multilabels); @@ -166,12 +166,37 @@ inline float predict_node(uint32_t n, learner& base, VW::example& ec) return sigmoid(ec.partial_prediction); } +inline float evaluate_node(uint32_t n, learner& base, VW::example& ec) +{ + base.predict(ec, n); + return ec.loss; +} + template void predict(plt& p, learner& base, VW::example& ec) { auto multilabels = std::move(ec.l.multilabels); VW::polyprediction pred = std::move(ec.pred); + // if true labels are present (e.g., predicting on holdout set), + // calculate training loss without updating base learner/weights + if (multilabels.label_v.size() > 0) + { + get_nodes_to_update(p, multilabels); + + float loss = 0; + ec.l.simple = {1.f}; + ec.ex_reduction_features.template get().reset_to_default(); + for (auto& n : p.positive_nodes) { loss += evaluate_node(n, base, ec); } + + ec.l.simple.label = -1.f; + for (auto& n : p.negative_nodes) { loss += evaluate_node(n, base, ec); } + + ec.loss = loss; + } + else { ec.loss = 0; } + + // start real prediction if (p.probabilities) { pred.a_s.clear(); } pred.multilabels.label_v.clear(); @@ -220,18 +245,21 @@ void predict(plt& p, learner& base, VW::example& ec) } } - // calculate evaluation measures - uint32_t tp = 0; - uint32_t pred_size = pred.multilabels.label_v.size(); - - for (uint32_t i = 0; i < pred_size; ++i) + // if there are true labels, calculate evaluation measures + if (p.true_labels.size() > 0) { - uint32_t pred_label = pred.multilabels.label_v[i]; - if (p.true_labels.count(pred_label)) { ++tp; } + uint32_t tp = 0; + uint32_t pred_size = pred.multilabels.label_v.size(); + + for (uint32_t i = 0; i < pred_size; ++i) + { + uint32_t pred_label = pred.multilabels.label_v[i]; + if (p.true_labels.count(pred_label)) { ++tp; } + } + p.tp += tp; + p.fp += static_cast(pred_size) - tp; + p.fn += static_cast(p.true_labels.size()) - tp; } - p.tp += tp; - p.fp += static_cast(pred_size) - tp; - p.fn += static_cast(p.true_labels.size()) - tp; } // top-k prediction @@ -270,21 +298,23 @@ void predict(plt& p, learner& base, VW::example& ec) } } - // calculate precision and recall at - float tp_at = 0; - for (size_t i = 0; i < p.top_k; ++i) + // if there are true labels, calculate precision and recall at k + if (p.true_labels.size() > 0) { - uint32_t pred_label = pred.multilabels.label_v[i]; - if (p.true_labels.count(pred_label)) { tp_at += 1; } - p.p_at[i] += tp_at / (i + 1); - if (p.true_labels.size() > 0) { p.r_at[i] += tp_at / p.true_labels.size(); } + float tp_at = 0; + for (size_t i = 0; i < p.top_k; ++i) + { + uint32_t pred_label = pred.multilabels.label_v[i]; + if (p.true_labels.count(pred_label)) { tp_at += 1; } + p.p_at[i] += tp_at / (i + 1); + if (p.true_labels.size() > 0) { p.r_at[i] += tp_at / p.true_labels.size(); } + } } } ++p.ec_count; p.node_queue.clear(); - ec.loss = 0; ec.pred = std::move(pred); ec.l.multilabels = std::move(multilabels); } @@ -455,7 +485,7 @@ std::shared_ptr VW::reductions::plt_setup(VW::setup_base_i tree->model_file_version = all.runtime_state.model_file_ver; - size_t ws = tree->t; + size_t feature_width = tree->t; std::string name_addition = ""; VW::prediction_type_t pred_type; void (*pred_ptr)(plt&, learner&, VW::example&); @@ -474,9 +504,9 @@ std::shared_ptr VW::reductions::plt_setup(VW::setup_base_i } else { pred_type = VW::prediction_type_t::MULTILABELS; } - auto l = make_reduction_learner(std::move(tree), require_singleline(stack_builder.setup_base_learner()), learn, - pred_ptr, stack_builder.get_setupfn_name(plt_setup) + name_addition) - .set_params_per_weight(ws) + auto l = make_reduction_learner(std::move(tree), require_singleline(stack_builder.setup_base_learner(feature_width)), + learn, pred_ptr, stack_builder.get_setupfn_name(plt_setup) + name_addition) + .set_feature_width(feature_width) .set_learn_returns_prediction(false) .set_input_label_type(VW::label_type_t::MULTILABEL) .set_output_label_type(VW::label_type_t::SIMPLE) @@ -489,8 +519,5 @@ std::shared_ptr VW::reductions::plt_setup(VW::setup_base_i .set_finish(::finish) .set_save_load(::save_load_tree) .build(); - - all.parser_runtime.example_parser->lbl_parser = VW::multilabel_label_parser_global; - return l; } diff --git a/vowpalwabbit/core/src/reductions/recall_tree.cc b/vowpalwabbit/core/src/reductions/recall_tree.cc index 6331d1c9b7f..73b9e48a1ef 100644 --- a/vowpalwabbit/core/src/reductions/recall_tree.cc +++ b/vowpalwabbit/core/src/reductions/recall_tree.cc @@ -548,10 +548,10 @@ std::shared_ptr VW::reductions::recall_tree_setup(VW::setu << std::endl; } - size_t ws = tree->max_routers + tree->k; - auto l = make_reduction_learner(std::move(tree), require_singleline(stack_builder.setup_base_learner()), learn, - predict, stack_builder.get_setupfn_name(recall_tree_setup)) - .set_params_per_weight(ws) + size_t feature_width = tree->max_routers + tree->k; + auto l = make_reduction_learner(std::move(tree), require_singleline(stack_builder.setup_base_learner(feature_width)), + learn, predict, stack_builder.get_setupfn_name(recall_tree_setup)) + .set_feature_width(feature_width) .set_update_stats(VW::details::update_stats_multiclass_label) .set_output_example_prediction(VW::details::output_example_prediction_multiclass_label) .set_print_update(VW::details::print_update_multiclass_label) @@ -561,8 +561,5 @@ std::shared_ptr VW::reductions::recall_tree_setup(VW::setu .set_input_label_type(VW::label_type_t::MULTICLASS) .set_output_label_type(VW::label_type_t::SIMPLE) .build(); - - all.parser_runtime.example_parser->lbl_parser = VW::multiclass_label_parser_global; - return l; } diff --git a/vowpalwabbit/core/src/reductions/search/search.cc b/vowpalwabbit/core/src/reductions/search/search.cc index 304d45cc158..6bbfef7c40c 100644 --- a/vowpalwabbit/core/src/reductions/search/search.cc +++ b/vowpalwabbit/core/src/reductions/search/search.cc @@ -190,7 +190,7 @@ class search_private size_t history_length = 0; // value of --search_history_length, used by some tasks, default 1 size_t A = 0; // NOLINT total number of actions, [1..A]; 0 means ldf - size_t num_learners = 0; // total number of learners; + size_t feature_width = 0; // total number of learners; bool cb_learner = false; // do contextual bandit learning on action (was "! rollout_all_actions" which was confusing) search_state state; // current state of learning size_t learn_learner_id = 0; // we allow user to use different learners for different states @@ -422,8 +422,7 @@ int select_learner(search_private& priv, int policy, size_t learner_id, bool is_ learner_id *= 3; if (!is_local) { learner_id += 1 + static_cast(is_training ^ (priv.all->sd->example_number % 2 == 1)); } } - int p = static_cast(policy * priv.num_learners + learner_id); - return p; + return static_cast(policy * priv.feature_width + learner_id); } } @@ -2474,7 +2473,7 @@ void search_initialize(VW::workspace* all, search& sch) priv.active_csoaa = false; priv.label_is_test = mc_label_is_test; - priv.num_learners = 1; + priv.feature_width = 1; priv.state = search_state::INITIALIZE; priv.mix_per_roll_policy = -2; @@ -2780,6 +2779,13 @@ void search::set_label_parser(VW::label_parser& lp, bool (*is_test)(const VW::po { priv->all->logger.err_warn("Task should not set label parser except in initialize function."); } + + // This will override the label parser object passed in, and it will persist. + // I don't know why Search needs to override is_test, and if it didn't then this would not be necessary. + // Remove the overriding of is_test breaks the unit tests. + // TODO: figure out why Search needs to override is_test and remove this. + lp.test_label = is_test; + this->priv->all->parser_runtime.example_parser->lbl_parser = lp; this->priv->all->parser_runtime.example_parser->lbl_parser.test_label = is_test; this->priv->label_is_test = is_test; @@ -2794,7 +2800,7 @@ void search::get_test_action_sequence(std::vector& V) } } -void search::set_num_learners(size_t num_learners) { this->priv->num_learners = num_learners; } +void search::set_feature_width(size_t feature_width) { this->priv->feature_width = feature_width; } uint64_t search::get_mask() { return this->priv->all->weights.mask(); } size_t search::get_stride_shift() { return this->priv->all->weights.stride_shift(); } @@ -3337,8 +3343,6 @@ std::shared_ptr VW::reductions::search_setup(VW::setup_bas cdbg << "active_csoaa = " << priv.active_csoaa << ", active_csoaa_verify = " << priv.active_csoaa_verify << endl; - auto base = stack_builder.setup_base_learner(); - // default to OAA labels unless the task wants to override this (which they can do in initialize) all.parser_runtime.example_parser->lbl_parser = VW::multiclass_label_parser_global; @@ -3348,6 +3352,11 @@ std::shared_ptr VW::reductions::search_setup(VW::setup_bas VW::label_type_t expected_label_type = all.parser_runtime.example_parser->lbl_parser.label_type; + auto stash_lbl_parser = all.parser_runtime.example_parser->lbl_parser; + if (priv.xv) { priv.feature_width *= 3; } + auto base = stack_builder.setup_base_learner(priv.total_number_of_policies * priv.feature_width); + all.parser_runtime.example_parser->lbl_parser = stash_lbl_parser; + if (options.was_supplied("search_allowed_transitions")) { read_allowed_transitions(static_cast(priv.A), search_allowed_transitions.c_str(), all.logger); @@ -3365,9 +3374,7 @@ std::shared_ptr VW::reductions::search_setup(VW::setup_bas priv.start_clock_time = clock(); - if (priv.xv) { priv.num_learners *= 3; } - - cdbg << "num_learners = " << priv.num_learners << endl; + cdbg << "feature_width = " << priv.feature_width << endl; // No normal prediction is produced so the base prediction type is used. That type is unlikely to be accessible // though. TODO: either let search return a prediction or add a NO_PRED type. @@ -3377,7 +3384,7 @@ std::shared_ptr VW::reductions::search_setup(VW::setup_bas VW::LEARNER::make_reduction_learner(std::move(sch), base, do_actual_learning, do_actual_learning, stack_builder.get_setupfn_name(search_setup)) .set_learn_returns_prediction(true) - .set_params_per_weight(priv.total_number_of_policies * priv.num_learners) + .set_feature_width(priv.total_number_of_policies * priv.feature_width) .set_print_update(print_update_search) .set_end_examples(end_examples) .set_finish(search_finish) diff --git a/vowpalwabbit/core/src/reductions/search/search_dep_parser.cc b/vowpalwabbit/core/src/reductions/search/search_dep_parser.cc index 17474597628..9eae44e7f26 100644 --- a/vowpalwabbit/core/src/reductions/search/search_dep_parser.cc +++ b/vowpalwabbit/core/src/reductions/search/search_dep_parser.cc @@ -99,8 +99,8 @@ void initialize(Search::search& sch, size_t& /*num_actions*/, options_i& options data->ex.interactions = &sch.get_vw_pointer_unsafe().fc.interactions; data->ex.extent_interactions = &sch.get_vw_pointer_unsafe().fc.extent_interactions; - if (data->one_learner) { sch.set_num_learners(1); } - else { sch.set_num_learners(3); } + if (data->one_learner) { sch.set_feature_width(1); } + else { sch.set_feature_width(3); } std::vector> newpairs{{'B', 'C'}, {'B', 'E'}, {'B', 'B'}, {'C', 'C'}, {'D', 'D'}, {'E', 'E'}, {'F', 'F'}, {'G', 'G'}, {'E', 'F'}, {'B', 'H'}, {'B', 'J'}, {'E', 'L'}, {'d', 'B'}, {'d', 'C'}, @@ -251,7 +251,7 @@ void extract_features(Search::search& sch, uint32_t idx, VW::multi_ex& ec) task_data* data = sch.get_task_data(); reset_ex(data->ex); uint64_t mask = sch.get_mask(); - uint64_t multiplier = static_cast(all.reduction_state.wpp) << all.weights.stride_shift(); + uint64_t multiplier = static_cast(all.reduction_state.total_feature_width) << all.weights.stride_shift(); auto& stack = data->stack; auto& tags = data->tags; diff --git a/vowpalwabbit/core/src/reductions/search/search_entityrelationtask.cc b/vowpalwabbit/core/src/reductions/search/search_entityrelationtask.cc index 8bc6308d7a3..666cf4d4c4d 100644 --- a/vowpalwabbit/core/src/reductions/search/search_entityrelationtask.cc +++ b/vowpalwabbit/core/src/reductions/search/search_entityrelationtask.cc @@ -93,8 +93,8 @@ void initialize(Search::search& sch, size_t& /*num_actions*/, options_i& options sch.set_options(Search::IS_LDF); } - sch.set_num_learners(2); - if (my_task_data->search_order == 4) { sch.set_num_learners(3); } + sch.set_feature_width(2); + if (my_task_data->search_order == 4) { sch.set_feature_width(3); } } bool check_constraints(size_t ent1_id, size_t ent2_id, size_t rel_id) diff --git a/vowpalwabbit/core/src/reductions/search/search_graph.cc b/vowpalwabbit/core/src/reductions/search/search_graph.cc index 1839c92afc1..7596eafb15c 100644 --- a/vowpalwabbit/core/src/reductions/search/search_graph.cc +++ b/vowpalwabbit/core/src/reductions/search/search_graph.cc @@ -77,9 +77,9 @@ class task_data // for adding new features uint64_t mask; // all->reg.weight_mask - uint64_t multiplier; // all.reduction_state.wpp << all.stride_shift + uint64_t multiplier; // all.reduction_state.total_feature_width << all.stride_shift size_t ss; // stride_shift - size_t wpp; + size_t total_feature_width; // per-example data uint32_t N; // NOLINT number of nodes @@ -130,7 +130,7 @@ void initialize(Search::search& sch, size_t& num_actions, options_i& options) D->true_counts_total = static_cast(D->K + 1); for (size_t k = 0; k <= D->K; k++) { D->true_counts[k] = 1.; } - if (D->separate_learners) { sch.set_num_learners(D->num_loops); } + if (D->separate_learners) { sch.set_feature_width(D->num_loops); } sch.set_task_data(D.release()); sch.set_options(0); // Search::AUTO_HAMMING_LOSS @@ -188,8 +188,8 @@ void run_bfs(task_data& D, VW::multi_ex& ec) void setup(Search::search& sch, VW::multi_ex& ec) { task_data& D = *sch.get_task_data(); // NOLINT - D.multiplier = D.wpp << D.ss; - D.wpp = sch.get_vw_pointer_unsafe().reduction_state.wpp; + D.multiplier = D.total_feature_width << D.ss; + D.total_feature_width = sch.get_vw_pointer_unsafe().reduction_state.total_feature_width; D.mask = sch.get_vw_pointer_unsafe().weights.mask(); D.ss = sch.get_vw_pointer_unsafe().weights.stride_shift(); D.N = 0; diff --git a/vowpalwabbit/core/src/reductions/search/search_multiclasstask.cc b/vowpalwabbit/core/src/reductions/search/search_multiclasstask.cc index 0a49aab47ce..26ef4eb711c 100644 --- a/vowpalwabbit/core/src/reductions/search/search_multiclasstask.cc +++ b/vowpalwabbit/core/src/reductions/search/search_multiclasstask.cc @@ -24,7 +24,10 @@ void initialize(Search::search& sch, size_t& num_actions, VW::config::options_i& { task_data* my_task_data = new task_data(); sch.set_options(0); - sch.set_num_learners(num_actions); + size_t num_learner_ceil = 1; + while (num_learner_ceil < num_actions) { num_learner_ceil *= 2; } + // TODO: temporary fix to ensure enough learners, look into case where learner is created outside bounds + sch.set_feature_width(num_learner_ceil); my_task_data->max_label = num_actions; my_task_data->num_level = static_cast(ceil(log(num_actions) / log(2))); my_task_data->y_allowed.push_back(1); diff --git a/vowpalwabbit/core/src/reductions/search/search_sequencetask.cc b/vowpalwabbit/core/src/reductions/search/search_sequencetask.cc index 6e9246b0935..f9a2aae79b6 100644 --- a/vowpalwabbit/core/src/reductions/search/search_sequencetask.cc +++ b/vowpalwabbit/core/src/reductions/search/search_sequencetask.cc @@ -191,7 +191,7 @@ void initialize(Search::search& sch, size_t& num_actions, options_i& options) Search::AUTO_HAMMING_LOSS | // please just use hamming loss on individual predictions -- we won't declare loss Search::EXAMPLES_DONT_CHANGE | // we don't do any internal example munging 0); - sch.set_num_learners(data->multipass); + sch.set_feature_width(num_actions); sch.set_task_data(data.release()); } diff --git a/vowpalwabbit/core/src/reductions/shared_feature_merger.cc b/vowpalwabbit/core/src/reductions/shared_feature_merger.cc index 775e9e7498a..340661924bd 100644 --- a/vowpalwabbit/core/src/reductions/shared_feature_merger.cc +++ b/vowpalwabbit/core/src/reductions/shared_feature_merger.cc @@ -122,7 +122,7 @@ std::shared_ptr VW::reductions::shared_feature_merger_setu if (options.was_supplied("large_action_space")) { data->store_shared_ex_in_reduction_features = true; } auto multi_base = VW::LEARNER::require_multiline(base); - data->label_type = all.parser_runtime.example_parser->lbl_parser.label_type; + data->label_type = base->get_input_label_type(); // Both label and prediction types inherit that of base. auto learner = VW::LEARNER::make_reduction_learner(std::move(data), multi_base, predict_or_learn, diff --git a/vowpalwabbit/core/src/reductions/slates.cc b/vowpalwabbit/core/src/reductions/slates.cc index 9e533b304a7..d1ceef92639 100644 --- a/vowpalwabbit/core/src/reductions/slates.cc +++ b/vowpalwabbit/core/src/reductions/slates.cc @@ -247,7 +247,6 @@ void learn_or_predict(VW::reductions::slates_data& data, VW::LEARNER::learner& b std::shared_ptr VW::reductions::slates_setup(VW::setup_base_i& stack_builder) { options_i& options = *stack_builder.get_options(); - VW::workspace& all = *stack_builder.get_all_pointer(); auto data = VW::make_unique(); bool slates_option = false; option_group_definition new_options("[Reduction] Slates"); @@ -262,7 +261,6 @@ std::shared_ptr VW::reductions::slates_setup(VW::setup_bas } auto base = require_multiline(stack_builder.setup_base_learner()); - all.parser_runtime.example_parser->lbl_parser = VW::slates::slates_label_parser; auto l = make_reduction_learner(std::move(data), base, learn_or_predict, learn_or_predict, stack_builder.get_setupfn_name(slates_setup)) .set_learn_returns_prediction(base->learn_returns_prediction) diff --git a/vowpalwabbit/core/src/reductions/stagewise_poly.cc b/vowpalwabbit/core/src/reductions/stagewise_poly.cc index e90e1b67969..d292adf5dd7 100644 --- a/vowpalwabbit/core/src/reductions/stagewise_poly.cc +++ b/vowpalwabbit/core/src/reductions/stagewise_poly.cc @@ -129,7 +129,7 @@ inline uint64_t wid_mask_un_shifted(const stagewise_poly& poly, uint64_t wid) inline uint64_t constant_feat(const stagewise_poly& poly) { - return stride_shift(poly, VW::details::CONSTANT * poly.all->reduction_state.wpp); + return stride_shift(poly, VW::details::CONSTANT * poly.all->reduction_state.total_feature_width); } inline uint64_t constant_feat_masked(const stagewise_poly& poly) { return wid_mask(poly, constant_feat(poly)); } diff --git a/vowpalwabbit/core/src/reductions/svrg.cc b/vowpalwabbit/core/src/reductions/svrg.cc index 2693cc8ac5d..4a8360e5107 100644 --- a/vowpalwabbit/core/src/reductions/svrg.cc +++ b/vowpalwabbit/core/src/reductions/svrg.cc @@ -167,12 +167,8 @@ void save_load(svrg& s, VW::io_buf& model_file, bool read, bool text) VW::details::bin_text_read_write_fixed( model_file, reinterpret_cast(&resume), sizeof(resume), read, msg, text); - double temp = 0.; - double temp_normalized_sum_norm_x = 0.; - if (resume) - { - VW::details::save_load_online_state_gd(*s.all, model_file, read, text, temp, temp_normalized_sum_norm_x); - } + std::vector temp_pms = {VW::reductions::details::gd_per_model_state()}; + if (resume) { VW::details::save_load_online_state_gd(*s.all, model_file, read, text, temp_pms); } else { VW::details::save_load_regressor_gd(*s.all, model_file, read, text); } } } @@ -196,7 +192,6 @@ std::shared_ptr VW::reductions::svrg_setup(VW::setup_base_ all.weights.stride_shift(2); auto l = make_bottom_learner(std::move(s), learn, predict, stack_builder.get_setupfn_name(svrg_setup), VW::prediction_type_t::SCALAR, VW::label_type_t::SIMPLE) - .set_params_per_weight(VW::details::UINT64_ONE << all.weights.stride_shift()) .set_output_example_prediction(VW::details::output_example_prediction_simple_label) .set_update_stats(VW::details::update_stats_simple_label) .set_print_update(VW::details::print_update_simple_label) diff --git a/vowpalwabbit/core/src/vw.cc b/vowpalwabbit/core/src/vw.cc index d0dfe0feefd..ca4ebb9a8fd 100644 --- a/vowpalwabbit/core/src/vw.cc +++ b/vowpalwabbit/core/src/vw.cc @@ -752,7 +752,7 @@ void VW::setup_example(VW::workspace& all, VW::example* ae) if (!all.fc.limit_strings.empty()) { feature_limit(all, ae); } - uint64_t multiplier = static_cast(all.reduction_state.wpp) << all.weights.stride_shift(); + uint64_t multiplier = static_cast(all.reduction_state.total_feature_width) << all.weights.stride_shift(); if (multiplier != 1) { // make room for per-feature information. @@ -911,6 +911,8 @@ void VW::empty_example(VW::workspace& /*all*/, example& ec) ec.is_newline = false; ec.ex_reduction_features.clear(); ec.num_features_from_interactions = 0; + ec.feature_space_hash = 0; + ec.is_set_feature_space_hash = false; } void VW::move_feature_namespace(example* dst, example* src, namespace_index c) diff --git a/vowpalwabbit/core/tests/automl_test.cc b/vowpalwabbit/core/tests/automl_test.cc index dd97220dbfc..a8d3ff7a47a 100644 --- a/vowpalwabbit/core/tests/automl_test.cc +++ b/vowpalwabbit/core/tests/automl_test.cc @@ -7,6 +7,7 @@ #include "simulator.h" #include "vw/core/automl_impl.h" #include "vw/core/estimators/confidence_sequence_robust.h" +#include "vw/core/interactions.h" #include "vw/core/metric_sink.h" #include "vw/core/vw_fwd.h" @@ -696,7 +697,7 @@ TEST(Automl, QbaseUnittestWIterations) } EXPECT_EQ(prio_queue.size(), 0); EXPECT_EQ(estimators.size(), 11); - const interaction_vec_t expected2 = { + interaction_vec_t expected2 = { {'B', 'C', 'C'}, {'A', 'A'}, {'A', 'B'}, @@ -705,6 +706,7 @@ TEST(Automl, QbaseUnittestWIterations) {'B', 'C'}, {'C', 'C'}, }; + std::sort(expected2.begin(), expected2.end(), VW::details::sort_interactions_comparator); EXPECT_EQ(estimators[2].first.config_index, 9); EXPECT_EQ(estimators[2].first.live_interactions, expected2); @@ -797,8 +799,9 @@ TEST(Automl, QuadcubicUnitTest) set_ns_list_t{{'A', 'A', 'A'}, {'B', 'B', 'B'}, {'C', 'C', 'C'}}, 4000, config_type::Interaction); ns_based_config::apply_config_to_interactions(false, ns_counter, "both", test_config_exclusion, interactions); - const interaction_vec_t expected2{{'A', 'A', 'A'}, {'B', 'B', 'B'}, {'C', 'C', 'C'}, {'A', 'A'}, {'A', 'B'}, - {'A', 'C'}, {'B', 'B'}, {'B', 'C'}, {'C', 'C'}}; + interaction_vec_t expected2{{'A', 'A', 'A'}, {'B', 'B', 'B'}, {'C', 'C', 'C'}, {'A', 'A'}, {'A', 'B'}, {'A', 'C'}, + {'B', 'B'}, {'B', 'C'}, {'C', 'C'}}; + std::sort(expected2.begin(), expected2.end(), VW::details::sort_interactions_comparator); EXPECT_EQ(interactions, expected2); } diff --git a/vowpalwabbit/core/tests/automl_weights_test.cc b/vowpalwabbit/core/tests/automl_weights_test.cc index a016a98f099..2497b011f7a 100644 --- a/vowpalwabbit/core/tests/automl_weights_test.cc +++ b/vowpalwabbit/core/tests/automl_weights_test.cc @@ -33,7 +33,7 @@ size_t get_hash_for_feature(VW::workspace& all, const std::string& ns, const std std::uint64_t hash_ft = VW::hash_feature(all, feature, VW::hash_space(all, ns)); std::uint64_t ft = hash_ft & all.runtime_state.parse_mask; // apply multiplier like setup_example - ft *= (static_cast(all.reduction_state.wpp) << all.weights.stride_shift()); + ft *= (static_cast(all.reduction_state.total_feature_width) << all.weights.stride_shift()); return ft; } @@ -98,7 +98,7 @@ bool weights_offset_test(cb_sim&, VW::workspace& all, VW::multi_ex&) // all weights of offset 1 will be set to zero VW::reductions::multi_model::clear_innermost_offset( - weights, offset_to_clear, all.reduction_state.wpp, all.reduction_state.wpp); + weights, offset_to_clear, all.reduction_state.total_feature_width, all.reduction_state.total_feature_width); for (auto index : feature_indexes) { @@ -116,7 +116,7 @@ bool weights_offset_test(cb_sim&, VW::workspace& all, VW::multi_ex&) // copy from offset 2 to offset 1 VW::reductions::multi_model::move_innermost_offsets( - weights, offset_to_clear + 1, offset_to_clear, all.reduction_state.wpp, all.reduction_state.wpp); + weights, offset_to_clear + 1, offset_to_clear, all.reduction_state.total_feature_width, all.reduction_state.total_feature_width); for (auto index : feature_indexes) { @@ -180,11 +180,11 @@ bool all_weights_equal_test(cb_sim&, VW::workspace& all, VW::multi_ex&) for (auto iter = weights.begin(); iter != weights.end(); ++iter) { size_t prestride_index = iter.index() >> weights.stride_shift(); - size_t current_offset = (iter.index() >> weights.stride_shift()) & (all.reduction_state.wpp - 1); + size_t current_offset = (iter.index() >> weights.stride_shift()) & (all.reduction_state.total_feature_width - 1); if (current_offset == 0) { float* first_weight = &weights.first()[(prestride_index + 0) << weights.stride_shift()]; - uint32_t till = 1; // instead of all.reduction_state.wpp, champdupe only uses 3 configs + uint32_t till = 1; // instead of all.reduction_state.total_feature_width, champdupe only uses 3 configs for (uint32_t i = 1; i <= till; ++i) { float* other = &weights.first()[(prestride_index + i) << weights.stride_shift()]; diff --git a/vowpalwabbit/core/tests/cb_las_one_pass_svd_test.cc b/vowpalwabbit/core/tests/cb_las_one_pass_svd_test.cc index 002bcf31932..8fb0e7ab840 100644 --- a/vowpalwabbit/core/tests/cb_las_one_pass_svd_test.cc +++ b/vowpalwabbit/core/tests/cb_las_one_pass_svd_test.cc @@ -21,7 +21,7 @@ using internal_action_space_op = VW::cb_explore_adf::cb_explore_adf_base>; -TEST(Las, CheckMatricsWithLASRunsOK) +TEST(Las, CheckMetricsWithLASRunsOK) { auto d = 3; std::vector args{"--cb_explore_adf", "--large_action_space", "--max_actions", std::to_string(d), @@ -204,6 +204,181 @@ TEST(Las, CheckAOLinearCombinationOfActions) } } +TEST(Las, CheckActionCacheWithCB) +{ + auto d = 3; + Eigen::MatrixXf U_wcache; + Eigen::MatrixXf U_wnocache; + + for (const int cache_slack : {50, -1}) + { + std::vector args{"--cb_explore_adf", "--large_action_space", "--max_actions", std::to_string(d), + "--quiet", "--action_cache_slack", std::to_string(cache_slack)}; + auto vw = VW::initialize(VW::make_unique(args)); + + std::vector e_r; + vw->l->get_enabled_learners(e_r); + if (std::find(e_r.begin(), e_r.end(), "cb_explore_adf_large_action_space") == e_r.end()) + { + FAIL() << "cb_explore_adf_large_action_space not found in enabled learners"; + } + + VW::LEARNER::learner* learner = + require_multiline(vw->l->get_learner_by_name_prefix("cb_explore_adf_large_action_space")); + + auto* action_space = (internal_action_space_op*)learner->get_internal_type_erased_data_pointer_test_use_only(); + + EXPECT_EQ(action_space != nullptr, true); + + action_space->explore._populate_all_testing_components(); + + { + VW::multi_ex examples; + + examples.push_back(VW::read_example(*vw, "shared |U b c")); + examples.push_back(VW::read_example(*vw, "| 1:0.1 2:0.12 3:0.13 b200:2 c500:9")); + examples.push_back(VW::read_example(*vw, "| a_1:0.5 a_2:0.65 a_3:0.12 a100:4 a200:33")); + examples.push_back(VW::read_example(*vw, "| a_1:0.8 a_2:0.32 a_3:0.15 a100:0.2 a200:0.2")); + examples.push_back(VW::read_example(*vw, "| a_1:2.1 a_2:1.29 a_3:0.42 a100:4.4 a200:33.4")); + examples.push_back(VW::read_example(*vw, "| a_4:0.8 a_5:0.32 a_6:0.15 d1:0.2 d10: 0.2")); + examples.push_back(VW::read_example(*vw, "| a_7 a_8 a_9 v1:0.99")); + examples.push_back(VW::read_example(*vw, "| a_10 a_11 a_12")); + examples.push_back(VW::read_example(*vw, "| a_13 a_14 a_15")); + examples.push_back(VW::read_example(*vw, "| a_16 a_17 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "| a_19 a_20 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "| a_21 a_22 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "| a_23 a_24 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "| a_25 a_26 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "| a_27 a_28 a_18:0.2")); + + vw->predict(examples); + vw->finish_example(examples); + } + + // same example second time to potentially excersise the cache + // but make them different order + { + VW::multi_ex examples; + + examples.push_back(VW::read_example(*vw, "shared |A a b")); + examples.push_back(VW::read_example(*vw, "| 1:0.1 2:0.12 3:0.13 b200:2 c500:9")); + examples.push_back(VW::read_example(*vw, "| a_7 a_8 a_9 v1:0.99")); + examples.push_back(VW::read_example(*vw, "| a_10 a_11 a_12")); + examples.push_back(VW::read_example(*vw, "| a_13 a_14 a_15")); + examples.push_back(VW::read_example(*vw, "| a_16 a_17 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "| a_19 a_20 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "| a_21 a_22 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "| a_23 a_24 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "| a_25 a_26 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "| a_27 a_28 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "| a_1:0.5 a_2:0.65 a_3:0.12 a100:4 a200:33")); + examples.push_back(VW::read_example(*vw, "| a_1:0.8 a_2:0.32 a_3:0.15 a100:0.2 a200:0.2")); + examples.push_back(VW::read_example(*vw, "| a_1:2.1 a_2:1.29 a_3:0.42 a100:4.4 a200:33.4")); + examples.push_back(VW::read_example(*vw, "| a_4:0.8 a_5:0.32 a_6:0.15 d1:0.2 d10: 0.2")); + + vw->predict(examples); + if (cache_slack == 50) { U_wcache = action_space->explore.U; } + else if (cache_slack == -1) { U_wnocache = action_space->explore.U; } + + vw->finish_example(examples); + } + } + + EXPECT_TRUE(U_wnocache.isApprox(U_wcache, vwtest::EXPLICIT_FLOAT_TOL)); +} + +TEST(Las, CheckActionCacheWithCCB) +{ + auto d = 3; + Eigen::MatrixXf U_wcache; + Eigen::MatrixXf U_wnocache; + + for (const int cache_slack : {50, -1}) + { + std::vector args{"--ccb_explore_adf", "--large_action_space", "--max_actions", std::to_string(d), + "--quiet", "--action_cache_slack", std::to_string(cache_slack)}; + auto vw = VW::initialize(VW::make_unique(args)); + + std::vector e_r; + vw->l->get_enabled_learners(e_r); + if (std::find(e_r.begin(), e_r.end(), "cb_explore_adf_large_action_space") == e_r.end()) + { + FAIL() << "cb_explore_adf_large_action_space not found in enabled learners"; + } + + VW::LEARNER::learner* learner = + require_multiline(vw->l->get_learner_by_name_prefix("cb_explore_adf_large_action_space")); + + auto* action_space = (internal_action_space_op*)learner->get_internal_type_erased_data_pointer_test_use_only(); + + EXPECT_EQ(action_space != nullptr, true); + + action_space->explore._populate_all_testing_components(); + + { + VW::multi_ex examples; + + examples.push_back(VW::read_example(*vw, "ccb shared |U b c")); + examples.push_back(VW::read_example(*vw, "ccb action | 1:0.1 2:0.12 3:0.13 b200:2 c500:9")); + examples.push_back(VW::read_example(*vw, "ccb action | a_1:0.5 a_2:0.65 a_3:0.12 a100:4 a200:33")); + examples.push_back(VW::read_example(*vw, "ccb action | a_1:0.8 a_2:0.32 a_3:0.15 a100:0.2 a200:0.2")); + examples.push_back(VW::read_example(*vw, "ccb action | a_1:2.1 a_2:1.29 a_3:0.42 a100:4.4 a200:33.4")); + examples.push_back(VW::read_example(*vw, "ccb action | a_4:0.8 a_5:0.32 a_6:0.15 d1:0.2 d10: 0.2")); + examples.push_back(VW::read_example(*vw, "ccb action | a_7 a_8 a_9 v1:0.99")); + examples.push_back(VW::read_example(*vw, "ccb action | a_10 a_11 a_12")); + examples.push_back(VW::read_example(*vw, "ccb action | a_13 a_14 a_15")); + examples.push_back(VW::read_example(*vw, "ccb action | a_16 a_17 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "ccb action | a_19 a_20 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "ccb action | a_21 a_22 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "ccb action | a_23 a_24 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "ccb action | a_25 a_26 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "ccb action | a_27 a_28 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "ccb action | a_44:0.8 a_5:0.32 a_6:0.15 d1:0.2 d10: 0.2")); + + examples.push_back(VW::read_example(*vw, "ccb slot 0:0:0.2 |Slot m")); + examples.push_back(VW::read_example(*vw, "ccb slot 1:0:0.25 |Slot n")); + + vw->predict(examples); + vw->finish_example(examples); + } + + // same example second time to potentially excersise the cache + // not affected by shared or slot features + // and shuffle order of actions + { + VW::multi_ex examples; + + examples.push_back(VW::read_example(*vw, "ccb shared |A a b")); + examples.push_back(VW::read_example(*vw, "ccb action | 1:0.1 2:0.12 3:0.13 b200:2 c500:9")); + examples.push_back(VW::read_example(*vw, "ccb action | a_7 a_8 a_9 v1:0.99")); + examples.push_back(VW::read_example(*vw, "ccb action | a_10 a_11 a_12")); + examples.push_back(VW::read_example(*vw, "ccb action | a_13 a_14 a_15")); + examples.push_back(VW::read_example(*vw, "ccb action | a_16 a_17 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "ccb action | a_19 a_20 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "ccb action | a_21 a_22 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "ccb action | a_23 a_24 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "ccb action | a_25 a_26 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "ccb action | a_27 a_28 a_18:0.2")); + examples.push_back(VW::read_example(*vw, "ccb action | a_1:0.5 a_2:0.65 a_3:0.12 a100:4 a200:33")); + examples.push_back(VW::read_example(*vw, "ccb action | a_1:0.8 a_2:0.32 a_3:0.15 a100:0.2 a200:0.2")); + examples.push_back(VW::read_example(*vw, "ccb action | a_1:2.1 a_2:1.29 a_3:0.42 a100:4.4 a200:33.4")); + examples.push_back(VW::read_example(*vw, "ccb action | a_4:0.8 a_5:0.32 a_6:0.15 d1:0.2 d10: 0.2")); + examples.push_back(VW::read_example(*vw, "ccb action | a_44:0.8 a_5:0.32 a_6:0.15 d1:0.2 d10: 0.2")); + + examples.push_back(VW::read_example(*vw, "ccb slot 0:0:0.2 |Slot l")); + examples.push_back(VW::read_example(*vw, "ccb slot 1:0:0.25 |Slot f")); + + vw->predict(examples); + if (cache_slack == 50) { U_wcache = action_space->explore.U; } + else if (cache_slack == -1) { U_wnocache = action_space->explore.U; } + + vw->finish_example(examples); + } + } + + EXPECT_TRUE(U_wnocache.isApprox(U_wcache, vwtest::EXPLICIT_FLOAT_TOL)); +} + #ifdef BUILD_LAS_WITH_SIMD TEST(Las, ComputeDotProdScalarAndSimdHaveSameResults) { diff --git a/vowpalwabbit/core/tests/cb_las_spanner_test.cc b/vowpalwabbit/core/tests/cb_las_spanner_test.cc index 2a9956d7cee..f280e958cca 100644 --- a/vowpalwabbit/core/tests/cb_las_spanner_test.cc +++ b/vowpalwabbit/core/tests/cb_las_spanner_test.cc @@ -36,7 +36,7 @@ TEST(Las, CheckFindingMaxVolume) VW::cb_explore_adf::one_rank_spanner_state> largecb( /*d=*/0, /*c=*/2, false, vw.get(), seed, 1 << vw->iwc.num_bits, - /*thread_pool_size*/ 0, /*block_size*/ 0, /*use_explicit_simd=*/use_simd, + /*thread_pool_size*/ 0, /*block_size*/ 0, /*cache slack size*/ 50, /*use_explicit_simd=*/use_simd, VW::cb_explore_adf::implementation_type::one_pass_svd); largecb.U = Eigen::MatrixXf{{1, 2, 3}, {4, 5, 6}, {7, 8, 9}, {0, 0, 0}, {7, 5, 3}, {6, 4, 8}}; Eigen::MatrixXf X{{1, 2, 3}, {3, 2, 1}, {2, 1, 3}}; @@ -842,6 +842,19 @@ TEST(Las, CheckSingularValueSumDiffForDiffRanksIsSmall) vw->finish_example(examples); } + { + action_space->explore._test_only_set_rank(d); + VW::multi_ex examples; + + for (const auto& ex : dexs) { examples.push_back(VW::read_example(*vw, ex)); } + for (const auto& ex : exs) { examples.push_back(VW::read_example(*vw, ex)); } + + vw->predict(examples); + + larger_rank_sum = action_space->explore.S.sum(); + vw->finish_example(examples); + } + { action_space->explore._test_only_set_rank(d + 10); VW::multi_ex examples; diff --git a/vowpalwabbit/core/tests/epsilon_decay_test.cc b/vowpalwabbit/core/tests/epsilon_decay_test.cc index f8706cdb6af..a7d8bc059c0 100644 --- a/vowpalwabbit/core/tests/epsilon_decay_test.cc +++ b/vowpalwabbit/core/tests/epsilon_decay_test.cc @@ -252,10 +252,10 @@ TEST(EpsilonDecay, ScoreBoundsUnit) { // Initialize epsilon_decay_data class with 5 models uint64_t num_models = 5; - uint32_t wpp = 8; + uint32_t feature_width = 8; VW::dense_parameters dense_weights(num_models); epsilon_decay_data ep_data( - num_models, 100, .05, .1, dense_weights, "", false, wpp, 0, 1.f, 0, false, 1e-6, "bisect", false); + num_models, 100, .05, .1, dense_weights, "", false, feature_width, 0, 1.f, 0, false, 1e-6, "bisect", false); // Set update counts to fixed values with expected horizon bound violation size_t score_idx = 0; @@ -338,10 +338,10 @@ TEST(EpsilonDecay, HorizonBoundsUnit) { // Initialize epsilon_decay_data class with 5 models uint64_t num_models = 5; - uint32_t wpp = 8; + uint32_t feature_width = 8; VW::dense_parameters dense_weights(num_models); epsilon_decay_data ep_data( - num_models, 100, .05, .1, dense_weights, "", false, wpp, 0, 1.f, 0, false, 1e-6, "bisect", false); + num_models, 100, .05, .1, dense_weights, "", false, feature_width, 0, 1.f, 0, false, 1e-6, "bisect", false); // Set update counts to fixed values with expected horizon bound violation size_t score_idx = 0; diff --git a/vowpalwabbit/core/tests/example_ft_hash_test.cc b/vowpalwabbit/core/tests/example_ft_hash_test.cc new file mode 100644 index 00000000000..a589ca201a7 --- /dev/null +++ b/vowpalwabbit/core/tests/example_ft_hash_test.cc @@ -0,0 +1,180 @@ +// Copyright (c) by respective owners including Yahoo!, Microsoft, and +// individual contributors. All rights reserved. Released under a BSD (revised) +// license as described in the file LICENSE. + +#include "vw/core/vw.h" +#include "vw/test_common/test_common.h" + +#include +#include + +TEST(ExampleFtHash, CheckLargeStringOneSmallDiff) +{ + auto vw = VW::initialize(vwtest::make_args("--quiet")); + uint64_t hash_1; + uint64_t hash_2; + + { + auto* ex = VW::read_example(*vw, + "|Action politics:3 sports:93 music:18 food:165 math:141 science:33 computers:93 literature:84 " + "social_media:153 |f dancing:105 news:81 celebrities:0 coffee:45 baking:138 ai:9 taxes:15 technology:120 " + "pandemic:135 |m world_war_i:93 world_war_ii:147 religion:120 vaccination:141 the_queen:3 immigration:51 " + "economics:117 fashion:90 NYC:51 LA:45 elections:81 brunch:18 space_exploration:45 climate_change:129 dogs:141 " + "cats:45 inflation:138 |o sourdough:102 vegan:117 vacation:114 asteroid:147 eyewear:147 stationary:21 " + "movies:36 " + "oscars:12 acupuncture:6 hiking:99 swimming:156 barrys:111"); + hash_1 = ex->get_or_calculate_order_independent_feature_space_hash(); + + VW::finish_example(*vw, *ex); + } + { + auto* ex = VW::read_example(*vw, + "|Action politics:3 sports:93 music:18 food:165 math:141 science:33 computers:93 literature:84 " + "social_media:153 |f dancing:105 news:81 celebrities:0 coffee:45 baking:138 ai:9 taxes:15 technology:120 " + "pandemic:135 |m world_war_i:93 world_war_ii:147 religion:120 vaccination:141 the_queen:3 immigration:51 " + "economics:117 fashion:90 NYC:51 LA:45 elections:81 brunch:18 space_exploration:45 climate_change:129 dogs:141 " + "cats:45 inflation:138 |o sourdough:102 vegan:117 vacation:114 asteroid:147 eyewear:147 stationary:21 " + "movies:36 " + "oscars:12 acupuncture:6 hiking:99 swimming:156 barrys:112"); + hash_2 = ex->get_or_calculate_order_independent_feature_space_hash(); + + VW::finish_example(*vw, *ex); + } + + EXPECT_NE(hash_1, hash_2); + EXPECT_NE(hash_1, 0); + EXPECT_NE(hash_2, 0); +} + +TEST(ExampleFtHash, CheckOrderDoesNotCount) +{ + auto vw = VW::initialize(vwtest::make_args("--quiet", "--chain_hash")); + + std::vector bagofwords = {"politics:3", "sports:93", "music:18", "food:165", "math:141", "science:33", + "computers:93", "literature:84", "social_media:153", "dancing:105", "news:81", "celebrities:0", "coffee:45", + "baking:138", "ai:9", "taxes:15", "technology:120", "pandemic:135", "world_war_i:93", "world_war_ii:147", + "religion:120", "vaccination:141", "the_queen:3", "immigration:51", "economics:117", "fashion:90", "NYC:51", + "LA:45", "elections:81", "brunch:18", "space_exploration:45", "climate_change:129", "dogs:141", "cats:45", + "inflation:138", "sourdough:102", "vegan:117", "vacation:114", "asteroid:147", "eyewear:147", "stationary:21", + "movies:36", "oscars:12", "acupuncture:6", "hiking:99", "swimming:156", "barrys:111"}; + + uint64_t hash_1; + uint64_t hash_2; + + std::string s1; + std::string s2; + + { + std::set picked; + srand(0); + std::stringstream ss; + ss << "|"; + while (picked.size() != bagofwords.size()) + { + size_t ind = rand() % bagofwords.size(); + if (picked.find(ind) == picked.end()) + { + picked.insert(ind); + ss << " " << bagofwords[ind]; + } + } + + s1 = ss.str(); + + auto* ex = VW::read_example(*vw, s1); + hash_1 = ex->get_or_calculate_order_independent_feature_space_hash(); + + VW::finish_example(*vw, *ex); + } + { + std::set picked; + srand(10); + std::stringstream ss; + ss << "|"; + while (picked.size() != bagofwords.size()) + { + size_t ind = rand() % bagofwords.size(); + if (picked.find(ind) == picked.end()) + { + picked.insert(ind); + ss << " " << bagofwords[ind]; + } + } + + s2 = ss.str(); + + auto* ex = VW::read_example(*vw, s2); + hash_2 = ex->get_or_calculate_order_independent_feature_space_hash(); + + VW::finish_example(*vw, *ex); + } + + EXPECT_NE(s1, s2); + EXPECT_EQ(hash_1, hash_2); + EXPECT_NE(hash_1, 0); +} + +TEST(ExampleFtHash, CheckDefaultValueMissingSameHash) +{ + auto vw = VW::initialize(vwtest::make_args("--quiet")); + uint64_t hash_1; + uint64_t hash_2; + + { + auto* ex = VW::read_example(*vw, "| a"); + hash_1 = ex->get_or_calculate_order_independent_feature_space_hash(); + + VW::finish_example(*vw, *ex); + } + { + auto* ex = VW::read_example(*vw, "| a:1"); + hash_2 = ex->get_or_calculate_order_independent_feature_space_hash(); + + VW::finish_example(*vw, *ex); + } + + EXPECT_EQ(hash_1, hash_2); + EXPECT_NE(hash_1, 0); +} + +TEST(ExampleFtHash, CheckZeroHash) +{ + auto vw = VW::initialize(vwtest::make_args("--quiet", "--noconstant")); + { + auto* ex = VW::read_example(*vw, "| a:0"); + EXPECT_EQ(ex->get_or_calculate_order_independent_feature_space_hash(), 0); + + VW::finish_example(*vw, *ex); + } + + { + auto* ex = VW::read_example(*vw, "shared |"); + EXPECT_EQ(ex->get_or_calculate_order_independent_feature_space_hash(), 0); + + VW::finish_example(*vw, *ex); + } +} + +TEST(ExampleFtHash, SmallDiffs) +{ + auto vw = VW::initialize(vwtest::make_args("--quiet", "--noconstant")); + uint64_t hash_1; + uint64_t hash_2; + + { + auto* ex = VW::read_example(*vw, "| a1:0.0000001 a2"); + hash_1 = ex->get_or_calculate_order_independent_feature_space_hash(); + + VW::finish_example(*vw, *ex); + } + { + auto* ex = VW::read_example(*vw, "| a1:0.000001 a2"); + hash_2 = ex->get_or_calculate_order_independent_feature_space_hash(); + + VW::finish_example(*vw, *ex); + } + + EXPECT_NE(hash_1, hash_2); + EXPECT_NE(hash_1, 0); + EXPECT_NE(hash_2, 0); +} \ No newline at end of file diff --git a/vowpalwabbit/core/tests/flat_example_test.cc b/vowpalwabbit/core/tests/flat_example_test.cc index 80641bb1dee..f5ecb35cd44 100644 --- a/vowpalwabbit/core/tests/flat_example_test.cc +++ b/vowpalwabbit/core/tests/flat_example_test.cc @@ -16,12 +16,12 @@ TEST(FlatExample, SansInteractionTest) auto vw = VW::initialize(vwtest::make_args("--quiet", "--noconstant")); auto* ex = VW::read_example(*vw, "1 |x a:2 |y b:3"); - auto& flat = *VW::flatten_sort_example(*vw, ex); + VW::features fs; + VW::flatten_features(*vw, *ex, fs); - EXPECT_THAT(flat.fs.values, testing::UnorderedElementsAre(2, 3)); - EXPECT_EQ(flat.total_sum_feat_sq, 13); + EXPECT_THAT(fs.values, testing::UnorderedElementsAre(2, 3)); + EXPECT_EQ(fs.sum_feat_sq, 13); - VW::free_flatten_example(&flat); VW::finish_example(*vw, *ex); } @@ -30,12 +30,12 @@ TEST(FlatExample, WithInteractionTest) auto vw = VW::initialize(vwtest::make_args("--interactions", "xy", "--quiet", "--noconstant")); auto* ex = VW::read_example(*vw, "1 |x a:2 |y b:3"); - auto& flat = *VW::flatten_sort_example(*vw, ex); + VW::features fs; + VW::flatten_features(*vw, *ex, fs); - EXPECT_THAT(flat.fs.values, testing::UnorderedElementsAre(2, 3, 6)); - EXPECT_EQ(flat.total_sum_feat_sq, 49); + EXPECT_THAT(fs.values, testing::UnorderedElementsAre(2, 3, 6)); + EXPECT_EQ(fs.sum_feat_sq, 49); - VW::free_flatten_example(&flat); VW::finish_example(*vw, *ex); } @@ -44,10 +44,10 @@ TEST(FlatExample, EmptyExampleTest) auto vw = VW::initialize(vwtest::make_args("--quiet", "--noconstant")); auto* ex = VW::read_example(*vw, "1 |x a:0"); - auto& flat = *VW::flatten_sort_example(*vw, ex); + VW::features fs; + VW::flatten_features(*vw, *ex, fs); - EXPECT_TRUE(flat.fs.empty()); + EXPECT_TRUE(fs.empty()); - VW::free_flatten_example(&flat); VW::finish_example(*vw, *ex); } diff --git a/vowpalwabbit/core/tests/merge_test.cc b/vowpalwabbit/core/tests/merge_test.cc index 74a92064f90..0b39e4461e9 100644 --- a/vowpalwabbit/core/tests/merge_test.cc +++ b/vowpalwabbit/core/tests/merge_test.cc @@ -194,10 +194,15 @@ TEST(Merge, MergeCbModel) auto* vw_merged_cb_adf = reinterpret_cast( result->l->get_learner_by_name_prefix("cb_adf")->get_internal_type_erased_data_pointer_test_use_only()); - EXPECT_EQ(vw_merged_cb_adf->get_gen_cs().event_sum, - vw1_cb_adf->get_gen_cs().event_sum + vw2_cb_adf->get_gen_cs().event_sum); - EXPECT_EQ(vw_merged_cb_adf->get_gen_cs().action_sum, - vw1_cb_adf->get_gen_cs().action_sum + vw2_cb_adf->get_gen_cs().action_sum); + for (size_t i = 0; i < vw1_cb_adf->get_gen_cs_mtr().per_model_state.size(); i++) + { + EXPECT_EQ(vw_merged_cb_adf->get_gen_cs_mtr().per_model_state[i].event_sum, + vw1_cb_adf->get_gen_cs_mtr().per_model_state[i].event_sum + + vw2_cb_adf->get_gen_cs_mtr().per_model_state[i].event_sum); + EXPECT_EQ(vw_merged_cb_adf->get_gen_cs_mtr().per_model_state[i].action_sum, + vw1_cb_adf->get_gen_cs_mtr().per_model_state[i].action_sum + + vw2_cb_adf->get_gen_cs_mtr().per_model_state[i].action_sum); + } } TEST(Merge, MergeCbModelDelta) @@ -309,11 +314,17 @@ TEST(Merge, MergeCbModelDelta) reinterpret_cast(result_delta_merge->l->get_learner_by_name_prefix("cb_adf") ->get_internal_type_erased_data_pointer_test_use_only()); EXPECT_FLOAT_EQ(result_delta_merge->sd->weighted_labeled_examples, 4.f); - EXPECT_EQ(delta_merged_cb_adf->get_gen_cs().event_sum, - vw1_cb_adf->get_gen_cs().event_sum + vw2_cb_adf->get_gen_cs().event_sum - vw_base_cb_adf->get_gen_cs().event_sum); - EXPECT_EQ(delta_merged_cb_adf->get_gen_cs().action_sum, - vw1_cb_adf->get_gen_cs().action_sum + vw2_cb_adf->get_gen_cs().action_sum - - vw_base_cb_adf->get_gen_cs().action_sum); + for (size_t i = 0; i < vw_base_cb_adf->get_gen_cs_mtr().per_model_state.size(); i++) + { + EXPECT_EQ(delta_merged_cb_adf->get_gen_cs_mtr().per_model_state[i].event_sum, + vw1_cb_adf->get_gen_cs_mtr().per_model_state[i].event_sum + + vw2_cb_adf->get_gen_cs_mtr().per_model_state[i].event_sum - + vw_base_cb_adf->get_gen_cs_mtr().per_model_state[i].event_sum); + EXPECT_EQ(delta_merged_cb_adf->get_gen_cs_mtr().per_model_state[i].action_sum, + vw1_cb_adf->get_gen_cs_mtr().per_model_state[i].action_sum + + vw2_cb_adf->get_gen_cs_mtr().per_model_state[i].action_sum - + vw_base_cb_adf->get_gen_cs_mtr().per_model_state[i].action_sum); + } // merge workspaces directly without deltas std::vector workspaces{vw1.get(), vw2.get()}; @@ -323,8 +334,13 @@ TEST(Merge, MergeCbModelDelta) reinterpret_cast(result_model_merge->l->get_learner_by_name_prefix("cb_adf") ->get_internal_type_erased_data_pointer_test_use_only()); EXPECT_FLOAT_EQ(result_delta_merge->sd->weighted_labeled_examples, result_model_merge->sd->weighted_labeled_examples); - EXPECT_EQ(delta_merged_cb_adf->get_gen_cs().event_sum, model_merged_cb_adf->get_gen_cs().event_sum); - EXPECT_EQ(delta_merged_cb_adf->get_gen_cs().action_sum, model_merged_cb_adf->get_gen_cs().action_sum); + for (size_t i = 0; i < vw_base_cb_adf->get_gen_cs_mtr().per_model_state.size(); i++) + { + EXPECT_EQ(delta_merged_cb_adf->get_gen_cs_mtr().per_model_state[i].event_sum, + model_merged_cb_adf->get_gen_cs_mtr().per_model_state[i].event_sum); + EXPECT_EQ(delta_merged_cb_adf->get_gen_cs_mtr().per_model_state[i].action_sum, + model_merged_cb_adf->get_gen_cs_mtr().per_model_state[i].action_sum); + } } TEST(Merge, SerializeDeserializeDelta) diff --git a/vowpalwabbit/json_parser/include/vw/json_parser/parse_example_json.h b/vowpalwabbit/json_parser/include/vw/json_parser/parse_example_json.h index 9644535b0dd..6f6a72f858f 100644 --- a/vowpalwabbit/json_parser/include/vw/json_parser/parse_example_json.h +++ b/vowpalwabbit/json_parser/include/vw/json_parser/parse_example_json.h @@ -34,11 +34,11 @@ void read_line_json(const VW::label_parser& lbl_parser, hash_func_t hash_func, u bool chain_hash, VW::label_parser_reuse_mem* reuse_mem, const VW::named_labels* ldict, VW::multi_ex& examples, char* line, size_t length, example_factory_t example_factory, VW::io::logger& logger, std::unordered_map>* ignore_features, - std::unordered_map* dedup_examples = nullptr); + const std::unordered_map* dedup_examples = nullptr); template void read_line_json(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, - example_factory_t example_factory, std::unordered_map* dedup_examples = nullptr); + example_factory_t example_factory, const std::unordered_map* dedup_examples = nullptr); // returns true if succesfully parsed, returns false if not and logs warning template @@ -57,17 +57,17 @@ extern template void read_line_json(const VW::label_parser& lbl_parser, ha uint64_t parse_mask, bool chain_hash, VW::label_parser_reuse_mem* reuse_mem, const VW::named_labels* ldict, VW::multi_ex& examples, char* line, size_t length, example_factory_t example_factory, VW::io::logger& logger, std::unordered_map>* ignore_features, - std::unordered_map* dedup_examples); + const std::unordered_map* dedup_examples); extern template void read_line_json(const VW::label_parser& lbl_parser, hash_func_t hash_func, uint64_t hash_seed, uint64_t parse_mask, bool chain_hash, VW::label_parser_reuse_mem* reuse_mem, const VW::named_labels* ldict, VW::multi_ex& examples, char* line, size_t length, example_factory_t example_factory, VW::io::logger& logger, std::unordered_map>* ignore_features, - std::unordered_map* dedup_examples); + const std::unordered_map* dedup_examples); extern template void read_line_json(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, - example_factory_t example_factory, std::unordered_map* dedup_examples); + example_factory_t example_factory, const std::unordered_map* dedup_examples); extern template void read_line_json(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, - example_factory_t example_factory, std::unordered_map* dedup_examples); + example_factory_t example_factory, const std::unordered_map* dedup_examples); extern template bool read_line_decision_service_json(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, bool copy_line, example_factory_t example_factory, diff --git a/vowpalwabbit/json_parser/include/vw/json_parser/parse_example_slates_json.h b/vowpalwabbit/json_parser/include/vw/json_parser/parse_example_slates_json.h index 517785694e6..58a2fd4df87 100644 --- a/vowpalwabbit/json_parser/include/vw/json_parser/parse_example_slates_json.h +++ b/vowpalwabbit/json_parser/include/vw/json_parser/parse_example_slates_json.h @@ -19,36 +19,38 @@ namespace details template void parse_slates_example_json(const VW::label_parser& lbl_parser, hash_func_t hash_func, uint64_t hash_seed, uint64_t parse_mask, bool chain_hash, VW::multi_ex& examples, char* line, size_t length, - VW::example_factory_t example_factory, std::unordered_map* dedup_examples = nullptr); + VW::example_factory_t example_factory, const std::unordered_map* dedup_examples = nullptr); template void parse_slates_example_json(const VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, - VW::example_factory_t example_factory, std::unordered_map* dedup_examples = nullptr); + VW::example_factory_t example_factory, const std::unordered_map* dedup_examples = nullptr); template void parse_slates_example_dsjson(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, VW::example_factory_t example_factory, VW::parsers::json::decision_service_interaction* data, - std::unordered_map* dedup_examples = nullptr); + const std::unordered_map* dedup_examples = nullptr); // Define extern template specializations so they don't get initialized when this file is included extern template void parse_slates_example_json(const VW::label_parser& lbl_parser, hash_func_t hash_func, uint64_t hash_seed, uint64_t parse_mask, bool chain_hash, VW::multi_ex& examples, char* line, size_t length, - VW::example_factory_t example_factory, std::unordered_map* dedup_examples); + VW::example_factory_t example_factory, const std::unordered_map* dedup_examples); extern template void parse_slates_example_json(const VW::label_parser& lbl_parser, hash_func_t hash_func, uint64_t hash_seed, uint64_t parse_mask, bool chain_hash, VW::multi_ex& examples, char* line, size_t length, - VW::example_factory_t example_factory, std::unordered_map* dedup_examples); + VW::example_factory_t example_factory, const std::unordered_map* dedup_examples); extern template void parse_slates_example_json(const VW::workspace& all, VW::multi_ex& examples, char* line, - size_t length, VW::example_factory_t example_factory, std::unordered_map* dedup_examples); + size_t length, VW::example_factory_t example_factory, + const std::unordered_map* dedup_examples); extern template void parse_slates_example_json(const VW::workspace& all, VW::multi_ex& examples, char* line, - size_t length, VW::example_factory_t example_factory, std::unordered_map* dedup_examples); + size_t length, VW::example_factory_t example_factory, + const std::unordered_map* dedup_examples); extern template void parse_slates_example_dsjson(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, VW::example_factory_t example_factory, VW::parsers::json::decision_service_interaction* data, - std::unordered_map* dedup_examples); + const std::unordered_map* dedup_examples); extern template void parse_slates_example_dsjson(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, VW::example_factory_t example_factory, VW::parsers::json::decision_service_interaction* data, - std::unordered_map* dedup_examples); + const std::unordered_map* dedup_examples); } // namespace details } // namespace json } // namespace parsers diff --git a/vowpalwabbit/json_parser/src/parse_example_json.cc b/vowpalwabbit/json_parser/src/parse_example_json.cc index 9a22ae69ace..2fc613d9747 100644 --- a/vowpalwabbit/json_parser/src/parse_example_json.cc +++ b/vowpalwabbit/json_parser/src/parse_example_json.cc @@ -1244,7 +1244,7 @@ class UIntDedupState : public BaseState if (ctx.dedup_examples->find(i) == ctx.dedup_examples->end()) { THROW("dedup id not found: " << i); } - auto* stored_ex = (*ctx.dedup_examples)[i]; + auto* stored_ex = ctx.dedup_examples->at(i); new_ex->indices = stored_ex->indices; for (auto& ns : new_ex->indices) { new_ex->feature_space[ns] = stored_ex->feature_space[ns]; } @@ -1512,7 +1512,7 @@ class Context std::vector> namespace_path; std::vector*> return_path; - std::unordered_map* dedup_examples = nullptr; + const std::unordered_map* dedup_examples; std::unordered_map>* ignore_features = nullptr; VW::multi_ex* examples; @@ -1633,7 +1633,7 @@ class VWReaderHandler : public rapidjson::BaseReaderHandler, V bool chain_hash, VW::label_parser_reuse_mem* reuse_mem, const VW::named_labels* ldict, VW::io::logger* logger, VW::multi_ex* examples, rapidjson::InsituStringStream* stream, const char* stream_end, VW::example_factory_t example_factory, std::unordered_map>* ignore_features, - std::unordered_map* dedup_examples = nullptr) + const std::unordered_map* dedup_examples = nullptr) { ctx.init(lbl_parser, hash_func, hash_seed, parse_mask, chain_hash, reuse_mem, ldict, logger); ctx.examples = examples; @@ -1693,7 +1693,7 @@ void VW::parsers::json::read_line_json(const VW::label_parser& lbl_parser, hash_ uint64_t parse_mask, bool chain_hash, VW::label_parser_reuse_mem* reuse_mem, const VW::named_labels* ldict, VW::multi_ex& examples, char* line, size_t length, example_factory_t example_factory, VW::io::logger& logger, std::unordered_map>* ignore_features, - std::unordered_map* dedup_examples) + const std::unordered_map* dedup_examples) { if (lbl_parser.label_type == VW::label_type_t::SLATES) { @@ -1731,7 +1731,7 @@ void VW::parsers::json::read_line_json(const VW::label_parser& lbl_parser, hash_ template void VW::parsers::json::read_line_json(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, - example_factory_t example_factory, std::unordered_map* dedup_examples) + example_factory_t example_factory, const std::unordered_map* dedup_examples) { return read_line_json(all.parser_runtime.example_parser->lbl_parser, all.parser_runtime.example_parser->hasher, all.runtime_config.hash_seed, all.runtime_state.parse_mask, all.parser_runtime.chain_hash_json, @@ -2006,17 +2006,17 @@ template void VW::parsers::json::read_line_json(const VW::label_parser& lb uint64_t hash_seed, uint64_t parse_mask, bool chain_hash, VW::label_parser_reuse_mem* reuse_mem, const VW::named_labels* ldict, VW::multi_ex& examples, char* line, size_t length, example_factory_t example_factory, VW::io::logger& logger, std::unordered_map>* ignore_features, - std::unordered_map* dedup_examples); + const std::unordered_map* dedup_examples); template void VW::parsers::json::read_line_json(const VW::label_parser& lbl_parser, hash_func_t hash_func, uint64_t hash_seed, uint64_t parse_mask, bool chain_hash, VW::label_parser_reuse_mem* reuse_mem, const VW::named_labels* ldict, VW::multi_ex& examples, char* line, size_t length, example_factory_t example_factory, VW::io::logger& logger, std::unordered_map>* ignore_features, - std::unordered_map* dedup_examples); + const std::unordered_map* dedup_examples); template void VW::parsers::json::read_line_json(VW::workspace& all, VW::multi_ex& examples, char* line, - size_t length, example_factory_t example_factory, std::unordered_map* dedup_examples); + size_t length, example_factory_t example_factory, const std::unordered_map* dedup_examples); template void VW::parsers::json::read_line_json(VW::workspace& all, VW::multi_ex& examples, char* line, - size_t length, example_factory_t example_factory, std::unordered_map* dedup_examples); + size_t length, example_factory_t example_factory, const std::unordered_map* dedup_examples); template bool VW::parsers::json::read_line_decision_service_json(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, bool copy_line, example_factory_t example_factory, diff --git a/vowpalwabbit/json_parser/src/parse_example_slates_json.cc b/vowpalwabbit/json_parser/src/parse_example_slates_json.cc index beb0dbfeb9a..f4d99f6c8f5 100644 --- a/vowpalwabbit/json_parser/src/parse_example_slates_json.cc +++ b/vowpalwabbit/json_parser/src/parse_example_slates_json.cc @@ -153,7 +153,7 @@ template void parse_context(const Value& context, const VW::label_parser& lbl_parser, VW::hash_func_t hash_func, uint64_t hash_seed, uint64_t parse_mask, bool chain_hash, VW::multi_ex& examples, VW::example_factory_t example_factory, VW::multi_ex& slot_examples, - std::unordered_map* dedup_examples = nullptr) + const std::unordered_map* dedup_examples = nullptr) { std::vector> namespaces; handle_features_value(" ", context, examples[0], namespaces, hash_func, hash_seed, parse_mask, chain_hash); @@ -184,7 +184,7 @@ void parse_context(const Value& context, const VW::label_parser& lbl_parser, VW: if (dedup_examples->find(dedup_id) == dedup_examples->end()) { THROW("dedup id not found: " << dedup_id); } - auto* stored_ex = (*dedup_examples)[dedup_id]; + auto* stored_ex = dedup_examples->at(dedup_id); ex->indices = stored_ex->indices; for (auto& ns : ex->indices) { ex->feature_space[ns] = stored_ex->feature_space[ns]; } ex->ft_offset = stored_ex->ft_offset; @@ -220,7 +220,7 @@ void parse_context(const Value& context, const VW::label_parser& lbl_parser, VW: template void VW::parsers::json::details::parse_slates_example_json(const VW::label_parser& lbl_parser, hash_func_t hash_func, uint64_t hash_seed, uint64_t parse_mask, bool chain_hash, VW::multi_ex& examples, char* line, size_t /*length*/, - VW::example_factory_t example_factory, std::unordered_map* dedup_examples) + VW::example_factory_t example_factory, const std::unordered_map* dedup_examples) { Document document; document.ParseInsitu(line); @@ -234,7 +234,8 @@ void VW::parsers::json::details::parse_slates_example_json(const VW::label_parse template void VW::parsers::json::details::parse_slates_example_json(const VW::workspace& all, VW::multi_ex& examples, char* line, - size_t length, VW::example_factory_t example_factory, std::unordered_map* dedup_examples) + size_t length, VW::example_factory_t example_factory, + const std::unordered_map* dedup_examples) { parse_slates_example_json(all.parser_runtime.example_parser->lbl_parser, all.parser_runtime.example_parser->hasher, all.runtime_config.hash_seed, all.runtime_state.parse_mask, @@ -244,7 +245,7 @@ void VW::parsers::json::details::parse_slates_example_json(const VW::workspace& template void VW::parsers::json::details::parse_slates_example_dsjson(VW::workspace& all, VW::multi_ex& examples, char* line, size_t /*length*/, VW::example_factory_t example_factory, VW::parsers::json::decision_service_interaction* data, - std::unordered_map* dedup_examples) + const std::unordered_map* dedup_examples) { Document document; document.ParseInsitu(line); @@ -325,24 +326,28 @@ void VW::parsers::json::details::parse_slates_example_dsjson(VW::workspace& all, // Explicitly instantiate templates only in this source file template void VW::parsers::json::details::parse_slates_example_json(const VW::label_parser& lbl_parser, hash_func_t hash_func, uint64_t hash_seed, uint64_t parse_mask, bool chain_hash, VW::multi_ex& examples, char* line, - size_t length, VW::example_factory_t example_factory, std::unordered_map* dedup_examples); + size_t length, VW::example_factory_t example_factory, + const std::unordered_map* dedup_examples); template void VW::parsers::json::details::parse_slates_example_json(const VW::label_parser& lbl_parser, hash_func_t hash_func, uint64_t hash_seed, uint64_t parse_mask, bool chain_hash, VW::multi_ex& examples, char* line, - size_t length, VW::example_factory_t example_factory, std::unordered_map* dedup_examples); + size_t length, VW::example_factory_t example_factory, + const std::unordered_map* dedup_examples); template void VW::parsers::json::details::parse_slates_example_json(const VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, VW::example_factory_t example_factory, - std::unordered_map* dedup_examples); + const std::unordered_map* dedup_examples); template void VW::parsers::json::details::parse_slates_example_json(const VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, VW::example_factory_t example_factory, - std::unordered_map* dedup_examples); + const std::unordered_map* dedup_examples); template void VW::parsers::json::details::parse_slates_example_dsjson(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, VW::example_factory_t example_factory, - VW::parsers::json::decision_service_interaction* data, std::unordered_map* dedup_examples); + VW::parsers::json::decision_service_interaction* data, + const std::unordered_map* dedup_examples); template void VW::parsers::json::details::parse_slates_example_dsjson(VW::workspace& all, VW::multi_ex& examples, char* line, size_t length, VW::example_factory_t example_factory, - VW::parsers::json::decision_service_interaction* data, std::unordered_map* dedup_examples); + VW::parsers::json::decision_service_interaction* data, + const std::unordered_map* dedup_examples); #ifdef VW_WINDOWS_GETOBJECT_MACRO_WAS_UNDEF # pragma pop_macro("GetObject")