diff --git a/.gitattributes b/.gitattributes index 60fc8fed5..e8db8d0bd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -10,6 +10,7 @@ *.PNG binary *.png binary *.jpg binary +*.pdf binary # # Exclude files from exporting diff --git a/.github/workflows/fate-push.yml b/.github/workflows/fate-push.yml deleted file mode 100644 index ae666455b..000000000 --- a/.github/workflows/fate-push.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Push fate images to DockerHub - -on: - push: - # Publish `master` as Docker `latest` image. - branches: - - master - - jenkins-integration - - # Publish `v1.2.3` tags as releases. - tags: - - v* - -jobs: - # no test is required - push: - runs-on: ubuntu-18.04 - if: github.event_name == 'push' - - steps: - - uses: actions/checkout@v2 - - - name: Prepare the TAG - id: prepare-the-tag - run: | - # strip git ref prefix from version - TAG="" - VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') - if [ $VERSION = "master" ]; then - TAG=latest - else - TAG=${VERSION##*v}-release - fi - echo "::notice col=5 title=print tag::TAG=$TAG" - echo "::set-output name=tag::$TAG" - - name: Build image - run: | - IMG_TAG=${{steps.prepare-the-tag.outputs.tag}} - cd docker-build - bash docker-build.sh all - - - name: Log into DockerHub - run: docker login -u ${{ secrets.DOCKERHUB_USERNAME }} -p ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Push image - run: | - IMG_TAG=${{steps.prepare-the-tag.outputs.tag}} - cd docker-build - bash docker-build.sh push diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md index 86a5229ae..4f3872671 100644 --- a/PULL_REQUEST_TEMPLATE.md +++ b/PULL_REQUEST_TEMPLATE.md @@ -1,11 +1,11 @@ -Fixes ISSUE#xxx - -Changes: - -1. - -2. - -3. +Fixes ISSUE #xxx +## Description +1. Tell the story why you need to make this change from the user's perspective. +2. What will be the pain point if you don't make this change? +## Tests +### Before fix +screenshots, console outputs or logs are acceptable. +### After fix +screenshots, console outputs or logs are acceptable. \ No newline at end of file diff --git a/README.md b/README.md index 01a2d1f58..4793de551 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,9 @@ $ make docker-build ``` ## Specifying an image repository -By default, the installation script pulls the images from [Docker Hub](https://hub.docker.com/u/federatedai) during the deployment. A user could also modify the file `.env` to specify a local registry (such as Harbor) to pull images from. A local registry can improve the efficiency of the deployment. +By default, the installation script pulls the images from [Docker Hub](https://hub.docker.com/u/federatedai) during the deployment. +* For docker compose mode, modify the .env file to specify the image registry. +* For K8s mode, check this [offical doc](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/). ## License [Apache License 2.0](https://github.com/FederatedAI/FATE/blob/master/LICENSE) diff --git a/build/ci/ansible/roles/kubefate/files/get_images.sh b/build/ci/ansible/roles/kubefate/files/get_images.sh index ad14b0ae2..6383192c9 100644 --- a/build/ci/ansible/roles/kubefate/files/get_images.sh +++ b/build/ci/ansible/roles/kubefate/files/get_images.sh @@ -20,11 +20,11 @@ main() { docker pull ${DOCKER_REGISTRY}/federatedai/python-spark:${FATE_VERSION} kind load docker-image ${DOCKER_REGISTRY}/federatedai/python-spark:${FATE_VERSION} - docker pull ${DOCKER_REGISTRY}/federatedai/hadoop-namenode:2.0.0-hadoop2.7.4-java8 - kind load docker-image ${DOCKER_REGISTRY}/federatedai/hadoop-namenode:2.0.0-hadoop2.7.4-java8 + docker pull ${DOCKER_REGISTRY}/federatedai/hadoop-namenode:2.0.0-hadoop3.2.1-java8 + kind load docker-image ${DOCKER_REGISTRY}/federatedai/hadoop-namenode:2.0.0-hadoop3.2.1-java8 - docker pull ${DOCKER_REGISTRY}/federatedai/hadoop-datanode:2.0.0-hadoop2.7.4-java8 - kind load docker-image ${DOCKER_REGISTRY}/federatedai/hadoop-datanode:2.0.0-hadoop2.7.4-java8 + docker pull ${DOCKER_REGISTRY}/federatedai/hadoop-datanode:2.0.0-hadoop3.2.1-java8 + kind load docker-image ${DOCKER_REGISTRY}/federatedai/hadoop-datanode:2.0.0-hadoop3.2.1-java8 docker pull ${DOCKER_REGISTRY}/federatedai/nginx:1.17 kind load docker-image ${DOCKER_REGISTRY}/federatedai/nginx:1.17 diff --git a/build/ci/ansible/roles/kubefate/files/kubefate-e2e/common.sh b/build/ci/ansible/roles/kubefate/files/kubefate-e2e/common.sh index efba699b1..38ebf5470 100644 --- a/build/ci/ansible/roles/kubefate/files/kubefate-e2e/common.sh +++ b/build/ci/ansible/roles/kubefate/files/kubefate-e2e/common.sh @@ -126,16 +126,7 @@ set_cluster_image() { sed -i "s#registry: .*#registry: ${DOCKER_REGISTRY}#g" cluster-spark.yaml sed -i "s#registry: .*#registry: ${DOCKER_REGISTRY}#g" cluster-serving.yaml fi - if [[ $FATE_VERSION != "" ]]; then - sed -i "s#imageTag: .*#imageTag: ${FATE_VERSION}#g" cluster.yaml - sed -i "s#imageTag: .*#imageTag: ${FATE_VERSION}#g" cluster-spark.yaml - fi - if [[ $FATE_VERSION != "" ]]; then - sed -i "s#imageTag: .*#imageTag: ${FATE_SERVING_VERSION}#g" cluster-serving.yaml - fi logdebug "REGISTRY=${DOCKER_REGISTRY}" - logdebug "FATE_IMG_TAG=${FATE_VERSION}" - logdebug "FATE_SERVING_IMG_TAG=${fate_serving_version}" } jobUUID="" diff --git a/docker-build/.env b/docker-build/.env deleted file mode 100644 index 8fe30b1e3..000000000 --- a/docker-build/.env +++ /dev/null @@ -1,2 +0,0 @@ -#PREFIX=federatedai -#IMG_TAG=1.8.0-release diff --git a/docker-build/README.md b/docker-build/README.md deleted file mode 100644 index 5a151f65a..000000000 --- a/docker-build/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# KubeFATE docker build - -This contains the builds of some images for KubeFATE to deploy FATE. - -- client -- nginx -- spark -- python-spark - -## Prerequisites - -1. A Linux host -2. Docker: 18+ - -## Build - -All images build. - -```bash -IMG_TAG=latest bash docker-build.sh all -``` - -## push - -Push images to DockerHub - -```bash -IMG_TAG=latest bash docker-build.sh all -``` diff --git a/docker-build/client/Dockerfile b/docker-build/client/Dockerfile deleted file mode 100644 index 4a32cddd5..000000000 --- a/docker-build/client/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -ARG SOURCE_PREFIX=federatedai -ARG SOURCE_TAG=1.5.0-release -FROM ${SOURCE_PREFIX}/python:${SOURCE_TAG} as data - -FROM python:3.6 - -COPY pipeline /data/projects/fate/pipeline -RUN pip install notebook fate-client pandas sklearn -RUN mkdir /data/projects/fate/logs -COPY --from=data /data/projects/fate/examples /data/projects/fate/examples -COPY --from=data /data/projects/fate/fateflow/examples /data/projects/fate/fateflow/examples - - -WORKDIR /data/projects/fate/ - -ENV FATE_FLOW_IP=fateflow -ENV FATE_FLOW_PORT=9380 - -CMD flow init --ip ${FATE_FLOW_IP} --port ${FATE_FLOW_PORT} && pipeline init --ip ${FATE_FLOW_IP} --port ${FATE_FLOW_PORT} && jupyter notebook --ip=0.0.0.0 --port=20000 --allow-root --debug --NotebookApp.notebook_dir='/data/projects/fate/' --no-browser --NotebookApp.token='' --NotebookApp.password='' diff --git a/docker-build/client/pipeline/pipeline_tutorial_hetero_sbt.ipynb b/docker-build/client/pipeline/pipeline_tutorial_hetero_sbt.ipynb deleted file mode 100644 index dd42feac8..000000000 --- a/docker-build/client/pipeline/pipeline_tutorial_hetero_sbt.ipynb +++ /dev/null @@ -1,838 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Pipeline Tutorial with HeteroSecureBoost" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### install" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "`Pipeline` is distributed along with [fate_client](https://pypi.org/project/fate-client/).\n", - "\n", - "```bash\n", - "pip install fate_client\n", - "```\n", - "\n", - "To use Pipeline, we need to first specify which `FATE Flow Service` to connect to. Once `fate_client` installed, one can find an cmd enterpoint name `pipeline`:" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Usage: pipeline [OPTIONS] COMMAND [ARGS]...\r\n", - "\r\n", - "Options:\r\n", - " --help Show this message and exit.\r\n", - "\r\n", - "Commands:\r\n", - " config pipeline config tool\r\n", - " init \b - DESCRIPTION: Pipeline Config Command.\r\n" - ] - } - ], - "source": [ - "!pipeline --help" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Assume we have a `FATE Flow Service` in 127.0.0.1:9380(defaults in standalone), then exec" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Pipeline configuration succeeded.\r\n" - ] - } - ], - "source": [ - "!pipeline init --ip fateflow --port 9380" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Hetero SecureBoost Example" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - " Before start a modeling task, data to be used should be uploaded. Please refer to this [guide](https://github.com/FederatedAI/FATE/blob/master/doc/tutorial/pipeline/pipeline_tutorial_upload.ipynb)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `pipeline` package provides components to compose a `FATE pipeline`." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2022-03-24 13:39:50.653168: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcudart.so.10.1'; dlerror: libcudart.so.10.1: cannot open shared object file: No such file or directory\n", - "2022-03-24 13:39:50.653332: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n" - ] - } - ], - "source": [ - "from pipeline.backend.pipeline import PipeLine\n", - "from pipeline.component import Reader, DataTransform, Intersection, HeteroSecureBoost, Evaluation\n", - "from pipeline.interface import Data" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Make a `pipeline` instance:\n", - "\n", - " - initiator: \n", - " * role: guest\n", - " * party: 9999\n", - " - roles:\n", - " * guest: 9999\n", - " * host: 10000\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "pipeline = PipeLine() \\\n", - " .set_initiator(role='guest', party_id=9999) \\\n", - " .set_roles(guest=9999, host=10000)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Define a `Reader` to load data" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "reader_0 = Reader(name=\"reader_0\")\n", - "# set guest parameter\n", - "reader_0.get_party_instance(role='guest', party_id=9999).component_param(\n", - " table={\"name\": \"breast_hetero_guest\", \"namespace\": \"experiment\"})\n", - "# set host parameter\n", - "reader_0.get_party_instance(role='host', party_id=10000).component_param(\n", - " table={\"name\": \"breast_hetero_host\", \"namespace\": \"experiment\"})" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Add a `DataTransform` component to parse raw data into Data Instance" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "data_transform_0 = DataTransform(name=\"data_transform_0\")\n", - "# set guest parameter\n", - "data_transform_0.get_party_instance(role='guest', party_id=9999).component_param(\n", - " with_label=True)\n", - "data_transform_0.get_party_instance(role='host', party_id=[10000]).component_param(\n", - " with_label=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Add a `Intersection` component to perform PSI for hetero-scenario" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "intersect_0 = Intersection(name=\"intersect_0\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now, we define the `HeteroSecureBoost` component. The following parameters will be set for all parties involved." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "hetero_secureboost_0 = HeteroSecureBoost(name=\"hetero_secureboost_0\",\n", - " num_trees=5,\n", - " bin_num=16,\n", - " task_type=\"classification\",\n", - " objective_param={\"objective\": \"cross_entropy\"},\n", - " encrypt_param={\"method\": \"iterativeAffine\"},\n", - " tree_param={\"max_depth\": 3})\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To show the evaluation result, an \"Evaluation\" component is needed." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "evaluation_0 = Evaluation(name=\"evaluation_0\", eval_type=\"binary\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Add components to pipeline, in order of execution:\n", - "\n", - " - data_transform_0 comsume reader_0's output data\n", - " - intersect_0 comsume data_transform_0's output data\n", - " - hetero_secureboost_0 consume intersect_0's output data\n", - " - evaluation_0 consume hetero_secureboost_0's prediciton result on training data\n", - "\n", - "Then compile our pipeline to make it ready for submission." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "pipeline.add_component(reader_0)\n", - "pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data))\n", - "pipeline.add_component(intersect_0, data=Data(data=data_transform_0.output.data))\n", - "pipeline.add_component(hetero_secureboost_0, data=Data(train_data=intersect_0.output.data))\n", - "pipeline.add_component(evaluation_0, data=Data(data=hetero_secureboost_0.output.data))\n", - "pipeline.compile();\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now, submit(fit) our pipeline:" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[32m2022-03-24 13:39:56.241\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m123\u001b[0m - \u001b[1mJob id is 202203241339527023860\n", - "\u001b[0m\n", - "\u001b[32m2022-03-24 13:39:56.269\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m144\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KJob is still waiting, time elapse: 0:00:00\u001b[0m\n", - "\u001b[32m2022-03-24 13:39:56.788\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m144\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KJob is still waiting, time elapse: 0:00:00\u001b[0m\n", - "\u001b[32m2022-03-24 13:39:57.306\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m144\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KJob is still waiting, time elapse: 0:00:01\u001b[0m\n", - "\u001b[0mm2022-03-24 13:39:58.404\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m173\u001b[0m - \u001b[1m\n", - "\u001b[32m2022-03-24 13:39:58.406\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:02\u001b[0m\n", - "\u001b[32m2022-03-24 13:39:58.956\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:02\u001b[0m\n", - "\u001b[32m2022-03-24 13:39:59.488\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:03\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:00.027\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:03\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:00.562\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:04\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:01.094\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:04\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:01.643\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:05\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:02.204\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:05\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:02.760\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:06\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:03.297\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:07\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:03.883\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:07\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:04.467\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:08\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:05.069\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:08\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:05.613\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:09\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:06.175\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:09\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:06.742\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:10\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:07.343\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:11\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:07.920\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:11\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:08.580\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:12\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:09.160\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:12\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:09.729\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:13\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:10.350\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:14\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:10.899\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:14\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:11.430\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:15\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:11.979\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:15\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:12.513\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:16\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:13.051\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_0, time elapse: 0:00:16\u001b[0m\n", - "\u001b[0mm2022-03-24 13:40:29.604\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m173\u001b[0m - \u001b[1m\n", - "\u001b[32m2022-03-24 13:40:29.607\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:33\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:30.138\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:33\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:30.665\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:34\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:31.208\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:34\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:31.770\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:35\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:32.389\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:36\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:32.931\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:36\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:33.528\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:37\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:34.071\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:37\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:34.634\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:38\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:35.209\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:38\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:35.770\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:39\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:36.319\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:40\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:36.850\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:40\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:37.403\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:41\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:37.943\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:41\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:38.504\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:42\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:39.077\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:42\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:39.622\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:43\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:40.273\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:44\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:40.816\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:44\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:41.372\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:45\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:41.913\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:45\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:42.449\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:46\u001b[0m\n", - "\u001b[0mm2022-03-24 13:40:55.864\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m173\u001b[0m - \u001b[1m\n", - "\u001b[32m2022-03-24 13:40:55.868\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:00:59\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:56.408\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:00\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:56.949\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:00\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:57.489\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:01\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:58.037\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:01\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:58.573\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:02\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:59.115\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:02\u001b[0m\n", - "\u001b[32m2022-03-24 13:40:59.650\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:03\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:00.185\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:03\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:00.720\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:04\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:01.259\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:05\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:01.799\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:05\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:02.377\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:06\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:02.976\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:06\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:03.527\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:07\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:04.089\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:07\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:04.636\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:08\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:05.191\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:08\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:05.749\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:09\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:06.860\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:10\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:07.420\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:11\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:07.987\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:11\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:08.541\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:12\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:09.075\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:12\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:09.607\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:13\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:10.135\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:13\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:10.663\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:14\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:11.207\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:14\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:11.742\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:15\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:12.277\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:16\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:12.806\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:16\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:13.367\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:17\u001b[0m\n", - "\u001b[0mm2022-03-24 13:41:26.106\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m173\u001b[0m - \u001b[1m\n", - "\u001b[32m2022-03-24 13:41:26.109\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:29\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:26.650\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:30\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:27.179\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:30\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:27.711\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:31\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:28.250\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:32\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:28.786\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:32\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:29.315\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:33\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:29.862\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:33\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:30.401\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:34\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:30.935\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:34\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:31.466\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:35\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:32.003\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:35\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:32.551\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:36\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:33.111\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:36\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:33.666\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:37\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:34.207\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:37\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:34.770\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:38\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:35.329\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:39\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:35.872\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:39\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:36.427\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:40\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:36.977\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:40\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:37.555\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:41\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:38.110\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:41\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:38.668\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:42\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:39.215\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:42\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:39.750\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:43\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:40.290\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:44\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:40.830\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:44\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:41.375\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:45\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:41.925\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:45\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:42.459\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:46\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:43.005\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:46\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:43.579\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:47\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:44.119\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:47\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:44.658\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:48\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:45.209\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:48\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:45.779\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:49\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:46.604\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:50\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:47.161\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:50\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:47.707\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:51\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:48.249\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:52\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:48.799\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:52\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:49.338\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:53\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:49.903\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:53\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:50.449\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:54\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:50.985\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:54\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:51.538\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:55\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:52.079\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:55\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:52.626\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:56\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:53.163\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:56\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:53.710\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:57\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:54.253\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:58\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:54.798\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:58\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:55.373\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:59\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:55.927\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:59\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:56.464\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:00\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:56.995\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:00\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:57.531\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:01\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:58.078\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:01\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:58.655\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:02\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:59.331\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:03\u001b[0m\n", - "\u001b[32m2022-03-24 13:41:59.888\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:03\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:00.435\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:04\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:00.990\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:04\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:01.562\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:05\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:02.107\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:05\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:02.726\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:06\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:03.295\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:07\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:03.860\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:07\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:04.393\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:08\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:04.959\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:08\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:05.501\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:09\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:06.031\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:09\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:06.579\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:10\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:07.125\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:10\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:07.670\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:11\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:08.220\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:11\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:08.766\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:12\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:09.813\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:13\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:10.656\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:14\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:11.202\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:14\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:11.744\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:15\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:12.281\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:16\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:12.849\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:16\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:13.404\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:17\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:13.964\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:17\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:14.510\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:18\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:15.046\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:18\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:15.740\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:19\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:16.710\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:20\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:17.260\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:21\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:17.807\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:21\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:18.588\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:22\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:19.126\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:22\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:19.658\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:23\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:20.203\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:23\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:20.746\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:24\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:21.294\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:25\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:21.836\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:25\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:22.374\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:26\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:22.919\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:26\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:23.464\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:27\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:24.051\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:27\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:24.638\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:28\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:25.195\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:28\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:25.748\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:29\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:26.563\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:30\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:27.109\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:30\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:27.646\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:31\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:28.220\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:31\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:28.759\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:32\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:29.292\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:33\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:29.822\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:33\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:30.379\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:34\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:30.916\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:34\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:31.473\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:35\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:32.019\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:35\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:32.656\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:36\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:33.201\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:36\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:33.761\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:37\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:34.478\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:38\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:35.023\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:38\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:35.566\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:39\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:36.101\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:39\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:36.646\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:40\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:37.195\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:40\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:37.759\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:41\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:38.301\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:42\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:38.846\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:42\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:39.661\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:43\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:40.215\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:43\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:40.775\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:44\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:41.378\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:45\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:41.947\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:45\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:42.495\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:46\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:43.034\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:46\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:43.590\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:47\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:44.127\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:47\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:44.663\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:48\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:45.214\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:48\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:45.753\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:49\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:46.288\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:50\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:46.826\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:50\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:47.397\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:51\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:47.945\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:51\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:48.486\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:52\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:49.025\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:52\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:49.597\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:53\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:50.158\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:53\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:50.703\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:54\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:51.232\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:54\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:51.772\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:55\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:52.312\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:56\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:52.844\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:56\u001b[0m\n", - "\u001b[32m2022-03-24 13:42:53.400\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:02:57\u001b[0m\n", - "\u001b[0mm2022-03-24 13:43:08.015\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m173\u001b[0m - \u001b[1m\n", - "\u001b[32m2022-03-24 13:43:08.018\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:03:11\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:08.557\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:03:12\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:09.100\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:03:12\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:09.629\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:03:13\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:10.159\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:03:13\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:10.701\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:03:14\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:11.260\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:03:15\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:11.804\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:03:15\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:12.366\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:03:16\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:12.904\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:03:16\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:13.442\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:03:17\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:13.992\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:03:17\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:14.597\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:03:18\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:15.160\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:03:18\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:15.695\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:03:19\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:16.229\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:03:19\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:16.786\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:03:20\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:17.320\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:03:21\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:17.858\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:03:21\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:18.399\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:03:22\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:32.831\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m131\u001b[0m - \u001b[1mJob is success!!! Job id is 202203241339527023860\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:32.834\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m132\u001b[0m - \u001b[1mTotal time: 0:03:36\u001b[0m\n" - ] - } - ], - "source": [ - "pipeline.fit()\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Once training is done, trained model may be used for prediction. Optionally, save the trained pipeline for future use." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "pipeline.dump(\"pipeline_saved.pkl\");" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, deploy needed components from train pipeline" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "pipeline = PipeLine.load_model_from_file('pipeline_saved.pkl')\n", - "pipeline.deploy_component([pipeline.data_transform_0, pipeline.intersect_0, pipeline.hetero_secureboost_0]);" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Define new `Reader` components for reading prediction data" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "reader_1 = Reader(name=\"reader_1\")\n", - "reader_1.get_party_instance(role=\"guest\", party_id=9999).component_param(table={\"name\": \"breast_hetero_guest\", \"namespace\": \"experiment\"})\n", - "reader_1.get_party_instance(role=\"host\", party_id=10000).component_param(table={\"name\": \"breast_hetero_host\", \"namespace\": \"experiment\"})" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Optionally, define new `Evaluation` component." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "evaluation_0 = Evaluation(name=\"evaluation_0\", eval_type=\"binary\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Add components to predict pipeline in order of execution:" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "predict_pipeline = PipeLine()\n", - "predict_pipeline.add_component(reader_1)\\\n", - " .add_component(pipeline, \n", - " data=Data(predict_input={pipeline.data_transform_0.input.data: reader_1.output.data}))\\\n", - " .add_component(evaluation_0, data=Data(data=pipeline.hetero_secureboost_0.output.data));\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Then, run prediction job" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[32m2022-03-24 13:43:34.211\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m123\u001b[0m - \u001b[1mJob id is 202203241343334274690\n", - "\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:34.227\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m144\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KJob is still waiting, time elapse: 0:00:00\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:34.748\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m144\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KJob is still waiting, time elapse: 0:00:00\u001b[0m\n", - "\u001b[0mm2022-03-24 13:43:35.300\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m173\u001b[0m - \u001b[1m\n", - "\u001b[32m2022-03-24 13:43:35.301\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:01\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:35.886\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:01\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:36.426\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:02\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:36.962\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:02\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:37.499\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:03\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:38.036\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:03\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:38.583\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:04\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:39.122\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:04\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:39.657\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:05\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:40.213\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:06\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:40.799\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:06\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:41.377\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:07\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:41.919\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:07\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:42.464\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:08\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:43.053\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:08\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:43.615\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:09\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:44.268\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:10\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:44.843\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:10\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:45.400\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:11\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:45.959\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:11\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:46.503\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:12\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:47.044\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:12\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:47.583\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:13\u001b[0m\n", - "\u001b[32m2022-03-24 13:43:48.121\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component reader_1, time elapse: 0:00:13\u001b[0m\n", - "\u001b[0mm2022-03-24 13:44:04.793\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m173\u001b[0m - \u001b[1m\n", - "\u001b[32m2022-03-24 13:44:04.799\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:30\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:05.344\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:31\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:05.886\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:31\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:06.422\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:32\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:06.951\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:32\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:07.491\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:33\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:08.049\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:33\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:08.593\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:34\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:09.183\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:34\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:09.754\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:35\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:10.298\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:36\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:10.851\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:36\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:11.417\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:37\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:11.963\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:37\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:12.498\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:38\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:13.070\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:38\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:13.634\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:39\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:14.187\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:39\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:14.727\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:40\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:15.267\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component data_transform_0, time elapse: 0:00:41\u001b[0m\n", - "\u001b[0mm2022-03-24 13:44:28.085\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m173\u001b[0m - \u001b[1m\n", - "\u001b[32m2022-03-24 13:44:28.088\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:00:53\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:28.640\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:00:54\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:29.171\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:00:54\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:29.706\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:00:55\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:30.237\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:00:56\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:30.772\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:00:56\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:31.307\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:00:57\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:31.856\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:00:57\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:32.394\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:00:58\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:32.948\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:00:58\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:33.527\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:00:59\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:34.088\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:00:59\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:34.620\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:00\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:35.165\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:00\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:35.731\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:01\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:36.275\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:02\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:36.872\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:02\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:37.405\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:03\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:37.936\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:03\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:38.468\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:04\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:39.002\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:04\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:39.531\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:05\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:40.061\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:05\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:40.593\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:06\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:41.124\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component intersect_0, time elapse: 0:01:06\u001b[0m\n", - "\u001b[0mm2022-03-24 13:44:53.264\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m173\u001b[0m - \u001b[1m\n", - "\u001b[32m2022-03-24 13:44:53.267\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:19\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:53.800\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:19\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:54.327\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:20\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:54.855\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:20\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:55.388\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:21\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:55.922\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:21\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:56.451\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:22\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:56.980\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:22\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:57.522\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:23\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:58.058\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:23\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:58.589\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:24\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:59.145\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:24\u001b[0m\n", - "\u001b[32m2022-03-24 13:44:59.705\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:25\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:00.268\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:26\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:00.813\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:26\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:01.344\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:27\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:01.951\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:27\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:02.525\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:28\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:03.083\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:28\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:03.684\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:29\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:04.261\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:30\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:04.799\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:30\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:05.383\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:31\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:05.968\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:31\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:06.537\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:32\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:07.096\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:32\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:07.659\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:33\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:08.219\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:34\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:08.787\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:34\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:09.378\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:35\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:09.932\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:35\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:10.467\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:36\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:10.996\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:36\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:11.525\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:37\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:12.054\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:37\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:12.591\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component hetero_secureboost_0, time elapse: 0:01:38\u001b[0m\n", - "\u001b[0mm2022-03-24 13:45:24.750\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m173\u001b[0m - \u001b[1m\n", - "\u001b[32m2022-03-24 13:45:24.752\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:01:50\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:25.292\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:01:51\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:25.827\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:01:51\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:26.358\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:01:52\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:26.886\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:01:52\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:27.412\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:01:53\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:27.949\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:01:53\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:28.483\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:01:54\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:29.012\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:01:54\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:29.652\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:01:55\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:30.208\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:01:55\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:30.773\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:01:56\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:31.350\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:01:57\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:31.889\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:01:57\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:32.466\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:01:58\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:33.006\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:01:58\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:33.535\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component evaluation_0, time elapse: 0:01:59\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:45.660\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m131\u001b[0m - \u001b[1mJob is success!!! Job id is 202203241343334274690\u001b[0m\n", - "\u001b[32m2022-03-24 13:45:45.662\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m132\u001b[0m - \u001b[1mTotal time: 0:02:11\u001b[0m\n" - ] - } - ], - "source": [ - "predict_pipeline.predict()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For more demo on using pipeline to submit jobs, please refer to [pipeline demos](https://github.com/FederatedAI/FATE/tree/master/examples/pipeline/demo)" - ] - } - ], - "metadata": { - "interpreter": { - "hash": "ad4309918fa4cd1705b305e369b2f64d901b1851e9144aef7b9b07ea3efcb1bb" - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.12" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/docker-build/client/pipeline/pipeline_tutorial_upload.ipynb b/docker-build/client/pipeline/pipeline_tutorial_upload.ipynb deleted file mode 100644 index d9504f247..000000000 --- a/docker-build/client/pipeline/pipeline_tutorial_upload.ipynb +++ /dev/null @@ -1,368 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Pipeline Upload Data Tutorial " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### install" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "`Pipeline` is distributed along with [fate_client](https://pypi.org/project/fate-client/).\n", - "\n", - "```bash\n", - "pip install fate_client\n", - "```\n", - "\n", - "To use Pipeline, we need to first specify which `FATE Flow Service` to connect to. Once `fate_client` installed, one can find an cmd enterpoint name `pipeline`:" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "import os" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Usage: pipeline [OPTIONS] COMMAND [ARGS]...\r\n", - "\r\n", - "Options:\r\n", - " --help Show this message and exit.\r\n", - "\r\n", - "Commands:\r\n", - " config pipeline config tool\r\n", - " init \b - DESCRIPTION: Pipeline Config Command.\r\n" - ] - } - ], - "source": [ - "!pipeline --help" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Assume we have a `FATE Flow Service` which name is `fateflow` and the port is `9380`, then exec" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Pipeline configuration succeeded.\r\n" - ] - } - ], - "source": [ - "!pipeline init --ip fateflow --port 9380" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### upload data" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - " Before start a modeling task, the data to be used should be uploaded. \n", - " Typically, a party is usually a cluster which include multiple nodes. Thus, when we upload these data, the data will be allocated to those nodes." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "from pipeline.backend.pipeline import PipeLine" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Make a `pipeline` instance:\n", - "\n", - " - initiator: \n", - " * role: guest\n", - " * party: 9999\n", - " - roles:\n", - " * guest: 9999\n", - "\n", - "note that only local party id is needed.\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "pipeline_upload = PipeLine().set_initiator(role='guest', party_id=9999).set_roles(guest=9999)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Define partitions for data storage" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "partition = 4" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Define table name and namespace, which will be used in FATE job configuration" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "dense_data_guest = {\"name\": \"breast_hetero_guest\", \"namespace\": f\"experiment\"}\n", - "dense_data_host = {\"name\": \"breast_hetero_host\", \"namespace\": f\"experiment\"}\n", - "tag_data = {\"name\": \"breast_hetero_host\", \"namespace\": f\"experiment\"}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now, we add data to be uploaded" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "data_base = \"/data/projects/fate/\"\n", - "pipeline_upload.add_upload_data(file=os.path.join(data_base, \"examples/data/breast_hetero_guest.csv\"),\n", - " table_name=dense_data_guest[\"name\"], # table name\n", - " namespace=dense_data_guest[\"namespace\"], # namespace\n", - " head=1, partition=partition) # data info\n", - "\n", - "pipeline_upload.add_upload_data(file=os.path.join(data_base, \"examples/data/breast_hetero_host.csv\"),\n", - " table_name=dense_data_host[\"name\"],\n", - " namespace=dense_data_host[\"namespace\"],\n", - " head=1, partition=partition)\n", - "\n", - "pipeline_upload.add_upload_data(file=os.path.join(data_base, \"examples/data/breast_hetero_host.csv\"),\n", - " table_name=tag_data[\"name\"],\n", - " namespace=tag_data[\"namespace\"],\n", - " head=1, partition=partition)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can then upload data" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " UPLOADING:||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||100.00%\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[32m2022-03-24 13:18:11.025\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m123\u001b[0m - \u001b[1mJob id is 202203241318106644180\n", - "\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:11.041\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m144\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KJob is still waiting, time elapse: 0:00:00\u001b[0m\n", - "\u001b[0mm2022-03-24 13:18:11.597\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m173\u001b[0m - \u001b[1m\n", - "\u001b[32m2022-03-24 13:18:11.599\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:00\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:12.134\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:01\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:12.667\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:01\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:13.201\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:02\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:13.747\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:02\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:14.282\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:03\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:14.826\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:03\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:15.361\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:04\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:15.895\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:04\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:16.432\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:05\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:16.967\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:05\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:17.506\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:06\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:18.041\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:07\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:18.575\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:07\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:19.146\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:08\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:19.719\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:08\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:20.285\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:09\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:20.824\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:09\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:21.361\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:10\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:21.900\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:10\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:29.293\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m131\u001b[0m - \u001b[1mJob is success!!! Job id is 202203241318106644180\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:29.295\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m132\u001b[0m - \u001b[1mTotal time: 0:00:18\u001b[0m\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " UPLOADING:||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||100.00%\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[32m2022-03-24 13:18:29.575\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m123\u001b[0m - \u001b[1mJob id is 202203241318293060140\n", - "\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:29.593\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m144\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KJob is still waiting, time elapse: 0:00:00\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:30.111\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m144\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KJob is still waiting, time elapse: 0:00:00\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:30.629\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m144\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KJob is still waiting, time elapse: 0:00:01\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:31.146\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m144\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KJob is still waiting, time elapse: 0:00:01\u001b[0m\n", - "\u001b[0mm2022-03-24 13:18:31.676\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m173\u001b[0m - \u001b[1m\n", - "\u001b[32m2022-03-24 13:18:31.679\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:02\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:32.208\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:02\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:32.739\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:03\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:33.270\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:03\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:33.804\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:04\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:34.328\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:04\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:34.861\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:05\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:35.391\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:05\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:35.921\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:06\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:36.456\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:06\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:36.983\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:07\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:37.543\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:07\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:38.107\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:08\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:38.671\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:09\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:39.226\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:09\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:39.765\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:10\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:40.295\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:10\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:40.824\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:11\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:41.357\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:11\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:41.919\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:12\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:48.837\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m131\u001b[0m - \u001b[1mJob is success!!! Job id is 202203241318293060140\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:48.840\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m132\u001b[0m - \u001b[1mTotal time: 0:00:19\u001b[0m\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " UPLOADING:||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||100.00%\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[32m2022-03-24 13:18:49.206\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m123\u001b[0m - \u001b[1mJob id is 202203241318488510330\n", - "\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:49.227\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m144\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KJob is still waiting, time elapse: 0:00:00\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:49.746\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m144\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KJob is still waiting, time elapse: 0:00:00\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:50.263\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m144\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KJob is still waiting, time elapse: 0:00:01\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:50.790\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m144\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KJob is still waiting, time elapse: 0:00:01\u001b[0m\n", - "\u001b[0mm2022-03-24 13:18:51.319\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m173\u001b[0m - \u001b[1m\n", - "\u001b[32m2022-03-24 13:18:51.321\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:02\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:51.852\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:02\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:52.384\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:03\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:52.915\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:03\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:53.451\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:04\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:53.979\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:04\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:54.507\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:05\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:55.036\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:05\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:55.570\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:06\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:56.099\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:06\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:56.637\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:07\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:57.215\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:08\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:57.795\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:08\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:58.336\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:09\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:58.869\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:09\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:59.400\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:10\u001b[0m\n", - "\u001b[32m2022-03-24 13:18:59.928\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:10\u001b[0m\n", - "\u001b[32m2022-03-24 13:19:00.459\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:11\u001b[0m\n", - "\u001b[32m2022-03-24 13:19:00.986\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m177\u001b[0m - \u001b[1m\u001b[80D\u001b[1A\u001b[KRunning component upload_0, time elapse: 0:00:11\u001b[0m\n", - "\u001b[32m2022-03-24 13:19:09.035\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m131\u001b[0m - \u001b[1mJob is success!!! Job id is 202203241318488510330\u001b[0m\n", - "\u001b[32m2022-03-24 13:19:09.037\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpipeline.utils.invoker.job_submitter\u001b[0m:\u001b[36mmonitor_job_status\u001b[0m:\u001b[36m132\u001b[0m - \u001b[1mTotal time: 0:00:19\u001b[0m\n" - ] - } - ], - "source": [ - "pipeline_upload.upload(drop=1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For more demo on using pipeline to submit jobs, please refer to [pipeline demos](https://github.com/FederatedAI/FATE/tree/master/examples/pipeline/demo)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.12" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/docker-build/docker-build.sh b/docker-build/docker-build.sh deleted file mode 100644 index 458322abc..000000000 --- a/docker-build/docker-build.sh +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2019-2020 VMware, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# you may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#!/bin/bash - -set -e - -if [ -z "$IMG_TAG" ]; then - IMG_TAG=latest -fi -if [ -z "$PREFIX" ]; then - PREFIX=federatedai -fi - -source .env - -buildModule() { - # nginx - docker build --build-arg SOURCE_PREFIX=${PREFIX} --build-arg SOURCE_TAG=${IMG_TAG} -t ${PREFIX}/nginx:${IMG_TAG} nginx - echo "Image: "${PREFIX}/nginx:${IMG_TAG}" Build Successful" - - # python-spark - docker build --build-arg SOURCE_PREFIX=${PREFIX} --build-arg SOURCE_TAG=${IMG_TAG} -t ${PREFIX}/python-spark:${IMG_TAG} python-spark - echo "Image: " ${PREFIX}/python-spark:${IMG_TAG}" Build Successful" - - # spark - docker build --build-arg SOURCE_PREFIX=${PREFIX} --build-arg SOURCE_TAG=${IMG_TAG} -t ${PREFIX}/spark-base:${IMG_TAG} spark/base - echo "Image: "${PREFIX}/spark-base:${IMG_TAG}" Build Successful" - - docker build --build-arg SOURCE_PREFIX=${PREFIX} --build-arg SOURCE_TAG=${IMG_TAG} -t ${PREFIX}/spark-master:${IMG_TAG} spark/master - echo "Image: "${PREFIX}/spark-master:${IMG_TAG}" Build Successful" - docker build --build-arg SOURCE_PREFIX=${PREFIX} --build-arg SOURCE_TAG=${IMG_TAG} -t ${PREFIX}/spark-worker:${IMG_TAG} spark/worker - echo "Image: "${PREFIX}/spark-worker:${IMG_TAG}" Build Successful" - - # client - docker build --build-arg SOURCE_PREFIX=${PREFIX} --build-arg SOURCE_TAG=${IMG_TAG} -t ${PREFIX}/client:${IMG_TAG} client - echo "Image: "${PREFIX}/client:${IMG_TAG}" Build Successful" -} - -pushImage() { - ## push image - for module in "nginx" "python-spark" "spark-base" "spark-master" "spark-worker" "client"; do - echo "### START PUSH ${module} ###" - docker push ${PREFIX}/${module}:${IMG_TAG} - echo "### FINISH PUSH ${module} ###" - echo "" - done -} - -while [ "$1" != "" ]; do - case $1 in - modules) - buildModule - ;; - all) - buildModule - ;; - push) - pushImage - ;; - --tag) - IMG_TAG=$2 - shift - ;; - *) - echo "Usage: bash docker-build.sh --tag \$TAG [modules|all|push]" - esac - shift -done diff --git a/docker-build/fate-operator-fatejob/Dockerfile b/docker-build/fate-operator-fatejob/Dockerfile deleted file mode 100644 index fa11e5133..000000000 --- a/docker-build/fate-operator-fatejob/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -ARG SOURCE_PREFIX=federatedai -ARG SOURCE_TAG=1.5.1-release -FROM ${SOURCE_PREFIX}/client:${SOURCE_TAG} - -COPY job.py . diff --git a/docker-build/fate-operator-fatejob/README.md b/docker-build/fate-operator-fatejob/README.md deleted file mode 100644 index 5c27e269a..000000000 --- a/docker-build/fate-operator-fatejob/README.md +++ /dev/null @@ -1,5 +0,0 @@ -**Build fatejob image for fate-operator.** - -```bash -docker build -t federatedai/job . -``` \ No newline at end of file diff --git a/docker-build/fate-operator-fatejob/fateJob-test.yaml b/docker-build/fate-operator-fatejob/fateJob-test.yaml deleted file mode 100644 index 0f096a310..000000000 --- a/docker-build/fate-operator-fatejob/fateJob-test.yaml +++ /dev/null @@ -1,68 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: fatejob-toy - namespace: fate-9999 -spec: - template: - metadata: - name: fatejob-toy - spec: - containers: - - name: fatejob-toy - image: federatedai/job:latest - imagePullPolicy: IfNotPresent - env: - - name: FateFlowServer - value: fateflow-client:9380 - command: - - /bin/bash - - -c - - | - set -eux; - python job.py --dsl='{ - "components": { - "secure_add_example_0": { - "module": "SecureAddExample" - } - } - }' --config='{ - "initiator": { - "role": "guest", - "party_id": 9999 - }, - "job_parameters": { - "work_mode": 1 - }, - "role": { - "guest": [ - 9999 - ], - "host": [ - 9999 - ] - }, - "role_parameters": { - "guest": { - "secure_add_example_0": { - "seed": [ - 123 - ] - } - }, - "host": { - "secure_add_example_0": { - "seed": [ - 321 - ] - } - } - }, - "algorithm_parameters": { - "secure_add_example_0": { - "partition": 10, - "data_num": 1000 - } - } - }' - restartPolicy: Never \ No newline at end of file diff --git a/docker-build/fate-operator-fatejob/job.py b/docker-build/fate-operator-fatejob/job.py deleted file mode 100644 index f475fb47d..000000000 --- a/docker-build/fate-operator-fatejob/job.py +++ /dev/null @@ -1,92 +0,0 @@ -import json -import time -import os -import argparse -import tarfile -import requests - - -fateflow_uri = os.getenv('FateFlowServer') -server_url = "http://{}/{}".format(fateflow_uri, "v1") -print(server_url) - - -def submit_job(dsl, config): - post_data = {'job_dsl': dsl, - 'job_runtime_conf': config} - response = requests.post( - "/".join([server_url, "job", "submit"]), json=post_data) - - return prettify(response) - - -def query_job(query_conditions): - response = requests.post( - "/".join([server_url, "job", "query"]), json=query_conditions) - return prettify(response) - - -def stop_job(self, job_id): - post_data = { - 'job_id': job_id - } - response = requests.post( - "/".join([server_url, "job", "stop"]), json=post_data) - return prettify(response) - - -def prettify(response, verbose=False): - if verbose: - if isinstance(response, requests.Response): - if response.status_code == 200: - print("Success!") - print(json.dumps(response.json(), indent=4, ensure_ascii=False)) - else: - print(response) - - return response - - -def run_job(job_dsl, config_data): - print(job_dsl) - print(config_data) - response = submit_job(job_dsl, config_data) - prettify(response, True) - stdout = json.loads(response.content) - print(stdout) - jobid = stdout["jobId"] - query_condition = { - "job_id": jobid - } - job_status = query_job(query_condition) - prettify(job_status, True) - - for i in range(500): - time.sleep(1) - job_detail = query_job(query_condition).json() - final_status = job_detail["data"][0]["f_status"] - print(final_status) - - if final_status == "failed": - print("Failed") - break - if final_status == "success": - print("Success") - break - - # r - # esponse = manager.fetch_job_log_new(jobid) - - -if __name__ == "__main__": - arg_parser = argparse.ArgumentParser() - arg_parser.add_argument("--dsl", type=str, help="please input dsl") - arg_parser.add_argument("--config", type=str, help="please input config") - args = arg_parser.parse_args() - print(type(args.dsl)) - print(args.config) - dsl = json.loads(args.dsl) - config = json.loads(args.config) - - run_job(dsl, config) - exit(0) diff --git a/docker-build/nginx/.env b/docker-build/nginx/.env deleted file mode 100644 index 11f6a42b6..000000000 --- a/docker-build/nginx/.env +++ /dev/null @@ -1 +0,0 @@ -export NGINX_VERSION=1.18.0 diff --git a/docker-build/nginx/Dockerfile b/docker-build/nginx/Dockerfile deleted file mode 100644 index e6016f8f1..000000000 --- a/docker-build/nginx/Dockerfile +++ /dev/null @@ -1,61 +0,0 @@ -FROM ubuntu:18.04 - -# Docker Build Arguments -ARG RESTY_VERSION="1.17.8.2" -ARG RESTY_LUAROCKS_VERSION="3.3.1" -ARG RESTY_OPENSSL_VERSION="1.1.1g" -ARG RESTY_OPENSSL_PATCH_VERSION="1.1.1f" -ARG RESTY_OPENSSL_URL_BASE="https://www.openssl.org/source" -ARG RESTY_PCRE_VERSION="8.44" -ARG RESTY_J="4" -ARG RESTY_PREFIX="/data/projects/fate/proxy" -ARG RESTY_CONFIG_OPTIONS="\ - --with-luajit \ - --with-http_ssl_module \ - --with-http_v2_module \ - --with-stream \ - --with-stream_ssl_module \ - " - -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - build-essential \ - ca-certificates \ - curl \ - gettext-base \ - libgd-dev \ - libgeoip-dev \ - libncurses5-dev \ - libperl-dev \ - libreadline-dev \ - libxslt1-dev \ - make \ - perl \ - unzip \ - zlib1g-dev \ - libssl-dev - -RUN cd /tmp \ - && curl -fSL https://openresty.org/download/openresty-${RESTY_VERSION}.tar.gz -o openresty-${RESTY_VERSION}.tar.gz \ - && tar xzf openresty-${RESTY_VERSION}.tar.gz \ - && cd /tmp/openresty-${RESTY_VERSION} \ - && eval ./configure --prefix=${RESTY_PREFIX} -j${RESTY_J} ${RESTY_CONFIG_OPTIONS} \ - && make -j${RESTY_J} \ - && make -j${RESTY_J} install \ - && cd /tmp \ - && rm -rf \ - openssl-${RESTY_OPENSSL_VERSION}.tar.gz openssl-${RESTY_OPENSSL_VERSION} \ - pcre-${RESTY_PCRE_VERSION}.tar.gz pcre-${RESTY_PCRE_VERSION} \ - openresty-${RESTY_VERSION}.tar.gz openresty-${RESTY_VERSION} \ - && apt-get autoremove -y - -WORKDIR /data/projects/fate/proxy/ -COPY proxy/conf ./nginx/conf -COPY proxy/lua ./nginx/lua - -ENV PATH=$PATH:/data/projects/fate/proxy/bin:/data/projects/fate/proxy/nginx/sbin - -CMD ["openresty", "-g", "daemon off;"] - -STOPSIGNAL SIGQUIT - diff --git a/docker-build/nginx/proxy/conf/nginx.conf b/docker-build/nginx/proxy/conf/nginx.conf deleted file mode 100644 index 4ca9c3733..000000000 --- a/docker-build/nginx/proxy/conf/nginx.conf +++ /dev/null @@ -1,68 +0,0 @@ - -#user nobody; -worker_processes 2; - -#error_log logs/error.log; -#error_log logs/error.log notice; -error_log logs/info.log info; -error_log logs/error.log error; - -#pid logs/nginx.pid; - - -events { - worker_connections 1024; -} - - -http { - include mime.types; - default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" "$http_host" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for" ' - '$upstream_status $upstream_addr ' - '$request_time $upstream_response_time' - ; - - access_log logs/access.log main; - - sendfile on; - #tcp_nopush on; - - keepalive_timeout 65; - underscores_in_headers on; - - #gzip on; - lua_package_path "$prefix/lua/?.lua;;"; - init_worker_by_lua_file 'lua/initialize.lua'; - - upstream http_cluster { - server 0.0.0.1; # just an invalid address as a place holder - balancer_by_lua_file 'lua/balancer.lua'; - } - - upstream grpc_cluster { - server 0.0.0.2; # just an invalid address as a place holder - balancer_by_lua_file 'lua/balancer.lua'; - } - - include vhost/*.conf; -} - -stream { - log_format tcp_proxy '$remote_addr [$time_local] ' - '$protocol $status $bytes_sent $bytes_received ' - '$session_time "$upstream_addr" ' - '"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"'; - - access_log logs/tcp-access.log tcp_proxy; - - server { - listen 9128; - proxy_connect_timeout 1s; - proxy_timeout 3s; - proxy_pass 127.0.0.1:3128; - } -} diff --git a/docker-build/nginx/proxy/conf/route_table.yaml b/docker-build/nginx/proxy/conf/route_table.yaml deleted file mode 100644 index 57ae70325..000000000 --- a/docker-build/nginx/proxy/conf/route_table.yaml +++ /dev/null @@ -1,27 +0,0 @@ -default: - proxy: - - host: 127.0.0.1 - http_port: 9300 - grpc_port: 9310 -local: - test_proxy: - - host: 127.0.0.1 - http_port: 9302 -9999: - proxy: - - host: 127.0.0.1 - http_port: 9300 - grpc_port: 9310 - fateflow: - - host: 127.0.0.1 - http_port: 9380 - grpc_port: 9360 -10000: - proxy: - - host: 127.0.0.1 - http_port: 9300 - grpc_port: 9310 - fateflow: - - host: 127.0.0.1 - http_port: 9380 - grpc_port: 9360 diff --git a/docker-build/nginx/proxy/conf/vhost/coordination_grpc_proxy.conf b/docker-build/nginx/proxy/conf/vhost/coordination_grpc_proxy.conf deleted file mode 100644 index eea756e61..000000000 --- a/docker-build/nginx/proxy/conf/vhost/coordination_grpc_proxy.conf +++ /dev/null @@ -1,18 +0,0 @@ -server { - listen 9310 http2; - server_name coordination_grpc_proxy; - - #charset koi8-r; - - access_log logs/coordination.grpc.access.log main; - - location / { - access_by_lua_file 'lua/router.lua'; - grpc_pass grpc://grpc_cluster; - } - - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root html; - } -} \ No newline at end of file diff --git a/docker-build/nginx/proxy/conf/vhost/coordination_http_proxy.conf b/docker-build/nginx/proxy/conf/vhost/coordination_http_proxy.conf deleted file mode 100644 index 93a940815..000000000 --- a/docker-build/nginx/proxy/conf/vhost/coordination_http_proxy.conf +++ /dev/null @@ -1,26 +0,0 @@ -server { - listen 9300; - server_name coordination_http_proxy; - - #charset koi8-r; - - access_log logs/coordination.http.access.log main; - - location / { - access_by_lua_file 'lua/router.lua'; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Real-PORT $remote_port; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Port $server_port; - proxy_redirect off; - proxy_set_header Connection ""; - proxy_next_upstream off; - proxy_pass http://http_cluster; - } - - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root html; - } -} \ No newline at end of file diff --git a/docker-build/nginx/proxy/conf/vhost/test.conf b/docker-build/nginx/proxy/conf/vhost/test.conf deleted file mode 100644 index fe69e40a3..000000000 --- a/docker-build/nginx/proxy/conf/vhost/test.conf +++ /dev/null @@ -1,13 +0,0 @@ -server { - listen 9302; - server_name test_proxy; - - location /test { - content_by_lua_file 'lua/check.lua'; - } - - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root html; - } -} diff --git a/docker-build/nginx/proxy/lua/balancer.lua b/docker-build/nginx/proxy/lua/balancer.lua deleted file mode 100644 index 383575516..000000000 --- a/docker-build/nginx/proxy/lua/balancer.lua +++ /dev/null @@ -1,33 +0,0 @@ --- --- Copyright 2019 The FATE Authors. All Rights Reserved. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- -local _M = { - _VERSION = '0.1' -} - -local ngx = ngx -local ngx_balancer = require "ngx.balancer" - -local function balance() - local dest_cluster = ngx.ctx.dest_cluster - local ok, err = ngx_balancer.set_current_peer(dest_cluster) - if not ok then - ngx.log(ngx.INFO, 'failed to set current peer: ' .. err, ngx.HTTP_SERVICE_UNAVAILABLE) - return ngx.ERROR - end -end - -balance() - diff --git a/docker-build/nginx/proxy/lua/check.lua b/docker-build/nginx/proxy/lua/check.lua deleted file mode 100644 index 8c65c1c6a..000000000 --- a/docker-build/nginx/proxy/lua/check.lua +++ /dev/null @@ -1,29 +0,0 @@ --- --- Copyright 2019 The FATE Authors. All Rights Reserved. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- -local _M = { - _VERSION = '0.1' -} - -local ngx = ngx -local math = require "math" -local string = require "string" - -local function check() - ngx.say("ok") -end - -check() - diff --git a/docker-build/nginx/proxy/lua/initialize.lua b/docker-build/nginx/proxy/lua/initialize.lua deleted file mode 100644 index d68820a59..000000000 --- a/docker-build/nginx/proxy/lua/initialize.lua +++ /dev/null @@ -1,20 +0,0 @@ --- --- Copyright 2019 The FATE Authors. All Rights Reserved. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- -local _M = { - _VERSION = '0.1' -} -local route_table = require "route_table" -route_table.start() diff --git a/docker-build/nginx/proxy/lua/route_table.lua b/docker-build/nginx/proxy/lua/route_table.lua deleted file mode 100644 index 80340924e..000000000 --- a/docker-build/nginx/proxy/lua/route_table.lua +++ /dev/null @@ -1,105 +0,0 @@ --- --- Copyright 2019 The FATE Authors. All Rights Reserved. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- -local ngx = ngx -local new_timer = ngx.timer.at -local yaml_parser = require "yaml_parser" -local io = io -local string = require "string" - -local _M = { - _VERSION = '0.1' -} - --- alternatively: local lrucache = require "resty.lrucache.pureffi" -local lrucache = require "resty.lrucache" - --- we need to initialize the cache on the lua module level so that --- it can be shared by all the requests served by each nginx worker process: -local route_cache, err = lrucache.new(500) -- allow up to 500 items in the cache -if not route_cache then - error("failed to create the cache: " .. (err or "unknown")) -end - -local function reload_route_table() - ngx.log(ngx.INFO, "start reload route table config") - local prefix_path = ngx.config.prefix() - local route_table_config_path = prefix_path.."conf/route_table.yaml" - local file = io.open(route_table_config_path, "r") - local content = file:read("*a") - file:close() - ngx.log(ngx.INFO, string.format("load route table config %s success", route_table_config_path)) - local yaml_table = yaml_parser.parse(content) - for k, v in pairs(yaml_table) do - route_cache:set(tostring(k), v) - end - ngx.log(ngx.INFO, "reload route table done") -end - -local function reload() - reload_route_table() - local ok, err = new_timer(5, reload) - if not ok then - if err ~= "process exiting" then - errlog("failed to create timer: ", err) - end - reload_route_table() - return - end -end - -local function get_server_address(server) - local port - if ngx.req.http_version() == 2 then - port = server["grpc_port"] - else - ngx.log(ngx.INFO, server["http_port"]) - port = server["http_port"] - end - return string.format("%s:%s", server["host"], port) -end - -function _M.get_dest_server(dest_env, dest_service) - ngx.log(ngx.INFO, string.format("try to get %s %s server", dest_env, dest_service)) - if dest_env ~= nil then - dest_env = tostring(dest_env) - else - return nil - end - - local route = _M.get_route() - local env_services = route:get(dest_env) - local server - if env_services ~= nil then - local service = env_services[dest_service] - server = get_server_address(service[math.random(1, #service)]) - ngx.log(ngx.INFO, string.format("get %s %s server: %s", dest_env, dest_service, server)) - else - local default_proxy = route:get("default")["proxy"] - server = get_server_address(default_proxy[math.random(1, #default_proxy)]) - ngx.log(ngx.INFO, string.format("get %s %s default server: %s", dest_env, dest_service, server)) - end - return server -end - -function _M.get_route() - return route_cache -end - -function _M.start() - reload() -end - -return _M diff --git a/docker-build/nginx/proxy/lua/router.lua b/docker-build/nginx/proxy/lua/router.lua deleted file mode 100644 index 57a28ae00..000000000 --- a/docker-build/nginx/proxy/lua/router.lua +++ /dev/null @@ -1,33 +0,0 @@ --- --- Copyright 2019 The FATE Authors. All Rights Reserved. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- -local _M = { - _VERSION = '0.1' -} - -local ngx = ngx -local route_table = require "route_table" - -local function routing() - local request_headers = ngx.req.get_headers() - local dest_env = request_headers["dest-party-id"] - if dest_env == nil then - dest_env = request_headers["dest-env"] - end - ngx.ctx.dest_cluster = route_table.get_dest_server(dest_env, request_headers["service"]) -end - -routing() - diff --git a/docker-build/nginx/proxy/lua/utils.lua b/docker-build/nginx/proxy/lua/utils.lua deleted file mode 100644 index b20470a4e..000000000 --- a/docker-build/nginx/proxy/lua/utils.lua +++ /dev/null @@ -1,39 +0,0 @@ --- --- Copyright 2019 The FATE Authors. All Rights Reserved. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- -local _M = { - _VERSION = '0.1' -} - -local string = require "string" - -function _M.string_startswith(str, start) - return string.sub(str, 1, string.len(start)) == start -end - -function _M.string_split(str, delimiter) - if str == nil or str == "" or delimiter==nil then - return nil - end - - local result = {} - for match in (str..delimiter):gmatch("(.-)"..delimiter) do - table.insert(result, match) - end - return result -end - -return _M - diff --git a/docker-build/nginx/proxy/lua/yaml_parser.lua b/docker-build/nginx/proxy/lua/yaml_parser.lua deleted file mode 100644 index 720bae8bf..000000000 --- a/docker-build/nginx/proxy/lua/yaml_parser.lua +++ /dev/null @@ -1,472 +0,0 @@ --- --- Copyright 2019 The FATE Authors. All Rights Reserved. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -local schar = string.char -local ssub, gsub = string.sub, string.gsub -local sfind, smatch = string.find, string.match -local tinsert, tremove = table.insert, table.remove - -local UNESCAPES = { - ['0'] = "\x00", z = "\x00", N = "\x85", - a = "\x07", b = "\x08", t = "\x09", - n = "\x0a", v = "\x0b", f = "\x0c", - r = "\x0d", e = "\x1b", ['\\'] = '\\', -} - --- help function -local function select(list, pred) - local selected = {} - for i = 0, #list do - local v = list[i] - if v and pred(v, i) then - tinsert(selected, v) - end - end - return selected -end - --- return: indent_count, left_string -local function count_indent(line) - local _, j = sfind(line, '^%s+') - if not j then - return 0, line - end - return j, ssub(line, j+1) -end - -local function trim(str) - return string.gsub(str, "^%s*(.-)%s*$", "%1") -end - -local function ltrim(str) - return smatch(str, "^%s*(.-)$") -end - -local function rtrim(str) - return smatch(str, "^(.-)%s*$") -end - -local function isemptyline(line) - return line == '' or sfind(line, '^%s*$') or sfind(line, '^%s*#') -end - -local function startswith(haystack, needle) - return ssub(haystack, 1, #needle) == needle -end - -local function startswithline(line, needle) - return startswith(line, needle) and isemptyline(ssub(line, #needle+1)) -end - --- class -local class = {__meta={}} -function class.__meta.__call(cls, ...) - local self = setmetatable({}, cls) - if cls.__init then - cls.__init(self, ...) - end - return self -end - -function class.def(base, type, cls) - base = base or class - local mt = {__metatable=base, __index=base} - for k, v in pairs(base.__meta) do mt[k] = v end - cls = setmetatable(cls or {}, mt) - cls.__index = cls - cls.__metatable = cls - cls.__type = type - cls.__meta = mt - return cls -end - -local types = { - null = class:def('null'), - map = class:def('map'), - seq = class:def('seq'), -} - -local Null = types.null -function Null.__tostring() return 'yaml.null' end -function Null.isnull(v) - if v == nil then return true end - if type(v) == 'table' and getmetatable(v) == Null then return true end - return false -end -local null = Null() - --- implement function -local function parse_string(line, stopper) - - stopper = stopper or '' - local q = ssub(line, 1, 1) - if q == ' ' or q == '\t' then - return parse_string(ssub(line, 2)) - end - - if q == "'" then - local i = sfind(line, "'", 2, true) - if not i then - return nil, line - end - return ssub(line, 2, i-1), ssub(line, i+1) - end - - if q == '"' then - local i, buf = 2, '' - while i < #line do - local c = ssub(line, i, i) - if c == '\\' then - local n = ssub(line, i+1, i+1) - if UNESCAPES[n] ~= nil then - buf = buf..UNESCAPES[n] - elseif n == 'x' then - local h = ssub(i+2,i+3) - if sfind(h, '^[0-9a-fA-F]$') then - buf = buf..schar(tonumber(h, 16)) - i = i + 2 - else - buf = buf..'x' - end - else - buf = buf..n - end - i = i + 1 - elseif c == q then - break - else - buf = buf..c - end - i = i + 1 - end - return buf, ssub(line, i+1) - end - - if q == '-' or q == ':' then - if ssub(line, 2, 2) == ' ' or #line == 1 then - return nil, line - end - end - - local buf = '' - while #line > 0 do - local c = ssub(line, 1, 1) - if sfind(stopper, c, 1, true) then - break - elseif c == ':' and (ssub(line, 2, 2) == ' ' or #line == 1) then - break - elseif c == '#' and (ssub(buf, #buf, #buf) == ' ') then - break - else - buf = buf..c - end - line = ssub(line, 2) - end - return rtrim(buf), line -end - -local function parse_flowstyle(line, lines) - local stack = {} - while true do - if #line == 0 then - if #lines == 0 then - break - else - line = tremove(lines, 1) - end - end - local c = ssub(line, 1, 1) - if c == '#' then - line = '' - elseif c == ' ' or c == '\t' or c == '\r' or c == '\n' then - line = ssub(line, 2) - elseif c == '{' or c == '[' then - tinsert(stack, {v={},t=c}) - line = ssub(line, 2) - elseif c == ':' then - local s = tremove(stack) - tinsert(stack, {v=s.v, t=':'}) - line = ssub(line, 2) - elseif c == ',' then - local value = tremove(stack) - if value.t == ':' or value.t == '{' or value.t == '[' then error() end - if stack[#stack].t == ':' then - -- map - local key = tremove(stack) - stack[#stack].v[key.v] = value.v - elseif stack[#stack].t == '{' then - -- set - stack[#stack].v[value.v] = true - elseif stack[#stack].t == '[' then - -- seq - tinsert(stack[#stack].v, value.v) - end - line = ssub(line, 2) - elseif c == '}' then - if stack[#stack].t == '{' then - if #stack == 1 then break end - stack[#stack].t = '}' - line = ssub(line, 2) - else - line = ','..line - end - elseif c == ']' then - if stack[#stack].t == '[' then - if #stack == 1 then break end - stack[#stack].t = ']' - line = ssub(line, 2) - else - line = ','..line - end - else - local s, rest = parse_string(line, ',{}[]') - if not s then - error('invalid flowstyle line: '..line) - end - tinsert(stack, {v=s, t='s'}) - line = rest - end - end - return stack[1].v, line -end - -local function parse_scalar(line, lines) - - line = ltrim(line) - line = gsub(line, '%s*#.*$', '') - - if line == '' or line == '~' then - return null - end - - if startswith(line, '{') or startswith(line, '[') then - return parse_flowstyle(line, lines) - end - - local s, _ = parse_string(line) - if s and s ~= line then - return s - end - - -- Special cases - if sfind('\'"!$', ssub(line, 1, 1), 1, true) then - error('unsupported line: '..line) - end - - if startswithline(line, '{}') then - return {} - end - if startswithline(line, '[]') then - return {} - end - - -- Regular unquoted string - local v = line - if v == 'null' or v == 'Null' or v == 'NULL'then - return null - elseif v == 'true' or v == 'True' or v == 'TRUE' then - return true - elseif v == 'false' or v == 'False' or v == 'FALSE' then - return false - elseif v == '.inf' or v == '.Inf' or v == '.INF' then - return math.huge - elseif v == '+.inf' or v == '+.Inf' or v == '+.INF' then - return math.huge - elseif v == '-.inf' or v == '-.Inf' or v == '-.INF' then - return -math.huge - elseif v == '.nan' or v == '.NaN' or v == '.NAN' then - return 0 / 0 - elseif sfind(v, '^[%+%-]?[0-9]+$') or sfind(v, '^[%+%-]?[0-9]+%.$')then - return tonumber(v) - elseif sfind(v, '^[%+%-]?[0-9]+%.[0-9]+$') then - return tonumber(v) - end - return v -end - -local parse_map - -local function parse_seq(line, lines, indent) - local seq = setmetatable({}, types.seq) - if line ~= '' then - error() - end - while #lines > 0 do - line = lines[1] - - local level = count_indent(line) - if level < indent and indent ~= -1 then - return seq - elseif level > indent and indent ~= -1 then - error("found bad indenting in line: ".. line) - end - - local i, j = sfind(line, '%-%s+') - if not i then - i, j = sfind(line, '%-$') - if not i then - return seq - end - end - local rest = ssub(line, j+1) - - if sfind(rest, '^[^\'\"%s]*:') then - local indent2 = j - lines[1] = string.rep(' ', indent2)..rest - tinsert(seq, parse_map('', lines, indent2)) - elseif isemptyline(rest) then - tremove(lines, 1) - if #lines == 0 then - tinsert(seq, null) - return seq - end - if sfind(lines[1], '^%s*%-') then - local nextline = lines[1] - local indent2 = count_indent(nextline) - if indent2 == indent then - tinsert(seq, null) - else - tinsert(seq, parse_seq('', lines, indent2)) - end - else - local nextline = lines[1] - local indent2 = count_indent(nextline) - tinsert(seq, parse_map('', lines, indent2)) - end - elseif rest then - tremove(lines, 1) - local tmp = parse_scalar(rest, lines) - tinsert(seq, tmp) - end - end - return seq -end - -function parse_map(line, lines, indent) - if not isemptyline(line) then - error('not map line: '..line) - end - local map = setmetatable({}, types.map) - while #lines > 0 do - line = lines[1] - - local level, _ = count_indent(line) - if level < indent then - return map - elseif level > indent then - error("found bad indenting in line: ".. line) - end - - local key - local s, rest = parse_string(line) - if s and startswith(rest, ':') then - local sc = parse_scalar(s, {}) - if sc and type(sc) ~= 'string' then - key = sc - else - key = s - end - line = ssub(rest, 2) - else - error("failed to classify line: "..line) - end - - if map[key] ~= nil then - print("found a duplicate key '"..key.."' in line: "..line) - local suffix = 1 - while map[key..'__'..suffix] do - suffix = suffix + 1 - end - key = key ..'_'..suffix - end - - line = ltrim(line) - - if not isemptyline(line) then - tremove(lines, 1) - line = ltrim(line) - map[key] = parse_scalar(line, lines) - else - tremove(lines, 1) - if #lines == 0 then - map[key] = null - return map; - end - if sfind(lines[1], '^%s*%-') then - local indent2 = count_indent(lines[1]) - map[key] = parse_seq('', lines, indent2) - else - local indent2 = count_indent(lines[1]) - if indent >= indent2 then - map[key] = null - else - map[key] = parse_map('', lines, indent2) - end - end - end - end - return map -end - -local function parse_documents(lines) - lines = select(lines, function(s) return not isemptyline(s) end) - - if #lines == 1 and not sfind(lines[1], '^%s*%-') then - local line = lines[1] - line = ltrim(line) - return parse_scalar(line, lines) - end - - local root = {} - while #lines > 0 do - local line = lines[1] - if sfind(line, '^%s*%-') then - tinsert(root, parse_seq('', lines, -1)) - elseif sfind(line, '^%s*[^%s]') then - local level = count_indent(line) - tinsert(root, parse_map('', lines, level)) - else - error('parse error: '..line) - end - end - - if #root > 1 and Null.isnull(root[1]) then - tremove(root, 1) - return root - end - - return root - -end - -local function parse(yaml) - local lines = {} - for line in string.gmatch(yaml..'\n', '(.-)\n') do - table.insert(lines, line) - end - - local docs = parse_documents(lines) - if #docs == 1 then - return docs[1] - end - return docs -end - -return { - null = null, - parse = parse, -} - diff --git a/docker-build/nginx/proxy/tests/client.py b/docker-build/nginx/proxy/tests/client.py deleted file mode 100644 index 52353ebc3..000000000 --- a/docker-build/nginx/proxy/tests/client.py +++ /dev/null @@ -1,71 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import json -from fate_flow.utils.proto_compatibility import basic_meta_pb2 -from fate_flow.utils.proto_compatibility import proxy_pb2, proxy_pb2_grpc -import grpc -from fate_flow.settings import FATEFLOW_SERVICE_NAME, IP, GRPC_PORT, DEFAULT_GRPC_OVERALL_TIMEOUT - - -def get_proxy_data_channel(): - channel = grpc.insecure_channel('{}:{}'.format("127.0.0.1", 9361)) - stub = proxy_pb2_grpc.DataTransferServiceStub(channel) - return channel, stub - - -def wrap_grpc_packet(_json_body, _method, _url, _src_party_id, _dst_party_id, job_id=None, overall_timeout=DEFAULT_GRPC_OVERALL_TIMEOUT): - _src_end_point = basic_meta_pb2.Endpoint(ip=IP, port=GRPC_PORT) - _src = proxy_pb2.Topic(name=job_id, partyId="{}".format(_src_party_id), role=FATEFLOW_SERVICE_NAME, callback=_src_end_point) - _dst = proxy_pb2.Topic(name=job_id, partyId="{}".format(_dst_party_id), role=FATEFLOW_SERVICE_NAME, callback=None) - _task = proxy_pb2.Task(taskId=job_id) - _command = proxy_pb2.Command(name=FATEFLOW_SERVICE_NAME) - _conf = proxy_pb2.Conf(overallTimeout=overall_timeout) - _meta = proxy_pb2.Metadata(src=_src, dst=_dst, task=_task, command=_command, operator=_method, conf=_conf) - _data = proxy_pb2.Data(key=_url, value=bytes(json.dumps(_json_body), 'utf-8')) - return proxy_pb2.Packet(header=_meta, body=_data) - - -def send(): - json_body = "dadadad" - method = "POST" - endpoint = "test" - src_party_id = 9999 - dest_party_id = 10000 - job_id = "121" - _packet = wrap_grpc_packet(json_body, method, endpoint, src_party_id, dest_party_id, job_id) - try: - channel, stub = get_proxy_data_channel() - _return, call = stub.unaryCall.with_call(_packet, metadata=( - ('k1', 'v1'), - ('k2', 'v2'), - )) - print("grpc api response: {}".format(_return)) - - for key, value in call.trailing_metadata(): - print(key, value) - """ - _return, call = stub.unaryCall.with_call(_packet) - """ - channel.close() - - json_body = json.loads(_return.body.value) - return json_body - except Exception as e: - raise Exception('rpc request error: {}'.format(e)) - - -if __name__ == '__main__': - send() diff --git a/docker-build/nginx/proxy/tests/server.py b/docker-build/nginx/proxy/tests/server.py deleted file mode 100644 index 99d2d9a95..000000000 --- a/docker-build/nginx/proxy/tests/server.py +++ /dev/null @@ -1,93 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import grpc -import json -import uuid -import collections -from concurrent import futures -from grpc._cython import cygrpc -from fate_flow.utils.proto_compatibility import proxy_pb2_grpc -from fate_flow.utils.grpc_utils import wrap_grpc_packet -import time - - -class _ClientCallDetails( - collections.namedtuple( - '_ClientCallDetails', - ('method', 'timeout', 'metadata', 'credentials', 'wait_for_ready')), - grpc.ClientCallDetails): - pass - - -class RequestIDClientInterceptor(grpc.UnaryUnaryClientInterceptor): - - def intercept_unary_unary(self, continuation, client_call_details, request): - rid = uuid.uuid1() - print(f"Sending RPC request, Method: {client_call_details.method}, Request ID: {rid}.") - - # Add request into client call details, aka, metadata. - metadata = [] - if client_call_details.metadata is not None: - metadata = list(client_call_details.metadata) - metadata.append(("request_id", rid)) - - client_call_details = _ClientCallDetails( - client_call_details.method, client_call_details.timeout, metadata, - client_call_details.credentials, client_call_details.wait_for_ready) - return continuation(client_call_details, request) - - -class UnaryServicer(proxy_pb2_grpc.DataTransferServiceServicer): - def unaryCall(self, _request, context): - packet = _request - header = packet.header - _suffix = packet.body.key - param_bytes = packet.body.value - param = bytes.decode(param_bytes) - job_id = header.task.taskId - src = header.src - dst = header.dst - method = header.operator - resp_json = "xczczcz" - - for key, value in context.invocation_metadata(): - print(key, value) - - context.set_trailing_metadata(( - ('retcode', "0"), - ('msg', 'ok'), - )) - - return wrap_grpc_packet(resp_json, method, _suffix, dst.partyId, src.partyId, job_id) - - -def serve(): - server = grpc.server(futures.ThreadPoolExecutor(max_workers=10), - options=[(cygrpc.ChannelArgKey.max_send_message_length, -1), - (cygrpc.ChannelArgKey.max_receive_message_length, -1)]) - - proxy_pb2_grpc.add_DataTransferServiceServicer_to_server(UnaryServicer(), server) - server.add_insecure_port("{}:{}".format("0.0.0.0", 9360)) - server.start() - try: - while True: - time.sleep(10) - except KeyboardInterrupt: - server.stop(0) - - -if __name__ == "__main__": - serve() diff --git a/docker-build/python-spark/Dockerfile b/docker-build/python-spark/Dockerfile deleted file mode 100644 index 88adec4b1..000000000 --- a/docker-build/python-spark/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -ARG SOURCE_PREFIX=federatedai -ARG SOURCE_TAG=1.5.0-release -FROM ${SOURCE_PREFIX}/python:${SOURCE_TAG} - -RUN rpm --rebuilddb && \ - rpm --import /etc/pki/rpm-gpg/RPM* && \ - yum install -y which java-1.8.0-openjdk wget && \ - yum clean all && \ - wget https://archive.apache.org/dist/hadoop/common/hadoop-2.7.4/hadoop-2.7.4.tar.gz && \ - tar -xf ./hadoop-2.7.4.tar.gz -C /data/projects/ && rm ./hadoop-2.7.4.tar.gz - -RUN wget https://archive.apache.org/dist/spark/spark-2.4.1/spark-2.4.1-bin-hadoop2.7.tgz && \ - tar -xf ./spark-2.4.1-bin-hadoop2.7.tgz -C /data/projects/ && rm ./spark-2.4.1-bin-hadoop2.7.tgz - -ENV JAVA_HOME=/usr/lib/jvm/jre-1.8.0-openjdk -ENV SPARK_HOME=/data/projects/spark-2.4.1-bin-hadoop2.7/ -ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/projects/hadoop-2.7.4/lib/native -ENV PATH=$PATH:/data/projects/spark-2.4.1-bin-hadoop2.7/bin:/data/projects/hadoop-2.7.4/bin - -RUN pip install pyspark \ No newline at end of file diff --git a/docker-build/spark/base/Dockerfile b/docker-build/spark/base/Dockerfile deleted file mode 100644 index 63c78a6fa..000000000 --- a/docker-build/spark/base/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -# Refer to https://github.com/big-data-europe/docker-spark/tree/2.4.1-hadoop2.7/base -ARG SOURCE_PREFIX=federatedai -ARG SOURCE_TAG=1.5.0-release -FROM ${SOURCE_PREFIX}/python:${SOURCE_TAG} - -ENV ENABLE_INIT_DAEMON true -ENV INIT_DAEMON_BASE_URI http://identifier/init-daemon -ENV INIT_DAEMON_STEP spark_master_init - -ENV SPARK_VERSION=2.4.1 -ENV HADOOP_VERSION=2.7 - -WORKDIR / - -RUN set -eux && \ - rpm --rebuilddb && \ - rpm --import /etc/pki/rpm-gpg/RPM* && \ - yum -y install gcc gcc-c++ make openssl-devel gmp-devel mpfr-devel libmpc-devel\ - libmpcdevel libaio numactl autoconf automake libtool libffi-devel \ - snappy snappy-devel zlib zlib-devel bzip2 bzip2-devel lz4-devel libasan lsof sysstat telnet psmisc wget && \ - yum install -y which java-1.8.0-openjdk java-1.8.0-openjdk-devel && \ - yum clean all && \ - wget https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz && \ - tar -xvzf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz && \ - mv spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} spark && \ - rm spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz && \ - cd / - -ENV PYTHONPATH=$PYTHONPATH:/data/projects/fate/python -ENV JAVA_HOME=/usr/lib/jvm/jre-1.8.0-openjdk diff --git a/docker-build/spark/master/Dockerfile b/docker-build/spark/master/Dockerfile deleted file mode 100644 index 517bb06e4..000000000 --- a/docker-build/spark/master/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -ARG SOURCE_PREFIX=federatedai -ARG SOURCE_TAG=1.5.0-release -FROM ${SOURCE_PREFIX}/spark-base:${SOURCE_TAG} - -COPY master.sh / - -ENV SPARK_MASTER_PORT 7077 -ENV SPARK_MASTER_WEBUI_PORT 8080 -ENV SPARK_MASTER_LOG /spark/logs - -EXPOSE 8080 7077 6066 - -CMD ["/bin/bash", "/master.sh"] \ No newline at end of file diff --git a/docker-build/spark/master/master.sh b/docker-build/spark/master/master.sh deleted file mode 100644 index 6bca75c03..000000000 --- a/docker-build/spark/master/master.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -export SPARK_MASTER_HOST=`hostname` - -. "/spark/sbin/spark-config.sh" - -. "/spark/bin/load-spark-env.sh" - -mkdir -p $SPARK_MASTER_LOG - -export SPARK_HOME=/spark - -ln -sf /dev/stdout $SPARK_MASTER_LOG/spark-master.out - -cd /spark/bin && /spark/sbin/../bin/spark-class org.apache.spark.deploy.master.Master \ - --ip $SPARK_MASTER_HOST --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT >> $SPARK_MASTER_LOG/spark-master.out diff --git a/docker-build/spark/worker/Dockerfile b/docker-build/spark/worker/Dockerfile deleted file mode 100644 index a97e6f9d7..000000000 --- a/docker-build/spark/worker/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -ARG SOURCE_PREFIX=federatedai -ARG SOURCE_TAG=1.5.0-release -FROM ${SOURCE_PREFIX}/spark-base:${SOURCE_TAG} - -COPY worker.sh / - -ENV SPARK_WORKER_WEBUI_PORT 8081 -ENV SPARK_WORKER_LOG /spark/logs -ENV SPARK_MASTER "spark://spark-master:7077" - -EXPOSE 8081 - -CMD ["/bin/bash", "/worker.sh"] \ No newline at end of file diff --git a/docker-build/spark/worker/worker.sh b/docker-build/spark/worker/worker.sh deleted file mode 100644 index 619f223e6..000000000 --- a/docker-build/spark/worker/worker.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -. "/spark/sbin/spark-config.sh" - -. "/spark/bin/load-spark-env.sh" - -mkdir -p $SPARK_WORKER_LOG - -export SPARK_HOME=/spark - -ln -sf /dev/stdout $SPARK_WORKER_LOG/spark-worker.out - -/spark/sbin/../bin/spark-class org.apache.spark.deploy.worker.Worker \ - --webui-port $SPARK_WORKER_WEBUI_PORT $SPARK_MASTER >> $SPARK_WORKER_LOG/spark-worker.out diff --git a/docker-deploy/.env b/docker-deploy/.env index fe6e37fc2..75857e184 100644 --- a/docker-deploy/.env +++ b/docker-deploy/.env @@ -1,7 +1,9 @@ RegistryURI= -TAG=1.8.0-release -SERVING_TAG=2.1.5-release +TAG=1.9.0-release +SERVING_TAG=2.1.6-release +SSH_PORT=22 # PREFIX: namespace on the registry's server. # RegistryURI: address of the local registry # TAG: tag of module images. +# SSH_PORT: port of SSH, default 22 diff --git a/docker-deploy/README.md b/docker-deploy/README.md index 246d0f76d..c27f8ef75 100644 --- a/docker-deploy/README.md +++ b/docker-deploy/README.md @@ -9,7 +9,9 @@ The nodes (target nodes) to install FATE must meet the following requirements: 1. A Linux host 2. Docker: 18+ 3. Docker-Compose: 1.24+ -4. Network connection to Internet to pull container images from Docker Hub. If network connection to Internet is not available, consider to set up [Harbor as a local registry](../registry/README.md) or use [offline images](https://github.com/FederatedAI/FATE/tree/master/docker-build). +4. The deployment machine have access to the Internet, so the hosts can communicate with each other; +5. Network connection to Internet to pull container images from Docker Hub. If network connection to Internet is not available, consider to set up [Harbor as a local registry](../registry/README.md) or use [offline images](https://github.com/FederatedAI/FATE/tree/master/build/docker-build). +6. A host running FATE is recommended to be with 8 CPUs and 16G RAM. ## Deploying FATE @@ -40,7 +42,7 @@ RegistryURI=192.168.10.1/federatedai ... ``` -**NOTE:** For Chinese user who has difficulty to access docker hub, you can set `RegistryURI` to `hub.c.163.com` to use the mirror of the registry within China. +**NOTE:** For Chinese user who has difficulty to access docker hub, you can set `RegistryURI` to `hub.c.163.com` to use the mirror of the registry within China, we have already pushed the images to the 163 registry. ### Configuring multiple parties of FATE @@ -50,7 +52,9 @@ The following steps illustrate how to generate necessary configuration files and Before deploying the FATE system, multiple parties should be defined in the configuration file: `docker-deploy/parties.conf`. -In the following sample of `docker-deploy/parties.conf` , two parities are specified by id as `10000` and `9999`. Their cluster are going to be deployed on hosts with IP addresses of *192.168.7.1* and *192.168.7.2*. By default, to save time for downloading images, KubeFATE will use images without neural network dependencies, set the `enabled_nn` to `true` in "parties.conf" if neural network workflow is required. +The meaning of the `parties.conf` configuration file configuration items see this document [parties.conf file introduction](../docs/configurations/Docker_compose_Partys_configuration.md) + +In the following sample of `docker-deploy/parties.conf` , two parities are specified by id as `10000` and `9999`. Their clusters are going to be deployed on hosts with IP addresses of *192.168.7.1* and *192.168.7.2*. ```bash user=fate @@ -59,34 +63,26 @@ party_list=(10000 9999) party_ip_list=(192.168.7.1 192.168.7.2) serving_ip_list=(192.168.7.1 192.168.7.2) -# backend could be eggroll, spark_rabbitmq and spark_pulsar spark_local_pulsar -backend=eggroll +computing=Eggroll +federation=Eggroll +storage=Eggroll -# true if you need python-nn else false, the default value will be false -enabled_nn=false +algorithm=Basic +device=IPCL -# default -exchangeip= +compute_core=4 -# modify if you are going to use an external db -mysql_ip=mysql -mysql_user=fate -mysql_password=fate_dev -mysql_db=fate_flow +...... -name_node=hdfs://namenode:9000 +``` -# Define fateboard login information -fateboard_username=admin -fateboard_password=admin +* For more details about FATE on Spark with Rebbitmq please refer to this [document](../docs/FATE_On_Spark.md). +* For more details about FATE on Spark with Pulsar, refer to this [document](../docs/FATE_On_Spark_With_Pulsar.md) +* For more details about FATE on Spark with local pulsar, refer to this [document](placeholder) -# Define serving admin login information -serving_admin_username=admin -serving_admin_password=admin -``` +Using Docker-compose to deploy FATE can support the combination of many different types of engines (choice of computing federation storage), for more details about different types of FATE see: [Architecture introduction of different types of FATE](../docs/Introduction_to_Engine_Architecture.md). -Spark was introduced in FATE v1.5 as the underlying computing backend, for more details -about FATE on Spark please refer to this [document](../docs/FATE_On_Spark.md). +**Note**: Exchange components are not deployed by default. For deployment, users can fill in the server IP into the `exchangeip` of the above configuration file. The default listening port of this component is 9371. On the host running FATE, the non-root user needs the owner permission of `/data/projects/fate` folder and Docker permission. No other action is required if the user is root. @@ -112,7 +108,7 @@ drwxr-xr-x. 2 fate docker 6 May 27 00:51 fate By default, the exchange service is not deployed. The exchange service runs on port 9371. If an exchange (co-locates on the host of the same party or runs standalone) service is needed, update the value of `exchangeip` to the IP address of the desired host. -After editting the above configuration file, use the following commands to generate configuration of target hosts. +After editing the above configuration file, use the following commands to generate configuration of target hosts. ```bash cd docker-deploy @@ -125,7 +121,7 @@ Now, tar files have been generated for each party including the exchange node (p **Note:** Before running the below commands, all target hosts must -* allow password-less SSH access with SSH key; +* allow password-less SSH access with SSH key (Otherwise we will need to enter the password for each host for multiple times). * meet the requirements specified in [Prerequisites](#Prerequisites). To deploy FATE to all configured target hosts, use the below command: @@ -170,18 +166,18 @@ CONTAINER ID IMAGE COMMAND 3dca43f3c9d5 federatedai/serving-admin:2.1.5-release "/bin/sh -c 'java -c…" 5 minutes ago Up 5 minutes 0.0.0.0:8350->8350/tcp, :::8350->8350/tcp serving-9999_serving-admin_1 fe924918509b federatedai/serving-proxy:2.1.5-release "/bin/sh -c 'java -D…" 5 minutes ago Up 5 minutes 0.0.0.0:8059->8059/tcp, :::8059->8059/tcp, 0.0.0.0:8869->8869/tcp, :::8869->8869/tcp, 8879/tcp serving-9999_serving-proxy_1 b62ed8ba42b7 bitnami/zookeeper:3.7.0 "/opt/bitnami/script…" 5 minutes ago Up 5 minutes 0.0.0.0:2181->2181/tcp, :::2181->2181/tcp, 8080/tcp, 0.0.0.0:49226->2888/tcp, :::49226->2888/tcp, 0.0.0.0:49225->3888/tcp, :::49225->3888/tcp serving-9999_serving-zookeeper_1 -3c643324066f federatedai/client:1.8.0-release "/bin/sh -c 'flow in…" 5 minutes ago Up 5 minutes 0.0.0.0:20000->20000/tcp, :::20000->20000/tcp confs-9999_client_1 -3fe0af1ebd71 federatedai/fateboard:1.8.0-release "/bin/sh -c 'java -D…" 5 minutes ago Up 5 minutes 0.0.0.0:8080->8080/tcp, :::8080->8080/tcp confs-9999_fateboard_1 -635b7d99357e federatedai/python:1.8.0-release "container-entrypoin…" 5 minutes ago Up 5 minutes (healthy) 0.0.0.0:9360->9360/tcp, :::9360->9360/tcp, 8080/tcp, 0.0.0.0:9380->9380/tcp, :::9380->9380/tcp confs-9999_python_1 -8b515f08add3 federatedai/eggroll:1.8.0-release "/tini -- bash -c 'j…" 5 minutes ago Up 5 minutes 8080/tcp, 0.0.0.0:9370->9370/tcp, :::9370->9370/tcp confs-9999_rollsite_1 -108cc061c191 federatedai/eggroll:1.8.0-release "/tini -- bash -c 'j…" 5 minutes ago Up 5 minutes 4670/tcp, 8080/tcp confs-9999_clustermanager_1 -f10575e76899 federatedai/eggroll:1.8.0-release "/tini -- bash -c 'j…" 5 minutes ago Up 5 minutes 4671/tcp, 8080/tcp confs-9999_nodemanager_1 +3c643324066f federatedai/client:1.9.0-release "/bin/sh -c 'flow in…" 5 minutes ago Up 5 minutes 0.0.0.0:20000->20000/tcp, :::20000->20000/tcp confs-9999_client_1 +3fe0af1ebd71 federatedai/fateboard:1.9.0-release "/bin/sh -c 'java -D…" 5 minutes ago Up 5 minutes 0.0.0.0:8080->8080/tcp, :::8080->8080/tcp confs-9999_fateboard_1 +635b7d99357e federatedai/fateflow:1.9.0-release "container-entrypoin…" 5 minutes ago Up 5 minutes (healthy) 0.0.0.0:9360->9360/tcp, :::9360->9360/tcp, 8080/tcp, 0.0.0.0:9380->9380/tcp, :::9380->9380/tcp confs-9999_fateflow_1 +8b515f08add3 federatedai/eggroll:1.9.0-release "/tini -- bash -c 'j…" 5 minutes ago Up 5 minutes 8080/tcp, 0.0.0.0:9370->9370/tcp, :::9370->9370/tcp confs-9999_rollsite_1 +108cc061c191 federatedai/eggroll:1.9.0-release "/tini -- bash -c 'j…" 5 minutes ago Up 5 minutes 4670/tcp, 8080/tcp confs-9999_clustermanager_1 +f10575e76899 federatedai/eggroll:1.9.0-release "/tini -- bash -c 'j…" 5 minutes ago Up 5 minutes 4671/tcp, 8080/tcp confs-9999_nodemanager_1 aa0a0002de93 mysql:8.0.28 "docker-entrypoint.s…" 5 minutes ago Up 5 minutes 3306/tcp, 33060/tcp confs-9999_mysql_1 ``` ### Verifying the deployment -On the target node of each party, a container named `confs-_python_1` should have been created and running the `fate-flow` service. For example, on Party 10000's node, run the following commands to verify the deployment: +On the target node of each party, a container named `confs-_fateflow_1` should have been created and running the `fate-flow` service. For example, on Party 10000's node, run the following commands to verify the deployment: ```bash docker exec -it confs-10000_client_1 bash @@ -201,13 +197,11 @@ If the test passed, the output may look like the following: "2019-08-29 07:21:34,118 - secure_add_guest.py[line:121] - INFO: success to calculate secure_sum, it is 2000.0000000000002" ``` -For more details about the testing result, please refer to `python/examples/toy_example/README.md` . - ### Verifying the serving service #### Steps on the host -##### Logging in to the python container +##### Logging in to the client container ```bash docker exec -it confs-10000_client_1 bash @@ -236,7 +230,7 @@ flow data upload -c fateflow/examples/upload/upload_host.json #### Steps on the guest -##### Getting in to the python container +##### Getting in to the client container ```bash docker exec -it confs-9999_client_1 bash diff --git a/docker-deploy/README_zh.md b/docker-deploy/README_zh.md index 5acc0085d..5877da6e3 100644 --- a/docker-deploy/README_zh.md +++ b/docker-deploy/README_zh.md @@ -20,7 +20,8 @@ Compose是用于定义和运行多容器Docker应用程序的工具。通过Comp 2. 所有主机安装Docker 版本 : 18+； 3. 所有主机安装Docker-Compose 版本: 1.24+； 4. 部署机可以联网，所以主机相互之间可以网络互通； -5. 运行机已经下载FATE 的各组件镜像（离线构建镜像参考文档[构建镜像](https://github.com/FederatedAI/FATE/tree/master/docker-build)）。 +5. 运行机已经下载FATE的各组件镜像，如果无法连接dockerhub，请考虑使用harbor（[Harbor 作为本地镜像源](../registry/README.md)）或者使用离线部署（离线构建镜像参考文档[构建镜像](https://github.com/FederatedAI/FATE/tree/master/build/docker-build)）。 +6. 运行FATE的主机推荐配置8CPUs和16G RAM。 ### 下载部署脚本 @@ -46,7 +47,7 @@ RegistryURI=hub.c.163.com ```bash docker pull federatedai/eggroll:-release docker pull federatedai/fateboard:-release -docker pull federatedai/python:-release +docker pull federatedai/fateflow:-release docker pull federatedai/serving-server:-release docker pull federatedai/serving-proxy:-release docker pull federatedai/serving-admin:-release @@ -61,7 +62,7 @@ $ docker images REPOSITORY TAG federatedai/eggroll -release federatedai/fateboard -release -federatedai/python -release +federatedai/fateflow -release federatedai/client -release federatedai/serving-server -release federatedai/serving-proxy -release @@ -93,7 +94,9 @@ RegistryURI=192.168.10.1/federatedai 根据需求修改配置文件`kubeFATE\docker-deploy\parties.conf`。 -下面是修改好的文件，`party 10000`的集群将部署在*192.168.7.1*上，而`party 9999`的集群将部署在*192.168.7.2*上。为了减少所需拉取镜像的大小，KubeFATE在默认情况下，会使用不带神经网络的“python”容器，若需要跑神经网络的算法则需把“parties.conf”中的`enabled_nn`设置成`true`。 +`parties.conf`配置文件配置项的含义查看这个文档[parties.conf文件介绍](../docs/configurations/Docker_compose_Partys_configuration.md) + +下面是修改好的文件，`party 10000`的集群将部署在*192.168.7.1*上，而`party 9999`的集群将部署在*192.168.7.2*上。 ```bash user=fate @@ -102,35 +105,28 @@ party_list=(10000 9999) party_ip_list=(192.168.7.1 192.168.7.2) serving_ip_list=(192.168.7.1 192.168.7.2) -# backend could be eggroll, spark_rabbitmq and spark_pulsar spark_local_pulsar -backend=eggroll +computing=Eggroll +federation=Eggroll +storage=Eggroll -# true if you need python-nn else false, the default value will be false -enabled_nn=false +algorithm=Basic +device=IPCL -# default -exchangeip= +compute_core=4 -# modify if you are going to use an external db -mysql_ip=mysql -mysql_user=fate -mysql_password=fate_dev -mysql_db=fate_flow +...... -name_node=hdfs://namenode:9000 +``` -# Define fateboard login information -fateboard_username=admin -fateboard_password=admin +* 使用Spark+Rabbitmq的部署方式的文档可以参考[这里](../docs/FATE_On_Spark.md). +* 使用Spark+Pulsar的部署方式的文档可以参考[这里](../docs/FATE_On_Spark_With_Pulsar.md). +* 使用Spark+local Pulsar的部署方式的文档可以参考[这里](TBD) -# Define serving admin login information -serving_admin_username=admin -serving_admin_password=admin -``` +使用Docker-compose部署FATE可以支持多种种不同的类型引擎的组合(对computing federation storage的选择)，关于不同类型的FATE的更多细节查看: [不同类型FATE的架构介绍](../docs/Introduction_to_Engine_Architecture_zh.md)。 -FATE v1.5 开始支持使用Spark作为底层的分布式计算引擎，Spark集群默认会通过容器的方式部署。相关的简介可以参考这个[链接](../docs/FATE_On_Spark.md). +`algorithm`和`device`的配置可以查看这里[FATE_Algorithm_and_Computational_Acceleration_Selection.md](../docs/FATE_Algorithm_and_Computational_Acceleration_Selection.md) -**注意**: 默认情况下不会部署exchange组件。如需部署，用户可以把服务器IP填入上述配置文件的`exchangeip`中，该组件的默认监听端口为9371 +**注意**: 默认情况下不会部署exchange组件。如需部署，用户可以把服务器IP填入上述配置文件的`exchangeip`中，该组件的默认监听端口为9371。 在运行部署脚本之前，需要确保部署机器可以ssh免密登录到两个运行节点主机上。user代表免密的用户。 @@ -189,12 +185,12 @@ CONTAINER ID IMAGE COMMAND 3dca43f3c9d5 federatedai/serving-admin:2.1.5-release "/bin/sh -c 'java -c…" 5 minutes ago Up 5 minutes 0.0.0.0:8350->8350/tcp, :::8350->8350/tcp serving-9999_serving-admin_1 fe924918509b federatedai/serving-proxy:2.1.5-release "/bin/sh -c 'java -D…" 5 minutes ago Up 5 minutes 0.0.0.0:8059->8059/tcp, :::8059->8059/tcp, 0.0.0.0:8869->8869/tcp, :::8869->8869/tcp, 8879/tcp serving-9999_serving-proxy_1 b62ed8ba42b7 bitnami/zookeeper:3.7.0 "/opt/bitnami/script…" 5 minutes ago Up 5 minutes 0.0.0.0:2181->2181/tcp, :::2181->2181/tcp, 8080/tcp, 0.0.0.0:49226->2888/tcp, :::49226->2888/tcp, 0.0.0.0:49225->3888/tcp, :::49225->3888/tcp serving-9999_serving-zookeeper_1 -3c643324066f federatedai/client:1.8.0-release "/bin/sh -c 'flow in…" 5 minutes ago Up 5 minutes 0.0.0.0:20000->20000/tcp, :::20000->20000/tcp confs-9999_client_1 -3fe0af1ebd71 federatedai/fateboard:1.8.0-release "/bin/sh -c 'java -D…" 5 minutes ago Up 5 minutes 0.0.0.0:8080->8080/tcp, :::8080->8080/tcp confs-9999_fateboard_1 -635b7d99357e federatedai/python:1.8.0-release "container-entrypoin…" 5 minutes ago Up 5 minutes (healthy) 0.0.0.0:9360->9360/tcp, :::9360->9360/tcp, 8080/tcp, 0.0.0.0:9380->9380/tcp, :::9380->9380/tcp confs-9999_python_1 -8b515f08add3 federatedai/eggroll:1.8.0-release "/tini -- bash -c 'j…" 5 minutes ago Up 5 minutes 8080/tcp, 0.0.0.0:9370->9370/tcp, :::9370->9370/tcp confs-9999_rollsite_1 -108cc061c191 federatedai/eggroll:1.8.0-release "/tini -- bash -c 'j…" 5 minutes ago Up 5 minutes 4670/tcp, 8080/tcp confs-9999_clustermanager_1 -f10575e76899 federatedai/eggroll:1.8.0-release "/tini -- bash -c 'j…" 5 minutes ago Up 5 minutes 4671/tcp, 8080/tcp confs-9999_nodemanager_1 +3c643324066f federatedai/client:1.9.0-release "/bin/sh -c 'flow in…" 5 minutes ago Up 5 minutes 0.0.0.0:20000->20000/tcp, :::20000->20000/tcp confs-9999_client_1 +3fe0af1ebd71 federatedai/fateboard:1.9.0-release "/bin/sh -c 'java -D…" 5 minutes ago Up 5 minutes 0.0.0.0:8080->8080/tcp, :::8080->8080/tcp confs-9999_fateboard_1 +635b7d99357e federatedai/fateflow:1.9.0-release "container-entrypoin…" 5 minutes ago Up 5 minutes (healthy) 0.0.0.0:9360->9360/tcp, :::9360->9360/tcp, 8080/tcp, 0.0.0.0:9380->9380/tcp, :::9380->9380/tcp confs-9999_fateflow_1 +8b515f08add3 federatedai/eggroll:1.9.0-release "/tini -- bash -c 'j…" 5 minutes ago Up 5 minutes 8080/tcp, 0.0.0.0:9370->9370/tcp, :::9370->9370/tcp confs-9999_rollsite_1 +108cc061c191 federatedai/eggroll:1.9.0-release "/tini -- bash -c 'j…" 5 minutes ago Up 5 minutes 4670/tcp, 8080/tcp confs-9999_clustermanager_1 +f10575e76899 federatedai/eggroll:1.9.0-release "/tini -- bash -c 'j…" 5 minutes ago Up 5 minutes 4671/tcp, 8080/tcp confs-9999_nodemanager_1 aa0a0002de93 mysql:8.0.28 "docker-entrypoint.s…" 5 minutes ago Up 5 minutes 3306/tcp, 33060/tcp confs-9999_mysql_1 ``` @@ -206,7 +202,7 @@ docker-compose上的FATE启动成功之后需要验证各个服务是否都正 ```bash #在192.168.7.1上执行下列命令 -$ docker exec -it confs-10000_client_1 bash #进入python组件容器内部 +$ docker exec -it confs-10000_client_1 bash #进入client组件容器内部 $ flow test toy --guest-party-id 10000 --host-party-id 9999 #验证 ``` @@ -223,8 +219,6 @@ $ flow test toy --guest-party-id 10000 --host-party-id 9999 #验证 "2019-08-29 07:21:34,118 - secure_add_guest.py[line:121] - INFO: success to calculate secure_sum, it is 2000.0000000000002" ``` -有关测试结果的更多详细信息，请参阅"python/examples/toy_example/README.md"这个文件 。 - ### 验证Serving-Service功能 #### Host方操作 @@ -751,18 +745,6 @@ rm -rf ../confs-/ # 删除docker-compose部署文件 ### 可能遇到的问题 -#### python容器退出 - -```bash -docker exec -it confs-10000_python_1 bash -``` - -进入docker容器后马上又弹出来了。 - -解决办法：稍等一会再尝试。 - -因为python服务依赖其他所有服务的正常运行，然而第一次启动的时候MySQL需要初始化数据库，python服务的容器会出现几次重启，当MySQL等其他服务都运行正常之后，就可以正常执行了。 - #### 采用docker hub下载镜像速度可能较慢 解决办法：可以自己构建镜像，自己构建镜像参考[这里](https://github.com/FederatedAI/FATE/tree/master/docker-build)。 diff --git a/docker-deploy/docker_deploy.sh b/docker-deploy/docker_deploy.sh index 7d75e2e4f..7ef6275c5 100755 --- a/docker-deploy/docker_deploy.sh +++ b/docker-deploy/docker_deploy.sh @@ -153,10 +153,10 @@ DeployPartyInternal() { return 0 fi - scp ${WORKINGDIR}/outputs/confs-$target_party_id.tar $user@$target_party_ip:~/ + scp -P ${SSH_PORT} ${WORKINGDIR}/outputs/confs-$target_party_id.tar $user@$target_party_ip:~/ #rm -f ${WORKINGDIR}/outputs/confs-$target_party_id.tar echo "$target_party_ip training cluster copy is ok!" - ssh -tt $user@$target_party_ip <#${proxy_port}#g" ./confs-$party_id/confs/eggroll/conf/eggroll.properties fi - # generate conf dir + if [ "$computing" == "Spark" ]; then + # computing + cp -r training_template/backends/spark/nginx confs-$party_id/confs/ + cp -r training_template/backends/spark/spark confs-$party_id/confs/ + # storage + if [ "$storage" == "HDFS" ]; then + cp training_template/docker-compose-spark.yml confs-$party_id/docker-compose.yml + cp -r training_template/backends/spark/hadoop confs-$party_id/confs/ + # federation + if [ "$federation" == "RabbitMQ" ]; then + cp -r training_template/backends/spark/rabbitmq confs-$party_id/confs/ + sed -i '201,215d' confs-$party_id/docker-compose.yml + elif [ "$federation" == "Pulsar" ]; then + cp -r training_template/backends/spark/pulsar confs-$party_id/confs/ + sed -i '182,199d' confs-$party_id/docker-compose.yml + fi + fi + fi + + if [ "$computing" == "Spark_local" ]; then + # computing + cp -r training_template/backends/spark/nginx confs-$party_id/confs/ + cp -r training_template/backends/spark/spark confs-$party_id/confs/ + # storage + if [ "$storage" == "LocalFS" ]; then + cp training_template/docker-compose-spark-slim.yml confs-$party_id/docker-compose.yml + # federation + if [ "$federation" == "RabbitMQ" ]; then + cp -r training_template/backends/spark/rabbitmq confs-$party_id/confs/ + sed -i '147,161d' confs-$party_id/docker-compose.yml + elif [ "$federation" == "Pulsar" ]; then + cp -r training_template/backends/spark/pulsar confs-$party_id/confs/ + sed -i '129,145d' confs-$party_id/docker-compose.yml + fi + fi + fi + cp ${WORKINGDIR}/.env ./confs-$party_id + + # Modify the configuration file + + # Images choose + Suffix="" + # computing + if [ "$computing" == "Spark" ] || [ "$computing" == "Spark_local" ]; then + Suffix=$Suffix"-spark" + fi + # algorithm + if [ "$algorithm" == "NN" ]; then + Suffix=$Suffix"-nn" + fi + # device + if [ "$device" == "IPCL" ]; then + Suffix=$Suffix"-ipcl" + fi + + # federatedai/fateflow-${computing}-${algorithm}-${device}:${version} + sed -i "s#image: \"federatedai/fateflow:\${TAG}\"#image: \"federatedai/fateflow${Suffix}:\${TAG}\"#g" ./confs-$party_id/docker-compose.yml - # check if use python-nn - if [ "$enabled_nn" = "true" ]; then - sed -i 's#image: "federatedai/python:${TAG}"#image: "federatedai/python-nn:${TAG}"#g' ./confs-$party_id/docker-compose.yml + # eggroll or spark-worker + if [ "$computing" == "Eggroll" ]; then + sed -i "s#image: \"federatedai/eggroll:\${TAG}\"#image: \"federatedai/eggroll${Suffix}:\${TAG}\"#g" ./confs-$party_id/docker-compose.yml + elif [ "$computing" == "Spark" ]; then + sed -i "s#image: \"federatedai/spark-worker:\${TAG}\"#image: \"federatedai/spark-worker${Suffix}:\${TAG}\"#g" ./confs-$party_id/docker-compose.yml fi + # RegistryURI if [ "$RegistryURI" != "" ]; then sed -i 's#federatedai#${RegistryURI}/federatedai#g' ./confs-$party_id/docker-compose.yml sed -i 's#image: "mysql:8"#image: ${RegistryURI}/federatedai/mysql:8#g' ./confs-$party_id/docker-compose.yml @@ -182,7 +296,7 @@ GenerateConfig() { echo "CREATE USER '${db_user}'@'%' IDENTIFIED BY '${db_password}';" >>./confs-$party_id/confs/mysql/init/insert-node.sql echo "GRANT ALL ON *.* TO '${db_user}'@'%';" >>./confs-$party_id/confs/mysql/init/insert-node.sql - if [[ "$backend" == "eggroll" ]]; then + if [[ "$computing" == "Eggroll" ]]; then echo 'USE `'${db_name}'`;' >>./confs-$party_id/confs/mysql/init/insert-node.sql echo "INSERT INTO server_node (host, port, node_type, status) values ('${clustermanager_ip}', '${clustermanager_port_db}', 'CLUSTER_MANAGER', 'HEALTHY');" >>./confs-$party_id/confs/mysql/init/insert-node.sql for ((j = 0; j < ${#nodemanager_ip[*]}; j++)); do @@ -205,35 +319,41 @@ GenerateConfig() { sed -i "s/127.0.0.1:8000/${serving_ip}:8000/g" ./confs-$party_id/confs/fate_flow/conf/service_conf.yaml - if [[ "$backend" == "spark_rabbitmq" ]]; then + if [[ "$computing" == "Spark" ]] || [[ "$computing" == "Spark_local" ]] ; then sed -i "s/proxy: rollsite/proxy: nginx/g" ./confs-$party_id/confs/fate_flow/conf/service_conf.yaml - # - sed -i "s/ computing: eggroll/ computing: spark/g" ./confs-$party_id/confs/fate_flow/conf/service_conf.yaml - sed -i "s/ federation: eggroll/ federation: rabbitmq/g" ./confs-$party_id/confs/fate_flow/conf/service_conf.yaml - sed -i "s/ storage: eggroll/ storage: hdfs/g" ./confs-$party_id/confs/fate_flow/conf/service_conf.yaml + sed -i "s/computing: eggroll/computing: spark/g" ./confs-$party_id/confs/fate_flow/conf/service_conf.yaml fi - - if [[ "$backend" == "spark_pulsar" ]]; then - sed -i "s/proxy: rollsite/proxy: nginx/g" ./confs-$party_id/confs/fate_flow/conf/service_conf.yaml - # - sed -i "s/ computing: eggroll/ computing: spark/g" ./confs-$party_id/confs/fate_flow/conf/service_conf.yaml + if [[ "$federation" == "Pulsar" ]]; then sed -i "s/ federation: eggroll/ federation: pulsar/g" ./confs-$party_id/confs/fate_flow/conf/service_conf.yaml - sed -i "s/ storage: eggroll/ storage: hdfs/g" ./confs-$party_id/confs/fate_flow/conf/service_conf.yaml + elif [[ "$federation" == "RabbitMQ" ]]; then + sed -i "s/ federation: eggroll/ federation: RabbitMQ/g" ./confs-$party_id/confs/fate_flow/conf/service_conf.yaml fi - if [ "$backend" == "spark_local_pulsar" ]; then - sed -i 's/proxy: rollsite/proxy: nginx/g' ./confs-$party_id/confs/fate_flow/conf/service_conf.yaml - - sed -i "s/ computing: eggroll/ computing: spark/g" ./confs-$party_id/confs/fate_flow/conf/service_conf.yaml - sed -i "s/ federation: eggroll/ federation: pulsar/g" ./confs-$party_id/confs/fate_flow/conf/service_conf.yaml + if [[ "$storage" == "HDFS" ]]; then + sed -i "s/ storage: eggroll/ storage: hdfs/g" ./confs-$party_id/confs/fate_flow/conf/service_conf.yaml + elif [[ "$storage" == "LocalFS" ]]; then sed -i "s/ storage: eggroll/ storage: localfs/g" ./confs-$party_id/confs/fate_flow/conf/service_conf.yaml fi + if [[ "$computing" == "Spark_local" ]] ; then + sed -i "s#spark.master .*#spark.master local[*]#g" ./confs-$party_id/confs/spark/spark-defaults.conf + fi + + # compute_core + sed -i "s/nodes: .*/nodes: 1/g" ./confs-$party_id/confs/fate_flow/conf/service_conf.yaml + sed -i "s/cores_per_node: .*/cores_per_node: $compute_core/g" ./confs-$party_id/confs/fate_flow/conf/service_conf.yaml + + if [[ "$federation" == "Eggroll" ]]; then + sed -i "s/eggroll.session.processors.per.node=.*/eggroll.session.processors.per.node=$compute_core/g" ./confs-$party_id/confs/eggroll/conf/eggroll.properties + fi + if [[ "$computing" == "Spark"* ]]; then + sed -i "s/spark.cores.max .*/spark.cores.max $compute_core/g" ./confs-$party_id/confs/spark/spark-defaults.conf + fi echo fate_flow module of $party_id done! - # now we handles the route table + # federation config # eggroll - if [[ "$backend" == "eggroll" ]]; then + if [[ "$federation" == "Eggroll" ]]; then cat >./confs-$party_id/confs/eggroll/conf/route_table.json <./confs-$party_id/confs/nginx/route_table.yaml <./confs-$party_id/confs/fate_flow/conf/pulsar_route_table.yaml <./confs-$party_id/confs/fate_flow/conf/rabbitmq_route_table.yaml <./confs-$party_id/confs/fate_flow/conf/pulsar_route_table.yaml <|serving-$party_id|g" ./serving-$party_id/docker-compose.yml + + if [ "$RegistryURI" != "" ]; then sed -i 's#federatedai#${RegistryURI}/federatedai#g' ./serving-$party_id/docker-compose.yml fi @@ -537,6 +646,7 @@ main() { exit 1 else CleanOutputDir + CheckConfig GenerateConfig fi diff --git a/docker-deploy/parties.conf b/docker-deploy/parties.conf index e0b9774a1..8812093fe 100644 --- a/docker-deploy/parties.conf +++ b/docker-deploy/parties.conf @@ -6,11 +6,20 @@ party_list=(10000 9999) party_ip_list=(192.168.1.1 192.168.1.2) serving_ip_list=(192.168.1.1 192.168.1.2) -# backend could be eggroll, spark_rabbitmq and spark_pulsar spark_local_pulsar -backend=eggroll - -# true if you need python-nn else false, the default value will be false -enabled_nn=false +# Engines: +# Computing : Eggroll, Spark, Spark_local +computing=Eggroll +# Federation: Eggroll(computing: Eggroll), Pulsar/RabbitMQ(computing: Spark/Spark_local) +federation=Eggroll +# Storage: Eggroll(computing: Eggroll), HDFS(computing: Spark), LocalFS(computing: Spark_local) +storage=Eggroll +# Algorithm: Basic, NN +algorithm=Basic +# Device: IPCL, CPU +device=CPU + +# spark and eggroll +compute_core=4 # default exchangeip= diff --git a/docker-deploy/serving_template/docker-compose-serving.yml b/docker-deploy/serving_template/docker-compose-serving.yml index cca2ce290..a1f13c507 100644 --- a/docker-deploy/serving_template/docker-compose-serving.yml +++ b/docker-deploy/serving_template/docker-compose-serving.yml @@ -17,6 +17,21 @@ networks: fate-serving-network: external: name: + +volumes: + data_dir_server: + driver: local + driver_opts: + type: none + o: bind + device: /data/server + data_dir_zookeeper: + driver: local + driver_opts: + type: none + o: bind + device: /data/zookeeper + services: serving-server: image: "federatedai/serving-server:${SERVING_TAG}" @@ -24,7 +39,8 @@ services: - "8000:8000" volumes: - ./confs/serving-server/conf/serving-server.properties:/data/projects/fate-serving/serving-server/conf/serving-server.properties - - ./confs/serving-server/cache:/root/.fate + - ./data/server:/data/projects/fate-serving/serving-server/.fate + - /etc/localtime:/etc/localtime:ro networks: - fate-serving-network @@ -38,15 +54,20 @@ services: volumes: - ./confs/serving-proxy/conf/application.properties:/data/projects/fate-serving/serving-proxy/conf/application.properties - ./confs/serving-proxy/conf/route_table.json:/data/projects/fate-serving/serving-proxy/conf/route_table.json + - /etc/localtime:/etc/localtime:ro networks: - fate-serving-network serving-zookeeper: image: "bitnami/zookeeper:3.7.0" + user: root ports: - "2181:2181" - "2888" - "3888" + volumes: + - ./data/zookeeper:/bitnami/zookeeper + - /etc/localtime:/etc/localtime:ro environment: ALLOW_ANONYMOUS_LOGIN: "yes" networks: @@ -58,5 +79,6 @@ services: - "8350:8350" volumes: - ./confs/serving-admin/conf/application.properties:/data/projects/fate-serving/serving-admin/conf/application.properties + - /etc/localtime:/etc/localtime:ro networks: - fate-serving-network diff --git a/docker-deploy/training_template/backends/spark/nginx/nginx.conf b/docker-deploy/training_template/backends/spark/nginx/nginx.conf index 14ca94e71..e448fd286 100644 --- a/docker-deploy/training_template/backends/spark/nginx/nginx.conf +++ b/docker-deploy/training_template/backends/spark/nginx/nginx.conf @@ -39,12 +39,12 @@ http { init_worker_by_lua_file 'lua/initialize.lua'; upstream http_cluster { - server python:9380; # just an invalid address as a place holder + server fateflow:9380; # just an invalid address as a place holder balancer_by_lua_file 'lua/balancer.lua'; } upstream grpc_cluster { - server python:9360; # just an invalid address as a place holder + server fateflow:9360; # just an invalid address as a place holder balancer_by_lua_file 'lua/balancer.lua'; } diff --git a/docker-deploy/training_template/backends/spark/spark/spark-defaults.conf b/docker-deploy/training_template/backends/spark/spark/spark-defaults.conf index 1384f7fbb..fdeaf62c4 100644 --- a/docker-deploy/training_template/backends/spark/spark/spark-defaults.conf +++ b/docker-deploy/training_template/backends/spark/spark/spark-defaults.conf @@ -1,3 +1,4 @@ spark.master spark://spark-master:7077 #spark.eventLog.enabled true #spark.eventLog.dir hdfs://namenode:9000/spark/logs +spark.cores.max 4 \ No newline at end of file diff --git a/docker-deploy/training_template/docker-compose-eggroll.yml b/docker-deploy/training_template/docker-compose-eggroll.yml index 9107bc536..49ef770f9 100644 --- a/docker-deploy/training_template/docker-compose-eggroll.yml +++ b/docker-deploy/training_template/docker-compose-eggroll.yml @@ -1,5 +1,5 @@ -# Copyright 2019-2020 VMware, Inc. -# +# Copyright 2019-2022 VMware, Inc. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # you may obtain a copy of the License at @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -version: '3' +version: '3.7' networks: fate-network: ipam: @@ -49,6 +49,8 @@ services: PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python volumes: - ./confs/eggroll/conf:/data/projects/fate/eggroll/conf + - /etc/localtime:/etc/localtime:ro + - shared_dir_federatedml:/data/projects/fate/fate/python/federatedml networks: - fate-network command: ["bash", "-c", "java -Dlog4j.configurationFile=$${EGGROLL_HOME}/conf/log4j2.properties -cp $${EGGROLL_HOME}/lib/*:$${EGGROLL_HOME}/conf/ com.webank.eggroll.rollsite.EggSiteBootstrap -c $${EGGROLL_HOME}/conf/eggroll.properties"] @@ -61,10 +63,11 @@ services: volumes: - ./confs/fateboard/conf:/data/projects/fate/fateboard/conf - fate_flow_logs:/data/projects/fate/fateflow/logs + - /etc/localtime:/etc/localtime:ro networks: - fate-network depends_on: - - python + - fateflow clustermanager: image: "federatedai/eggroll:${TAG}" @@ -73,6 +76,8 @@ services: - 4670 volumes: - ./confs/eggroll/conf:/data/projects/fate/eggroll/conf + - /etc/localtime:/etc/localtime:ro + - shared_dir_federatedml:/data/projects/fate/fate/python/federatedml environment: PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python networks: @@ -88,15 +93,18 @@ services: - ./confs/eggroll/conf:/data/projects/fate/eggroll/conf - ./confs/fate_flow/conf/service_conf.yaml:/data/projects/fate/conf/service_conf.yaml - ./shared_dir/data/nodemanager:/data/projects/fate/eggroll/data + - /etc/localtime:/etc/localtime:ro + - shared_dir_federatedml:/data/projects/fate/fate/python/federatedml networks: - fate-network command: ["bash", "-c", "java -Dlog4j.configurationFile=$${EGGROLL_HOME}/conf/log4j2.properties -cp $${EGGROLL_HOME}/lib/*: com.webank.eggroll.core.Bootstrap --bootstraps com.webank.eggroll.core.resourcemanager.NodeManagerBootstrap -c $${EGGROLL_HOME}/conf/eggroll.properties -p 4671 -s 'EGGROLL_DEAMON'"] - python: - image: "federatedai/python:${TAG}" + fateflow: + image: "federatedai/fateflow:${TAG}" environment: - PYTHONPATH: "$PYTHONPATH:/data/projects/fate/fate/python:/data/projects/fate/eggroll/python:/data/projects/fate/fateflow/python:/data/projects/fate/fate/python/fate_client" + PYTHONPATH: "$$PYTHONPATH:/data/projects/fate/fate/python:/data/projects/fate/eggroll/python:/data/projects/fate/fateflow/python:/data/projects/fate/fate/python/fate_client" FATE_PROJECT_BASE: "/data/projects/fate" + FATE_LOG_LEVEL: "INFO" ports: - "9360:9360" - "9380:9380" @@ -105,10 +113,16 @@ services: - shared_dir_federatedml:/data/projects/fate/fate/python/federatedml - shared_dir_examples:/data/projects/fate/examples - download_dir:/data/projects/fate/python/download_dir - - ./confs/fate_flow/conf:/data/projects/fate/conf + - ./confs/fate_flow/conf/service_conf.yaml:/data/projects/fate/conf/service_conf.yaml + - ./confs/fate_flow/conf/pulsar_route_table.yaml:/data/projects/fate/conf/pulsar_route_table.yaml + - ./confs/fate_flow/conf/rabbitmq_route_table.yaml:/data/projects/fate/conf/rabbitmq_route_table.yaml + - ./confs/fate_flow/conf/component_registry.json:/data/projects/fate/fateflow/conf/component_registry.json + - ./confs/fate_flow/conf/job_default_config.yaml:/data/projects/fate/fateflow/conf/job_default_config.yaml + - ./confs/fate_flow/conf/template_info.yaml:/data/projects/fate/fateflow/conf/template_info.yaml - ./confs/eggroll/conf:/data/projects/fate/eggroll/conf - fate_flow_logs:/data/projects/fate/fateflow/logs - ./shared_dir/data/model_local_cache:/data/projects/fate/fateflow/model_local_cache + - /etc/localtime:/etc/localtime:ro depends_on: - mysql - rollsite @@ -136,14 +150,15 @@ services: - "20000:20000" restart: always environment: - FATE_FLOW_IP: "python" + FATE_FLOW_IP: "fateflow" FATE_FLOW_PORT: "9380" FATE_SERVING_HOST: "fate-serving:8059" volumes: - download_dir:/data/projects/fate/download_dir - shared_dir_examples:/data/projects/fate/examples + - /etc/localtime:/etc/localtime:ro depends_on: - - python + - fateflow networks: - fate-network @@ -154,6 +169,7 @@ services: volumes: - ./confs/mysql/init:/docker-entrypoint-initdb.d/ - ./shared_dir/data/mysql:/var/lib/mysql + - /etc/localtime:/etc/localtime:ro restart: always environment: MYSQL_ALLOW_EMPTY_PASSWORD: "yes" diff --git a/docker-deploy/training_template/docker-compose-exchange.yml b/docker-deploy/training_template/docker-compose-exchange.yml index 5d16f0ed5..b6cb916ba 100644 --- a/docker-deploy/training_template/docker-compose-exchange.yml +++ b/docker-deploy/training_template/docker-compose-exchange.yml @@ -1,4 +1,16 @@ -version: '3' +# Copyright 2019-2022 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# you may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +version: '3.7' services: exchange: @@ -8,4 +20,5 @@ services: - "9371:9370" volumes: - ./conf:/data/projects/fate/eggroll/conf + - /etc/localtime:/etc/localtime:ro command: ["bash", "-c", "java -Dlog4j.configurationFile=$${EGGROLL_HOME}/conf/log4j2.properties -cp $${EGGROLL_HOME}/lib/*:$${EGGROLL_HOME}/conf/ com.webank.eggroll.rollsite.EggSiteBootstrap -c $${EGGROLL_HOME}/conf/eggroll.properties"] diff --git a/docker-deploy/training_template/docker-compose-spark-slim.yml b/docker-deploy/training_template/docker-compose-spark-slim.yml index 195201693..228e3d321 100644 --- a/docker-deploy/training_template/docker-compose-spark-slim.yml +++ b/docker-deploy/training_template/docker-compose-spark-slim.yml @@ -1,4 +1,4 @@ -# Copyright 2019-2020 VMware, Inc. +# Copyright 2019-2022 VMware, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -version: "3" +version: "3.7" networks: fate-network: @@ -48,26 +48,33 @@ services: volumes: - ./confs/fateboard/conf:/data/projects/fate/fateboard/conf - fate_flow_logs:/data/projects/fate/fateflow/logs + - /etc/localtime:/etc/localtime:ro restart: always networks: - fate-network depends_on: - - python + - fateflow - python: - image: "federatedai/python-spark:${TAG}" + fateflow: + image: "federatedai/fateflow:${TAG}" restart: always ports: - 9380:9380 - 9360:9360 volumes: - - ./confs/spark/spark-defaults.conf:/data/projects/spark-2.4.1-bin-hadoop2.7/conf/spark-defaults.conf - - ./confs/fate_flow/conf:/data/projects/fate/conf + - ./confs/spark/spark-defaults.conf:/data/projects/spark-3.1.2-bin-hadoop3.2/conf/spark-defaults.conf + - ./confs/fate_flow/conf/service_conf.yaml:/data/projects/fate/conf/service_conf.yaml + - ./confs/fate_flow/conf/pulsar_route_table.yaml:/data/projects/fate/conf/pulsar_route_table.yaml + - ./confs/fate_flow/conf/rabbitmq_route_table.yaml:/data/projects/fate/conf/rabbitmq_route_table.yaml + - ./confs/fate_flow/conf/component_registry.json:/data/projects/fate/fateflow/conf/component_registry.json + - ./confs/fate_flow/conf/job_default_config.yaml:/data/projects/fate/fateflow/conf/job_default_config.yaml + - ./confs/fate_flow/conf/template_info.yaml:/data/projects/fate/fateflow/conf/template_info.yaml - shared_dir_federatedml:/data/projects/fate/fate/python/federatedml - shared_dir_examples:/data/projects/fate/examples - download_dir:/data/projects/fate/python/download_dir - fate_flow_logs:/data/projects/fate/fateflow/logs - ./shared_dir/data/model_local_cache:/data/projects/fate/fateflow/model_local_cache + - /etc/localtime:/etc/localtime:ro networks: fate-network: ipv4_address: 192.167.0.100 @@ -84,10 +91,11 @@ services: set -x sleep 5 && python fateflow/python/fate_flow/fate_flow_server.py environment: - PYTHONPATH: "$PYTHONPATH:/data/projects/fate/fate/python:/data/projects/fate/eggroll/python:/data/projects/fate/fateflow/python:/data/projects/fate/fate/python/fate_client" + PYTHONPATH: "$$PYTHONPATH:/data/projects/fate/fate/python:/data/projects/fate/eggroll/python:/data/projects/fate/fateflow/python:/data/projects/fate/fate/python/fate_client" FATE_PROJECT_BASE: "/data/projects/fate" FATE_FLOW_UPLOAD_MAX_NUM: "1000000" FATE_FLOW_UPLOAD_MAX_BYTES: "104868093952" + FATE_LOG_LEVEL: "INFO" mysql: image: "mysql:8.0.28" @@ -96,6 +104,7 @@ services: volumes: - ./confs/mysql/init:/docker-entrypoint-initdb.d/ - ./shared_dir/data/mysql:/var/lib/mysql + - /etc/localtime:/etc/localtime:ro restart: always environment: MYSQL_ALLOW_EMPTY_PASSWORD: "yes" @@ -110,11 +119,30 @@ services: volumes: - ./confs/nginx/route_table.yaml:/data/projects/fate/proxy/nginx/conf/route_table.yaml - ./confs/nginx/nginx.conf:/data/projects/fate/proxy/nginx/conf/nginx.conf + - /etc/localtime:/etc/localtime:ro restart: always networks: - fate-network depends_on: - - python + - fateflow + + rabbitmq: + image: federatedai/rabbitmq:3.8.3-management + ports: + - "5672:5672" + - "15672:15672" + environment: + RABBITMQ_DEFAULT_USER: fate + RABBITMQ_DEFAULT_PASS: fate + RABBITMQ_USER: fate + RABBITMQ_PASSWORD: fate + RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "-rabbit max_message_size 536870912" + volumes: + - ./confs/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins + - ./shared_dir/data/rabbitmq:/var/lib/rabbitmq + restart: always + networks: + - fate-network pulsar: image: "federatedai/pulsar:2.7.0" @@ -122,12 +150,10 @@ services: - "6650:6650" - "6651:6651" - "8001:8080" - deploy: - restart_policy: - condition: any volumes: - ./confs/pulsar/standalone.conf:/pulsar/conf/standalone.conf - ./shared_dir/data/pulsar:/pulsar/data + - /etc/localtime:/etc/localtime:ro command: ["/bin/bash", "-c", "bin/pulsar standalone -nss"] restart: always @@ -140,13 +166,14 @@ services: - "20000:20000" restart: always environment: - FATE_FLOW_IP: "python" + FATE_FLOW_IP: "fateflow" FATE_FLOW_PORT: "9380" FATE_SERVING_HOST: "fate-serving:8059" volumes: - download_dir:/data/projects/fate/download_dir - shared_dir_examples:/data/projects/fate/examples + - /etc/localtime:/etc/localtime:ro depends_on: - - python + - fateflow networks: - fate-network \ No newline at end of file diff --git a/docker-deploy/training_template/docker-compose-spark.yml b/docker-deploy/training_template/docker-compose-spark.yml index 0945858ed..2f9f0c8b5 100644 --- a/docker-deploy/training_template/docker-compose-spark.yml +++ b/docker-deploy/training_template/docker-compose-spark.yml @@ -1,4 +1,4 @@ -# Copyright 2019-2020 VMware, Inc. +# Copyright 2019-2022 VMware, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -version: "3" +version: "3.7" networks: fate-network: @@ -47,26 +47,33 @@ services: volumes: - ./confs/fateboard/conf:/data/projects/fate/fateboard/conf - fate_flow_logs:/data/projects/fate/fateflow/logs + - /etc/localtime:/etc/localtime:ro networks: - fate-network restart: always depends_on: - - python + - fateflow - python: - image: "federatedai/python-spark:${TAG}" + fateflow: + image: "federatedai/fateflow:${TAG}" restart: always ports: - 9380:9380 - 9360:9360 volumes: - - ./confs/spark/spark-defaults.conf:/data/projects/spark-2.4.1-bin-hadoop2.7/conf/spark-defaults.conf - - ./confs/fate_flow/conf:/data/projects/fate/conf + - ./confs/spark/spark-defaults.conf:/data/projects/spark-3.1.2-bin-hadoop3.2/conf/spark-defaults.conf + - ./confs/fate_flow/conf/service_conf.yaml:/data/projects/fate/conf/service_conf.yaml + - ./confs/fate_flow/conf/pulsar_route_table.yaml:/data/projects/fate/conf/pulsar_route_table.yaml + - ./confs/fate_flow/conf/rabbitmq_route_table.yaml:/data/projects/fate/conf/rabbitmq_route_table.yaml + - ./confs/fate_flow/conf/component_registry.json:/data/projects/fate/fateflow/conf/component_registry.json + - ./confs/fate_flow/conf/job_default_config.yaml:/data/projects/fate/fateflow/conf/job_default_config.yaml + - ./confs/fate_flow/conf/template_info.yaml:/data/projects/fate/fateflow/conf/template_info.yaml - shared_dir_federatedml:/data/projects/fate/fate/python/federatedml - shared_dir_examples:/data/projects/fate/examples - download_dir:/data/projects/fate/python/download_dir - fate_flow_logs:/data/projects/fate/fateflow/logs - ./shared_dir/data/model_local_cache:/data/projects/fate/fateflow/model_local_cache + - /etc/localtime:/etc/localtime:ro networks: fate-network: ipv4_address: 192.167.0.100 @@ -83,21 +90,22 @@ services: set -x sleep 5 && python fateflow/python/fate_flow/fate_flow_server.py environment: - PYTHONPATH: "$PYTHONPATH:/data/projects/fate/fate/python:/data/projects/fate/eggroll/python:/data/projects/fate/fateflow/python:/data/projects/fate/fate/python/fate_client" + PYTHONPATH: "$$PYTHONPATH:/data/projects/fate/fate/python:/data/projects/fate/eggroll/python:/data/projects/fate/fateflow/python:/data/projects/fate/fate/python/fate_client" FATE_PROJECT_BASE: "/data/projects/fate" FATE_FLOW_UPLOAD_MAX_NUM: "1000000" FATE_FLOW_UPLOAD_MAX_BYTES: "104868093952" + FATE_LOG_LEVEL: "INFO" namenode: - image: federatedai/hadoop-namenode:2.0.0-hadoop2.7.4-java8 + image: federatedai/hadoop-namenode:2.0.0-hadoop3.2.1-java8 restart: always ports: - 9000:9000 - 9870:9870 - - 50070:50070 volumes: - ./shared_dir/data/namenode:/hadoop/dfs/name - ./confs/hadoop/core-site.xml:/etc/hadoop/core-site.xml + - /etc/localtime:/etc/localtime:ro env_file: - ./confs/hadoop/hadoop.env environment: @@ -105,11 +113,37 @@ services: networks: - fate-network - datanode: - image: federatedai/hadoop-datanode:2.0.0-hadoop2.7.4-java8 + datanode-0: + image: federatedai/hadoop-datanode:2.0.0-hadoop3.2.1-java8 + restart: always + volumes: + - ./shared_dir/data/datanode-0:/hadoop/dfs/data + environment: + SERVICE_PRECONDITION: "namenode:9000" + env_file: + - ./confs/hadoop/hadoop.env + networks: + - fate-network + + datanode-1: + image: federatedai/hadoop-datanode:2.0.0-hadoop3.2.1-java8 + restart: always + volumes: + - ./shared_dir/data/datanode-1:/hadoop/dfs/data + environment: + SERVICE_PRECONDITION: "namenode:9000" + env_file: + - ./confs/hadoop/hadoop.env + networks: + - fate-network + + datanode-2: + image: federatedai/hadoop-datanode:2.0.0-hadoop3.2.1-java8 restart: always volumes: - ./shared_dir/data/datanode:/hadoop/dfs/data + - /etc/localtime:/etc/localtime:ro + - ./shared_dir/data/datanode-2:/hadoop/dfs/data environment: SERVICE_PRECONDITION: "namenode:9000" env_file: @@ -123,6 +157,8 @@ services: ports: - "8888:8080" - "7077:7077" + volumes: + - /etc/localtime:/etc/localtime:ro environment: INIT_DAEMON_STEP: setup_spark networks: @@ -139,6 +175,7 @@ services: SPARK_MASTER: "spark://spark-master:7077" volumes: - ./confs/fate_flow/conf:/data/projects/fate/conf + - /etc/localtime:/etc/localtime:ro networks: - fate-network @@ -156,6 +193,7 @@ services: volumes: - ./confs/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins - ./shared_dir/data/rabbitmq:/var/lib/rabbitmq + - /etc/localtime:/etc/localtime:ro restart: always networks: - fate-network @@ -166,12 +204,10 @@ services: - "6650:6650" - "6651:6651" - "8001:8080" - deploy: - restart_policy: - condition: any volumes: - ./confs/pulsar/standalone.conf:/pulsar/conf/standalone.conf - ./shared_dir/data/pulsar:/pulsar/data + - /etc/localtime:/etc/localtime:ro command: ["/bin/bash", "-c", "bin/pulsar standalone -nss"] restart: always @@ -185,6 +221,7 @@ services: volumes: - ./confs/mysql/init:/docker-entrypoint-initdb.d/ - ./shared_dir/data/mysql:/var/lib/mysql + - /etc/localtime:/etc/localtime:ro restart: always environment: MYSQL_ALLOW_EMPTY_PASSWORD: "yes" @@ -199,11 +236,12 @@ services: volumes: - ./confs/nginx/route_table.yaml:/data/projects/fate/proxy/nginx/conf/route_table.yaml - ./confs/nginx/nginx.conf:/data/projects/fate/proxy/nginx/conf/nginx.conf + - /etc/localtime:/etc/localtime:ro restart: always networks: - fate-network depends_on: - - python + - fateflow client: image: "federatedai/client:${TAG}" @@ -211,13 +249,14 @@ services: - "20000:20000" restart: always environment: - FATE_FLOW_IP: "python" + FATE_FLOW_IP: "fateflow" FATE_FLOW_PORT: "9380" FATE_SERVING_HOST: "fate-serving:8059" volumes: - download_dir:/data/projects/fate/download_dir - shared_dir_examples:/data/projects/fate/examples + - /etc/localtime:/etc/localtime:ro depends_on: - - python + - fateflow networks: - fate-network diff --git a/docker-deploy/training_template/public/fate_flow/conf/job_default_config.yaml b/docker-deploy/training_template/public/fate_flow/conf/job_default_config.yaml index e0c58862f..a6362a079 100644 --- a/docker-deploy/training_template/public/fate_flow/conf/job_default_config.yaml +++ b/docker-deploy/training_template/public/fate_flow/conf/job_default_config.yaml @@ -19,9 +19,11 @@ auto_retries: 0 auto_retry_delay: 1 #seconds # It can also be specified in the job configuration using the federated_status_collect_type parameter federated_status_collect_type: PUSH +detect_connect_max_retry_count: 3 +detect_connect_long_retry_count: 2 # upload -upload_max_bytes: 4194304000 # bytes +upload_max_bytes: 104857600 # bytes #component output output_data_summary_count_limit: 100 \ No newline at end of file diff --git a/docker-deploy/training_template/public/fate_flow/conf/service_conf.yaml b/docker-deploy/training_template/public/fate_flow/conf/service_conf.yaml index 0bd4d69c1..7ec13f019 100644 --- a/docker-deploy/training_template/public/fate_flow/conf/service_conf.yaml +++ b/docker-deploy/training_template/public/fate_flow/conf/service_conf.yaml @@ -4,13 +4,28 @@ dependent_distribution: false encrypt_password: false encrypt_module: fate_arch.common.encrypt_utils#pwdecrypt private_key: +party_id: +hook_module: + client_authentication: fate_flow.hook.flow.client_authentication + site_authentication: fate_flow.hook.flow.site_authentication + permission: fate_flow.hook.flow.permission +hook_server_name: +authentication: + client: + switch: false + http_app_key: + http_secret_key: + site: + switch: false +permission: + switch: false + component: false + dataset: false fateflow: # you must set real ip address, 127.0.0.1 and 0.0.0.0 is not supported host: 192.167.0.100 http_port: 9380 grpc_port: 9360 - # http_app_key: - # http_secret_key: # support rollsite/nginx/fateflow as a coordination proxy # rollsite support fate on eggroll, use grpc protocol # nginx support fate on eggroll and fate on spark, use http or grpc protocol, default is http @@ -42,7 +57,7 @@ zookeeper: # use_acl: false # user: fate # password: fate -# # engine services + # engine services default_engines: computing: eggroll federation: eggroll @@ -61,7 +76,7 @@ fate_on_eggroll: fate_on_spark: spark: # default use SPARK_HOME environment variable - home: /data/projects/spark-2.4.1-bin-hadoop2.7/ + home: /data/projects/spark-3.1.2-bin-hadoop3.2/ cores_per_node: 20 nodes: 2 linkis_spark: @@ -92,6 +107,9 @@ fate_on_spark: password: fate # default conf/rabbitmq_route_table.yaml route_table: + # mode: replication / client, default: replication + mode: replication + max_message_size: 1048576 pulsar: host: pulsar mng_port: 8080 @@ -103,6 +121,9 @@ fate_on_spark: topic_ttl: 5 # default conf/pulsar_route_table.yaml route_table: + # mode: replication / client, default: replication + mode: replication + max_message_size: 1048576 nginx: host: nginx http_port: 9300 @@ -112,10 +133,6 @@ fateboard: host: fateboard port: 8080 -# on API `/model/load` and `/model/load/do` -# automatic upload models to the model store if it exists locally but does not exist in the model storage -# or download models from the model store if it does not exist locally but exists in the model storage -# this config will not affect API `/model/store` or `/model/restore` enable_model_store: false model_store_address: # use mysql as the model store engine @@ -128,14 +145,7 @@ model_store_address: # other optional configs send to the engine # max_connections: 10 # stale_timeout: 10 - # use redis as the model store engine -# storage: redis -# host: 127.0.0.1 -# port: 6379 -# db: 0 -# password: - # the expiry time of keys, in seconds. defaults None (no expiry time) -# ex: + # use tencent cos as model store engine storage: tencent_cos Region: diff --git a/docker-deploy/training_template/public/fate_flow/conf/template_info.yaml b/docker-deploy/training_template/public/fate_flow/conf/template_info.yaml new file mode 100644 index 000000000..ae5140e78 --- /dev/null +++ b/docker-deploy/training_template/public/fate_flow/conf/template_info.yaml @@ -0,0 +1,10 @@ +# base dir: fateflow +template_path: + fate_examples: ../examples + fateflow_examples: examples +template_data: + base_dir: ../examples/data + min_data: ['breast_hetero_guest.csv', 'breast_hetero_host.csv', 'default_credit_hetero_guest.csv', 'default_credit_hetero_host.csv'] + +delete_path: + fateflow_examples: [data] \ No newline at end of file diff --git a/docker-deploy/training_template/public/fateboard/conf/application.properties b/docker-deploy/training_template/public/fateboard/conf/application.properties index 9ff1f0591..31e06e8a7 100644 --- a/docker-deploy/training_template/public/fateboard/conf/application.properties +++ b/docker-deploy/training_template/public/fateboard/conf/application.properties @@ -1,5 +1,8 @@ server.port=8080 fateflow.url=http://localhost:9380 +#priority is higher than {fateflow.url}, split by ; +#below config can support configuring more than one fate flow for this fate board +fateflow.url-list= fateflow.http_app_key= fateflow.http_secret_key= spring.http.encoding.charset=UTF-8 @@ -17,4 +20,6 @@ server.board.login.username= server.board.login.password= server.servlet.session.timeout=4h server.servlet.session.cookie.max-age=4h -management.endpoints.web.exposure.exclude=* \ No newline at end of file +management.endpoints.web.exposure.exclude=* +feign.client.config.default.connectTimeout=10000 +feign.client.config.default.readTimeout=10000 \ No newline at end of file diff --git a/docs/Customize_KubeFATE_Chart.md b/docs/Customize_KubeFATE_Chart.md index 1fcaec3fe..fe18b09ef 100644 --- a/docs/Customize_KubeFATE_Chart.md +++ b/docs/Customize_KubeFATE_Chart.md @@ -3,7 +3,7 @@ KubeFATE is designed to deploy and manage clusters based on Helm Chart. In KubeF 1. receives the customized setting through `cluster.yaml` or `cluster-serving.yaml`; 2. render `values.yaml` in Chart Template with `Go Template`. -* Note: The content of `values.yaml` will persistent into MySQL in the KubeFATE service. If you going to do some hacks on the above process, make sure you understand this. +* Note: The content of `values.yaml` will be persisted into MySQL in the KubeFATE service. If you are going to do some hacks on the above process, make sure you understand this. In another word, the Helm Chart of KubeFATE is a twofold chart, which is specified for FATE and FATE-Serving. @@ -17,34 +17,30 @@ They are both complicated and large but take it easy. There are only small featu # KubeFATE's Chart Structure All charts using in KubeFATE are located at https://github.com/FederatedAI/KubeFATE/tree/gh-pages/package. And you can find the developed one from https://github.com/FederatedAI/KubeFATE/tree/master/helm-charts. We suggested you develop your version based on one chart in https://github.com/FederatedAI/KubeFATE/tree/gh-pages/package, which are fine verified. -Unzip one KubeFATE's Chart, you can find a `templates` folder and 4 files" +Unzip one KubeFATE's Chart, you can find a `templates` folder and 4 files: 1. `Chart.yaml`: a YAML file containing information about the chart; 2. `value.yaml`: the default values for chart according to Helm Chart standard; 3. `values-template-example.yaml`: the example files of what values-template will look like; -4. `value-template.yaml`: core file for developed a customized KubeFATE's chart. It bridges the `cluster.yaml`/`cluser-serving.yaml` and templates. The values set in `cluster.yaml`/`cluser-serving.yaml` will passed to here, and set all the variables using in the templates. It follows the `Go Template` standard. +4. `value-template.yaml`: core file for developed a customized KubeFATE's chart. It bridges the `cluster.yaml`/`cluser-serving.yaml` and templates. The values set in `cluster.yaml`/`cluser-serving.yaml` will be passed to here, and set all the variables using in the templates. It follows the `Go Template` standard. * Note: `value.yaml` and the rendered `value-template.yaml` will be merged as the "VALUEs" to the chart templates. ## `templates` folder In `templates` folder, the template yaml file combined with values will generate valid Kubernetes manifest files for each `FATE` or `FATE-Serving` component. -e.g. For `FATE` v1.4.2, there are following templates locating in `template` folder: -1. clustermanager-module.yaml: cluster-manager module -2. eggroll-config.yaml: eggroll module -3. mysql-module.yaml: MySQL module including the creating SQL -4. nodemanager.yaml: node-manager module -5. rollsite-module.yaml: roll-site module -6. python-module.yaml: python, fate-flow, fate-board, notebook/client module and related python libs. -we supported both ingress and istio gateway for HTTP traffic, -7. ingress.yaml: ingress for fate-board and notebook/client -8. istio.yaml: istio compatible service gateway for fate-board and notebook/client. But this feature is **experimental**. -and in some KubeFATE's charts, there is an optional NOTE.txt file locating in `template` folder to describe short usage notes. +e.g. For `FATE` v1.9.0, there are following templates locating in `template` folder: +1. eggroll: eggroll module, including 3 eggroll related components: clustermanager, nodemanager and rollsite/lb-rollsite. +2. spark: spark module, including spark, hdfs, nginx, pulsar/rabbitmq. People just need to pick one module from spark and eggroll. +3. client: the module for the jupyter notebook client. +4. fateboard: the module for fateboard. +5. fateflow: the core component of FATE, note that the pod name of this module is called "python". +6. mysql: the mysql module, used to store the metadata for FATE jobs. All the config of the `FATE` and `FATE-Serving` are setting as ConfigMap in each template yaml. If you are going to change the default config of the components, also find the corresponding template yaml. **Note: the template files just how we construct the resources, not how the pod or service looks like in Kubernetes.** # Build KubeFATE's chart -We provides a `Makefile` in the repo, but it is very straghtforward to call the helm command: +We provides a `Makefile` in the repo, but it is very straightforward to call the helm command: ``` release: lint helm package ./FATE diff --git a/docs/Deploy_FATE_Cluster_with_Admin_Role_in_Certain_Namespace.md b/docs/Deploy_FATE_Cluster_with_Admin_Role_in_Certain_Namespace.md new file mode 100644 index 000000000..dcf769fd4 --- /dev/null +++ b/docs/Deploy_FATE_Cluster_with_Admin_Role_in_Certain_Namespace.md @@ -0,0 +1,140 @@ +# Deploy FATE Cluster with Admin Role in Certain Namespace + +## Background + +When deploying KubeFATE and FATE cluster with Kubernetes, user may not have full control over every resource in the cluster. Kubernetes provides [Role-based access control(RBAC) authorization](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) to restrict the actions a user can take. In default case, we ask user to create a new role with a wide set of permissions and create clusterrolebinding to grant access within the whole cluster. But it may not work when user only has access to some limited resources in particular namespace. We studied all the [user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles)(`cluster-admin`, `admin`, `edit`, `view`) in Kubernetes and found that to deploy KubeFATE and run FATE jobs, `admin` is necessary because of the privilege to create roles and role bindings within the namespace. And in this case, some configuration files need to be modified. + +## Deploy Steps + +### Environment + +Start with a K8s cluster which does not have KubeFATE preinstalled, but have 3 namespaces (`fate-exchange`，`fate-9999`，`fate-10000`) and 2 users: + +- User `9999` which bound with `admin` role in namespace `fate-9999` and `fate-exchange` +- User `10000` which bound with `admin` role in namespace `fate-10000` + +Similar to [Deploy an exchange central multi parties federated learning network with KubeFATE](https://github.com/FederatedAI/KubeFATE/wiki/Deploy-an-exchange-central-multi-parties-federated-learning-network-with-KubeFATE), in this document we will deploy two Parties with an exchange and run FATE test job. + +| party | party ID | owner | namespace | K8s version | KubeFATE version | FATE version | +|-------------|----------|------------|---------------|-------------|------------------|--------------| +| exchange | 1 | user-9999 | fate-exchange | v1.24.3 | v1.4.4 | v1.8.0 | +| party-9999 | 9999 | user-9999 | fate-9999 | v1.24.3 | v1.4.4 | v1.8.0 | +| party-10000 | 10000 | user-10000 | fate-10000 | v1.24.3 | v1.4.4 | v1.8.0 | + +### Install KubeFATE + +Because there is no KubeFATE in the cluster, each user needs to [install KubeFATE](https://github.com/FederatedAI/KubeFATE/tree/master/k8s-deploy#readme) separately. Default configuration files needs to be modified beforehand. + +User `9999` will use the `rbac-config.yaml` file below: + +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubefate-admin + namespace: fate-9999 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: sc-9999-edit-binding + namespace: fate-exchange +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: edit +subjects: + - kind: ServiceAccount + name: kubefate-admin + namespace: fate-9999 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: sc-edit-binding + namespace: fate-9999 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: edit +subjects: + - kind: ServiceAccount + name: kubefate-admin + namespace: fate-9999 +--- +apiVersion: v1 +kind: Secret +metadata: + name: kubefate-secret + namespace: fate-9999 +type: Opaque +stringData: + kubefateUsername: admin + kubefatePassword: admin + mariadbUsername: kubefate + mariadbPassword: kubefate +``` + +Compare to default `rbac-config.yaml`, all the namespaces should be modified because KubeFATE should be deployed to user's own namespace. In general, we need to remove any clusterrole or clusterrolebinding and create only role and rolebinding in the corresponding namespaces. PodSecurityPolicy was also removed because `admin` role cannot access the resource at cluster scope. + +User `10000` use the `rbac-config.yaml` similar to user `9999` but doesn't create rolebinding in `fate-exchange` namespace. + +Default namespace in `kubefate.yaml` should be changed to the namespace with privileges correspondingly. + +Execute the commands below after all files are ready: + +```bash +$ kubectl apply -f ./rbac-config.yaml +$ kubectl apply -f ./kubefate.yaml +``` + +### Deploy Exchange + +In this document we use Nginx and ATS to deploy Exchange, so the certificate configuration between each Party and Exchange should be solved first. Refer to [pulsar and certificate generation of ATS](https://github.com/FederatedAI/FATE/blob/v1.6.0/cluster-deploy/doc/fate_on_spark/FATE_on_spark_with_pulsar_zh.md) for more information. + +The template configuration file of Exchange is [trafficServer.yaml](https://github.com/FederatedAI/KubeFATE/blob/master/k8s-deploy/examples/party-exchange/trafficServer.yaml). And `route_table` of nginx and trafficServer will be configured according to the services address and port of the Party after the FATE clusters are up and running. + +When `cluster-exchange.yaml` is configured, user 9999 will deploy it to `fate-exchange` namespace with KubeFATE: + +```sh +kubefate cluster install -f ./cluster-exchange.yaml +``` + +Check the status of Exchange cluster to make sure it is up and running: + +```sh +kubefate cluster ls +``` + +### Add Parties + +We use Spark + Pulsar as the backend of FATE. So the certificates for Pulsar should be installed beforehand like exchange. + +Refer to [cluster-spark-pulsar.yaml](https://github.com/FederatedAI/KubeFATE/blob/master/k8s-deploy/examples/party-9999/cluster-spark-pulsar.yaml) to get the configuration template of FATE cluster. Don't forget to change the ip and ports of Exchange service. + +`cluster-exchange.yaml` should be modified after Party 9999 and Party 1000 are all set. Add hosts and ports to route tables to connect each Party. And update the Exchange cluster: + +```sh +kubefate cluster update -f ./cluster-exchange.yaml +``` + +### Test + +When all deployments are success, there is a federated learning network in the K8s cluster which contains two Parties and an Exchange. Create a new terminal in the notebook of Party 9999 and run command below to check if it works smoothly: + +```sh +flow test toy -gid 9999 -hid 10000 +``` + +## Tips + +When installing multiple KubeFATE services in one K8s cluster, the domains should be unique, or they may conflict with each other. Change the `serviceurl` in `config.yaml` under `./kubefate` directory to access different KubeFATE service. + +## Limits + +There will be some limits when user only has admin privilege in certain namespace: + +1. Some commands of KubeFATE won't work smoothly: + - `kubefate namespace ls`(cannot get namespaces) + - `kubefate cluster describe`(cannot get node ip) +2. PodSecurityPolicy can not be enabled. diff --git a/docs/Eggroll_with_TLS.md b/docs/Eggroll_with_TLS.md new file mode 100644 index 000000000..de8690af0 --- /dev/null +++ b/docs/Eggroll_with_TLS.md @@ -0,0 +1,178 @@ +# Eggroll with TLS + +Since KubeFATE release v1.9.0, we can leverage KubeFATE to deploy eggroll-based FATE clusters who communicate with each other by TLS. + +## Generate certificates +Preparations: + +In a place where you can access your K8s cluster, run +```bash +mkdir my-ca +cd my-ca +wget https://raw.githubusercontent.com/apache/pulsar/master/tests/certificate-authority/openssl.cnf +export CA_HOME=$(pwd) +mkdir certs crl newcerts private +chmod 700 private/ +touch index.txt +echo 1000 > serial +``` +Generate the private key of the root cert +```bash +openssl genrsa -aes256 -out private/ca.key.pem 4096 +chmod 400 private/ca.key.pem +``` +private/ca.key.pem is a key you should not share with anyone. + +Then generate the root certification using the private key: +```bash +openssl req -config openssl.cnf -key private/ca.key.pem \ + -new -x509 -days 7300 -sha256 -extensions v3_ca \ + -out certs/ca.cert.pem +chmod 444 certs/ca.cert.pem +``` +When it prompts the requirement of the common name, you need to come up with one. In this example we use: +``` +example.com +``` +This root certificate is also the CA certificate, once it has been created, you can create certificate requests and sign them with this CA, with above common name as the suffix. + +For one rollsite, we need 2 pairs of certifications and private keys, one pair for acting as a client and another one pair for acting as a server. + +Suppose a site is called party-9999. + +For client: + +Generate the private key for the rollsite party, because rollsite doesn't support RSA formatted private key, we need to change to format to pkcs8. +```bash +mkdir fate-9999 +openssl genrsa -out fate-9999/client_rsa.key 2048 +openssl pkcs8 -topk8 -inform PEM -in fate-9999/client_rsa.key -outform PEM -out fate-9999/client.key -nocrypt +``` +Generate the certificate request with the private key: +``` +openssl req -config openssl.cnf -key fate-9999/client.key -new -sha256 -out fate-9999/client.csr +``` +When it prompts the request common name, you can type in something like ```party-9999-client.example.com```. + +Generate the client certification for rollsite party-9999: +``` +openssl ca -config openssl.cnf -days 10000 -notext -md sha256 -in fate-9999/client.csr -out fate-9999/client.crt +``` + +For server: + +The steps are similar, example: +```bash +openssl genrsa -out fate-9999/server_rsa.key 2048 +openssl pkcs8 -topk8 -inform PEM -in fate-9999/server_rsa.key -outform PEM -out fate-9999/server.key -nocrypt +openssl req -config openssl.cnf -key fate-9999/server.key -new -sha256 -out fate-9999/server.csr +``` +Type ```party-9999-server.example.com``` as the common name. +```bash +openssl ca -config openssl.cnf -days 10000 -extensions server_cert -notext -md sha256 -in fate-9999/server.csr -out fate-9999/server.crt +``` +The last step is to create a K8s secret based on the generated files: +```bash +kubectl -n fate-9999 create secret generic eggroll-certs \ + --from-file=ca.pem=certs/ca.cert.pem \ + --from-file=client.key=fate-9999/client.key \ + --from-file=client.crt=fate-9999/client.crt \ + --from-file=server.key=fate-9999/server.key \ + --from-file=server.crt=fate-9999/server.crt +``` + +## Enable TLS for rollsite in cluster.yaml file + +Enable this switch under the rollsite module. +```yaml +rollsite: + enableTLS: true +``` + +## P2P mode + +In this mode, a FATE cluster's rollsite will communicate with the rollsite of another FATE cluster. + +So we need to: +1. Generate the certs files for each party +2. Create the K8s secrets for each party, in the corresponding K8s namespace +3. Turn on "enableTLS" for each party's rollsite + +## Exchange mode +In this mode, every FATE cluster's rollsite will talk to the rollsite of the FATE-Exchange. + +So we need to: +1. Generate the certs files for each party, and the FATE exchange +2. Create the K8s secrets for each party, in the corresponding K8s namespace +3. Turn on "enableTLS" for each party's rollsite, and also the exchange's rollsite + +An example of the steps for exchange: + +For client cert: +```bash +mkdir fate-exchange + +openssl genrsa -out fate-exchange/client_rsa.key 2048 +openssl pkcs8 -topk8 -inform PEM -in fate-exchange/client_rsa.key -outform PEM -out fate-exchange/client.key -nocrypt +openssl req -config openssl.cnf -key fate-exchange/client.key -new -sha256 -out fate-exchange/client.csr +``` +Type in for example, ```exchange-client.example.com``` as the common name. +```bash +openssl ca -config openssl.cnf -days 10000 -notext -md sha256 -in fate-exchange/client.csr -out fate-exchange/client.crt +``` + +For server cert: +```bash +openssl genrsa -out fate-exchange/server_rsa.key 2048 +openssl pkcs8 -topk8 -inform PEM -in fate-exchange/server_rsa.key -outform PEM -out fate-exchange/server.key -nocrypt +openssl req -config openssl.cnf -key fate-exchange/server.key -new -sha256 -out fate-exchange/server.csr +``` +Type in for example, ```exchange-server.example.com``` as the common name. +```bash +openssl ca -config openssl.cnf -days 10000 -extensions server_cert -notext -md sha256 -in fate-exchange/server.csr -out fate-exchange/server.crt +``` + +Create K8s secret: +```bash +kubectl -n fate-exchange create secret generic eggroll-certs \ + --from-file=ca.pem=certs/ca.cert.pem \ + --from-file=client.key=fate-exchange/client.key \ + --from-file=client.crt=fate-exchange/client.crt \ + --from-file=server.key=fate-exchange/server.key \ + --from-file=server.crt=fate-exchange/server.crt +``` + +Then in the cluster.yaml file of FATE-Exchange, turn on the ```enableTLS``` switch under the rollsite module. + +## Docker-Compose mode + +In KubeFATE release v1.9.0, we will not provide a switch for enabling TLS for rollsite. This can be done in below manual steps: + +1. Generate the certs, as above documents shows, for every FATE cluster and for the FATE Exchange if needed. +2. Run `docker ps` to get the container id of the rollsite. +3. Run `docker exec -it bash` to get into the rollsite container. +4. In dir `/data/projects/fate/eggroll/conf`, run `mkdir cert`. +5. Edit 5 files: `ca.pem client.crt client.key server.crt server.key`, input the contents of the certifications and the private keys you have generated. `ca.pem` is the CA's certification, we assume you use this certification to sign both the client certification and the server certification of rollsite. +6. In dir `/data/projects/fate/eggroll/conf`, edit `eggroll.properties`, tail below contents: + ``` + eggroll.core.security.secure.cluster.enabled=true + eggroll.core.security.secure.client.auth.enabled=true + eggroll.core.security.ca.crt.path=conf/cert/ca.pem + eggroll.core.security.crt.path=conf/cert/server.crt + eggroll.core.security.key.path=conf/cert/server.key + eggroll.core.security.client.ca.crt.path=conf/cert/ca.pem + eggroll.core.security.client.crt.path=conf/cert/client.crt + eggroll.core.security.client.key.path=conf/cert/client.key + ``` +7. Run `docker restart ` + +After above steps, your eggroll deployed on Docker Compose should start to communicate with each other by TLS. + +One way to verify that is to check the log of the rollsite container by `docker logs `, if you see logs like: + +``` +[INFO ][2046][2022-08-03 06:41:48,543][main,pid:1,tid:1][c.w.e.c.t.GrpcServerUtils:107] - gRPC server at port=9380 starting in secure mode. server private key path: /data/projects/fate/eggroll/conf/cert/server.key, key crt path: /data/projects/fate/eggroll/conf/cert/server.crt, ca crt path: /data/projects/fate/eggroll/conf/cert/ca.pem +[INFO ][2051][2022-08-03 06:41:48,548][main,pid:1,tid:1][c.w.e.r.EggSiteBootstrap:107] - secure server started at 9380 +``` + +Then it indicates that you have enabled TLS on rollsite successfully on Docker Compose. diff --git a/docs/FATE_Algorithm_and_Computational_Acceleration_Selection.md b/docs/FATE_Algorithm_and_Computational_Acceleration_Selection.md new file mode 100644 index 000000000..3f820589c --- /dev/null +++ b/docs/FATE_Algorithm_and_Computational_Acceleration_Selection.md @@ -0,0 +1,24 @@ +# FATE Algorithm and Computational Acceleration Selection + +As a federated learning framework, FATE supports many algorithms. It is also the needs of many enterprises to select the corresponding algorithm and accelerator card according to the business. Currently, KubeFATE supports the selection of algorithms and accelerator cards. + +Whether in docker-compose or k8s deployment, the following two parameters can be selected: + +- `algorithm` Algorithm choice +- `device` Computing Device Selection + +## Algorithm + +The choice of algorithm consists of two options: + +- `Basic` + Basic is the default option, which includes dependencies related to the removal of nn (including homo_nn and hetero_nn) algorithms. +- `NN` + NN contains all the dependencies required for nn to include (homo_nn and hetero_nn). ***NN can only be used when computing is Eggroll*** + +## Device + +Device selection consists of an option: + +- `CPU` + The CPU is a computing device that uses the CPU as a FATE computing device. diff --git a/docs/FATE_On_Spark.md b/docs/FATE_On_Spark.md index 6ded953e4..d83c59621 100644 --- a/docs/FATE_On_Spark.md +++ b/docs/FATE_On_Spark.md @@ -1,22 +1,21 @@ # Overview Originally, the FATE use the underlying [EggRoll]("https://github.com/WeBankFinTech/eggroll") as the underlying computing engine, the -following picture illustrates the overview architecture. +following picture illustrates the overall architecture.
-As the above figure show, the EggRoll provide both computing and storage resource. However it will be little different while using different backend. +As the above figure show, the EggRoll provide both computing and storage resource. -In FATE v1.5.0 a user can select Spark as the underlying computing engine, however, spark -it self is a in-memory computing engine without the ability to persist data. Thus to use FATE on Spark a HDFS is also needed to be included to provide persistence capability. For example, a user need to upload their data to HDFS through FATE before further processing; the output data of every component will be also stored to the HDFS. +Since FATE v1.5.0 a user can select Spark as the underlying computing engine, however, spark itself is an in-memory computing engine without the data persistence. Thus, HDFS is also needed to be deployed to help on data persistence. For example, a user need to upload their data to HDFS through FATE before doing any training job, and the output data of each component will also be stored in the HDFS module. -**Currently the verifed Spark version is [2.4.1](https://archive.apache.org/dist/spark/spark-2.4.1/spark-2.4.1-bin-hadoop2.7.tgz) and the Hadoop is [2.7.4](https://archive.apache.org/dist/hadoop/common/hadoop-2.7.4/hadoop-2.7.4.tar.gz)** +**Currently the verifed Spark version is [3.1.2](https://archive.apache.org/dist/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz) and the Hadoop is [3.2.1](https://archive.apache.org/dist/hadoop/common/hadoop-3.2.1/hadoop-3.2.1.tar.gz)** The following picture shows the architecture of FATE on Spark:
- +
In current implementation, the `fate_flow` service uses the `spark-submit` binary tool to submit job to the Spark cluster. With the configuration of the fate's job, a user can also specify the configuration for the spark application, here is an example: diff --git a/docs/FATE_On_Spark_With_Pulsar.md b/docs/FATE_On_Spark_With_Pulsar.md index a4ce9b0d6..8559582ac 100644 --- a/docs/FATE_On_Spark_With_Pulsar.md +++ b/docs/FATE_On_Spark_With_Pulsar.md @@ -11,7 +11,7 @@ FATE supports using [Spark](https://spark.apache.org/) as a computing engine sin ## Configuration and Usage ### Update Config of the FATE Flow service -For more details of the Pulsar Deployment, a user can refer to this [Deployment Guide](https://github.com/FederatedAI/FATE/blob/develop-1.6/cluster-deploy/doc/fate_on_spark/pulsar_deployment_guide.md). After the Pulsar cluster is running, a user needs to update the configuration of the "fate flow" service. The update splits into two parts. They are: +For more details of the Pulsar Deployment, a user can refer to this [Deployment Guide](https://github.com/FederatedAI/FATE/blob/master/deploy/cluster-deploy/doc/fate_on_spark/common/pulsar_deployment_guide.md). After the Pulsar cluster is running, a user needs to update the configuration of the "fate flow" service. The update splits into two parts. They are: - "conf/service_conf.yaml" ``` yml @@ -86,8 +86,6 @@ When submitting a task, the user can declare in the config file to use Pulsar as } ``` -In above configuration, the `backend: 2` specifies the use of Pulsar as the transmission service. In `pulsar_run`, a user can also specify the parameters when creating "producer" and "consumer". - Generally, there is no need to set such a configuration. As for the available parameters, please refer to the [`create_producer`](https://pulsar.apache.org/api/python/2.7.0-SNAPSHOT/#pulsar.Client.create_producer) and [`subscribe`](https://pulsar.apache.org/api/python/2.7.0-SNAPSHOT/#pulsar.Client.subscribe) methods in the Pulsar python client. ## Deployment of Star Network @@ -104,30 +102,30 @@ Using Pulsar as a transmission engine can support star deployment. Its central n Next, we will create a federated learning network based on the SNI proxy model. Since certificates are required, a user can unify domain name suffix for this network, such as "fate.org". In this way, each entity in the network can be identified by `${party_id}.fate.org`. For example, the CN of the certificate used by party 10000 is "10000.fate.org". #### Deployment Plan -Host | IP | OS | Application | Service ---- | --- | --- | --- | --- | -proxy.fate.org | 192.168.0.1 | CentOS 7.2/Ubuntu 16.04 | ats | ats -10000.fate.org | 192.168.0.2 | CentOS 7.2/Ubuntu 16.04 | pulsar | pulsar -9999.fate.org | 192.168.0.3 | CentOS 7.2/Ubuntu 16.04 | pulsar | pulsar +| Host | IP | OS | Application | Service | +|----------------|-------------|-------------------------|-------------|---------| +| proxy.fate.org | 192.168.0.1 | CentOS 7.2/Ubuntu 16.04 | ats | ats | +| 10000.fate.org | 192.168.0.2 | CentOS 7.2/Ubuntu 16.04 | pulsar | pulsar | +| 9999.fate.org | 192.168.0.3 | CentOS 7.2/Ubuntu 16.04 | pulsar | pulsar | -The architecture is shown in the following diagram. The Pulsar service "10000.fate.org" belongs to the organization with ID 10000, and the pulsar service "9999.fate.org" belongs to the organization with ID 9999, and "proxy.fate.org" is ats Service is the center of the star network. +The architecture is shown in the following diagram. The Pulsar service "10000.fate.org" belongs to the organization with ID 10000, and the pulsar service "9999.fate.org" belongs to the organization with ID 9999, and "proxy.fate.org" is the ATS Service and is the center of the star network.
-#### Certificate Generation +### Certificate Generation Since the SNI proxy is based on TLS, it is necessary to configure the certificate for the ATS and Pulsar services. -##### Generate CA's Certificate +#### Generate CA's Certificate Enter the following command to create a directory for the CA, and put the openssl configuration file in the directory. ``` bash $ mkdir my-ca $ cd my-ca -$ wget https://raw.githubusercontent.com/apache/pulsar/master/site2/website/static/examples/openssl.cnf +$ wget https://raw.githubusercontent.com/apache/pulsar/master/tests/certificate-authority/openssl.cnf $ export CA_HOME=$(pwd) ``` @@ -151,7 +149,7 @@ Once the above commands are completed, the CA-related certificates and keys have - certs/ca.cert.pem the certification of CA - private/ca.key.pem the key of CA -##### Generate a certificate for "10000.fate.org" +#### Generate a certificate for "10000.fate.org" 1. Create the directory @@ -188,27 +186,27 @@ $ openssl ca -config openssl.cnf -extensions server_cert \ At this time, the certificate "broker.cert.pem" and a key "broker.key-pk8.pem" are stored in the "10000.fate.org" directory. -##### Generate a certificate for "9999.fate.org" +#### Generate a certificate for "9999.fate.org" The generation of the "9999.fate.org" certificate is consistent with the above steps, however, the Common Name in the step 4 is "9999.fate.org". The following operation assume that the certificate of "9999.fate.org" has been generated. -##### Generate a certificate for "proxy.fate.org" +#### Generate a certificate for "proxy.fate.org" The generation of the "proxy.fate.org" certificate is the same as the above steps, the conversion in step 3 can be omitted, and the Common Name in step 4 is "proxy.fate.org". The following operation assume that the certificate of "proxy.fate.org" has been generated. -#### Deploy Apache Traffic Server +### Deploy Apache Traffic Server -##### Install Apache Traffic Server +#### Install Apache Traffic Server 1. Log in to the "proxy.fate.org" host and prepare the dependencies according to this [document](https://github.com/apache/trafficserver/tree/9.0.0). 2. Download ``` bash -$ wget https://apache.claz.org/trafficserver/trafficserver-9.0.0.tar.bz2 +$ wget https://archive.apache.org/dist/trafficserver/trafficserver-9.0.0.tar.bz2 ``` 3. Install @@ -224,7 +222,7 @@ $ echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/ts/lib' >> ~/.profile $ source ~/.profile ``` -Once the processes were finished, the traffic server will be installed in the "/opt/ts" directory and the configuration file path is "/opt/ts/etc/trafficserver/". +Once the processes are finished, the traffic server will be installed in the "/opt/ts" directory and the configuration file path is "/opt/ts/etc/trafficserver/". ##### Start Apache Traffic Server @@ -261,7 +259,7 @@ Once the processes were finished, the traffic server will be installed in the "/ tunnel_route: 192.168.0.3:6651 ``` - For a more details of the configurations, please refer to the official [documents](https://docs.trafficserver.apache.org/en/9.0.x/admin-guide/configuring-traffic-server.en.html). + For more details of the configurations, please refer to the official [documents](https://docs.trafficserver.apache.org/en/9.0.x/admin-guide/configuring-traffic-server.en.html). 2. Start the service @@ -273,9 +271,9 @@ Copy the certificate, private key, and CA certificate generated for ATS in the p #### Deploy Pulsar - The details of how Pulsar deploy can be find in [pulsar_deployment_guide.md](https://github.com/FederatedAI/FATE/blob/develop-1.6-pulsar/cluster-deploy/doc/fate_on_spark/pulsar_deployment_guide.md). A user only need to add a certificate for the broker and open the security service port. The specific operations are as follows: + The details of how Pulsar deploy can be find in [pulsar_deployment_guide.md](https://github.com/FederatedAI/FATE/blob/master/deploy/cluster-deploy/doc/fate_on_spark/common/pulsar_deployment_guide.md). A user only need to add a certificate for the broker and open the security service port. The specific operations are as follows: 1. Log in to the corresponding host and copy the certificate, private key, and CA certificate generated for 10000.fate.org to the "/opt/pulsar/certs" directory -2. Modify the conf/standalone.conf file in the pulsar installation directory and append the following content +2. Modify the conf/standalone.conf file in the pulsar installation directory and append the following content: ``` bash brokerServicePortTls=6651 @@ -324,7 +322,7 @@ default: ``` -FATE will fill in the host and proxy parameters of the cluster according to the content of the "default" field. For example, the Pulsar cluster used for synchronization with party 9999 is as follow: +FATE will fill in the host and proxy parameters of the cluster according to the content of the "default" field. For example, the Pulsar cluster used for synchronization with party 9999 is as follows: ``` json diff --git a/docs/FATE_Upgrade.md b/docs/FATE_Upgrade.md new file mode 100644 index 000000000..68f15a540 --- /dev/null +++ b/docs/FATE_Upgrade.md @@ -0,0 +1,284 @@ +# Using KubeFATE to upgrade a FATE cluster + +## Overview +Since KubeFATE v1.4.5, KubeFATE can help to upgrade a FATE cluster. +This document is going to illustrate how to do that, and also show the limitations for this new feature due to historical reasons. + +## Which versions are supported? + +"Supported" means that support using KubeFATE CLI to upgrade a FATE cluster automatically. + +The supporting matrices depend on the configuration of the data persistence of the FATE cluster, in specific: + +### Case 1: When using existing persistence volume claim for MySQL +Support FATE versions + +| From/To | v1.7.2 | v1.8.0 | v1.9.0 | future versions | +|-----------------|--------|--------|--------|-----------------| +| v1.7.1 or lower | no | no | no | no | +| v1.7.2 | | yes | yes | yes | +| v1.8.0 | | | yes | yes | +| v1.9.0 | | | | yes | + +#### Q&A +1. What does "yes" or "no" means in the form? + 1. "yes" means that the FATE cluster can work properly after upgrading, also, the data produced during the previous version, such as the job info or model info, can still be accessed in the new version's FATE cluster. +2. Why "v1.7.1 or lower" is not supported? + 1. This is because in v1.7.1 and previous versions, the MySQL image version in our helm chart is set to 8, which is like setting to the latest version of the MySQL 8 series. Since v1.7.2, we set the MySQL image version to 8.0.28. So for example if you install a v1.7.1 cluster now, your MySQL version should be larger than 8.0.28, however, downgrade a MySQL database is not supported. [reference](https://dev.mysql.com/doc/refman/8.0/en/downgrading.html). +3. Is there a workaround for "v1.7.1 or lower"? + 1. Yes, please check [workaround](#workarounds). + + +### Case 2: When using storage class to auto-provision the persistence +Support FATE versions + +| From/To | v1.7.2 | v1.8.0 | v1.9.0 | future versions | +|-----------------|--------|--------|--------|-----------------| +| v1.7.1 or lower | no | no | no | no | +| v1.7.2 | | yes | no | no | +| v1.8.0 | | | no | no | +| v1.9.0 | | | | yes | + +#### Q&A +1. Why "v1.7.1 or lower" is not supported? + 1. Same with above. +2. Why upgrade from v1.7.2 to v1.8.0 is supported, but upgrade from v1.8.0 to v1.9.0 is not supported? + 1. In v1.9.0, we make some adjustments to the app type of several FATE K8s components. Basically, for each "deployment" who will have a PVC/PV when persistence is enabled, we change the app type from "deployment" to "statefulSet". We believe this is the right thing to do, but the side effect is that the data from v1.8.0 cannot be carried forward. The good thing is, in the future versions we will not change the app type once again, so this gap will happen only once. + +### Case 3: When disabling persistence + +This is similar with destroy an old-versioned FATE cluster and create a new-versioned one. +The new one would be a brand new one with no data from the previous version. +It can work properly, but it will abandon all the history data. + + +## Usage guidance + +In this section, we will introduce how to use KubeFATE to conduct an upgrade. +We will upgrade from v1.7.2 to v1.8.0 to illustrate the processes. + +### Install a FATE cluster of v1.7.2 + +The cluster.yaml: + +```yaml +name: fate-9999 +namespace: fate-9999 +chartName: fate +chartVersion: v1.7.2 +partyId: 9999 +registry: "" +pullPolicy: +imagePullSecrets: + - name: myregistrykey +persistence: true +istio: + enabled: false +podSecurityPolicy: + enabled: false +ingressClassName: nginx +modules: + - rollsite + - clustermanager + - nodemanager + - mysql + - python + - fateboard + - client + +backend: eggroll + +ingress: + fateboard: + hosts: + - name: party9999.fateboard.example.com + client: + hosts: + - name: party9999.notebook.example.com + +rollsite: + type: NodePort + nodePort: 30091 + partyList: + - partyId: 10000 + partyIp: 192.168.10.1 + partyPort: 30101 + +python: + type: NodePort + httpNodePort: 30097 + grpcNodePort: 30092 + logLevel: INFO + storageClass: nfs-client + +client: + storageClass: nfs-client + +servingIp: 192.168.9.1 +servingPort: 30095 + +mysql: + storageClass: nfs-client + +nodemanager: + count: 2 + sessionProcessorsPerNode: 4 + storageClass: nfs-client + accessMode: ReadWriteOnce + size: 2Gi + list: + - name: nodemanager + nodeSelector: + sessionProcessorsPerNode: 4 + subPath: "nodemanager" + existingClaim: "" + storageClass: nfs-client + accessMode: ReadWriteOnce + size: 1Gi +``` + +In this example, we have already installed [nfs-subdir-external-provisioner](https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner) as the storage class. +For the background of how to set up a nfs server, you could check [here](https://www.itzgeek.com/how-tos/linux/centos-how-tos/how-to-setup-nfs-server-on-centos-7-rhel-7-fedora-22.html) + +After installation, the pods are like: +``` +client-75f67f5d47-svfh2 1/1 Running 0 2m30s +clustermanager-7bd6fb46c8-bnx57 1/1 Running 0 2m30s +mysql-77f95d4844-8dfjj 1/1 Running 0 2m30s +nodemanager-0-6dbcc56dd4-ddldx 2/2 Running 0 2m30s +nodemanager-1-646ddbf48c-2lllz 2/2 Running 0 2m30s +python-5bd5d69779-c4t5r 2/2 Running 0 2m30s +rollsite-5d45c85c4d-dbpnt 1/1 Running 0 2m30s +``` +The PVC/PC are provisioned by the storage class, they are like: + +PVC: +``` +NAMESPACE NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +fate-9999 client-data Bound pvc-e7ea7c79-c5e3-4824-8269-4eac390dcae8 1Gi RWO nfs-client 3m27s +fate-9999 mysql-data Bound pvc-0865a743-9bfb-4990-bd26-84b0241f0084 1Gi RWO nfs-client 3m27s +fate-9999 nodemanager-0-data Bound pvc-eaac241e-4d0f-4d06-96d0-30e85e284adc 2Gi RWO nfs-client 3m27s +fate-9999 nodemanager-1-data Bound pvc-eed13a83-442e-4c9d-bb00-e6a7ce7c05f2 2Gi RWO nfs-client 3m27s +fate-9999 python-data Bound pvc-5fc77a70-fe67-4b10-8db5-d3b0ef76d8f7 1Gi RWO nfs-client 3m27s +``` + +PV: +``` +NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE +pvc-0865a743-9bfb-4990-bd26-84b0241f0084 1Gi RWO Retain Bound fate-9999/mysql-data nfs-client 3m44s +pvc-5fc77a70-fe67-4b10-8db5-d3b0ef76d8f7 1Gi RWO Retain Bound fate-9999/python-data nfs-client 3m43s +pvc-e7ea7c79-c5e3-4824-8269-4eac390dcae8 1Gi RWO Retain Bound fate-9999/client-data nfs-client 3m44s +pvc-eaac241e-4d0f-4d06-96d0-30e85e284adc 2Gi RWO Retain Bound fate-9999/nodemanager-0-data nfs-client 3m44s +pvc-eed13a83-442e-4c9d-bb00-e6a7ce7c05f2 2Gi RWO Retain Bound fate-9999/nodemanager-1-data nfs-client 3m44s +``` + +### Backup the Mysql database +Before any upgrade, we should back up the database. Currently, KubeFATE doesn't support back up the database automatically. This need to be done by the mysqldump tool manually. In the future this could be another feature of KubeFATE. + +There are several ways to do the backup work, in this example, we do that in the mysql pod: + +``` +$ kubectl exec -it mysql-77f95d4844-8dfjj -n fate-9999 bash +kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead. +root@mysql-77f95d4844-8dfjj:/# +``` + +Note that "mysql-77f95d4844-8dfjj" is the pod id which must be different in your FATE cluster. Check the pod id by ```kubectl get pods -n ```. + +We need to change the directory to "/var/lib/mysql/", because that is where the PV is mounted to. +If we generate a mysql snapshot there, the file will be persisted in the nfs server even the pod is gone. +``` +root@mysql-77f95d4844-8dfjj:/# cd /var/lib/mysql/ +root@mysql-77f95d4844-8dfjj:/var/lib/mysql# +``` +Then we can run the mysqldump command to create a snapshot of the current status of the database: +``` +root@mysql-77f95d4844-8dfjj:/var/lib/mysql# mysqldump -h localhost -u root eggroll_meta > snapshot.sql +root@mysql-77f95d4844-8dfjj:/var/lib/mysql# ls | grep snapshot +snapshot.sql +``` +Now we can quit the container of mysql. login to the nfs server and manage the snapshot file. +``` +nfsServer ~ → cd /mnt/myshareddir/ +nfsServer myshareddir → ls +fate-9999-client-data-pvc-e7ea7c79-c5e3-4824-8269-4eac390dcae8 fate-9999-nodemanager-1-data-pvc-eed13a83-442e-4c9d-bb00-e6a7ce7c05f2 +fate-9999-mysql-data-pvc-0865a743-9bfb-4990-bd26-84b0241f0084 fate-9999-python-data-pvc-5fc77a70-fe67-4b10-8db5-d3b0ef76d8f7 +fate-9999-nodemanager-0-data-pvc-eaac241e-4d0f-4d06-96d0-30e85e284adc +nfsServer myshareddir → ls fate-9999-mysql-data-pvc-0865a743-9bfb-4990-bd26-84b0241f0084 | grep snapshot +snapshot.sql +``` + +### Using KubeFATE CLI to do the upgrade +We just need to change `chartVersion` in the cluster.yaml file: +``` +chartVersion: v1.7.2 -> chartVersion: v1.8.0 +``` +Then execute: +``` +kubefate cluster update -f cluster.yaml +``` +A KubeFATE job will be created. At this time, a K8s one-time job will be launched, it will do 2 things: +1. Shut down the "python" pod, which includes FATE-Flow and FATE-Board. +2. It executes one or more .sql script(s) to help upgrade the schema of the MySQL database. + +After that, the K8s job will be deleted by default. For debugging purpose, to keep the finished job on the K8s cluster, we can run: +``` +kubefate cluster update -f cluster.yaml --keepUpgradeJob +``` +Soon, the new pods will be spawned: +``` +fate-9999 client-7dd4fc8cbf-jtz2x 1/1 Running 0 5m4s +fate-9999 clustermanager-7987759b84-7bgbf 1/1 Running 0 5m26s +fate-9999 mysql-fc76db6db-rdvw5 1/1 Running 0 5m26s +fate-9999 nodemanager-0-54845ccd98-jjptt 2/2 Running 0 5m25s +fate-9999 nodemanager-1-cdd975587-cn96q 2/2 Running 0 5m26s +fate-9999 python-57f8c857d8-ckp2z 2/2 Running 0 5m36s +fate-9999 rollsite-6b67d75c78-bngqb 1/1 Running 0 5m26s +``` +At this time, the cluster has already been upgrade to v1.8.0, which means the upgrade has been done: +``` +➜ kubefate cluster list +UUID NAME NAMESPACE REVISION STATUS CHART ChartVERSION AGE +4a8372d1-a15e-4653-8420-60627d89f858 fate-9999 fate-9999 2 Running fate v1.8.0 3h35m +``` + +## Workarounds + +As mentioned, due to a historical reason, upgrading from a FATE cluster whose version is less or equal than v1.7.1 is not supported by KubeFATE. However, there is a manual workaround for that purpose: + +Suppose we have a v1.7.1 FATE cluster, how to upgrade to v1.7.2 manually without data loss in the MySQL database? +### Export the data in the MySQL database +Get into the container of MySQL by ```kubectl exec -it -n bash```, go to "/var/lib/mysql" and run below command: + +``` +mysqldump -h localhost -u root --no-create-info \ +--ignore-table=eggroll_meta.server_node \ +--ignore-table=eggroll_meta.t_component_info \ +--ignore-table=eggroll_meta.t_component_provider_info \ +--ignore-table=eggroll_meta.t_component_registry \ +--ignore-table=eggroll_meta.t_engine_registry \ +eggroll_meta > data.sql +``` +The reason we ignore the tables is: the information are version-specific. The data in those tables will be re-generated when a new "python" container of the new version is spawned. It means nothing to carry these data from v1.7.1 to v1.7.2. + +Also, we need to prepare the snapshot for rollback, in case of upgrade failure: +``` +mysqldump -h localhost -u root eggroll_meta > snapshot.sql +``` + +### Destroy the v1.7.1 cluster and install a fresh v1.7.2 cluster +Use ```kubefate cluster delete ``` and ```kubefate cluster install -f cluster.yaml``` to make this happen. + +### Import the data into the new MySQL database +Get into the container of MySQL by ```kubectl exec -it -n bash```. Make sure before that the dumped data file "data.sql" has been put into the corresponding nfs folder. + +Change directory to "/var/lib/mysql" and run below command: +``` +mysql -u root eggroll_meta < data.sql +``` +After that, we have "upgraded" the FATE cluster from v1.7.1 to v1.7.2. + +## Other notes +1. We also support upgrade over multiple versions, just simply change the version number in the cluster.yaml file, then KubeFATE can figure out which sql scripts it needs to execute for the MySQL database. Currently, if you try to upgrade from v1.7.2 to v1.9.0, skipping v1.8.0, this will only work when you have configured an existing PVC/PV. However, in the future, if you would like to upgrade from v1.9.0 and skip several in-the-middle versions, it will also work even when you are using a storage class. +2. We support changing the architecture during upgrade, for example, you can upgrade from an Eggroll based FATE cluster of version v1.7.2 to a Spark based FATE cluster of version v1.8.0. The new cluster will work properly, although the temporary data in the Eggroll PV will be lost. +3. Again, at current stage, KubeFATE v1.4.5, you are responsible to do the backup of the MySQL database before any kind of upgrade. \ No newline at end of file diff --git a/docs/Introduction_to_Engine_Architecture.md b/docs/Introduction_to_Engine_Architecture.md new file mode 100644 index 000000000..4618bb766 --- /dev/null +++ b/docs/Introduction_to_Engine_Architecture.md @@ -0,0 +1,151 @@ +# Introducing KubeFATE to deploy FATE supported engines + +FATE supports two computing engines: eggroll and spark + +## Supported engine + +FATE supports multiple computing engines, storage engines and federation engines, and KubeFATE supports quick deployment of FATE clusters with different combination of engines. + +List of supported engine combinations: + +- **Eggroll** +- **Spark + hdfs + RabbitMQ** +- **Spark + hdfs + Pulsar** +- **Spark-local + localfs + Pulsar** + +### Eggroll + +Eggroll is a computing storage engine natively supported by FATE, including three components: rollsite, nodemanager, and clustermanager. + +Rollsite is responsible for data transmission, nodemanager is responsible for storage and computing, and clustermanager is responsible for managing nodemanager. + +Modify the `parties.conf` configuration when using docker-compose: + +```bash +computing=Eggroll +federation=Eggroll +storage=Eggroll +``` + +Modify the `cluster.yaml` configuration when using Kubernetes: + +```yaml +computing: Eggroll +federation: Eggroll +storage: Eggroll +``` + +Architecture diagram: + +
+ +
+ +To enabled TLS for the eggroll federation between different FATE parties, check this [doc](/docs/Eggroll_with_TLS.md). + +### spark_rabbitmq + +spark_rabbitmq, a FATE cluster of spark + hdfs + rabbitmq engine combination will be deployed. + +spark is the computing component, hdfs is the storage component, rabbitmq is the federation component. + +Modify the `parties.conf` configuration when using docker-compose: + +```bash +computing=Spark +federation=RabbitMQ +storage=HDFS +``` + +Modify the `cluster.yaml` configuration when using Kubernetes: + +```yaml +computing: Spark +federation: RabbitMQ +storage: HDFS +``` + +Architecture diagram: + +
+ +
+ +### spark_pulsar + +spark_pulsar, it will deploy a FATE cluster of Spark + Hdfs + Pulsar engine combination. + +Spark is the computing component, Hdfs is the storage component, and Pulsar is the federation component. + +Modify the `parties.conf` configuration when using docker-compose: + +```bash +computing=Spark +federation=Pulsar +storage=HDFS +``` + +Modify the `cluster.yaml` configuration when using Kubernetes: + +```yaml +computing: Spark +federation: Pulsar +storage: HDFS +``` + +Architecture diagram: + +
+ +
+ +### spark_local_pulsar (slim FATE) + +spark_local_pulsar, a slim FATE cluster is deployed. All computing and storage parts are together, implemented through Spark local and localfs, and data transfer is done through Pulsar. + +Spark local is the computing component, localfs is the storage component, and Pulsar is the federation component. + +Modify the `parties.conf` configuration when using docker-compose: + +```bash +computing=Spark_local +federation=Pulsar +storage=LocalFS +``` + +Modify the `cluster.yaml` configuration when using Kubernetes: + +```yaml +computing: Spark_local +federation: Pulsar +storage: LocalFS +``` + +Architecture diagram: + +
+ +
+ +## Comparison of different engines + +Eggroll can be used as FATE's computing, storage and transmission engine. Eggroll is a simple and high-performance [federation] machine learning computing framework that supports multiple federated network architectures: straight-chain mode, star and ring, etc. Eggroll also supports being used between different organizations with TLS, Eggroll is an open source project on github owned by WeBank. +repository: + +Rabbitmq is a simple and easy-to-use Message Queue. It has a long history and is supported on many cloud platform. It is suitable for FATE with Spark as the computing engine. + +Compared with Rabbitmq, Pulsar can support large-scale cluster deployment, so it is suitable for larger-scale federated learning. This mode also supports "star mode" (deployment with FATE-Exchange). + +Compared with other modes, spark_local_pulsar can use this mode to minimize the number of components required for the cluster. It can also run FATE clusters with less resources, and can be used in small-scale federated learning, on IOT devices, etc. + +## Exchange Architecture + +When using the rollsite or Pulsar architecture, a federated learning framework can be deployed using FATE-Exchange. + +That is to say, when the federation is Eggroll and Pulsar, it can support the use of Exchange. + +**Through FATE-Exchange, rollsite parties can only do FML with other rollsite parties, and Pulsar parties can only do FML with other Pulsar parties.** + +Reference: + +- [Deploy an exchange central multi parties federated learning network with KubeFATE](https://github.com/FederatedAI/KubeFATE/wiki/Deploy-an-exchange-central-multi-parties-federated-learning-network-with-KubeFATE) diff --git a/docs/Introduction_to_Engine_Architecture_zh.md b/docs/Introduction_to_Engine_Architecture_zh.md new file mode 100644 index 000000000..3d781a8ca --- /dev/null +++ b/docs/Introduction_to_Engine_Architecture_zh.md @@ -0,0 +1,154 @@ +# KubeFATE 部署FATE支持引擎介绍 + +FATE支持eggroll和spark两种计算引擎 + +## 支持引擎组合 + +FATE支持选择不同的计算引擎，存储引擎和数据传输引擎，现在KubeFATE支持快捷的部署不同引擎组合的FATE集群。 + +目前支持的组合包括以下几种： + +- **Eggroll** +- **Spark + HDFS + RabbitMQ** +- **Spark + HDFS + Pulsar** +- **Spark-local + LocalFS + Pulsar** +- **Spark-local + LocalFS + RabbitMQ** + +### Eggroll + +Eggroll是FATE原生支持的计算存储引擎，包括了rollsite，nodemanager，clustermanager三个组件 + +rollsite负责数据传输，nodemanager负责存储和计算，clustermanager负责管理nodemanager。 + +docker-compose使用的时候修改`parties.conf`配置 + +```bash +computing=Eggroll +federation=Eggroll +storage=Eggroll +``` + +***当使用Eggroll计算引擎的时候，federation和storage也必须是Eggroll*** + +k8s使用的时候修改`cluster.yaml`配置 + +```yaml +computing: Eggroll +federation: Eggroll +storage: Eggroll +``` + +架构图： + +
+ +
+ +### spark_rabbitmq + +当使用Spark + HDFS + RabbitMQ的时候，会部署Spark + HDFS + RabbitMQ的引擎组合的FATE集群。 + +spark是计算组件，HDFS是存储组件，RabbitMQ是数据传输组件。 + +docker-compose使用的时候修改`parties.conf`配置 + +```bash +computing=Spark +federation=RabbitMQ +storage=HDFS +``` + +k8s使用的时候修改`cluster.yaml`配置 + +```yaml +computing: Spark +federation: RabbitMQ +storage: HDFS +``` + +架构图： + +
+ +
+ +### spark_pulsar + +当使用Spark + HDFS + Pulsar的时候，会部署Spark + HDFS + Pulsar的引擎组合的FATE集群。 + +Spark是计算组件，HDFS是存储组件，Pulsar是数据传输组件。 + +docker-compose使用的时候修改`parties.conf`配置 + +```bash +computing=Spark +federation=Pulsar +storage=HDFS +``` + +k8s使用的时候修改`cluster.yaml`配置 + +```yaml +computing: Spark +federation: Pulsar +storage: HDFS +``` + +架构图： + +
+ +
+ +### spark_local (Slim FATE) + +当使用Spark_local + LocalFS + Pulsar/RabbitMQ的时候，会部署一个slim的FATE集群。所有的计算和存储部分都在一起，通过Spark_local和LocalFS来实现，数据传输是通过Pulsar或者RabbitMQ完成。 + +Spark local是计算组件，localfs是存储组件，Pulsar或者RabbitMQ是数据传输组件。 + +docker-compose使用的时候修改`parties.conf`配置 + +```bash +computing=Spark_local +federation=Pulsar +storage=LocalFS +``` + +***这里federation也可以替换为RabbitMQ*** + +k8s使用的时候修改`cluster.yaml`配置 + +```yaml +computing: Spark_local +federation: Pulsar +storage: LocalFS +``` + +架构图： + +
+ +
+ +## 不同引擎的比较 + +Eggroll可以作为FATE的计算，存储和传输引擎，Eggroll是一个简单高性能[联邦]机器学习的计算框架，支持多种联邦网络架构：直链模式、星型和环形等，支持不同组织之间使用证书加密，Eggroll是微众银行主导的github的开源项目。 +项目地址： + +RabbitMQ是一个简单易上手的MQ(Message Queue)，发展较早，有较多的云平台支持，适合上手Spark计算引擎的FATE进行联邦学习的时候使用。 + +Pulsar相比RabbitMQ，可以支持更大规模的集群化部署，也支持exchange模式的网络结构。使用集群化部署，适合较大规模的联邦学习计算。 + +Slim FATE相比其他模式，使用这个模式可以最大化减少集群所需的组件，在较少资源多情况下也可以运行FATE集群，可以使用在小规模联邦学习计算，IOT设备等情况。 + +## Exchange架构模式 + +当使用rollsite和Pulsar传输数据的时候，可以使用Exchange的架构来部署联邦学习网络。 + +也就是说federation是Eggroll和Pulsar的时候可以支持Exchange的使用。 + +**rollsite只能和其他rollsite的FATE通过exchange链接，Pulsar也只能和其他Pulsar的FATE通过exchange链接。** + +参考文档： + +- [使用KubeFATE部署一个多成员参与的联邦学习网络](https://github.com/FederatedAI/KubeFATE/wiki/%E4%BD%BF%E7%94%A8KubeFATE%E9%83%A8%E7%BD%B2%E4%B8%80%E4%B8%AA%E5%A4%9A%E6%88%90%E5%91%98%E5%8F%82%E4%B8%8E%E7%9A%84%E8%81%94%E9%82%A6%E5%AD%A6%E4%B9%A0%E7%BD%91%E7%BB%9C) diff --git a/docs/KubeFATE_command_line_user_guide.md b/docs/KubeFATE_CLI_user_guide.md similarity index 78% rename from docs/KubeFATE_command_line_user_guide.md rename to docs/KubeFATE_CLI_user_guide.md index 5b80c1ecd..b52199762 100644 --- a/docs/KubeFATE_command_line_user_guide.md +++ b/docs/KubeFATE_CLI_user_guide.md @@ -1,23 +1,23 @@ -# KubeFATE Command Line User Guide +# KubeFATE CLI User Guide -### What is KubeFATE command line +## What is KubeFATE CLI -KubeFATE command line is a command line tool that connects KubeFATE service to deploy FATE on Kubernetes. +KubeFATE CLI is a CLI tool that connects KubeFATE service to deploy FATE on Kubernetes. -### Install KubeFATE command line +## Install KubeFATE CLI -Before using `kubefate` command line, you need to deploy KubeFATE service. +Before using `kubefate` CLI, you need to deploy KubeFATE service. -#### Deploy KubeFATE service on Kubernetes +### Deploy KubeFATE service on Kubernetes -Get the code +Get the source code from GitHub: ```bash git clone https://github.com/FederatedAI/KubeFATE.git cd KubeFATE/k8s-deploy ``` -Deploy KubeFATE service on Kubernetes +Deploy KubeFATE service on Kubernetes: ``` kubectl apply -f ./rbac-config.yaml @@ -26,25 +26,25 @@ kubectl apply -f ./kubefate.yaml *A more detailed deployment process is here([deploy KubeFATE in Kubernetes](https://github.com/FederatedAI/KubeFATE/tree/master/k8s-deploy#initial-a-new-fate-deployment)).* -#### Install command line +### Install CLI -`kubefate` command line is developed by go and can run easily on Linux, Mac OS and Windows. +`kubefate` CLI is developed by go and can run easily on Linux, Mac OS and Windows. In addition to downloading and using the release package, it can also be compiled and installed on different platforms. -##### Linux +#### Linux ```bash go build -o bin/kubefate kubefate.go ``` -##### Mac OS +#### Mac OS ```bash go build -o bin/kubefate kubefate.go ``` -##### Windows +#### Windows ```bash go build -o bin/kubefate.exe -buildmode=exe kubefate.go @@ -52,7 +52,7 @@ go build -o bin/kubefate.exe -buildmode=exe kubefate.go Add `./bin` to the 'PATH' environment variable. -#### Modify configuration +### Modify configuration Configuration in current working directory `config.yaml` file. @@ -66,21 +66,19 @@ user: serviceurl: example.com ``` -#### Check +### Verify the KubeFATE CLI works properly Use `kubefate version` to verify that the installation is successful. -### KubeFATE command line +## KubeFATE CLI commands -If you have successfully installed `kubefate` command line, you can use these commands. +If you have successfully installed `kubefate` CLI, you can use these commands. The `kubefate` command contains command actions and parameters. -#### cluster +### cluster operations -Related operations of cluster command - -##### install +#### install Install a cluster @@ -100,7 +98,9 @@ If it runs successfully, a job_UUID will be returned. The cluster installation s *`` means `cluster.yaml` `cluster- spart.yaml` `cluster- serving.yaml` and so on.* -##### update +Note that although this command can return a job UUID, it doesn't mean that the job runs successfully. + +#### update Update a cluster @@ -117,7 +117,7 @@ OPTIONS: If it runs successfully, a job_UUID will be returned. According to the job_UUID, the cluster update status can be obtained. -##### delete +#### delete Delete a cluster @@ -126,7 +126,7 @@ kubefate cluster delete ``` If it runs successfully, a job_UUID will be returned. According to the job_UUID, the cluster deletion status can be obtained. -##### list +#### list Get the list of currently running clusters. @@ -139,7 +139,7 @@ OPTIONS: --all, -A List all clusters including deleted ones (default: false) --help, -h show help (default: false) ``` -##### describe +#### describe Get the description information of the given cluster. @@ -147,7 +147,7 @@ Get the description information of the given cluster. kubefate cluster describe ``` -##### logs +#### logs Gets the component log for a given cluster. (If no component is specified, all logs will be obtained.) @@ -170,11 +170,11 @@ rwise 10, if a selector is provided. (default: -1) --limit-bytes value Maximum bytes of logs to return. Defaults to no limit. (default: 0) --help, -h show help (default: false) ``` -#### job +### job Through the install, update and delete of the cluster, the corresponding jobs will be generated. -##### list +#### list Get the list of all jobs @@ -182,7 +182,7 @@ Get the list of all jobs kubefate job list ``` -##### stop +#### stop Cancel the Running job @@ -192,7 +192,7 @@ kubefate job stop *This only works for Running jobs of type ClusterInstall* -##### describe +#### describe Get the description information of the given job @@ -200,7 +200,7 @@ Get the description information of the given job kubefate job describe ``` -##### delete +#### delete Delete the record of the given job @@ -208,11 +208,11 @@ Delete the record of the given job kubefate job delete ``` -#### chart +### chart Chart is the management of the chart needed to install cluster. -##### upload +#### upload Upload chart file to KubeFATE service. The chart file must be generated by the `helm package` @@ -220,7 +220,7 @@ Upload chart file to KubeFATE service. The chart file must be generated by the ` kubefate chart upload -f ``` -##### list +#### list Get the chart list of existing KubeFATE services. @@ -228,7 +228,7 @@ Get the chart list of existing KubeFATE services. kubefate chart list ``` -##### delete +#### delete Delete chart file from KubeFATE service. @@ -236,9 +236,9 @@ Delete chart file from KubeFATE service. kubefate chart delete ``` -#### namespace +### namespace -##### list +#### list Get the namespace list of Kubernetes. @@ -246,9 +246,9 @@ Get the namespace list of Kubernetes. kubefate namespace list ``` -#### user +### user -##### list +#### list Get the user list of KubeFATE service. @@ -256,7 +256,7 @@ Get the user list of KubeFATE service. kubefate user list ``` -##### describe +#### describe Obtain the specific user description information of KubeFATE service. @@ -264,26 +264,23 @@ Obtain the specific user description information of KubeFATE service. kubefate user describe ``` -#### version +### version -View the corresponding version of KubeFATE service of the current command line and connection. +View the corresponding version of KubeFATE service of the current CLI and connection. ```bash kubefate version ``` -#### help +### help -Get command line help. +Get CLI help. ```bash kubefate help ``` -All commands can be add OPTIONS of `--help` to view help information. - - - +All commands supports adding `--help` to show help information. -If you have any questions, you can get help through issue. \ No newline at end of file +If you have any questions with regarding to KubeFATE CLI, you can get help through creating an issue [here](https://github.com/FederatedAI/KubeFATE/issues/new/choose). \ No newline at end of file diff --git a/docs/Manage_FATE_and_FATE-Serving_Version.md b/docs/Manage_FATE_and_FATE-Serving_Version.md index fdff26324..6b20a7304 100644 --- a/docs/Manage_FATE_and_FATE-Serving_Version.md +++ b/docs/Manage_FATE_and_FATE-Serving_Version.md @@ -1,5 +1,5 @@ # Manage FATE and FATE-Serving Version -In KubeFATE, we can deploy and manage multiple clusters including FATE and FATE-serving with multiple versions. We reuse the features of [Helm](https://helm.sh/), and fully compatiable with Helm Chart. It means the charts can be: +By using one certain version of KubeFATE CLI and KubeFATE Service, we can deploy and manage multiple FATE/FATE-Serving clusters including having multiple versions. We reuse the features of [Helm](https://helm.sh/), and fully compatible with Helm Chart. It means the charts can be: 1. Deploy directly with Helm; 2. Can be customized to fit different requirements. @@ -12,13 +12,13 @@ To set the default chart repo in kubefate.yml, value: "https://federatedai.github.io/KubeFATE/" ``` -In the sample config file is set to our address hosted in github. It can be customized to self repo which follow [The Chart Repository Guide](https://helm.sh/docs/topics/chart_repository/). +In the sample config file is set to our address hosted in GitHub. It can be customized to self repo which follow [The Chart Repository Guide](https://helm.sh/docs/topics/chart_repository/). ## Manage charts in KubeFATE manually KubeFATE provides commands to manage the charts, which include: 1. Upload a new chart; 2. List existed charts; -3. Delete a existed chart; +3. Delete an existed chart; 4. Show the details of a chart. The chart can be downloaded in each KubeFATE release, with name `fate-{release_version}.tgz`: https://github.com/FederatedAI/KubeFATE/releases @@ -30,18 +30,18 @@ The chart can be downloaded in each KubeFATE release, with name `fate-{release_v Download it and copy it to the folder to upload. ``` -$ kubefate chart upload -f ./fate-v1.8.0.tgz +$ kubefate chart upload -f ./fate-v1.9.0.tgz Upload file success $ kubefate chart ls -UUID NAME VERSION APPVERSION -ca3f7843-749a-4f69-9f6b-4c544a7623ac fate v1.8.0 v1.8.0 +UUID NAME VERSION APPVERSION +ca3f7843-749a-4f69-9f6b-4c544a7623ac fate v1.9.0 v1.9.0 ``` -Then, we can deploy the fate cluster of v1.8.0 version. The detail of cluster.yaml please refer to: [FATE Cluster Configuration](./configurations/FATE_cluster_configuration.md) +Then, we can deploy the fate cluster of v1.9.0 version. The detail of cluster.yaml please refer to: [FATE Cluster Configuration](./configurations/FATE_cluster_configuration.md) ``` chartName: fate -chartVersion: v1.8.0 +chartVersion: v1.9.0 ``` We can delete the chart with: diff --git a/docs/Pulsar_Exchange.md b/docs/Pulsar_Exchange.md index a6156743d..3b173cec5 100644 --- a/docs/Pulsar_Exchange.md +++ b/docs/Pulsar_Exchange.md @@ -1,6 +1,6 @@ -# Deploy an ATS exchange to pulsar +# Deploy an ATS exchange for pulsar -#### Create an Exchange Cluster Yaml +## Create an Exchange Cluster Yaml ```bash $ cat exchange.yaml @@ -10,7 +10,6 @@ chartName: fate-exchange chartVersion: v1.6.0 partyId: 1 registry: "" -imageTag: "1.6.0-release" pullPolicy: imagePullSecrets: - name: myregistrykey @@ -32,11 +31,9 @@ trafficServer: ``` +## Import the secret keys to Kubernetes - -#### Import the secret key to Kubernetes - -Before that, you need to finish generating the secret key. https://github.com/FederatedAI/KubeFATE/blob/master/docs/FATE_On_Spark_With_Pulsar.md#generate-cas-certificate +Before that, you need to finish generating the secret key by folling this [doc](https://github.com/FederatedAI/KubeFATE/blob/master/docs/FATE_On_Spark_With_Pulsar.md#generate-cas-certificate). Execute the corresponding command in the corresponding cluster. @@ -61,12 +58,9 @@ kubectl -n fate-exchange create secret generic traffic-server-cert \ --from-file=ca.cert.pem=certs/ca.cert.pem ``` - - -#### Configure cluster to connect to exchange +## Configure the clusters to connect to the exchange service ```bash -$ cat cluster.yaml ... pulsar: ... @@ -75,4 +69,4 @@ pulsar: port: 30000 ``` -Then you can deploy and test. \ No newline at end of file +Then you can deploy and start to use the FATE cluster. \ No newline at end of file diff --git a/docs/Secure_Gateways_with_Istio b/docs/Secure_Gateways_with_Istio new file mode 100644 index 000000000..d3c11036b --- /dev/null +++ b/docs/Secure_Gateways_with_Istio @@ -0,0 +1,140 @@ +# Secure Gateways with Istio + +## Prerequisite + +### Install Istio + +Please refer to [install Istio](https://istio.io/latest/docs/setup/getting-started/#install) + +### Install openssl + +Please refer to [install openssl](https://github.com/openssl/openssl#download) + +### Enable Istio + +Before installing a cluster yaml file with kubefate, please make sure you have enabled istio. + +```yaml +istio: + enabled: true +``` + +Please NOTE: Istio is not compatible with Spark, so please choose Eggroll as the computing if you would like to enable Istio. + +### Prepare root certificate + +Create a root certificate and private key to sign the certificates for your services. + +```bash +mkdir my-ca +cd my-ca +export CA_HOME=$(pwd) +mkdir fateboard notebook +openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -subj '/O=ca Inc./CN=ca.com' -keyout ca.key -out ca.crt +``` + +Now we get `ca.key` and `ca.crt` as CA to sign the certificates. + +### Set ingress IP and port + +If your environment has an external load balancer, set the ingress IP and ports: + +```bash +export INGRESS_HOST=$(kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip}') +export INGRESS_PORT=$(kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.spec.ports[?(@.name=="http2")].port}') +export SECURE_INGRESS_PORT=$(kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.spec.ports[?(@.name=="https")].port}') +``` + +If you're using node port, run: + +```bash +export INGRESS_HOST=$(minikube ip) +export INGRESS_PORT=$(kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.spec.ports[?(@.name=="http2")].nodePort}') +export SECURE_INGRESS_PORT=$(kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.spec.ports[?(@.name=="https")].nodePort}') +``` + +Please NOTE: enable istio will disable the ingresses for fateboard and notebook, so you have to access them with $INGRESS_HOST and $SECURE_INGRESS_PORT. + +## Generate credential for your services + +1. Create a certificate and a private key for `*.fateboard.example.com`: + + ```bash + cd $CA_HOME/fateboard + openssl req -out fateboard.csr -newkey rsa:2048 -nodes -keyout fateboard.key -subj "/CN=*.fateboard.example.com/O=example organization" + openssl x509 -req -sha256 -days 365 -CA $CA_HOME/ca.crt -CAkey $CA_HOME/ca.key -set_serial 0 -in fateboard.csr -out fateboard.crt + ``` + `/CN=*.fateboard.example.com` means the CA is requested to sign a wildcard certificate of which the common name is *.fateboard.example.com. Then the certificate could be used for all the *.fateboard.example.com subdomains. + +2. Create a tls secret for the fateboard gateway + + ```bash + kubectl create -n istio-system secret tls fateboard-credential --key=fateboard.key --cert=fateboard.crt + ``` + + The name of the secret of fateboard is fixed to `fateboard-credential` + +3. Create a certificate and a private key for `*.notebook.example.com`: + + ```bash + cd $CA_HOME/notebook + openssl req -out notebook.csr -newkey rsa:2048 -nodes -keyout notebook.key -subj "/CN=*.notebook.example.com/O=example organization" + openssl x509 -req -sha256 -days 365 -CA $CA_HOME/ca.crt -CAkey $CA_HOME/ca.key -set_serial 0 -in notebook.csr -out notebook.crt + ``` + `/CN=*.notebook.example.com` means the CA is requested to sign a wildcard certificate of which the common name is *.notebook.example.com. Then the certificate could be used for all the *.notebook.example.com subdomains. + +4. Create a tls secret for the client gateway + + ```bash + kubectl create -n istio-system secret tls client-credential --key=notebook.key --cert=notebook.crt + ``` + + The name of the secret of notebook is fixed to `client-credential` + +Now you have enabled https access for the services gateways! + +## Access Service with HTTPS + +If your partyID is 9999, access fateboard with istio $SECURE_INGRESS_PORT and cacert which signed certificates for your services. + +Test https accessibility of fateboard: + +```bash +❯ curl https://party9999.fateboard.example.com:$SECURE_INGRESS_PORT/ --cacert $CA_HOME/ca.crt -I +HTTP/1.1 200 OK +x-frame-options: DENY +last-modified: Mon, 18 Apr 2022 14:26:51 GMT +accept-ranges: bytes +vary: accept-encoding +content-type: text/html;charset=UTF-8 +content-language: en-US +content-length: 3734 +date: Tue, 09 Aug 2022 07:19:25 GMT +x-envoy-upstream-service-time: 8 +server: istio-envoy +``` + +Test https accessibility of client: + +```bash +❯ curl https://party9999.notebook.example.com:$SECURE_INGRESS_PORT/ --cacert $CA_HOME/ca.crt -I +HTTP/1.1 405 Method Not Allowed +server: istio-envoy +content-type: text/html; charset=UTF-8 +date: Tue, 09 Aug 2022 07:24:18 GMT +content-length: 87 +x-envoy-upstream-service-time: 6 +``` + +The above results show that the HTTPS connection has been established. + + + +If you have not set `party9999.fateboard.example.com` and `$INGRESS_HOST` the same IP, you have to access gateway with resolve option: + +```bash +curl https://party9999.fateboard.example.com:$SECURE_INGRESS_PORT/ --cacert $CA_HOME/ca.crt -I --resolve "party9999.fateboard.example.com:$SECURE_INGRESS_PORT:$INGRESS_HOST" +``` + + + diff --git a/docs/Update_and_Delete_a_FATE_Cluster.md b/docs/Update_and_Delete_a_FATE_Cluster.md index 129151983..be4653b70 100644 --- a/docs/Update_and_Delete_a_FATE_Cluster.md +++ b/docs/Update_and_Delete_a_FATE_Cluster.md @@ -1,8 +1,8 @@ # Update and Delete a FATE Cluster -Besides install a new cluster, KubeFATE provides command to update, delete and describe a cluster. And the cluster being managed, not only FATE, but also FATE-Serving, even other clusters to add supports with chart(Refer to [Manage FATE and FATE-Serving Version](./Manage_FATE_and_FATE-Serving_Version.md)). +In addition to installing a new cluster, KubeFATE provides command to update, delete and describe a cluster. And the cluster being managed, not only FATE, but also FATE-Serving, even other clusters to add supports with chart(Refer to [Manage FATE and FATE-Serving Version](./Manage_FATE_and_FATE-Serving_Version.md)). ## Show the detail of a cluster -With the command `kubefate cluster describe ${cluster_id}`, the details of cluster deployed can be shown, including information of how to access the cluster. +With the command `kubefate cluster describe ${cluster_id}`, the details of cluster deployed can be seen, including information of how to access the cluster. ``` $ kubefate cluster ls @@ -101,6 +101,8 @@ UUID NAME NAMESPACE REVISION and find the nodemanager has been scaled out to 2 instances. And the revision have been updated to 2. +**Note: The Update cluster functionality will not work when there is no change in the cluster.yaml file. For example, only change the helm chart will not make update happen.** + ## Delete a cluster To delete a cluster can be use command `kubefate cluster delete ${cluster_id}` ``` diff --git a/docs/Use_image_pull_secrets.md b/docs/Use_image_pull_secrets.md index affa4dc71..3432455a0 100644 --- a/docs/Use_image_pull_secrets.md +++ b/docs/Use_image_pull_secrets.md @@ -1,6 +1,6 @@ -## How to configure username and password for a images repository/registry +## How to configure username and password for an image repository/registry ### 1. Create `imagePullSecrets` -This solution can be used in both [Dockerhub](https://hub.docker.com/) or other private/public image repositories/registries. The following example take Dockerhub as an example, +This solution can be used in both [Dockerhub](https://hub.docker.com/) or other private/public image repositories/registries. The following example takes Dockerhub as an example, ```bash DOCKER_REGISTRY_SERVER= DOCKER_USER= @@ -12,13 +12,15 @@ kubectl create secret docker-registry myregistrykey \ --docker-username=$DOCKER_USER \ --docker-password=$DOCKER_PASSWORD \ --docker-email=$DOCKER_EMAIL + -n ``` +**Note that the secret will only take effect when the namespace is the same with the corresponding production resources.** + > Note: The URL of Dockerhub is：https://index.docker.io/v1/ ### 2. Add the created secret to KubeFATE config -Make sure the secrete is created in the same namespace going to deploy FATE, add it in the `imagePullSecrets` of `cluster.yaml` as following, - +Make sure the secrete is created in the same namespace going to deploy FATE, add it in the `imagePullSecrets` of `cluster.yaml` as following: ```bash imagePullSecrets: - name: myregistrykey diff --git a/docs/configurations/Docker_compose_Partys_configuration.md b/docs/configurations/Docker_compose_Partys_configuration.md new file mode 100644 index 000000000..706e8005e --- /dev/null +++ b/docs/configurations/Docker_compose_Partys_configuration.md @@ -0,0 +1,27 @@ +# `partys.conf` file introduction + +The partys.conf file is the main configuration file for docker-compose to deploy FATE. The meaning of each configuration will be described in detail here. + +| Name | Description | default | +| --- | ---- | --- | +| user | Deploy user | fate | +| dir | Deploy PATH | /data/projects/fate | +| party_list | Deploy party list | (10000 9999) | +| party_ip_list | FATE Partys IP | (192.168.1.1 192.168.1.2) | +| serving_ip_list | FATE-Serving Partys IP | (192.168.1.1 192.168.1.2) | +| computing | Computing engine | Eggroll | +| federation | Federation engine | Eggroll | +| storage | Storage engine | Eggroll | +| algorithm | Algorithm | Basic | +| device | Device | CPU | +| compute_core | Cluster compute_core number, it is recommended to be less than the number of cpu cores | 4 | +| exchangeip | Deploy exchange cluster host IP | NULL | +| mysql_ip | External mysql IP | mysql | +| mysql_user | External mysql user | fate | +| mysql_password | External mysql password | fate_dev | +| mysql_db | External mysql database | fate_flow | +| name_node | External hdfs namenode | hdfs://namenode:9000 | +| fateboard_username | Define fateboard login information username | admin | +| fateboard_password | Define fateboard login information password | admin | +| serving_admin_username | Define serving_admin login information username | admin | +| serving_admin_password | Define serving_admin login information password | admin | diff --git a/docs/configurations/FATE_Exchange_cluster_configuration.md b/docs/configurations/FATE_Exchange_cluster_configuration.md index 623a204ac..6f908c544 100644 --- a/docs/configurations/FATE_Exchange_cluster_configuration.md +++ b/docs/configurations/FATE_Exchange_cluster_configuration.md @@ -2,68 +2,66 @@ `cluster-serving.yaml` declares information about the FATE-Serving cluster to be deployed, which KubeFATE CLI uses to deploy the FATE-Serving cluster. ## cluster-exchange.yaml -| Name | Type | Description | -| ------------------------- | --------- | ------------------------------------------------------------ | -| name | scalars | FATE-Exchange cluster name. | -| namespace | scalars | Kubernetes namespace for FATE-Exchange cluster. | -| chartName | scalars | FATE chart name. (fate/fate-serving/fate-exchange) | -| chartVersion | scalars | FATE chart corresponding version. | -| partyId | scalars | FATE-Exchange cluster party id. | -| registry | scalars | Other fate images sources. | -| imageTag | scalars | Image TAG | -| pullPolicy | scalars | Kubernetes images pull policy. | -| persistence | bool | Redis and servingServer data persistence. | -| podSecurityPolicy.enabled | bool | if `true`, create & use Pod Security Policy resources | -| modules | sequences | Modules to be deployed in the FATE-Serving cluster. | -| rollsite | mappings | Configuration of FATE cluster `rollsite` module. | -| trafficServer | mappings | Configuration of FATE cluster `nodemanager` module. | +| Name | Type | Description | +|---------------------------|-----------|---------------------------------------------------------------------------------------------------------| +| name | scalars | FATE-Exchange cluster name. | +| namespace | scalars | Kubernetes namespace for FATE-Exchange cluster. | +| chartName | scalars | FATE chart name. (fate/fate-serving/fate-exchange) | +| chartVersion | scalars | FATE chart corresponding version. | +| partyId | scalars | FATE-Exchange cluster party id. | +| registry | scalars | Other fate images sources. | +| pullPolicy | scalars | Kubernetes images pull policy. | +| persistence | bool | Redis and servingServer data persistence. | +| podSecurityPolicy.enabled | bool | if `true`, create & use Pod Security Policy resources | +| modules | sequences | Modules to be deployed in the FATE-Serving cluster. | +| rollsite | mappings | Configuration of FATE cluster `rollsite` module. | +| trafficServer | mappings | Configuration of FATE cluster `nodemanager` module. | | nginx | mappings | Configuration of FATE cluster `python` module.
If you use your own redis, please delete this item. | -| externalRedisIp | mappings | Access your own Redis. | -| externalRedisPort | scalars | Access your own Redis. | -| externalRedisDatabase | scalars | Access your own Redis. | -| externalRedisPassword | scalars | Access your own Redis. | +| externalRedisIp | mappings | Access your own Redis. | +| externalRedisPort | scalars | Access your own Redis. | +| externalRedisDatabase | scalars | Access your own Redis. | +| externalRedisPassword | scalars | Access your own Redis. | ### rollsite mappings It is used to declare the `servingProxy` module in the FATE cluster to be deployed. -| Name | subitem | Type | Description | -| ------------ | --------- | --------- | ------------------------------------------------------------ | -| nodePort | | scalars | The port used by `proxy` module's kubernetes service, default range: 30000-32767. | -| ingerssHost | | scalars | The entrance of FATE-Service api. | +| Name | subitem | Type | Description | +|--------------|-----------|-----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| nodePort | | scalars | The port used by `proxy` module's kubernetes service, default range: 30000-32767. | | partyList | | sequences | If this FATE cluster is exchange cluster, partyList is all party's sequences of all parties proxy address. If this FATE cluster is one of participants, delete this configuration item. | -| partyList | partyId | scalars | Participant FATE cluster party ID. | -| partyList | partyIp | scalars | Participant FATE cluster IP. | -| partyList | partyPort | scalars | Participant FATE cluster port. | -| exchange | | mappings | FATE cluster `exchange` module's ip and port. | -| exchange | ip | mappings | FATE cluster `exchange` module's ip. . | -| exchange | port | mappings | FATE cluster `exchange` module's port. | -| nodeSelector | | mappings | kubernetes nodeSelector. | +| partyList | partyId | scalars | Participant FATE cluster party ID. | +| partyList | partyIp | scalars | Participant FATE cluster IP. | +| partyList | partyPort | scalars | Participant FATE cluster port. | +| exchange | | mappings | FATE cluster `exchange` module's ip and port. | +| exchange | ip | mappings | FATE cluster `exchange` module's ip. . | +| exchange | port | mappings | FATE cluster `exchange` module's port. | +| nodeSelector | | mappings | kubernetes nodeSelector. | ### servingServer mappings -| Name | SubItem | Type | Description | -| ------------- | ------- | -------- | ------------------------------------------------------------ | -| type | | scalars | Kubernetes ServiceTypes, default is NodePort. | -| nodePort | | scalars | The port used by `proxy` module's kubernetes service, default range: 30000-32767. | -| fateflow | | mappings | FATE cluster `python` module's fateflowIp and fateflowPort. | -| fateflow | ip | scalars | FATE cluster `python` module's fateflowIp. | -| fateflow | port | scalars | FATE cluster `python` module's fateflowPort. | -| subPath | | scalars | Path of data persistence, specify the "subPath" if the PVC is shared with other components. | -| existingClaim | | scalars | Use the existing PVC which must be created manually before bound. | +| Name | SubItem | Type | Description | +|---------------|---------|----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| type | | scalars | Kubernetes ServiceTypes, default is NodePort. | +| nodePort | | scalars | The port used by `proxy` module's kubernetes service, default range: 30000-32767. | +| fateflow | | mappings | FATE cluster `python` module's fateflowIp and fateflowPort. | +| fateflow | ip | scalars | FATE cluster `python` module's fateflowIp. | +| fateflow | port | scalars | FATE cluster `python` module's fateflowPort. | +| subPath | | scalars | Path of data persistence, specify the "subPath" if the PVC is shared with other components. | +| existingClaim | | scalars | Use the existing PVC which must be created manually before bound. | | storageClass | | scalars | Specify the "storageClass" used to provision the volume. Or the default. StorageClass will be used(the default). Set it to "-" to disable dynamic provisioning. | -| accessMode | | scalars | Kubernetes Persistent Volume Access Modes:
ReadWriteOnce
ReadOnlyMany
ReadWriteMany. | -| size | | scalars | Match the volume size of PVC. | +| accessMode | | scalars | Kubernetes Persistent Volume Access Modes:
ReadWriteOnce
ReadOnlyMany
ReadWriteMany. | +| size | | scalars | Match the volume size of PVC. | ### servingRedis mappings Configuration of kubernetes deployment redis. -| Name | Type | Description | -| ------------ | -------- | ------------------------------------------------------------ | -| password | scalars | Kubernetes ServiceTypes, default is NodePort.
Other modules can connect to the fateflow. | -| nodeSelector | mappings | kubernetes nodeSelector. | -| subPath | scalars | Path of data persistence, specify the "subPath" if the PVC is shared with other components. | -| existingClaim | scalars | Use the existing PVC which must be created manually before bound. | +| Name | Type | Description | +|---------------|----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| password | scalars | Kubernetes ServiceTypes, default is NodePort.
Other modules can connect to the fateflow. | +| nodeSelector | mappings | kubernetes nodeSelector. | +| subPath | scalars | Path of data persistence, specify the "subPath" if the PVC is shared with other components. | +| existingClaim | scalars | Use the existing PVC which must be created manually before bound. | | storageClass | scalars | Specify the "storageClass" used to provision the volume. Or the default. StorageClass will be used(the default). Set it to "-" to disable dynamic provisioning. | -| accessMode | scalars | Kubernetes Persistent Volume Access Modes:
ReadWriteOnce
ReadOnlyMany
ReadWriteMany. | -| size | scalars | Match the volume size of PVC. | +| accessMode | scalars | Kubernetes Persistent Volume Access Modes:
ReadWriteOnce
ReadOnlyMany
ReadWriteMany. | +| size | scalars | Match the volume size of PVC. | diff --git a/docs/configurations/FATE_cluster_configuration.md b/docs/configurations/FATE_cluster_configuration.md index ee8b01dfa..74aab32d2 100644 --- a/docs/configurations/FATE_cluster_configuration.md +++ b/docs/configurations/FATE_cluster_configuration.md @@ -17,7 +17,11 @@ | podSecurityPolicy.enabled | bool | if `true`, create & use Pod Security Policy resources | | ingressClassName | scalars | The Ingress class name, such as "nginx". | | * modules | sequences | Modules to be deployed in the FATE cluster. | -| backend | set(eggroll,spark) | Configure cluster computing engine( eggroll or spark) | +| computing | set(Eggroll, Spark, Spark_local) | Configure cluster computing engine( eggroll, spark or spark_local) | +| federation | set(Eggroll,Pulsar,RabbitMQ) | Configure cluster federation engine( eggroll,pulsar or rabbitmq) | +| storage | set(Eggroll,HDFS,LocalFS) | Configure cluster storage engine( eggroll, hdfs or spark) | +| algorithm | set(Basic, NN) | Configure cluster algorithm ( basic or NeuralNetwork) | +| device | set(IPCL, CPU) | Configure cluster device( ipcl or cpu) | | ingress | mappings | Custom domain of FATE UI component | | rollsite | mappings | Configuration of FATE cluster `rollsite` module. | | nodemanager | mappings | Configuration of FATE cluster `nodemanager` module. | @@ -36,8 +40,10 @@ | hdfs | mappings | Configuration of FATE cluster `hdfs` module. | | nginx | mappings | Configuration of FATE cluster `nginx` module. | | rabbitmq | mappings | Configuration of FATE cluster `rabbitmq` module. | +| pulsar | mappings | Configuration of FATE cluster `pulsar` module. | +| skippedKeys | sequences | you can customize some keys which will be ignored in yaml validation | - +***Computing federation storage algorithm device configuration introduction reference [Introduction to Engine Architecture](../Introduction_to_Engine_Architecture.md) 和 [FATE Algorithm and Computational Acceleration Selection](../FATE_Algorithm_and_Computational_Acceleration_Selection.md)*** ### list of modules @@ -53,6 +59,7 @@ - hdfs - nginx - rabbitmq +- pulsar @@ -70,6 +77,7 @@ | `client.tls` | sequences | Set this to enable TLS on the ingress record | | `spark` | mappings | Configuration of spark UI domain | | `rabbitmq` | mappings | Configuration of Rabbitmq UI domain | +| `pulsar` | mappings | Configuration of Pulsar UI domain | @@ -143,6 +151,7 @@ The parties are directly connected. | hdfs | mappings | If you use the existing hdfs, you can set this configuration | | rabbitmq | mappings | If you use the existing rabbitmq, you can set this configuration | | nginx | mappings | If you use the existing nginx, you can set this configuration | +| logLevel | scalars | The log level of the Python process, default level is Info | ### fateboard mappings @@ -204,12 +213,11 @@ Configuration of kubernetes deployment spark. Configuration of kubernetes deployment hdfs. -| Name | SubItem | Type | Description | -|-----------------------|--------------|----------|--------------------------------------------------| -| namenode/
datanode | nodeSelector | mappings | kubernetes nodeSelector. | -| | type | scalars | Kubernetes ServiceTypes, default is `ClusterIp`. | - - +| Name | SubItem | Type | Description | +|------------------------|--------------|----------|--------------------------------------------------| +| namenode/
datanode | nodeSelector | mappings | kubernetes nodeSelector. | +| | type | scalars | Kubernetes ServiceTypes, default is `ClusterIp`. | +| datanode | replicas | scalars | The replicas of the HDFS datanode pods | ### nginx mappings @@ -279,3 +287,51 @@ Configuration of kubernetes deployment rabbitmq . port: 30084 ``` + +### pulsar mappings + +Configuration of kubernetes deployment pulsar . + +| Name | Type | Description | +| ---------------- | --------- | ------------------------------------------------------------ | +| nodeSelector | mappings | kubernetes nodeSelector. | +| type | scalars | Kubernetes ServiceTypes, default is `ClusterIp`. | +| nodePort | scalars | Kubernetes Service NodePort. | +| skippedKeys | sequences | you can customize some keys which will be ignored in yaml validation | +| tolerations | | Kubernetes tolerations | +| affinity | | Kubernetes affinity | +| env | mappings | env of pulsar. | +| confs | mappings | configuration of pulsar. | +| httpNodePort | scalars | Pulsar HttpNodePort | +| httpsNodePort | scalars | Pulsar HttpsNodePort | +| loadBalancerIP | scalars | Ip of loadBalancer | +| storageClass | scalars | Specify the "storageClass" used to provision the volume. Or the default. StorageClass will be used(the default). Set it to "-" to disable dynamic provisioning. | +| existingClaim | scalars | Kubernetes existingClaim | +| accessMode | scalars | Kubernetes Persistent Volume Access Modes:
ReadWriteOnce
ReadOnlyMany
ReadWriteMany. | +| size | scalars | Match the volume size of PVC. | +| publicLB.enabled | bool | if `true`, enable publicLB | +| exchange | mappings | FATE cluster `exchange` module's ip and port. | +| resources | mappings | resources of Pod | +| route_table | mappings | route table of pulsar. | + +*example of route_table*: + +```bash +10000: + host: 192.168.0.1 + port: 30104 +9999: + host: 192.168.0.2 + port: 30094 +8888: + host: 192.168.0.3 + port: 30084 +``` + + + +*available customized confs*: + +- backlogQuotaDefaultLimitGB +- brokerDeleteInactiveTopicsFrequencySeconds + diff --git a/docs/configurations/FATE_serving_cluster_configuration.md b/docs/configurations/FATE_serving_cluster_configuration.md index d7f813f8a..026d2e180 100644 --- a/docs/configurations/FATE_serving_cluster_configuration.md +++ b/docs/configurations/FATE_serving_cluster_configuration.md @@ -2,29 +2,29 @@ `cluster-serving.yaml` declares information about the FATE-Serving cluster to be deployed, which KubeFATE CLI uses to deploy the FATE-Serving cluster. ## cluster-serving.yaml -| Name | Type | Description | -| ------------------------- | --------- | ------------------------------------------------------------ | -| name | scalars | FATE-Serving cluster name. | -| namespace | scalars | Kubernetes namespace for FATE-Serving cluster. | -| chartName | scalars | FATE chart name. (fate/fate-serving) | -| chartVersion | scalars | FATE chart corresponding version. | -| partyId | scalars | FATE-Serving cluster party id. | -| registry | scalars | Other fate images sources. | -| pullPolicy | scalars | Kubernetes images pull policy. | +| Name | Type | Description | +|---------------------------|-----------|---------------------------------------------------------------------------------------------------------------------------------------------------| +| name | scalars | FATE-Serving cluster name. | +| namespace | scalars | Kubernetes namespace for FATE-Serving cluster. | +| chartName | scalars | FATE chart name. (fate/fate-serving) | +| chartVersion | scalars | FATE chart corresponding version. | +| partyId | scalars | FATE-Serving cluster party id. | +| registry | scalars | Other fate images sources. | +| pullPolicy | scalars | Kubernetes images pull policy. | | imagePullSecrets | slice | An **imagePullSecrets** is an authorization token, also known as a secret, that stores Docker credentials that are used for accessing a registry. | -| persistence | bool | Redis and servingServer data persistence. | -| istio | mappings | Whether to open istio | -| podSecurityPolicy.enabled | bool | if `true`, create & use Pod Security Policy resources | -| modules | sequences | Modules to be deployed in the FATE-Serving cluster. | -| ingress | mappings | Custom domain of FATE-Serving UI component | -| servingAdmin | mappings | Configuration of FATE cluster `servingAdmin` module. | -| servingZookeeper | mappings | Configuration of FATE cluster `servingZookeeper` module. | -| servingProxy | mappings | Configuration of FATE cluster `rollsite` module. | -| servingServer | mappings | Configuration of FATE cluster `nodemanager` module. | -| servingRedis | mappings | Configuration of FATE cluster `python` module.
If you use your own redis, please delete this item. | -| externalRedisIp | scalars | Access your own Redis. | -| externalRedisPort | scalars | Access your own Redis. | -| externalRedisPassword | scalars | Access your own Redis. | +| persistence | bool | Redis and servingServer data persistence. | +| istio | mappings | Whether to open istio | +| podSecurityPolicy.enabled | bool | if `true`, create & use Pod Security Policy resources | +| modules | sequences | Modules to be deployed in the FATE-Serving cluster. | +| ingress | mappings | Custom domain of FATE-Serving UI component | +| servingAdmin | mappings | Configuration of FATE cluster `servingAdmin` module. | +| servingZookeeper | mappings | Configuration of FATE cluster `servingZookeeper` module. | +| servingProxy | mappings | Configuration of FATE cluster `rollsite` module. | +| servingServer | mappings | Configuration of FATE cluster `nodemanager` module. | +| servingRedis | mappings | Configuration of FATE cluster `python` module.
If you use your own redis, please delete this item. | +| externalRedisIp | scalars | Access your own Redis. | +| externalRedisPort | scalars | Access your own Redis. | +| externalRedisPassword | scalars | Access your own Redis. | ### list of modules @@ -37,7 +37,7 @@ ### ingress mappings | Name | Type | Description | -| --------------------------- | --------- | -------------------------------------------- | +|-----------------------------|-----------|----------------------------------------------| | `servingProxy` | mappings | Configuration of Fateboard UI domain | | `servingProxy.annotations` | mappings | The annotations used commonly for ingresses | | `servingProxy.hosts` | sequences | Set hosts list of ingress record | @@ -54,7 +54,7 @@ This is the UI display component of FATE-Serving. | Name | subitem | Type | Description | -| ------------ | ------- | -------- | ------------------------ | +|--------------|---------|----------|--------------------------| | nodeSelector | | mappings | kubernetes nodeSelector. | | username | | scalars | username | | password | | scalars | password | @@ -63,57 +63,57 @@ This is the UI display component of FATE-Serving. It is used to declare the `servingProxy` module in the FATE cluster to be deployed. -| Name | subitem | Type | Description | -| ------------ | --------- | --------- | ------------------------------------------------------------ | -| nodePort | | scalars | The port used by `proxy` module's kubernetes service, default range: 30000-32767. | +| Name | subitem | Type | Description | +|--------------|-----------|-----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| nodePort | | scalars | The port used by `proxy` module's kubernetes service, default range: 30000-32767. | | partyList | | sequences | If this FATE cluster is exchange cluster, partyList is all party's sequences of all parties proxy address. If this FATE cluster is one of participants, delete this configuration item. | -| partyList | partyId | scalars | Participant FATE cluster party ID. | -| partyList | partyIp | scalars | Participant FATE cluster IP. | -| partyList | partyPort | scalars | Participant FATE cluster port. | -| exchange | | mappings | FATE cluster `exchange` module's ip and port. | -| exchange | ip | mappings | FATE cluster `exchange` module's ip. . | -| exchange | port | mappings | FATE cluster `exchange` module's port. | -| nodeSelector | | mappings | kubernetes nodeSelector. | +| partyList | partyId | scalars | Participant FATE cluster party ID. | +| partyList | partyIp | scalars | Participant FATE cluster IP. | +| partyList | partyPort | scalars | Participant FATE cluster port. | +| exchange | | mappings | FATE cluster `exchange` module's ip and port. | +| exchange | ip | mappings | FATE cluster `exchange` module's ip. . | +| exchange | port | mappings | FATE cluster `exchange` module's port. | +| nodeSelector | | mappings | kubernetes nodeSelector. | ### servingServer mappings -| Name | SubItem | Type | Description | -| ------------- | ------- | -------- | ------------------------------------------------------------ | -| type | | scalars | Kubernetes ServiceTypes, default is NodePort. | -| nodePort | | scalars | The port used by `proxy` module's kubernetes service, default range: 30000-32767. | -| fateflow | | mappings | FATE cluster `python` module's fateflowIp and fateflowPort. | -| fateflow | ip | scalars | FATE cluster `python` module's fateflowIp. | -| fateflow | port | scalars | FATE cluster `python` module's fateflowPort. | -| subPath | | scalars | Path of data persistence, specify the "subPath" if the PVC is shared with other components. | -| existingClaim | | scalars | Use the existing PVC which must be created manually before bound. | +| Name | SubItem | Type | Description | +|---------------|---------|----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| type | | scalars | Kubernetes ServiceTypes, default is NodePort. | +| nodePort | | scalars | The port used by `proxy` module's kubernetes service, default range: 30000-32767. | +| fateflow | | mappings | FATE cluster `python` module's fateflowIp and fateflowPort. | +| fateflow | ip | scalars | FATE cluster `python` module's fateflowIp. | +| fateflow | port | scalars | FATE cluster `python` module's fateflowPort. | +| subPath | | scalars | Path of data persistence, specify the "subPath" if the PVC is shared with other components. | +| existingClaim | | scalars | Use the existing PVC which must be created manually before bound. | | storageClass | | scalars | Specify the "storageClass" used to provision the volume. Or the default. StorageClass will be used(the default). Set it to "-" to disable dynamic provisioning. | -| accessMode | | scalars | Kubernetes Persistent Volume Access Modes:
ReadWriteOnce
ReadOnlyMany
ReadWriteMany. | -| size | | scalars | Match the volume size of PVC. | +| accessMode | | scalars | Kubernetes Persistent Volume Access Modes:
ReadWriteOnce
ReadOnlyMany
ReadWriteMany. | +| size | | scalars | Match the volume size of PVC. | ### servingRedis mappings Configuration of kubernetes deployment redis. -| Name | Type | Description | -| ------------ | -------- | ------------------------------------------------------------ | -| password | scalars | Kubernetes ServiceTypes, default is NodePort.
Other modules can connect to the fateflow. | -| nodeSelector | mappings | kubernetes nodeSelector. | -| subPath | scalars | Path of data persistence, specify the "subPath" if the PVC is shared with other components. | -| existingClaim | scalars | Use the existing PVC which must be created manually before bound. | +| Name | Type | Description | +|---------------|----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| password | scalars | Kubernetes ServiceTypes, default is NodePort.
Other modules can connect to the fateflow. | +| nodeSelector | mappings | kubernetes nodeSelector. | +| subPath | scalars | Path of data persistence, specify the "subPath" if the PVC is shared with other components. | +| existingClaim | scalars | Use the existing PVC which must be created manually before bound. | | storageClass | scalars | Specify the "storageClass" used to provision the volume. Or the default. StorageClass will be used(the default). Set it to "-" to disable dynamic provisioning. | -| accessMode | scalars | Kubernetes Persistent Volume Access Modes:
ReadWriteOnce
ReadOnlyMany
ReadWriteMany. | -| size | scalars | Match the volume size of PVC. | +| accessMode | scalars | Kubernetes Persistent Volume Access Modes:
ReadWriteOnce
ReadOnlyMany
ReadWriteMany. | +| size | scalars | Match the volume size of PVC. | ### servingZookeeper mappings Configuration of kubernetes deployment zookeeper. -| Name | Type | Description | -| ------------- | -------- | ------------------------------------------------------------ | -| nodeSelector | mappings | kubernetes nodeSelector. | -| subPath | scalars | Path of data persistence, specify the "subPath" if the PVC is shared with other components. | -| existingClaim | scalars | Use the existing PVC which must be created manually before bound. | +| Name | Type | Description | +|---------------|----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| nodeSelector | mappings | kubernetes nodeSelector. | +| subPath | scalars | Path of data persistence, specify the "subPath" if the PVC is shared with other components. | +| existingClaim | scalars | Use the existing PVC which must be created manually before bound. | | storageClass | scalars | Specify the "storageClass" used to provision the volume. Or the default. StorageClass will be used(the default). Set it to "-" to disable dynamic provisioning. | -| accessMode | scalars | Kubernetes Persistent Volume Access Modes:
ReadWriteOnce
ReadOnlyMany
ReadWriteMany. | -| size | scalars | Match the volume size of PVC. | +| accessMode | scalars | Kubernetes Persistent Volume Access Modes:
ReadWriteOnce
ReadOnlyMany
ReadWriteMany. | +| size | scalars | Match the volume size of PVC. | diff --git a/docs/configurations/kubefate_service_configuration.md b/docs/configurations/kubefate_service_configuration.md index 1c8ef1f0d..68d542443 100644 --- a/docs/configurations/kubefate_service_configuration.md +++ b/docs/configurations/kubefate_service_configuration.md @@ -31,21 +31,21 @@ env: - name: FATECLOUD_LOG_LEVEL value: "debug" ``` -| Name | Description | -| ------------------------ | ------------------------------------------------------------ | -| FATECLOUD_DB_TYPE | Database types that support kubefate. At present, there is only "mysql". | -| FATECLOUD_DB_HOST | DB host, for example "localhost". | -| FATECLOUD_DB_PORT | DB port, for example "3306". | -| FATECLOUD_DB_NAME | Database name. | -| FATECLOUD_DB_USERNAME | Database user. | -| FATECLOUD_DB_PASSWORD | Database password. | +| Name | Description | +|--------------------------|-------------------------------------------------------------------------------------| +| FATECLOUD_DB_TYPE | Database types that support kubefate. At present, there is only "mysql". | +| FATECLOUD_DB_HOST | DB host, for example "localhost". | +| FATECLOUD_DB_PORT | DB port, for example "3306". | +| FATECLOUD_DB_NAME | Database name. | +| FATECLOUD_DB_USERNAME | Database user. | +| FATECLOUD_DB_PASSWORD | Database password. | | FATECLOUD_REPO_NAME | Remote helm [chart repository](https://helm.sh/docs/topics/chart_repository/) name. | -| FATECLOUD_REPO_URL | Remote helm chart repository url. | -| FATECLOUD_USER_USERNAME | Username of KubeFATE service default user. | -| FATECLOUD_USER_PASSWORD | Password of KubeFATE service default user. | -| FATECLOUD_SERVER_ADDRESS | KubeFATE service address. | -| FATECLOUD_SERVER_PORT | KubeFATE service port. | -| FATECLOUD_LOG_LEVEL | KubeFATE service log level. | +| FATECLOUD_REPO_URL | Remote helm chart repository url. | +| FATECLOUD_USER_USERNAME | Username of KubeFATE service default user. | +| FATECLOUD_USER_PASSWORD | Password of KubeFATE service default user. | +| FATECLOUD_SERVER_ADDRESS | KubeFATE service address. | +| FATECLOUD_SERVER_PORT | KubeFATE service port. | +| FATECLOUD_LOG_LEVEL | KubeFATE service log level. | ### Service ``` @@ -69,10 +69,12 @@ rules: serviceName: kubefate servicePort: 8080 ``` -|Name |Description | -|--------|-------------------------------------| -|host |Defining the domain name of ingress. | -|backend |Backend depends on KubeFATE service. | +| Name | Description | +|---------|--------------------------------------| +| host | Defining the domain name of ingress. | +| backend | Backend depends on KubeFATE service. | + +You can config the nginx-ingress to enable the service with TLS. More info please refer to [kubefate_service_tls_enable](./kubefate_service_tls_enable.md) ## MySQL ### Deployment diff --git a/docs/configurations/kubefate_service_tls_enable.md b/docs/configurations/kubefate_service_tls_enable.md new file mode 100644 index 000000000..12605b3cd --- /dev/null +++ b/docs/configurations/kubefate_service_tls_enable.md @@ -0,0 +1,64 @@ +# How to enable TLS for KubeFATE service + +This artical will discuss some method to config the KubeFATE service to support calling its API, both RESTFul API and the cli, with TLS (HTTPS). + +## The Best Practice +The best way to turn KubeFATE service into HTTPS from HTTP is using CDN or reverse proxy. +- **CDN** +Take the Cloudflare as an example, you need to make sure you've aleady exposed the kubefate's nginx-ingress to 80 port. Then you can add item in the Cloudflare DNS and set the TLS option as below: +![cf_domain_setting](../images/enable_tls_cf_domain_setting.png) +![cf_tls_setting](../images/enable_tls_cf_tls_setting.png) + +- **Reverse Proxy** +You can set up a reverse proxy ([Nginx](https://www.nginx.com), [Caddy](https://caddyserver.com), etc) in front of your nginx-ingress. You need using them to proxy your domain's 443 port to the nginx-ingress exposed port. After that, config the TLS/SSL with server certificate. You can get that from CA, [Let's Encrypt](https://letsencrypt.org) or [sign by yourself]() +There're many useful tutorials on the Internet and you can also use some tools ([NginxProxyManager](https://nginxproxymanager.com)) to help you finishing the configuration. + +## Config in Nginx-ingress +This section we'll talk about how to config the TLS in your YAML of nginx-ingress. +All below happens in Linux enviroment and takes *example.com* as the domain. +1. Sign yourself a server certificate + ```bash + openssl genrsa -out ca.key 2048 + openssl req -sha256 -new -x509 -days 3650 -key ca.key -out ca.crt -subj "/C=CN/ST=BJ/L=BJ/O=ca/OU=test/CN=demoCA" + openssl genrsa -out server.key 2048 + openssl req -new -sha256 -key server.key -subj "/C=CN/ST=BJ/L=BJ/O=ca/OU=test/CN=example.com" -reqexts SAN -config <(cat /etc/ssl/openssl.cnf <(printf "[SAN]\nsubjectAltName=DNS:example.com")) -out server.csr + mkdir demoCA/newcerts -p + touch demoCA/index.txt + echo "01" > demoCA/serial + openssl ca -in server.csr -md sha256 -keyfile ca.key -cert ca.crt -extensions SAN -config <(cat /etc/ssl/openssl.cnf <(printf "[SAN]\nsubjectAltName=DNS:example.com")) -out server.crt + ``` + You'll get the server.key & server.crt under your current directory. +2. Set the TLS with certificate in nginx-ingress + ```bash + kubectl create secret -n kube-fate tls ingress-cert --key ./server.key --cert ./server.crt + ``` + Then add the TLS config in your kubefate.yaml ingress part as below (last 4 lines): + ```yaml + spec: + ingressClassName: nginx + rules: + - host: example.com + http: + paths: + - backend: + service: + name: kubefate + port: + number: 8080 + path: / + pathType: Prefix + tls: + - hosts: + - example.com + secretName: ingress-cert + ``` +## CLI config +Before we run the cli, due to it's a self-signed cert, we need to add our own CA root cert into the trust-store of the Linux where the kubefate cli will be run. +```bash +cp ca.crt /etc/ssl/certs/ +``` +Then we set the *safeconnect* to be **true** in the config.yaml of kubefate cli. +```yaml +safeconnect: true +``` +About the kubefate cli config file, you can refer to [Preparing cluster configuration and deploying FATE](../../k8s-deploy/README.md#preparing-cluster-configuration-and-deploying-fate). \ No newline at end of file diff --git a/docs/images/arch_eggroll.png b/docs/images/arch_eggroll.png index d822f3b8c..2d3b38844 100644 Binary files a/docs/images/arch_eggroll.png and b/docs/images/arch_eggroll.png differ diff --git a/docs/images/arch_slim.png b/docs/images/arch_slim.png new file mode 100644 index 000000000..9fd0c614a Binary files /dev/null and b/docs/images/arch_slim.png differ diff --git a/docs/images/arch_spark.png b/docs/images/arch_spark.png deleted file mode 100644 index 4e303157d..000000000 Binary files a/docs/images/arch_spark.png and /dev/null differ diff --git a/docs/images/arch_spark_pulsar.png b/docs/images/arch_spark_pulsar.png new file mode 100644 index 000000000..d8e2deada Binary files /dev/null and b/docs/images/arch_spark_pulsar.png differ diff --git a/docs/images/arch_spark_rabbitmq.png b/docs/images/arch_spark_rabbitmq.png new file mode 100644 index 000000000..26cdc6537 Binary files /dev/null and b/docs/images/arch_spark_rabbitmq.png differ diff --git a/docs/images/enable_tls_cf_domain_setting.png b/docs/images/enable_tls_cf_domain_setting.png new file mode 100644 index 000000000..8672120aa Binary files /dev/null and b/docs/images/enable_tls_cf_domain_setting.png differ diff --git a/docs/images/enable_tls_cf_tls_setting.png b/docs/images/enable_tls_cf_tls_setting.png new file mode 100644 index 000000000..ebcdbd3a1 Binary files /dev/null and b/docs/images/enable_tls_cf_tls_setting.png differ diff --git a/docs/tutorials/Build_Two_Parties_FATE_Cluster_in_One_Linux_Machine_with_MiniKube.md b/docs/tutorials/Build_Two_Parties_FATE_Cluster_in_One_Linux_Machine_with_MiniKube.md index bd97b7984..0b32f61a9 100644 --- a/docs/tutorials/Build_Two_Parties_FATE_Cluster_in_One_Linux_Machine_with_MiniKube.md +++ b/docs/tutorials/Build_Two_Parties_FATE_Cluster_in_One_Linux_Machine_with_MiniKube.md @@ -6,7 +6,7 @@ learning with these two parties, and check FATE-Dashboard for the status of the After the tutorial, the deployment architecture looks like the following diagram.
- +
# Prerequisites @@ -19,13 +19,17 @@ After the tutorial, the deployment architecture looks like the following diagram 4. Configure username and password for a images repository/registry after the docker has been installed, please refer to [use image pull secrets](https://github.com/federatedai/KubeFATE/blob/master/docs/Use_image_pull_secrets.md). 5. Network connectivity to dockerhub or 163 Docker Image Registry, and google gcr. -6. Setup the global KubeFATE version using in the tutorial and create a folder for the whole tutorial. We use - KubeFATE v1.8.0 in this tutorial, other versions should be similar. +6. Setup the global KubeFATE version using in the tutorial and create a folder for the whole tutorial. ``` -export release_version=v1.8.0 && export kubefate_version=v1.4.4 && cd ~ && mkdir demo && cd demo +export fate_version=v1.9.0 && export kubefate_version=v1.4.5 && cd ~ && mkdir demo && cd demo ``` -**!!!Note: in this tutorial, the IP of the machine we used is 192.168.100.123. Please change it to your machine's IP in all the following commands and config files.** +Notes: +* When talking about KubeFATE version, usually there are 3 notions: + * The KubeFATE CLI version, in this tutorial, it is v1.4.5. + * The KubeFATE service version, in this tutorial, it is v1.4.5. + * The FATE version, in this tutorial, it is v1.9.0, it also means the version of the helm chart of FATE, currently we use this version to tag the KubeFATE GitHub master branch. +* **In this tutorial, the IP of the machine we used is 192.168.100.123. Please change it to your machine's IP in all the following commands and config files.** # Start Tutorial ## Install the dependencies @@ -50,8 +54,8 @@ curl -LO https://github.com/kubernetes/minikube/releases/download/v1.19.0/miniku Try to verify if MiniKube installed, ``` kubefate@machine:~/demo$ minikube version -minikube version: v1.21.0 -commit: 76d74191d82c47883dc7e1319ef7cebd3e00ee11 +minikube version: v1.19.0 +commit: 15cede53bdc5fe242228853e737333b09d4336b5 ``` ### Install Kubernetes with MiniKube @@ -59,7 +63,7 @@ In a Linux machine, we suggest using Docker as the hypervisor, which is easy. Th [Install MiniKube - Install a Hypervisor](https://kubernetes.io/docs/tasks/tools/install-minikube/#install-a-hypervisor). It is only one command, ``` -sudo minikube start --vm-driver=none --kubernetes-version v1.19.0 +sudo minikube start --vm-driver=none --kubernetes-version v1.19.0 --cni=flannel ``` Wait a few seconds until the command finishes, then try to verify if Kubernetes installed, ``` @@ -76,14 +80,16 @@ another command. ``` sudo minikube addons enable ingress ``` -Till now, Kubernetes have been ready. + +Check the pod status by `kubectl get pods -A` +When all the pods are in the ready state, it means your Kubernetes cluster is ready. ## Setup Kubefate ### Install KubeFATE CLI Go to [KubeFATE Release](https://github.com/FederatedAI/KubeFATE/releases), and find the latest kubefate-k8s release -pack, which is `v1.8.0` as set to ENVs before. (replace ${release_version} with the newest version available) +pack, which is `v1.9.0` as set to ENVs before. (replace ${fate_version} with the newest version available) ``` -curl -LO https://github.com/FederatedAI/KubeFATE/releases/download/${release_version}/kubefate-k8s-${release_version}.tar.gz && tar -xzf ./kubefate-k8s-${release_version}.tar.gz +curl -LO https://github.com/FederatedAI/KubeFATE/releases/download/${fate_version}/kubefate-k8s-${fate_version}.tar.gz && tar -xzf ./kubefate-k8s-${fate_version}.tar.gz ``` Then we will get the release pack of KubeFATE, verify it, ``` @@ -92,15 +98,15 @@ kubefate@machine:~/kubefate ls cluster-serving.yaml cluster-spark-rabbitmq.yaml cluster.yaml examples rbac-config.yaml cluster-spark-pulsar.yaml cluster-spark-slim.yaml config.yaml kubefate.yaml ``` -Move the kubefate executable binary to path, +Move the KubeFATE executable binary to path, ``` chmod +x ./kubefate && sudo mv ./kubefate /usr/bin ``` -Try to verify if kubefate works, +Try to verify if the KubeFATE CLI works, ``` kubefate@machine:~/kubefate$ kubefate version -* kubefate commandLine version=v1.4.4 -* kubefate service connection error, resp.StatusCode=404, error: +* kubefate commandLine version=v1.4.5 +* kubefate service connection error, resp¬.StatusCode=404, error: @@ -113,14 +119,14 @@ kubefate@machine:~/kubefate$ kubefate version ``` -It is fine only the command line version shows and get an error on KubeFATE service's version because we have not +It is fine that only the command line version shows up but get an error on KubeFATE service's version because we have not deployed the KubeFATE service yet. ### Deploy KubeFATE service #### 1. Load the docker image of the KubeFATE service -Download the KubeFATE Server v1.4.4's docker image, +Download the KubeFATE Server v1.4.5's docker image, ``` -curl -LO https://github.com/FederatedAI/KubeFATE/releases/download/${release_version}/kubefate-${kubefate_version}.docker +curl -LO https://github.com/FederatedAI/KubeFATE/releases/download/${fate_version}/kubefate-${kubefate_version}.docker ``` and load into local Docker. Please note that, because we are using MiniKube, which is an all-in-one deployment of Kubernetes, loading image to local is work for this tutorial. If you are running a cluster-installed Kubernetes, @@ -128,7 +134,7 @@ the image needs to be loaded into [Docker Registry](https://docs.docker.com/regi [Harbor](https://goharbor.io/). For the details of using Harbor as a local image registry, please refer to: https://github.com/FederatedAI/KubeFATE/blob/master/registry/README.md. ``` -docker load < kubefate-v1.4.4.docker +docker load < kubefate-v1.4.5.docker ``` #### 2. Create kube-fate namespace and account for KubeFATE service We have prepared the yaml for creating kube-fate namespace, as well as creating a service account in rbac-config.yaml in your working folder. Just apply it, @@ -208,8 +214,8 @@ rtt min/avg/max/mdev = 0.054/0.067/0.080/0.013 ms When `example.com` well set, KubeFATE service version can be shown, ``` kubefate@machine:~/kubefate$ kubefate version -* kubefate service version=v1.4.4 -* kubefate commandLine version=v1.4.4 +* kubefate service version=v1.4.5 +* kubefate commandLine version=v1.4.5 ``` Note: The `kubefate` CLI can only work in the same directory of config.yaml @@ -223,17 +229,36 @@ kubectl create namespace fate-9999 kubectl create namespace fate-10000 ``` -We have 2 preset examples in `/kubefate/examples/party-9999/` and `/kubefate/examples/party-10000`. +If you are using dockerhub as the registry, you can create the docker credentials for these 2 namespaces because we will have several images to download. + +``` +DOCKER_REGISTRY_SERVER=docker.io +DOCKER_USER= +DOCKER_PASSWORD= + +kubectl -n fate-9999 create secret docker-registry myregistrykey \ + --docker-server=$DOCKER_REGISTRY_SERVER \ + --docker-username=$DOCKER_USER \ + --docker-password=$DOCKER_PASSWORD + +kubectl -n fate-10000 create secret docker-registry myregistrykey \ + --docker-server=$DOCKER_REGISTRY_SERVER \ + --docker-username=$DOCKER_USER \ + --docker-password=$DOCKER_PASSWORD +``` + +We have several preset examples in `/kubefate/examples/party-9999/` and `/kubefate/examples/party-10000`. + +In this tutorial, we will take the Spark+Pulsar architecture as the example -For `/kubefate/examples/party-9999/cluster.yaml`, modify it as following: +For `/kubefate/examples/party-9999/cluster-spark-pulsar.yaml`, modify it as following: ``` name: fate-9999 namespace: fate-9999 chartName: fate -chartVersion: v1.8.0 +chartVersion: v1.9.0 partyId: 9999 -registry: "hub.c.163.com/federatedai" -imageTag: "1.8.0-release" +registry: "" pullPolicy: imagePullSecrets: - name: myregistrykey @@ -244,31 +269,34 @@ podSecurityPolicy: enabled: false ingressClassName: nginx modules: - - rollsite - - clustermanager - - nodemanager - - mysql - python + - mysql - fateboard - client + - spark + - hdfs + - nginx + - pulsar -backend: eggroll +computing: Spark +federation: Pulsar +storage: HDFS +algorithm: Basic +device: CPU ingress: fateboard: hosts: - name: party9999.fateboard.example.com - client: + client: hosts: - name: party9999.notebook.example.com - -rollsite: - type: NodePort - nodePort: 30091 - partyList: - - partyId: 10000 - partyIp: 192.168.100.123 - partyPort: 30101 + spark: + hosts: + - name: party9999.spark.example.com + pulsar: + hosts: + - name: party9999.pulsar.example.com python: type: NodePort @@ -278,51 +306,81 @@ python: servingIp: 192.168.100.123 servingPort: 30095 + +nginx: + type: NodePort + httpNodePort: 30093 + grpcNodePort: 30098 + route_table: + 10000: + fateflow: + - host: 192.168.100.123 + http_port: 30103 + grpc_port: 30108 + +pulsar: + type: NodePort + httpNodePort: 30094 + httpsNodePort: 30099 + publicLB: + enabled: false + route_table: + 9999: + host: pulsar + port: 6650 + sslPort: 6651 + 10000: + host: 192.168.100.123 + port: 30104 + sslPort: 30109 + proxy: "" ``` and for fate-10000: ``` name: fate-10000 namespace: fate-10000 chartName: fate -chartVersion: v1.8.0 +chartVersion: v1.9.0 partyId: 10000 -registry: "hub.c.163.com/federatedai" -imageTag: "1.8.0-release" +registry: "" pullPolicy: imagePullSecrets: - name: myregistrykey persistence: false istio: enabled: false +ingressClassName: nginx podSecurityPolicy: enabled: false -ingressClassName: nginx modules: - - rollsite - - clustermanager - - nodemanager - - mysql - python + - mysql - fateboard - client + - spark + - hdfs + - nginx + - pulsar -backend: eggroll +computing: Spark +federation: Pulsar +storage: HDFS +algorithm: Basic +device: CPU ingress: fateboard: hosts: - name: party10000.fateboard.example.com - client: + client: hosts: - name: party10000.notebook.example.com - -rollsite: - type: NodePort - nodePort: 30101 - partyList: - - partyId: 9999 - partyIp: 192.168.100.123 - partyPort: 30091 + spark: + hosts: + - name: party10000.spark.example.com + pulsar: + hosts: + - name: party10000.pulsar.example.com python: type: NodePort @@ -332,6 +390,34 @@ python: servingIp: 192.168.100.123 servingPort: 30105 + +nginx: + type: NodePort + httpNodePort: 30103 + grpcNodePort: 30108 + route_table: + 9999: + fateflow: + - host: 192.168.100.123 + http_port: 30093 + grpc_port: 30098 + +pulsar: + type: NodePort + httpNodePort: 30104 + httpsNodePort: 30109 + publicLB: + enabled: false + route_table: + 9999: + host: 192.168.100.123 + port: 30094 + sslPort: 30099 + proxy: "" + 10000: + host: pulsar + port: 6650 + sslPort: 6651 ``` For the two files, pay extra attention of modify the partyId to the correct number otherwise you are not able to access the notebook or the fateboard. @@ -343,9 +429,9 @@ https://githubcom/FederatedAI/KubeFATE/blob/master/docs/configurations/FATE_clus ### Install the FATE clusters Okay, we can start to install these two FATE cluster via KubeFATE with the following command: ``` -kubefate@machine:~/kubefate$ kubefate cluster install -f examples/party-9999/cluster.yaml +kubefate@machine:~/kubefate$ kubefate cluster install -f examples/party-9999/cluster-spark-pulsar.yaml create job success, job id=2c1d926c-bb57-43d3-9127-8cf3fc6deb4b -kubefate@machine:~/kubefate$ kubefate cluster install -f examples/party-10000/cluster.yaml +kubefate@machine:~/kubefate$ kubefate cluster install -f examples/party-10000/cluster-spark-pulsar.yaml create job success, job id=7752db70-e368-41fa-8827-d39411728d1b ``` @@ -354,8 +440,8 @@ or watch the clusters till their STATUS changing to `Running`: ``` kubefate@machine:~/kubefate$ watch kubefate cluster ls UUID NAME NAMESPACE REVISION STATUS CHART ChartVERSION AGE -51476469-b473-4d41-b2d5-ea7241d5eac7 fate-9999 fate-9999 1 Running fate v1.8.0 88s -dacc0549-b9fc-463f-837a-4e7316db2537 fate-10000 fate-10000 1 Running fate v1.8.0 69s +29878fa9-aeee-4ae5-a5b7-fd4e9eb7c1c3 fate-9999 fate-9999 1 Running fate v1.9.0 88s +dacc0549-b9fc-463f-837a-4e7316db2537 fate-10000 fate-10000 1 Running fate v1.9.0 69s ``` We have about 10G Docker images that need to be pulled, this step will take a while for the first time. An alternative way is offline loading the images to the local environment. @@ -368,111 +454,142 @@ kubectl get po -n fate-10000 When finished applying the image, the result will be similar to this, ``` -NAME READY STATUS RESTARTS AGE -clustermanager-bcfc6866d-nfs6c 1/1 Running 0 12m -mysql-c77b7b94b-zblt5 1/1 Running 0 12m -nodemanager-0-5599db57f4-2khcg 2/2 Running 0 12m -nodemanager-1-7c986f9454-qcscd 2/2 Running 0 12m -python-57b66d96bd-vj8kq 3/3 Running 0 12m -rollsite-7846898d6d-j2gb9 1/1 Running 0 12m +NAME READY STATUS RESTARTS AGE +client-0 1/1 Running 0 53m +datanode-0 1/1 Running 0 53m +datanode-1 1/1 Running 0 40m +datanode-2 1/1 Running 0 40m +mysql-0 1/1 Running 0 53m +namenode-0 1/1 Running 0 53m +nginx-75b7565846-kpj86 1/1 Running 5 53m +pulsar-0 1/1 Running 1 53m +python-0 2/2 Running 0 53m +spark-master-fc67d9b57-99sjx 1/1 Running 1 53m +spark-worker-f74f94fdb-44248 1/1 Running 1 53m +spark-worker-f74f94fdb-bx2jv 1/1 Running 1 53m ``` ### Verify the deployment From above `kubefate cluster ls` command, we know the cluster UUID of `fate-9999` is -`51476469-b473-4d41-b2d5-ea7241d5eac7`, while cluster UUID of `fate-10000` is `dacc0549-b9fc-463f-837a-4e7316db2537`. +`29878fa9-aeee-4ae5-a5b7-fd4e9eb7c1c3`, while cluster UUID of `fate-10000` is `dacc0549-b9fc-463f-837a-4e7316db2537`. Thus, we can query there access information by: ``` -kubefate@machine:~/demo$ kubefate cluster describe 51476469-b473-4d41-b2d5-ea7241d5eac7 -UUID 51476469-b473-4d41-b2d5-ea7241d5eac7 -Name fate-9999 -NameSpace fate-9999 -ChartName fate -ChartVersion v1.8.0 -Revision 1 -Age 15h -Status Running -Spec backend: eggroll - chartName: fate - chartVersion: v1.8.0 - imagePullSecrets: - - name: myregistrykey - imageTag: 1.8.0-release - ingress: - client: - annotations: - kubernetes.io/ingress.class: nginx - hosts: - - name: party9999.notebook.example.com - fateboard: - annotations: - kubernetes.io/ingress.class: nginx - hosts: - - name: party9999.fateboard.example.com - istio: - enabled: false - modules: - - rollsite - - clustermanager - - nodemanager - - mysql - - python - - fateboard - - client - name: fate-9999 - namespace: fate-9999 - partyId: 9999 - persistence: false - podSecurityPolicy: - enabled: false - pullPolicy: null - python: - grpcNodePort: 30092 - httpNodePort: 30097 - type: NodePort - registry: "" - rollsite: - nodePort: 30091 - partyList: - - partyId: 10000 - partyIp: 10.192.173.64 - partyPort: 30101 - type: NodePort - servingIp: 10.192.173.64 - servingPort: 30095 - -Info dashboard: - - party9999.notebook.example.com - - party9999.fateboard.example.com - ip: 10.192.173.64 - port: 30091 - status: - containers: - client: Running - clustermanager: Running - fateboard: Running - mysql: Running - nodemanager-0: Running - nodemanager-0-eggrollpair: Running - nodemanager-1: Running - nodemanager-1-eggrollpair: Running - python: Running - rollsite: Running - deployments: - client: Available - clustermanager: Available - mysql: Available - nodemanager-0: Available - nodemanager-1: Available - python: Available - rollsite: Available -``` -In `Info->dashboard` field, we can see there are two dashboards in the current deployment: +kubefate@machine:~/demo$ kubefate cluster describe 29878fa9-aeee-4ae5-a5b7-fd4e9eb7c1c3 +UUID 29878fa9-aeee-4ae5-a5b7-fd4e9eb7c1c3 +Name fate-9999 +NameSpace fate-9999 +ChartName fate +ChartVersion v1.9.0 +Revision 1 +Age 54m +Status Running +Spec algorithm: Basic + chartName: fate + chartVersion: v1.9.0 + computing: Spark + device: CPU + federation: Pulsar + imagePullSecrets: + - name: myregistrykey + ingress: + client: + hosts: + - name: party9999.notebook.example.com + fateboard: + hosts: + - name: party9999.fateboard.example.com + pulsar: + hosts: + - name: party9999.pulsar.example.com + spark: + hosts: + - name: party9999.spark.example.com + ingressClassName: nginx + istio: + enabled: false + modules: + - python + - mysql + - fateboard + - client + - spark + - hdfs + - nginx + - pulsar + name: fate-9999 + namespace: fate-9999 + nginx: + grpcNodePort: 30098 + httpNodePort: 30093 + route_table: + "10000": + fateflow: + - grpc_port: 30108 + host: 192.168.100.123 + http_port: 30103 + type: NodePort + partyId: 9999 + persistence: false + podSecurityPolicy: + enabled: false + pullPolicy: null + pulsar: + httpNodePort: 30094 + httpsNodePort: 30099 + publicLB: + enabled: false + route_table: + "9999": + host: pulsar + port: 6650 + sslPort: 6651 + "10000": + host: 192.168.100.123 + port: 30104 + proxy: "" + sslPort: 30109 + type: NodePort + python: + grpcNodePort: 30092 + httpNodePort: 30097 + logLevel: INFO + type: NodePort + registry: "" + servingIp: 192.168.100.123 + servingPort: 30095 + storage: HDFS + +Info dashboard: + - party9999.notebook.example.com + - party9999.fateboard.example.com + - party9999.pulsar.example.com + - party9999.spark.example.com + ip: 192.168.100.124 + status: + containers: + client: Running + datanode: Running + fateboard: Running + fateflow: Running + mysql: Running + namenode: Running + nginx: Running + pulsar: Running + spark-master: Running + spark-worker: Running + deployments: + nginx: Available + spark-master: Available + spark-worker: Available +``` +In `Info->dashboard` field, we can see there are 4 dashboards in the current deployment: * Notebook in `party9999.notebook.example.com`, which is the Jupyter Notebook integrated, where data scientists can write python or access shell. We have pre-installed FATE-clients to the Notebook. * FATEBoard in `party9999.fateboard.example.com`, which we can use to check the status, job flows in FATE. +* Pulsar in `party9999.pulsar.example.com`, which is the UI console of Pulsar, the message queue for transferring the gradients during FML. +* Spark in `party9999.spark.example.com`, which is the UI console of Spark, -With similar command, we can see that the Notebook for `fate-10000` is `party10000.notebook.example.com`, -and the FATEBoard for `fate-10000` is `party10000.fateboard.example.com`. +With similar command, we can check the dashboards for fate-10000. ### (Optional) Configure the dashboards' URLs in hosts #### Note: if we have the dns service setup, this step can be skipped. @@ -534,8 +651,11 @@ Also the data output. This means that the job is successfully processed and KubeFate is running properly. ## Next Steps -1. The example showed above is the simplest of FATE's example. Please explore other Job examples in Notebook. -2. Now you have deployed your first FATE cluster based on eggroll. We also have prepared example YAML files +1. The example showed above is just one of FATE's examples. Please explore other Job examples in Notebook. +2. Now you have deployed your first FATE cluster based on Spark and Pulsar. We also have prepared example YAML files (https://github.com/FederatedAI/KubeFATE/tree/master/k8s-deploy/examples) for: * Deploy FATE-Serving - * Deploy Spark-based FATE cluster, and try different message queues: rabbitmq and pulsar. \ No newline at end of file + * Deploy Eggroll based FATE cluster. + * Try Rabbitmq as the message queue. + * Deploy Spark-local+LocalFS based FATE cluster. + * Deploy FATE-Exchange to deploy the star-mode federation. \ No newline at end of file diff --git a/docs/tutorials/Build_Two_Parties_FATE_Cluster_in_One_Linux_Machine_with_MiniKube_zh.md b/docs/tutorials/Build_Two_Parties_FATE_Cluster_in_One_Linux_Machine_with_MiniKube_zh.md index f47b5af3a..81f2f97c0 100644 --- a/docs/tutorials/Build_Two_Parties_FATE_Cluster_in_One_Linux_Machine_with_MiniKube_zh.md +++ b/docs/tutorials/Build_Two_Parties_FATE_Cluster_in_One_Linux_Machine_with_MiniKube_zh.md @@ -3,7 +3,7 @@ 我们的总体架构如下: -![总体架构](https://github.com/FederatedAI/KubeFATE/raw/master/docs/tutorials/images/goal.png) +![总体架构](../images/fate_on_spark_with_pulsar.png) 本文共出现两台机器： 1. 用来做Demo的机器，是一台Linux机器，参照前置条件第一点； @@ -17,10 +17,15 @@ 5. 要保证安装机器可以正常访问Docker Hub或者网易云镜像仓库，以及Google gcr； 6. 预先创建一个目录，以便整个过程使用该目录作为工作目录，命令如下： ``` -export release_version=v1.8.0 && export kubefate_version=v1.4.4 && cd ~ && mkdir demo && cd demo +export fate_version=v1.9.0 && export kubefate_version=v1.4.5 && cd ~ && mkdir demo && cd demo ``` -**注意：下文介绍的MiniKube机器IP地址是192.168.100.123。请修改为你准备的实验机器IP地址！！！** +Notes: +* 当我们提到"KubeFATE的版本"，通常来讲会有三个概念： + * KubeFATE命令行工具的版本，在本教程中为v1.4.5。 + * KubeFATE服务版本，在本教程中为v1.4.5。 + * FATE版本，在本教程中v1.9.0，它也意味着FATE的Helm Chart的版本, 值得注意的是我们用这个版本来给GitHub上的KubeFATE的发布打tag。 +* **下文介绍的MiniKube机器IP地址是192.168.100.123。请修改为你准备的实验机器IP地址** # 开始安装 ## 安装需要的工具 @@ -44,14 +49,14 @@ curl -LO https://github.com/kubernetes/minikube/releases/download/v1.19.0/miniku 验证安装结果: ``` kubefate@machine:~/demo$ minikube version -minikube version: v1.21.0 -commit: 76d74191d82c47883dc7e1319ef7cebd3e00ee11 +minikube version: v1.19.0 +commit: 15cede53bdc5fe242228853e737333b09d4336b5 ``` ### 使用MiniKube安装Kubernetes MiniKube支持使用不同的虚拟机来部署Kubernetes，但是在Linux环境下，我们建议直接使用Docker方式。更多的说明参考：[Install MiniKube - Install a Hypervisor](https://kubernetes.io/docs/tasks/tools/install-minikube/#install-a-hypervisor). ``` -sudo minikube start --vm-driver=none --kubernetes-version v1.19.0 +sudo minikube start --vm-driver=none --kubernetes-version v1.19.0 --cni=flannel ``` 待到命令执行完成，我们可以验证下， ``` @@ -66,13 +71,15 @@ kubeconfig: Configured ``` sudo minikube addons enable ingress ``` -到此，我们的Kubernetes也准备好了。 + +使用命令`kubectl get pods -A`来查看所有的状态，当所有的pod的状态都变成ready的时候，表示K8s集群已经准备就绪。 + ## 安装Kubefate ### 下载KubeFATE命令行工具 -我们从Github上 [KubeFATE Release](https://github.com/FederatedAI/KubeFATE/releases)页面找到Kuberetes部署的下载包，并下载对应版本，如前面环境变量设置`v1.8.0`， +我们从Github上 [KubeFATE Release](https://github.com/FederatedAI/KubeFATE/releases)页面找到Kuberetes部署的下载包，并下载对应版本，如前面环境变量设置`v1.9.0`， ``` -curl -LO https://github.com/FederatedAI/KubeFATE/releases/download/${release_version}/kubefate-k8s-${release_version}.tar.gz && tar -xzf ./kubefate-k8s-${release_version}.tar.gz +curl -LO https://github.com/FederatedAI/KubeFATE/releases/download/${fate_version}/kubefate-k8s-${fate_version}.tar.gz && tar -xzf ./kubefate-k8s-${fate_version}.tar.gz ``` 然后我们可以查看下下载解压的安装包内容， ``` @@ -85,10 +92,10 @@ cluster-spark-pulsar.yaml cluster-spark-slim.yaml config.yaml kubefate.yaml ``` chmod +x ./kubefate && sudo mv ./kubefate /usr/bin ``` -然后我们测试下kubefate命令是否可用， +然后我们测试下KubeFATE CLI命令是否可用， ``` kubefate@machine:~/kubefate$ kubefate version -* kubefate commandLine version=v1.4.4 +* kubefate commandLine version=v1.4.5 * kubefate service connection error, resp.StatusCode=404, error: @@ -106,13 +113,13 @@ kubefate@machine:~/kubefate$ kubefate version ### 部署Kubefate服务 #### 1.载入Kubefate服务镜像 -接着，我们下载KubeFATE服务镜像v1.4.4： +接着，我们下载KubeFATE服务镜像v1.4.5： ``` -curl -LO https://github.com/FederatedAI/KubeFATE/releases/download/${release_version}/kubefate-${kubefate_version}.docker +curl -LO https://github.com/FederatedAI/KubeFATE/releases/download/${fate_version}/kubefate-${kubefate_version}.docker ``` 然后读入本地Docker环境。请注意，因为本文使用的MiniKube，是all-in-one的Kubernetes环境，所以只需要导入本地Docker环境即可给kubelet拉取。如果你运行的是集群形式的Kubernetes，容器镜像需要读入[Docker Registry](https://docs.docker.com/registry/introduction/)或者[Harbor](https://goharbor.io/)。关于如何集成使用Harbor，可以参考：https://github.com/FederatedAI/KubeFATE/blob/master/registry/README.md ``` -docker load < kubefate-v1.4.4.docker +docker load < kubefate-v1.4.5.docker ``` #### 2.创建kube-fate的命名空间以及账号 @@ -189,8 +196,8 @@ rtt min/avg/max/mdev = 0.054/0.067/0.080/0.013 ms 当 `example.com` 顺利设置, KubeFATE服务的版本号应该就可以正常显示, ``` kubefate@machine:~/kubefate$ kubefate version -* kubefate service version=v1.4.4 -* kubefate commandLine version=v1.4.4 +* kubefate service version=v1.4.5 +* kubefate commandLine version=v1.4.5 ``` 到此，所有准备工作完毕，下面我们可以开始安装FATE了。需要注意的是，上面的工作只需要做一次，后面如果添加、删除、更新FATE集群，上面的不需要重新执行。 @@ -203,16 +210,36 @@ kubectl create namespace fate-9999 kubectl create namespace fate-10000 ``` -在exmaple目录下，我们已经预先设置了两个例子：`/kubefate/examples/party-9999/` 和 `/kubefate/examples/party-10000` -对于`/kubefate/examples/party-9999/cluster.yaml`，我们可以将其修改如下： +如果你是用dockerhub上的镜像仓库，可以先在相应的域名下注册登录信息，因为后面有些镜像需要从dockerhub下载。 + +``` +DOCKER_REGISTRY_SERVER=docker.io +DOCKER_USER= +DOCKER_PASSWORD= + +kubectl -n fate-9999 create secret docker-registry myregistrykey \ + --docker-server=$DOCKER_REGISTRY_SERVER \ + --docker-username=$DOCKER_USER \ + --docker-password=$DOCKER_PASSWORD + +kubectl -n fate-10000 create secret docker-registry myregistrykey \ + --docker-server=$DOCKER_REGISTRY_SERVER \ + --docker-username=$DOCKER_USER \ + --docker-password=$DOCKER_PASSWORD +``` + +在exmaple目录下我们有一些预设的例子：`/kubefate/examples/party-9999/` 和 `/kubefate/examples/party-10000` + +本教程中我们使用Spark+Pulsar的组合来做例子。 + +对于`/kubefate/examples/party-9999/cluster-spark-pulsar.yaml`，我们可以将其修改如下： ``` name: fate-9999 namespace: fate-9999 chartName: fate -chartVersion: v1.8.0 +chartVersion: v1.9.0 partyId: 9999 -registry: "hub.c.163.com/federatedai" -imageTag: "1.8.0-release" +registry: "" pullPolicy: imagePullSecrets: - name: myregistrykey @@ -223,31 +250,34 @@ podSecurityPolicy: enabled: false ingressClassName: nginx modules: - - rollsite - - clustermanager - - nodemanager - - mysql - python + - mysql - fateboard - client + - spark + - hdfs + - nginx + - pulsar -backend: eggroll +computing: Spark +federation: Pulsar +storage: HDFS +algorithm: Basic +device: CPU ingress: fateboard: hosts: - name: party9999.fateboard.example.com - client: + client: hosts: - name: party9999.notebook.example.com - -rollsite: - type: NodePort - nodePort: 30091 - partyList: - - partyId: 10000 - partyIp: 192.168.100.123 - partyPort: 30101 + spark: + hosts: + - name: party9999.spark.example.com + pulsar: + hosts: + - name: party9999.pulsar.example.com python: type: NodePort @@ -257,52 +287,82 @@ python: servingIp: 192.168.100.123 servingPort: 30095 -``` -对于`/kubefate/examples/party-10000/cluster.yaml`，我们可以将其修改如下： +nginx: + type: NodePort + httpNodePort: 30093 + grpcNodePort: 30098 + route_table: + 10000: + fateflow: + - host: 192.168.100.123 + http_port: 30103 + grpc_port: 30108 + +pulsar: + type: NodePort + httpNodePort: 30094 + httpsNodePort: 30099 + publicLB: + enabled: false + route_table: + 9999: + host: pulsar + port: 6650 + sslPort: 6651 + 10000: + host: 192.168.100.123 + port: 30104 + sslPort: 30109 + proxy: "" +``` + +对于fate-10000，我们可以将其修改如下： ``` name: fate-10000 namespace: fate-10000 chartName: fate -chartVersion: v1.8.0 +chartVersion: v1.9.0 partyId: 10000 -registry: "hub.c.163.com/federatedai" -imageTag: "1.8.0-release" +registry: "" pullPolicy: imagePullSecrets: - name: myregistrykey persistence: false istio: enabled: false +ingressClassName: nginx podSecurityPolicy: enabled: false -ingressClassName: nginx modules: - - rollsite - - clustermanager - - nodemanager - - mysql - python + - mysql - fateboard - client + - spark + - hdfs + - nginx + - pulsar -backend: eggroll +computing: Spark +federation: Pulsar +storage: HDFS +algorithm: Basic +device: CPU ingress: fateboard: hosts: - name: party10000.fateboard.example.com - client: + client: hosts: - name: party10000.notebook.example.com - -rollsite: - type: NodePort - nodePort: 30101 - partyList: - - partyId: 9999 - partyIp: 192.168.100.123 - partyPort: 30091 + spark: + hosts: + - name: party10000.spark.example.com + pulsar: + hosts: + - name: party10000.pulsar.example.com python: type: NodePort @@ -312,6 +372,34 @@ python: servingIp: 192.168.100.123 servingPort: 30105 + +nginx: + type: NodePort + httpNodePort: 30103 + grpcNodePort: 30108 + route_table: + 9999: + fateflow: + - host: 192.168.100.123 + http_port: 30093 + grpc_port: 30098 + +pulsar: + type: NodePort + httpNodePort: 30104 + httpsNodePort: 30109 + publicLB: + enabled: false + route_table: + 9999: + host: 192.168.100.123 + port: 30094 + sslPort: 30099 + proxy: "" + 10000: + host: pulsar + port: 6650 + sslPort: 6651 ``` **注意： 我们强烈建议阅读以下文档** @@ -320,9 +408,9 @@ servingPort: 30105 ### 安装FATE集群 如果一切没有问题，那就可以使用`kubefate cluster install`来部署两个fate集群了， ``` -kubefate@machine:~/kubefate$ kubefate cluster install -f examples/party-9999/cluster.yaml +kubefate@machine:~/kubefate$ kubefate cluster install -f examples/party-9999/cluster-spark-pulsar.yaml create job success, job id=2c1d926c-bb57-43d3-9127-8cf3fc6deb4b -kubefate@machine:~/kubefate$ kubefate cluster install -f examples/party-10000/cluster.yaml +kubefate@machine:~/kubefate$ kubefate cluster install -f examples/party-10000/cluster-spark-pulsar.yaml create job success, job id=7752db70-e368-41fa-8827-d39411728d1b ``` @@ -330,8 +418,8 @@ create job success, job id=7752db70-e368-41fa-8827-d39411728d1b ``` kubefate@machine:~/kubefate$ watch kubefate cluster ls UUID NAME NAMESPACE REVISION STATUS CHART ChartVERSION AGE -51476469-b473-4d41-b2d5-ea7241d5eac7 fate-9999 fate-9999 1 Running fate v1.8.0 88s -dacc0549-b9fc-463f-837a-4e7316db2537 fate-10000 fate-10000 1 Running fate v1.8.0 69s +29878fa9-aeee-4ae5-a5b7-fd4e9eb7c1c3 fate-9999 fate-9999 1 Running fate v1.9.0 88s +dacc0549-b9fc-463f-837a-4e7316db2537 fate-10000 fate-10000 1 Running fate v1.9.0 69s ``` 因为这个步骤需要到网易云镜像仓库去下载约10G的镜像，所以第一次执行视乎你的网络情况需要一定时间。 检查下载的进度可以用 @@ -351,97 +439,123 @@ rollsite-7846898d6d-j2gb9 1/1 Running 0 12m ``` ### 验证FATE的部署 -通过以上的 `kubefate cluster ls` 命令, 我们得到 `fate-9999` 的集群ID是 `51476469-b473-4d41-b2d5-ea7241d5eac7`, 而 `fate-10000` 的集群ID是 `dacc0549-b9fc-463f-837a-4e7316db2537`. 我们可以通过`kubefate cluster describe`查询集群的具体访问信息， -``` -kubefate@machine:~/demo$ kubefate cluster describe 51476469-b473-4d41-b2d5-ea7241d5eac7 -UUID 51476469-b473-4d41-b2d5-ea7241d5eac7 -Name fate-9999 -NameSpace fate-9999 -ChartName fate -ChartVersion v1.8.0 -Revision 1 -Age 15h -Status Running -Spec backend: eggroll - chartName: fate - chartVersion: v1.8.0 - imagePullSecrets: - - name: myregistrykey - imageTag: 1.8.0-release - ingress: - client: - annotations: - kubernetes.io/ingress.class: nginx - hosts: - - name: party9999.notebook.example.com - fateboard: - annotations: - kubernetes.io/ingress.class: nginx - hosts: - - name: party9999.fateboard.example.com - istio: - enabled: false - modules: - - rollsite - - clustermanager - - nodemanager - - mysql - - python - - fateboard - - client - name: fate-9999 - namespace: fate-9999 - partyId: 9999 - persistence: false - podSecurityPolicy: - enabled: false - pullPolicy: null - python: - grpcNodePort: 30092 - httpNodePort: 30097 - type: NodePort - registry: "" - rollsite: - nodePort: 30091 - partyList: - - partyId: 10000 - partyIp: 10.192.173.64 - partyPort: 30101 - type: NodePort - servingIp: 10.192.173.64 - servingPort: 30095 - -Info dashboard: - - party9999.notebook.example.com - - party9999.fateboard.example.com - ip: 10.192.173.64 - port: 30091 - status: - containers: - client: Running - clustermanager: Running - fateboard: Running - mysql: Running - nodemanager-0: Running - nodemanager-0-eggrollpair: Running - nodemanager-1: Running - nodemanager-1-eggrollpair: Running - python: Running - rollsite: Running - deployments: - client: Available - clustermanager: Available - mysql: Available - nodemanager-0: Available - nodemanager-1: Available - python: Available - rollsite: Available +通过以上的 `kubefate cluster ls` 命令, 我们得到 `fate-9999` 的集群ID是 `29878fa9-aeee-4ae5-a5b7-fd4e9eb7c1c3`, 而 `fate-10000` 的集群ID是 `dacc0549-b9fc-463f-837a-4e7316db2537`. 我们可以通过`kubefate cluster describe`查询集群的具体访问信息， +``` +kubefate@machine:~/demo$ kubefate cluster describe 29878fa9-aeee-4ae5-a5b7-fd4e9eb7c1c3 +UUID 29878fa9-aeee-4ae5-a5b7-fd4e9eb7c1c3 +Name fate-9999 +NameSpace fate-9999 +ChartName fate +ChartVersion v1.9.0 +Revision 1 +Age 54m +Status Running +Spec algorithm: Basic + chartName: fate + chartVersion: v1.9.0 + computing: Spark + device: CPU + federation: Pulsar + imagePullSecrets: + - name: myregistrykey + ingress: + client: + hosts: + - name: party9999.notebook.example.com + fateboard: + hosts: + - name: party9999.fateboard.example.com + pulsar: + hosts: + - name: party9999.pulsar.example.com + spark: + hosts: + - name: party9999.spark.example.com + ingressClassName: nginx + istio: + enabled: false + modules: + - python + - mysql + - fateboard + - client + - spark + - hdfs + - nginx + - pulsar + name: fate-9999 + namespace: fate-9999 + nginx: + grpcNodePort: 30098 + httpNodePort: 30093 + route_table: + "10000": + fateflow: + - grpc_port: 30108 + host: 192.168.100.123 + http_port: 30103 + type: NodePort + partyId: 9999 + persistence: false + podSecurityPolicy: + enabled: false + pullPolicy: null + pulsar: + httpNodePort: 30094 + httpsNodePort: 30099 + publicLB: + enabled: false + route_table: + "9999": + host: pulsar + port: 6650 + sslPort: 6651 + "10000": + host: 192.168.100.123 + port: 30104 + proxy: "" + sslPort: 30109 + type: NodePort + python: + grpcNodePort: 30092 + httpNodePort: 30097 + logLevel: INFO + type: NodePort + registry: "" + servingIp: 192.168.100.123 + servingPort: 30095 + storage: HDFS + +Info dashboard: + - party9999.notebook.example.com + - party9999.fateboard.example.com + - party9999.pulsar.example.com + - party9999.spark.example.com + ip: 192.168.100.124 + status: + containers: + client: Running + datanode: Running + fateboard: Running + fateflow: Running + mysql: Running + namenode: Running + nginx: Running + pulsar: Running + spark-master: Running + spark-worker: Running + deployments: + nginx: Available + spark-master: Available + spark-worker: Available ``` 从返回的内容中，我们看到`Info->dashboard`里包含了: * Jupyter Notebook的访问地址： `party9999.notebook.example.com`。这个是我们准备让数据科学家进行建模分析的平台。已经集成了FATE-Clients； * FATEBoard的访问地址： `party9999.fateboard.example.com`。我们可以通过FATEBoard来查询当前训练的状态。 +* Pulsar的UI地址，`party9999.pulsar.example.com`，可以使用此地址查看Pulsar的UI界面。 +* Spark的UI地址，`party9999.spark.example.com`，可以使用此地址查看Spark的UI界面。 -类似的命令我们得到，`fate-10000`的Jupyter Notebook和FATEBoard地址分别是：`party10000.notebook.example.com` 以及`party10000.fateboard.example.com`。 +遇此类推，我们也可以查看fate-10000的各种地址 ### 在浏览器访问FATE集群的机器上配置相关的Host信息 **注意: 如果DNS已经配置了相关的解析，这步可以跳过** @@ -501,6 +615,8 @@ notebook截图: ## 后续 1. 上面的联邦学习例子是最简单的一个例子，请自行探索其他的例子。 -2. 本文只介绍了基于eggroll的FATE集群架构，在`https://github.com/FederatedAI/KubeFATE/tree/master/k8s-deploy/examples`目录中，我们还准备了其他的YAML文件，用来： +2. 本文只介绍了基于Spark和Pulsar的FATE集群架构，在`https://github.com/FederatedAI/KubeFATE/tree/master/k8s-deploy/examples`目录中，我们还准备了其他的YAML文件，用来： * 部署FATE-Serving。 -* 部署基于Spark的FATE集群，以及基于两种消息队列的FATE集群：rabbitmq和pulsar。 \ No newline at end of file +* 部署基于Eggroll的FATE集群。 +* 部署Rabbitmq做消息队列的FATE集群。 +* 部署FATE-Exchange并尝试星型的联邦。 \ No newline at end of file diff --git a/fml_manager/Dockerfile b/fml_manager/Dockerfile deleted file mode 100644 index 2607e05f0..000000000 --- a/fml_manager/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM python:3.7 - -WORKDIR /fml_manager -COPY . /fml_manager - -RUN pip install notebook fate-client -RUN python setup.py sdist bdist_wheel && pip install dist/*.whl -RUN mkdir /fml_manager/Examples/Pipeline/logs - -CMD flow init -c /data/projects/fate/conf/service_conf.yaml && pipeline init -c /data/projects/fate/conf/pipeline_conf.yaml && jupyter notebook --ip=0.0.0.0 --port=20000 --allow-root --debug --NotebookApp.notebook_dir='/fml_manager/Examples' --no-browser --NotebookApp.token='' --NotebookApp.password='' diff --git a/fml_manager/Examples/Cluster_Management/update_route_table.ipynb b/fml_manager/Examples/Cluster_Management/update_route_table.ipynb deleted file mode 100644 index 064911ccb..000000000 --- a/fml_manager/Examples/Cluster_Management/update_route_table.ipynb +++ /dev/null @@ -1,146 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from fml_manager import ClusterManager\n", - "from fml_manager import Party, PartyType" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## For more details about the FMLManager, please refer to this [document](https://kubefate.readthedocs.io/README.html)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cluster_manager = ClusterManager(cluster_namespace='fate-10000', cluster_name='fate')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# get route table\n", - "route_table = cluster_manager.get_route_table()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# show all party\n", - "print(route_table.get_parties())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# delete route table party\n", - "party_id_1 = '9999'\n", - "party_id_2 = '8888'\n", - "route_table.remove_parties(party_id_1, party_id_2)\n", - "print(route_table.get_parties())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# define normal party\n", - "party = Party(p_id='9999', ip='192.168.2.2', port=30010)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# append normal party to route table\n", - "route_table.add_parties(party)\n", - "print(route_table.get_parties())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# define exchange\n", - "party = Party(ip='192.168.2.2', port=30009, p_type=PartyType.EXCHANGE)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# append exchange to route table\n", - "route_table.add_parties(party)\n", - "print(route_table.get_parties())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# update route table of configmap\n", - "cluster_manager.set_route_table(route_table)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# get entrypoint\n", - "print(cluster_manager.get_entry_point())" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.9" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/fml_manager/Examples/Data_Management/data_management_example.ipynb b/fml_manager/Examples/Data_Management/data_management_example.ipynb deleted file mode 100644 index db1653528..000000000 --- a/fml_manager/Examples/Data_Management/data_management_example.ipynb +++ /dev/null @@ -1,102 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Data Management Example" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import json \n", - "import time \n", - "import os\n", - "import requests\n", - "\n", - "from fml_manager import *" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## For more details about the FMLManager, please refer to this [document](https://kubefate.readthedocs.io/README.html)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Step 2. Upload data to FATE" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "manager = fml_manager.FMLManager()\n", - "response = manager.load_data(url='https://raw.githubusercontent.com/FederatedAI/FATE/master/examples/data/breast_homo_guest.csv', namespace='homo_breast_guest', table_name='homo_breast_guest', work_mode=1, head=1, partition=10)\n", - "output = json.loads(response.content)\n", - "guest_job_id = output['jobId']\n", - "guest_query_condition = QueryCondition(job_id=guest_job_id)\n", - "\n", - "manager.query_job_status(guest_query_condition)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Step 3. Query the data information just uploaded " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = manager.query_data(guest_job_id, limit=1000)\n", - "manager.prettify(response, verbose=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = manager.download_data(namespace='homo_breast_guest', table_name='homo_breast_guest', filename='homo_data_test', work_mode=1, delimitor=',')\n", - "manager.prettify(response, verbose=True)\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.9" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/fml_manager/Examples/Hetero_LR/data/breast.csv b/fml_manager/Examples/Hetero_LR/data/breast.csv deleted file mode 100644 index 7ce6e1be5..000000000 --- a/fml_manager/Examples/Hetero_LR/data/breast.csv +++ /dev/null @@ -1,570 +0,0 @@ -id,y,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11,x12,x13,x14,x15,x16,x17,x18,x19,x20,x21,x22,x23,x24,x25,x26,x27,x28,x29 -133,1,0.254879,-1.046633,0.20965599999999998,0.074214,-0.441366,-0.377645,-0.48593400000000003,0.347072,-0.28757,-0.733474,0.44951199999999997,-1.247226,0.413178,0.30378099999999997,-0.123848,-0.184227,-0.21907600000000002,0.26853699999999997,0.015996,-0.789267,-0.33736,-0.728193,-0.442587,-0.272757,-0.6080180000000001,-0.577235,-0.5011260000000001,0.143371,-0.46643100000000004,-0.5541020000000001 -273,1,-1.1429280000000002,-0.781198,-1.166747,-0.923578,0.6282300000000001,-1.021418,-1.111867,-0.9595229999999999,-0.09667200000000001,-0.121683,-1.245485,-0.842317,-1.255026,-1.038066,-0.42630100000000004,-1.088781,-0.976392,-0.898898,0.9834959999999999,0.045702,-0.49363900000000005,0.34862,-0.552483,-0.526877,2.253098,-0.82762,-0.7807390000000001,-0.37699699999999997,-0.310239,0.17630099999999999 -175,1,-1.451067,-1.406518,-1.456564,-1.0923370000000001,-0.708765,-1.168557,-1.305831,-1.745063,-0.49949899999999997,-0.30289299999999997,-1.549664,-1.126219,-1.5466520000000001,-1.216392,-0.35442399999999996,-1.167051,-1.114873,-1.2618200000000002,-0.32719299999999996,0.6297550000000001,-0.6668810000000001,-0.779358,-0.708418,-0.637545,0.7103689999999999,-0.976454,-1.057501,-1.9134470000000001,0.795207,-0.149751 -551,1,-0.879933,0.42058900000000005,-0.8775270000000001,-0.7804840000000001,-1.037534,-0.48388000000000003,-0.5554979999999999,-0.7685810000000001,0.43396,-0.200928,-0.851273,0.733108,-0.843535,-0.786363,-0.049836,-0.42453199999999996,-0.509221,-0.679649,0.797298,0.385927,-0.451772,0.45385200000000003,-0.431696,-0.49475399999999997,-1.182041,0.28122800000000003,0.084759,-0.25242,1.038575,0.351054 -199,0,0.42675799999999997,0.723479,0.316885,0.287273,1.0008350000000001,0.9627020000000001,1.077099,1.0535860000000001,2.996525,0.9616959999999999,0.091654,0.21649899999999997,0.10383900000000001,-0.034667,0.16793,0.308132,0.366614,0.280661,0.505223,0.264013,-0.707304,-1.026834,-0.702973,-0.460212,-0.999033,-0.531406,-0.39436,-0.72883,-0.644416,-0.6880029999999999 -274,0,0.963102,1.467675,0.829202,0.772457,-0.038076,-0.468613,-0.307946,-0.015321000000000001,-0.641864,-0.247477,1.080023,1.20783,0.956888,0.978402,-0.555822,-0.645696,-0.399365,-0.038153,-0.998966,-1.091216,0.057848000000000004,0.392164,-0.050026999999999995,0.12041400000000001,-0.5323479999999999,-0.770613,-0.519694,-0.531097,-0.769127,-0.394858 -420,1,-0.662496,0.21214899999999998,-0.620475,-0.632995,-0.327392,-0.385278,-0.077665,-0.730362,0.217178,-0.061279999999999994,-0.7263069999999999,-0.05809500000000001,-0.7319100000000001,-0.697343,-0.7757229999999999,-0.513983,-0.426233,-0.893482,0.800949,-0.018090000000000002,-0.42867299999999997,0.40486500000000003,-0.32675,-0.44085,0.07901,-0.279903,0.41699200000000003,-0.48616499999999996,-0.22548400000000002,-0.17244600000000002 -76,1,-0.45334300000000005,-2.147457,-0.473631,-0.483572,0.5580930000000001,-0.740244,-0.89617,-0.617229,-0.308601,-0.666975,-0.169639,-1.9430189999999998,-0.167192,-0.27215,2.3299369999999997,0.006804000000000001,-0.251467,0.429234,2.1591,0.5120939999999999,0.017786,-0.368046,-0.105966,-0.169129,2.11976,0.162743,-0.672216,-0.577002,0.626908,0.896114 -315,1,-0.606584,-0.971725,-0.678558,-0.591332,-0.9630129999999999,-1.302401,-1.212855,-1.3211540000000002,-1.591501,-1.2305540000000001,-0.46501400000000004,-0.567723,-0.526371,-0.49285200000000007,-0.8006310000000001,-1.250816,-1.058714,-1.0961450000000001,-2.1782209999999997,-0.8601469999999999,-0.8430110000000001,-0.9103530000000001,-0.90049,-0.608283,-0.7043550000000001,-1.255622,-0.9706290000000001,-1.363557,-0.800607,-0.9270579999999999 -399,1,-0.583805,-0.193332,-0.633283,-0.560041,-0.34931,-0.519504,-0.6106689999999999,-0.9295260000000001,-0.196974,-0.151608,-0.660984,-0.47231300000000004,-0.688248,-0.634204,-0.390718,-0.7963600000000001,-0.75668,-0.8393139999999999,0.129175,-0.369656,-0.221505,-0.139439,-0.317344,-0.336122,-0.526014,-0.326291,-0.368166,-1.0378399999999999,-0.698901,-0.273818 -238,1,-0.10751500000000001,2.420311,-0.141817,-0.204943,-1.063835,-0.074206,0.164131,-0.49358900000000006,-1.635181,-0.33170900000000003,0.026330000000000003,1.9920509999999998,0.02393,-0.088136,-1.005588,-0.008357,0.26994,-0.124821,-1.714551,-0.213719,-0.251822,2.0087450000000002,-0.376748,-0.22831300000000002,-0.24466999999999997,0.166096,0.219045,-0.273508,-1.052451,-0.07788300000000001 -246,1,-0.482335,0.348938,-0.565371,-0.489725,-0.9761639999999999,-0.6581819999999999,-0.20335999999999999,-0.988301,-0.216387,-0.663096,-0.263364,-0.432753,-0.322891,-0.322206,-1.7229349999999999,-1.1200510000000001,-0.570489,-0.9767959999999999,-1.1851639999999999,-0.9140159999999999,-0.87405,0.696974,-0.986625,-0.589142,-0.260004,-0.5470550000000001,-0.036596,-1.040273,-0.111671,-0.5843619999999999 -253,0,0.7415229999999999,-0.095626,0.704101,0.6001810000000001,0.404667,-0.087565,0.314773,1.082516,0.383809,-0.156041,0.9010940000000001,-0.5142,0.8662700000000001,0.777324,0.31595500000000004,-0.004567,0.47458599999999995,0.892752,0.005043,-0.945203,-0.346022,-0.653261,-0.333185,-0.147567,-0.761357,-0.583942,-0.284279,0.133639,-0.799396,-0.541998 -550,1,-0.954483,-0.14773599999999998,-0.9883299999999999,-0.8232010000000001,-1.414523,-1.150045,-1.305831,-1.745063,-0.716282,-0.998915,-0.9279569999999999,0.509709,-0.966282,-0.8372729999999999,-1.569218,-1.1763370000000002,-1.114873,-1.2618200000000002,-0.5499,-0.47030600000000006,-0.320758,0.158114,-0.371797,-0.432709,0.8460409999999999,-0.806941,-1.057501,-1.9134470000000001,1.149967,-0.592683 -208,1,-0.356014,0.567149,-0.23176999999999998,-0.424155,0.11096600000000001,1.182806,0.211146,-0.030548000000000002,1.985412,1.310816,-0.288925,0.756379,-0.203852,-0.356904,0.273255,0.8330879999999999,-0.021963,0.054189,0.140128,1.4604709999999999,-0.765413,-0.534421,-0.680696,-0.555479,-0.596684,0.27563899999999997,-0.21995399999999998,-0.566458,0.556683,0.152472 -185,1,-0.9109950000000001,-0.732345,-0.949311,-0.77978,0.864944,-0.969255,-1.272632,-1.586402,0.052164,-0.386571,-1.1494879999999998,-0.9726319999999999,-1.161936,-0.959569,-0.26261999999999996,-1.087644,-1.0948229999999999,-1.199811,-0.39655999999999997,-0.328545,0.069759,0.092797,-0.09210499999999999,-0.30598000000000003,2.449773,-0.753287,-1.004549,-1.523495,0.585742,-0.122895 -156,0,0.869914,-0.092369,0.7636729999999999,0.7408140000000001,0.413434,0.6077359999999999,0.41312200000000004,0.561767,-0.7081930000000001,-0.36385,1.00902,0.337507,1.047507,0.87829,1.077425,1.178005,1.2140739999999999,1.456866,0.581892,-0.161268,1.4658010000000001,0.332291,1.323683,1.178696,0.665367,1.344806,0.668654,1.072832,-0.334455,0.443725 -0,0,1.8866900000000002,-1.359293,2.303601,2.0012369999999997,1.3076860000000001,2.6166650000000002,2.109526,2.296076,2.750622,1.9370150000000002,1.097064,-2.0733349999999997,1.2699340000000001,0.984375,1.568466,3.283515,2.652874,2.532475,2.217515,2.255747,2.489734,-0.565265,2.833031,2.487578,-0.214002,1.316862,0.7240260000000001,0.66082,1.148757,0.907083 -70,0,1.779007,0.14701199999999998,1.746605,1.732277,-0.572873,-0.131459,-0.016736,0.978975,-0.565828,-1.000578,1.3668770000000001,0.470149,1.302886,1.351264,-0.44622700000000004,-0.027308999999999996,0.241064,0.78906,-0.838325,-1.160679,1.384594,-0.760851,1.296951,1.2257799999999999,-0.8656950000000001,-0.5006659999999999,-0.305168,0.308825,-0.809083,-0.793157 -293,1,-0.664567,0.011851,-0.68243,-0.637741,0.198638,-0.499147,-0.674477,-0.353352,0.32395100000000004,-0.76894,-0.646783,-0.425771,-0.676715,-0.631929,-0.899551,-0.9081739999999999,-0.777395,-0.673717,0.231402,-0.800607,-0.715245,0.038367,-0.8079189999999999,-0.582101,0.18468099999999998,-0.5856180000000001,-0.589324,-0.5220130000000001,-0.317504,-0.760627 -287,1,-0.548601,-1.650784,-0.591583,-0.533673,-1.587236,-0.8878290000000001,-0.7368439999999999,-0.928004,-0.9573309999999999,-0.8199219999999999,-0.351408,-1.435719,-0.415157,-0.39529899999999996,-1.907966,-1.270715,-0.8311299999999999,-0.959772,-1.732806,-0.990566,-0.9094200000000001,-1.356863,-0.866828,-0.608503,-0.7700239999999999,-0.672247,-0.5097470000000001,-0.955599,-0.5269699999999999,-0.6482859999999999 -222,1,-1.0559530000000001,-0.462024,-1.052072,-0.887716,0.360831,-0.70144,-0.9905370000000001,-0.89618,0.249533,0.223003,-1.1210870000000002,-0.409482,-1.105917,-0.972083,0.693131,-0.366161,-0.8929010000000001,-0.7678649999999999,0.359185,0.890594,-0.571959,0.000266,-0.606442,-0.556359,0.286019,-0.641508,-0.774539,-0.676761,0.705609,-0.012824 -262,0,0.853348,0.254488,0.912602,0.728509,-0.8315049999999999,0.20633200000000002,-0.20335999999999999,0.581561,0.268947,-0.504606,0.8982540000000001,0.6609689999999999,0.9239360000000001,0.8330690000000001,-0.45334399999999997,0.43510699999999997,0.102583,0.674535,1.082072,-1.156426,1.550617,0.755033,1.6236709999999999,1.113351,-0.10799700000000001,1.9187919999999998,0.480322,1.572438,0.8387959999999999,0.772804 -309,1,-0.318739,-1.3478940000000001,-0.396188,-0.36596799999999996,-1.348769,-1.24553,-1.218324,-1.207259,-1.284123,-1.005565,-0.30596599999999996,-1.26817,-0.38138099999999997,-0.353491,-0.913784,-1.269578,-1.057635,-1.0340850000000001,-1.309296,-1.079875,-0.02769,-0.704607,-0.148043,-0.161208,-0.9643649999999999,-1.160666,-0.963633,-1.131435,-0.765494,-0.575662 -534,1,-0.962766,0.135613,-0.918334,-0.8316389999999999,0.45726999999999995,-0.02077,-0.28731599999999996,-0.24356799999999998,-0.9896870000000001,-0.064605,-0.8995559999999999,-0.388538,-0.8723690000000001,-0.822768,0.036274,-0.129268,-0.45410500000000004,-0.542683,-0.70324,0.181792,-0.918443,0.664316,-0.842076,-0.6654869999999999,0.82004,0.463987,0.40174,0.269895,-0.8865729999999999,-0.098687 -54,0,0.379129,0.979143,0.31092800000000004,0.262662,0.28631,-0.308942,-0.0047420000000000006,0.584607,-0.365223,-0.289039,0.27626300000000004,0.635371,0.21793600000000002,0.164705,-0.41278000000000004,-0.635462,-0.45536000000000004,-0.401848,-0.7141930000000001,-0.8445530000000001,-0.34169099999999997,-0.694809,-0.380708,-0.22941399999999998,-0.7886920000000001,-0.848299,-0.525994,-0.41755,-1.1614209999999998,-0.819256 -172,0,0.522016,-1.406518,0.528365,0.389232,0.9087799999999999,0.661808,1.491126,1.036837,0.509996,0.9450709999999999,0.378508,-1.721948,0.43377299999999996,0.23324699999999998,2.087974,0.9695389999999999,1.436297,1.56778,0.563638,1.1188280000000002,0.056764999999999996,-1.013408,-0.030226,0.094672,-0.549349,-0.12676500000000002,0.369246,0.22771999999999998,-0.317504,0.141124 -484,1,0.153409,-1.868995,0.15604200000000001,-0.046202999999999994,0.952616,0.277579,0.6150979999999999,0.46583900000000006,-0.556121,-0.11835799999999999,0.455192,-1.8638990000000002,0.44613,0.262542,0.565034,0.48438100000000006,0.380424,0.34024499999999996,-0.10083400000000001,-0.029431,-0.87405,-1.505458,-0.852967,-0.582321,-0.335673,-0.40677199999999997,0.048618,-0.18267,-1.064558,-0.582092 -102,1,-0.606584,1.1664139999999998,-0.675579,-0.585004,-0.879725,-1.053734,-0.756514,-0.613574,-0.334485,-0.840426,-0.5530579999999999,0.28631100000000004,-0.607516,-0.557982,-1.155035,-1.2121549999999999,-0.815688,-0.805266,-0.265127,-0.854476,-0.767939,0.642544,-0.833166,-0.5645,-0.653686,-1.0831469999999999,-0.703052,-0.810908,-0.735225,-0.8559459999999999 -458,1,-0.399502,1.010084,-0.482567,-0.44314,-0.46328400000000003,-0.9221799999999999,-1.0917649999999999,-0.8434959999999999,-0.962184,-1.1657170000000001,-0.32016700000000003,1.359089,-0.3855,-0.38306999999999997,-0.9016860000000001,-1.0160069999999999,-0.96346,-0.80733,-0.527994,-1.17769,-0.5163770000000001,0.027481000000000002,-0.598521,-0.42126800000000003,-0.329006,-0.922409,-0.869136,-0.8856870000000001,-1.016127,-0.8627549999999999 -495,1,-0.053674,0.45641499999999996,-0.100117,-0.170488,-0.472051,-0.7345189999999999,-0.490252,-0.196518,-0.8602639999999999,-0.995036,0.21094000000000002,0.21417199999999997,0.170979,0.073978,-0.034891000000000005,-0.395915,-0.25812199999999996,0.01524,-1.1851639999999999,-0.753827,-0.6239319999999999,0.760476,-0.628718,-0.406967,-0.5420149999999999,-0.657716,-0.34363,0.005493,-0.115303,-0.7500359999999999 -158,1,-0.648001,-1.183422,-0.690472,-0.611372,-0.213419,-0.833757,-0.891517,-0.6753939999999999,-0.625686,-0.275185,-0.58714,-1.524147,-0.623168,-0.586707,-0.23130799999999999,-0.9841690000000001,-0.867289,-0.755484,-0.809117,-0.5284270000000001,-0.8047529999999999,-0.886041,-0.8391059999999999,-0.595962,-0.504347,-0.876859,-0.7809050000000001,-0.895906,-0.713431,-0.47845200000000004 -160,1,-0.610726,0.086759,-0.546606,-0.59186,0.150419,-0.413905,-0.367435,-0.540791,0.43234300000000003,-0.225865,-0.675185,0.207191,-0.6536489999999999,-0.668618,0.892395,0.184948,-0.255736,-0.297641,0.662213,0.24558400000000002,0.34875100000000003,0.863894,0.5247029999999999,-0.043939,0.797373,-0.079818,0.324152,-0.044792,1.6258059999999999,0.47890299999999997 -292,1,-0.523751,-0.9359,-0.549585,-0.5189060000000001,0.698367,-0.301944,-0.230706,-0.13713399999999998,0.775312,0.659126,-0.334368,-0.7608689999999999,-0.36367,-0.40155599999999997,0.294605,-0.4721,-0.342114,-0.39256199999999997,-0.297985,0.26968400000000003,-0.7065819999999999,-0.822358,-0.809404,-0.49871400000000005,0.5613630000000001,-0.304494,-0.28328400000000004,-0.077234,0.69108,0.352188 -493,1,-0.637646,-1.517252,-0.715492,-0.609263,-1.664826,-1.205453,-1.2255200000000002,-1.33699,-1.004247,-0.757302,-0.47353500000000004,-1.503204,-0.541199,-0.5050819999999999,-1.611206,-1.211208,-1.024816,-0.9654469999999999,-0.7251449999999999,-0.378161,-0.279974,0.488324,-0.375263,-0.34624299999999997,1.116386,-0.863389,-0.844765,-0.6328020000000001,0.986511,0.38812199999999997 -394,1,-0.561026,0.019993,-0.563882,-0.5644359999999999,0.474804,-0.489605,-0.536788,-0.790964,0.23982699999999998,-0.727932,-0.5757800000000001,-0.365268,-0.572504,-0.593533,0.46540200000000004,-0.128131,-0.514369,-0.403912,0.45776000000000006,-0.168356,-0.436974,0.789506,-0.493575,-0.398607,0.36835500000000004,-0.5124029999999999,-0.446417,-0.694766,-0.047499,-0.755332 -528,1,-0.341518,-1.676839,-0.379508,-0.399544,0.308228,-0.749786,-0.557897,-0.19956300000000002,-1.19838,-0.632617,-0.053194000000000005,-1.424083,-0.06833600000000001,-0.17260599999999998,2.023925,-0.128699,0.153179,0.44445200000000007,0.600147,0.251255,0.5086390000000001,2.5730060000000003,0.606383,0.096872,0.999714,0.39077199999999995,0.521768,2.714394,-0.410734,0.655546 -335,0,0.9775969999999999,1.216895,1.070467,0.846289,0.549325,-0.311486,0.574799,1.036837,-0.44934799999999997,-0.440878,0.832931,0.39801,0.816841,0.750021,1.105891,0.02386,0.778417,1.300555,-0.308938,-0.29594000000000004,1.483125,1.6549479999999999,1.589019,1.030404,-0.19533399999999998,-0.419626,0.43522799999999995,0.915489,-0.27270500000000003,-0.023415000000000002 -311,1,0.039513,-0.639524,-0.106074,-0.069935,-1.370687,-1.166649,-1.0781399999999999,-0.859941,-0.599801,-1.495995,0.137096,-0.8376620000000001,0.029285000000000002,0.028472000000000004,-1.4361389999999998,-1.311272,-0.9332020000000001,-0.777667,-0.655777,-1.452705,-0.321841,-0.554016,-0.45149700000000004,-0.251635,-0.67002,-1.087451,-0.88094,-0.887634,-0.678318,-1.096968 -25,0,1.238521,-0.696519,1.3444969999999998,1.020322,0.97015,0.8946350000000001,0.542655,2.1377200000000003,1.8851099999999998,1.216609,0.8556520000000001,-0.672441,0.9898399999999999,0.733241,1.582699,2.335941,1.68363,2.351917,4.484751,1.606484,2.3128830000000002,-0.43699099999999996,2.183056,1.5635059999999998,0.329354,0.699282,0.179919,1.974718,0.307261,1.380276 -522,1,-0.8985700000000001,0.12258499999999999,-0.919823,-0.781714,-0.9454790000000001,-1.12619,-1.184309,-1.3138459999999998,-0.556121,-0.43312,-0.8143520000000001,0.125743,-0.8513620000000001,-0.758776,-0.8006310000000001,-1.1410870000000002,-1.0512569999999999,-1.1157219999999999,-0.637523,0.089648,-0.977273,-0.242856,-0.9330629999999999,-0.6821649999999999,-0.947031,-1.0942129999999999,-0.9299790000000001,-1.254065,-0.686793,-0.567341 -512,0,0.029158999999999997,0.64857,0.17987,-0.06360700000000001,1.097274,0.8354739999999999,1.143785,1.377912,1.106957,1.493688,-0.20656100000000002,0.28631100000000004,-0.137124,-0.27926,1.013376,0.806556,0.6993199999999999,0.846065,1.1112790000000001,1.481735,-0.05259400000000001,-0.519362,0.11234300000000001,-0.14668699999999998,-0.5423479999999999,-0.158063,0.08707999999999999,0.250429,-0.422842,0.07946900000000001 -230,0,0.687682,-0.128194,0.7815439999999999,0.5421699999999999,1.662757,0.885093,1.101567,2.127061,0.336894,0.36930100000000005,0.830091,-0.048787,0.8827459999999999,0.6829,1.2624549999999999,1.001757,1.2831270000000001,1.5497239999999999,1.166043,0.064131,-0.394386,-0.975851,-0.352986,-0.18387,-0.503013,-0.301699,-0.044554,0.331534,-1.062137,-0.551832 -548,1,-1.105653,-0.014204,-1.136664,-0.9077559999999999,-0.5465720000000001,-1.010222,-0.8572620000000001,-1.159448,-0.56421,-0.262993,-1.262242,0.011717,-1.2735610000000002,-1.0500120000000002,-0.814864,-1.024157,-0.8214629999999999,-1.01381,-0.845627,-0.063453,-0.395108,0.26516,-0.401994,-0.486174,0.133012,-0.796322,-0.282621,-0.353802,0.18012899999999998,0.135829 -213,0,0.37291599999999997,0.38964899999999997,0.39135,0.24684099999999998,-0.353694,-0.476882,0.038915,-0.07166,-2.099484,-0.873676,0.935176,1.4591530000000001,0.928055,0.8336370000000001,0.301721,0.194424,0.996873,0.439809,-1.838683,-0.586549,0.449087,0.816721,0.44599300000000003,0.400275,8.029999,3.357389,3.7104790000000003,4.456526,0.146227,3.315409 -390,1,-1.012466,-1.632871,-1.0136479999999999,-0.854492,0.084665,-0.56785,-0.892524,-0.72549,0.058635,-0.372717,-1.098366,-1.645155,-1.079967,-0.947908,0.256176,-0.548095,-0.873441,-0.7541939999999999,-0.042419,0.410026,-0.7726310000000001,-1.2140739999999999,-0.751486,-0.626104,-0.45301099999999994,-0.6610689999999999,-0.775799,-0.789497,-0.140729,-0.5385939999999999 -275,1,-0.801242,-1.088973,-0.828082,-0.71755,0.154802,-1.085159,-0.962664,-0.382891,-1.101313,-1.30869,-0.635423,-0.449042,-0.64953,-0.623681,1.8602450000000001,-0.6110140000000001,-0.370489,0.647967,0.742534,-0.5737909999999999,0.851874,1.9525,0.571731,0.187079,2.37977,-0.591766,-0.637401,1.184757,0.755252,-0.299917 -219,0,2.408538,3.2133599999999998,2.172543,2.806362,0.369598,0.988783,0.61078,0.729259,-0.303748,-0.458057,1.534446,3.067156,1.4841229999999999,1.615766,-0.865392,0.164101,0.322671,0.45012700000000005,-1.40057,-1.370484,1.2055770000000001,0.188958,0.918747,1.5305030000000002,-0.50068,0.05375800000000001,-0.174198,-0.164827,-0.8744649999999999,-0.582092 -236,0,3.052564,1.4383629999999998,2.941018,3.627307,0.6896,1.007232,1.486328,2.203194,0.327187,0.156504,2.579618,1.7872689999999998,2.5344729999999998,2.88708,-0.0904,1.210223,1.3333469999999998,1.928896,0.355534,0.041449,2.356193,-0.45966999999999997,2.1687,2.540382,-0.204335,0.176156,0.433571,0.8700700000000001,-0.562083,-0.28062600000000004 -336,1,-0.5278930000000001,-1.4276879999999998,-0.592179,-0.535431,-0.796437,-0.361105,-0.61019,-0.8543069999999999,-0.7583439999999999,0.93953,-0.32300700000000004,-1.177414,-0.32495100000000005,-0.39985,-0.123848,-0.088901,-0.645568,-0.720662,-0.582759,1.3584040000000002,-0.8076399999999999,-1.044433,-0.962418,-0.57044,-0.603351,-0.135708,-0.025323,-0.726884,-0.502754,0.720983 -20,1,-0.36636799999999997,-0.844707,-0.332744,-0.43962399999999996,-0.051226,0.14844300000000002,-0.399099,-0.6361100000000001,0.45822700000000005,-0.11725,-0.297446,-0.833008,-0.261106,-0.38363800000000003,0.792763,0.429422,-0.541362,-0.45962700000000006,0.567289,0.753087,-0.793925,-0.8512059999999999,-0.73416,-0.56472,-0.981366,-0.363178,-0.49449399999999993,-0.860707,-0.455534,-0.5181680000000001 -193,0,-0.128223,2.224898,-0.16564500000000001,-0.196329,2.022211,1.376193,0.817074,0.476498,0.508378,2.02568,-0.507616,1.7616720000000001,-0.44563800000000003,-0.504797,0.500985,0.586719,0.247342,-0.08509800000000001,0.479666,0.9317049999999999,0.000462,1.074358,-0.11091600000000001,-0.129746,0.685701,0.7249909999999999,0.190198,0.22934200000000002,-0.213376,0.710014 -200,1,-0.378793,0.436874,-0.4501,-0.425737,0.461654,-0.318484,-0.645212,-0.10059,-0.376548,-0.12223699999999998,-0.538858,0.062913,-0.553145,-0.551441,-0.035602999999999996,-0.44481000000000004,-0.589196,-0.202461,0.6111,-0.378161,-0.186856,0.19802899999999998,-0.276256,-0.28815799999999997,0.15768,-0.429687,-0.592971,-0.06425700000000001,-0.667421,-0.172825 -83,0,0.840923,1.146872,1.013874,0.7337819999999999,0.29946100000000003,0.174525,-0.139073,1.058154,-0.954095,0.447992,1.41232,1.629029,1.529432,1.356952,1.789079,1.416794,1.317025,2.527316,-0.648476,1.338557,0.410829,3.07195,1.4528860000000001,0.58883,0.168014,1.957356,-0.34993,1.076077,1.212928,2.4946040000000003 -376,1,-1.12222,-0.46528100000000006,-0.9159510000000001,-0.9293790000000001,-0.792053,0.684709,1.587076,0.485634,-0.49141,1.997972,-1.0103209999999998,0.21649899999999997,-0.8987299999999999,-0.9004120000000001,-0.400681,1.1685299999999998,1.7476610000000001,0.27060100000000004,1.374147,3.07654,-1.059924,0.025667000000000002,-0.24903000000000003,-0.728456,0.48602700000000004,2.847676,4.032102,2.821453,-0.529391,3.179239 -295,1,-0.331164,-1.424431,-0.389933,-0.385832,-0.673696,-0.9355389999999999,-1.126787,-0.8616159999999999,-0.12579200000000001,-0.886975,-0.101476,-1.4008129999999999,-0.16101400000000002,-0.20531300000000002,-0.311725,-0.7984439999999999,-0.9814139999999999,-0.767349,-0.8018149999999999,-0.521339,-0.671573,-0.9475469999999999,-0.6866369999999999,-0.497174,-0.8976959999999999,-0.9682940000000001,-0.9158540000000001,-0.805718,0.1208,-0.7538189999999999 -77,0,1.267513,-1.102,1.275989,1.282251,0.676449,1.96653,0.510512,1.455568,1.375509,1.488146,1.114105,-0.730617,1.162839,0.9985950000000001,0.721598,2.089571,0.999384,1.523931,1.242713,0.557457,2.0768400000000002,-1.2089940000000001,1.7053509999999998,2.078346,0.299686,1.839429,0.48695299999999997,1.444293,0.649913,1.2327569999999999 -441,0,0.8512770000000001,1.593064,0.760694,0.709172,0.492339,1.004687,1.1102020000000001,0.902842,-0.648335,-0.249694,0.892574,1.426574,0.841556,0.779031,-0.9287290000000001,0.124303,0.396746,0.217723,-1.258183,-1.2372299999999998,0.378346,0.838493,0.206398,0.396975,0.356021,0.9837600000000001,0.5811189999999999,0.9122450000000001,-0.557239,-0.021146 -109,1,-0.674921,0.56552,-0.693153,-0.6372140000000001,1.6452229999999999,-0.220518,0.190997,-0.484605,-0.116085,0.24239899999999998,-0.7916300000000001,0.45851400000000003,-0.8027569999999999,-0.734885,-0.624141,-0.731356,-0.470426,-0.771992,-1.1851639999999999,0.35332199999999997,-0.616352,-0.418666,-0.628223,-0.526437,0.690702,-0.553761,-0.247475,-0.8696280000000001,-0.58872,-0.49849899999999997 -319,1,-0.6977,-0.8903030000000001,-0.759575,-0.641081,-2.116335,-1.317732,-1.19851,-1.3138459999999998,-1.617385,-1.36466,-0.482055,-0.532817,-0.550673,-0.5050819999999999,-1.47955,-1.322832,-0.946385,-0.82358,-1.2399280000000001,-1.018918,-0.09879099999999999,1.783766,-0.187646,-0.20191099999999998,0.10534500000000001,-0.8209129999999999,-0.72729,-0.09669900000000001,1.6693939999999998,-0.315425 -349,1,-0.716338,-1.295784,-0.71996,-0.675712,-0.134515,-0.418358,-1.156052,-1.019363,0.36116,-0.445866,-0.618382,-1.0075379999999998,-0.607104,-0.648993,1.383436,0.308132,-0.9678540000000001,-0.8008810000000001,2.363553,0.42703800000000003,-0.159426,-0.30272899999999997,-0.203487,-0.301139,-0.41367600000000004,-0.073111,-0.798345,-0.207002,0.823055,-0.25755300000000003 -89,1,0.014663999999999998,-1.211106,0.063706,-0.13533,-0.204652,0.347555,-0.056555999999999995,0.38209299999999996,0.40484000000000003,0.043456,0.145616,-0.9423799999999999,0.156563,-0.008501,1.1984059999999999,0.560187,0.136356,0.560267,1.1112790000000001,0.093901,0.38376,-0.8702559999999999,0.46926,0.053308,-0.5110140000000001,1.041885,0.41334499999999996,0.7192149999999999,0.45134399999999997,0.395687 -419,1,-0.809525,0.528066,-0.83404,-0.7428640000000001,-0.18273399999999998,-0.912638,-1.133983,-1.0894059999999999,0.12172899999999999,-0.704657,-0.8427530000000001,0.49341999999999997,-0.865778,-0.78096,0.38711999999999996,-0.8444969999999999,-1.002443,-0.984276,-0.717843,-0.192455,-0.42831199999999997,0.836678,-0.444567,-0.46967200000000003,-0.044328,-0.896252,-0.8263299999999999,-0.9143979999999999,0.774624,-0.65623 -562,0,0.259021,2.786709,0.638572,0.060502,0.40905,3.4188370000000003,4.307272,1.842324,1.9223189999999999,3.1561630000000003,0.310345,2.6366490000000002,0.470844,0.17636500000000002,0.600616,1.9777580000000001,2.086645,1.170295,1.15509,1.2364899999999999,-0.523235,-0.021506,-0.249525,-0.389146,-0.805359,1.283328,1.382525,0.6948840000000001,0.100217,0.887792 -370,0,0.6441939999999999,0.8716659999999999,0.6564439999999999,0.49998000000000004,0.400283,1.350111,2.094174,1.677876,3.1113869999999997,0.675751,0.6312810000000001,0.930909,0.701509,0.527612,0.075415,0.85962,1.1588319999999999,1.0010860000000001,1.3266850000000001,-0.08755299999999999,0.09394,-0.353531,0.052444000000000005,0.113593,-0.468679,0.765232,0.955794,0.772745,1.383649,0.109729 -546,1,-1.039387,-0.636267,-1.076496,-0.871368,-0.16958299999999998,-1.0550059999999999,-1.095507,-1.3825180000000001,-0.355517,-0.55171,-1.081325,-0.684076,-1.098091,-0.938523,-0.143774,-1.030979,-0.9878170000000001,-1.120082,0.267911,-0.111652,-0.702973,-0.45332,-0.747526,-0.602123,0.015008,-1.01893,-0.721952,-1.0221049999999998,-0.598406,-0.449704 -49,1,-0.231765,1.000313,-0.246067,-0.31955900000000004,-0.708765,-0.529046,-0.211036,0.20698699999999998,-0.048138,-0.818814,-0.181,0.700529,-0.20838299999999998,-0.26703000000000005,-0.629123,-0.5185310000000001,-0.518386,-0.38895100000000005,-0.009559999999999999,-0.796355,-0.618518,0.24701700000000001,-0.559909,-0.44305,-0.862028,-0.651568,-0.362861,0.007115000000000001,-0.500333,-0.6955680000000001 -412,1,-1.305488,0.376621,-1.2108299999999999,-1.018857,-1.0419180000000001,-0.41708500000000004,-0.409653,-1.354653,-0.8489399999999999,0.449654,-1.3434700000000002,0.556251,-1.327108,-1.098077,-1.186348,-0.830283,-0.645945,-1.129548,-1.9628150000000002,0.6297550000000001,-1.034299,-0.063236,-0.837621,-0.737829,-0.50868,0.070525,0.18091400000000002,-1.081636,-0.12498900000000002,0.298098 -417,0,1.4290370000000001,0.321254,1.4844899999999999,1.524843,0.847409,0.92835,0.714407,1.504293,0.165409,1.1556520000000001,0.389869,0.416627,0.45024899999999995,0.42152700000000004,1.113008,0.9998610000000001,0.795994,0.925768,0.9981,0.8282200000000001,3.482267,-0.006991,3.246381,2.9958169999999997,0.385689,0.749583,0.43224399999999996,1.556217,-0.043867,0.764104 -440,1,-0.809525,0.194236,-0.50997,-0.710519,0.295077,0.979241,0.986906,0.6226729999999999,-0.583624,0.63031,-0.896716,-0.486275,-0.83365,-0.805988,-0.513123,0.131884,0.072451,-0.329883,-1.1778620000000002,0.510676,-0.5333399999999999,0.288747,-0.029731,-0.488154,0.5080279999999999,1.168195,1.074496,0.954419,-0.6516810000000001,0.446751 -482,1,-0.298031,-1.198078,-0.366998,-0.387414,0.303844,-0.027768,-0.41924799999999995,0.287688,0.527792,0.516153,-0.18667999999999998,-1.216974,-0.191495,-0.308839,0.764297,0.21148000000000003,-0.388442,0.096492,-0.11908900000000001,0.509259,-0.889208,-1.167627,-0.8732629999999999,-0.6049829999999999,-0.8636950000000001,-0.612445,-0.615186,-0.487462,-0.43131800000000003,-0.381241 -291,1,-0.003974,0.083503,0.054770000000000006,-0.124431,-0.046843,0.31002199999999996,-0.440357,0.5221779999999999,0.09908,0.042901999999999996,0.23650100000000002,-0.044132,0.20846199999999998,0.09218,-0.458325,-0.11581199999999998,-0.369108,-0.018808000000000002,0.246006,-0.606396,-0.42398100000000005,-0.48779300000000003,-0.34407600000000005,-0.340302,-0.5696829999999999,-0.241898,-0.5482090000000001,0.011981,-0.644416,-0.37065 -198,0,1.468383,1.039396,1.761498,1.419368,-0.007390000000000001,1.9455380000000002,0.548412,0.85564,0.472787,0.457966,1.435041,0.744743,1.4635280000000002,1.402458,-0.792091,0.728855,0.28375100000000003,0.48494899999999996,-0.1629,-1.064282,0.11018199999999999,-0.261,0.47866499999999995,0.305448,-0.50568,0.642834,-0.160272,0.067133,-0.7739699999999999,-0.150508 -542,1,0.049868,1.07685,0.0041340000000000005,-0.095249,-1.155891,-0.742153,-0.53295,-0.07775,-0.289188,-0.7972020000000001,0.174018,1.426574,0.112489,0.038995,-0.9685819999999999,-0.610256,-0.599491,-0.48103599999999996,0.103619,-0.850224,-0.368399,0.305076,-0.341105,-0.284418,-0.755357,-0.768936,-0.411933,0.144993,-0.223063,-0.44213900000000006 -320,1,-1.033174,-0.825166,-1.064284,-0.861699,0.34329699999999996,-0.116191,-0.39526100000000003,-0.261383,-0.47361499999999995,0.724517,-1.101206,-0.723636,-1.048251,-0.940514,0.693131,0.128093,-0.270425,-0.23908800000000002,-0.250523,1.416525,-0.135245,0.461109,-0.628223,-0.388486,1.14972,0.959727,0.27010700000000004,0.59107,0.804894,1.43777 -94,0,0.40605,-0.235671,0.48368599999999995,0.253872,0.996451,1.056214,1.190321,1.475363,-0.10799600000000001,-0.088988,0.26490199999999997,0.125743,0.343154,0.144227,0.536567,0.965749,1.019472,1.0119200000000002,0.158383,0.0060090000000000005,0.25851999999999997,-0.45803699999999997,0.415796,0.149676,0.73637,0.652335,0.5562520000000001,1.089053,-0.673475,-0.10398299999999999 -116,1,-1.41959,-1.401633,-1.30823,-1.073352,-0.6342439999999999,-0.422174,-0.5650930000000001,-1.159448,-2.020213,-0.372717,-1.470424,-0.8213729999999999,-1.368711,-1.165198,-0.123848,0.378253,0.048094,-0.6664939999999999,-1.849636,1.252083,-0.331946,-0.431729,0.204913,-0.514776,3.769827,2.354171,2.013834,1.830352,0.399281,0.683158 -429,1,-0.507184,-0.768171,-0.547798,-0.5164449999999999,-1.120822,-1.006469,-1.139405,-1.195078,-1.190291,-1.313123,-0.39969099999999996,-0.37690300000000004,-0.45264,-0.43682299999999996,-1.238299,-1.12043,-0.938852,-0.7885,-1.287391,-1.043018,-0.39619,-0.604636,-0.374767,-0.376165,0.098678,-0.767818,-0.879016,-1.001343,-0.138308,-0.9890909999999999 -518,1,-0.252473,-0.212873,-0.23683400000000002,-0.361925,0.58001,0.266129,-0.708059,-0.076228,-0.515677,0.27620300000000003,-0.354249,-0.248914,-0.30971,-0.46014499999999997,1.810429,1.170425,-0.509095,0.10603499999999999,-0.374655,1.3796680000000001,0.13508499999999998,-0.086823,0.15342999999999998,-0.13128599999999999,-0.58935,-0.122294,-0.591645,0.104441,-0.288445,-0.187576 -323,0,1.870123,1.0068270000000001,1.9014919999999997,1.8588470000000001,1.176179,1.240059,1.257966,2.343279,4.298838,1.0226540000000002,1.7644970000000002,0.516691,1.809525,1.732374,1.4688350000000001,1.575986,2.105477,2.617595,2.765157,0.553204,0.595621,-0.351717,0.567276,0.631953,-0.5186810000000001,-0.065287,0.00020800000000000001,0.307203,0.8642219999999999,-0.170177 -248,1,-0.832304,1.549097,-0.872165,-0.746907,0.768505,-0.728158,-0.766109,-0.810759,0.8222280000000001,-0.137199,-0.9876,1.380033,-0.986877,-0.875668,0.014925,-0.606466,-0.81619,-0.845247,0.31172300000000003,0.069801,-0.561131,0.501025,-0.677726,-0.5213760000000001,0.049342000000000004,-0.845505,-0.6990729999999999,-0.9004479999999999,0.125643,-0.444787 -280,0,1.5429329999999999,1.664716,1.564912,1.482653,2.009061,0.825932,1.454664,1.105356,0.577943,0.734491,1.429361,1.701168,1.40998,1.374017,0.401353,0.776234,1.296937,1.230911,0.329977,-0.084717,0.833467,-0.39163200000000004,0.72024,0.644934,0.11701199999999999,-0.055227,0.264801,0.183924,-0.749754,-0.131973 -391,1,-1.263036,-0.468538,-1.288274,-0.9907299999999999,0.597545,-0.784138,-1.305831,-1.745063,-0.737313,0.26068600000000003,-1.531771,-0.57005,-1.511641,-1.196199,0.536567,-0.5697,-1.114873,-1.2618200000000002,0.6330060000000001,1.1599389999999998,0.40325,1.56423,0.148975,-0.252735,2.926459,-0.325173,-1.057501,-1.9134470000000001,-0.22911700000000002,1.112474 -462,1,-0.17999400000000002,1.026368,-0.20436700000000002,-0.256626,-1.344385,-0.688717,-0.599635,-0.898007,-0.899091,-1.069848,0.077453,1.791923,0.011573,-0.024997,-1.8794990000000003,-0.9875799999999999,-0.678462,-0.813778,-0.381957,-1.200372,-0.626819,-0.55456,-0.563869,-0.43601,-0.561682,-0.485576,-0.403976,-0.880172,-0.775181,-0.719776 -406,1,0.298367,-0.9928950000000001,0.257314,0.11833699999999998,-0.515887,-0.522048,-0.197603,-0.02598,-0.19859200000000002,-0.766169,0.571638,-1.030809,0.507915,0.41271,-0.100363,-0.36635100000000004,-0.424349,-0.093868,-0.27973000000000003,-0.5737909999999999,-0.600833,-1.051691,-0.562879,-0.40718699999999997,-1.027701,-0.727578,-0.450396,-0.4946,-0.671053,-0.822282 -416,1,-1.12222,0.9058639999999999,-1.147684,-0.9161940000000001,0.886862,-0.8585659999999999,-1.011215,-1.1710200000000002,-0.04652,-0.050197000000000006,-1.341198,0.560905,-1.333287,-1.091251,0.57215,-0.810194,-0.857873,-0.9375889999999999,0.779043,0.45538999999999996,0.090331,3.013891,-0.052998,-0.333702,2.5664439999999997,-0.488371,-0.604244,-0.506441,1.198399,0.300746 -411,1,-0.799171,0.12421300000000002,-0.814083,-0.719308,0.198638,-0.674722,-0.7939350000000001,-0.613574,0.15732000000000002,-0.28460599999999997,-0.876835,-0.572377,-0.867014,-0.801153,0.806996,-0.498443,-0.732448,-0.622129,-0.3564,0.085395,-0.752419,0.308704,-0.754456,-0.5895819999999999,-0.627685,-0.901115,-0.707363,-0.699633,-0.407102,-0.604031 -474,1,-0.8965,-1.030349,-0.7887649999999999,-0.786636,0.036445,0.862192,0.30853600000000003,-0.532112,-0.517295,1.332982,-0.922277,-0.853952,-0.888021,-0.841823,0.308838,0.048497000000000005,-0.47268599999999994,-0.8565959999999999,0.180289,0.789944,-0.927466,-1.231674,-0.774752,-0.676334,-0.8556940000000001,0.304702,-0.143362,-0.828103,-1.019759,0.33176300000000003 -96,1,-0.712196,-0.774684,-0.748256,-0.67747,-0.805204,-1.022181,-1.066914,-0.849434,-1.0899889999999999,-0.564455,-0.5530579999999999,-0.337343,-0.584037,-0.579597,0.579267,-0.6400100000000001,-0.802254,-0.503219,0.322675,0.503588,-0.141019,0.533683,-0.22576300000000002,-0.349763,-0.5360149999999999,-0.7650239999999999,-0.682495,0.550517,0.200712,-0.146347 -232,1,-0.809525,2.6222369999999997,-0.8584639999999999,-0.720187,-1.4215360000000001,-1.179499,-1.194624,-1.28796,0.016572999999999997,-0.602692,-0.8257120000000001,3.3789830000000003,-0.8723690000000001,-0.762473,-1.320851,-1.30009,-1.0525120000000001,-1.095861,0.121873,-0.640418,-0.654249,0.780434,-0.681686,-0.547338,-0.8940290000000001,-1.0431860000000002,-0.950636,-1.358853,-0.16736700000000002,-0.47693900000000006 -354,1,-0.859225,-1.6051879999999998,-0.823317,-0.750775,-1.916882,-0.818489,-0.729648,-1.1478760000000001,-0.525383,-0.762844,-0.848433,-1.214647,-0.853833,-0.76873,-1.6809479999999999,-0.828198,-0.549271,-0.88239,-0.44402299999999995,-0.278929,0.061457000000000005,-0.739624,0.229665,-0.25295500000000004,-0.500013,0.469017,0.43622299999999997,0.472657,1.268624,0.391905 -240,1,-0.293889,-1.079202,-0.39171999999999996,-0.34663099999999997,-0.200268,-0.796225,-0.5703699999999999,-0.340866,-0.599801,-1.0443559999999998,-0.138398,-0.858606,-0.189023,-0.22635999999999998,-0.151602,-0.720933,-0.524161,-0.299446,-0.345447,-0.878576,-0.292245,-1.008146,-0.43070600000000003,-0.289258,-0.190334,-0.72646,-0.457359,-0.124275,-0.189161,-0.756845 -378,1,-0.358085,-0.983124,-0.277044,-0.39304,-0.213419,0.357097,-0.07334700000000001,-0.140179,0.786637,0.68905,-0.132717,-0.963324,-0.152364,-0.211286,-0.973563,-0.5469579999999999,-0.581412,-0.62445,-0.071627,-0.542604,-0.95634,-1.224961,-0.873758,-0.6377649999999999,-0.609684,0.24378200000000003,-0.247475,-0.5580229999999999,-0.286024,0.4059 -177,0,0.314933,0.451529,0.48368599999999995,0.176876,0.400283,1.351383,1.506478,1.35355,0.247916,0.623106,0.662522,0.190901,0.713866,0.5062810000000001,0.138753,0.971434,1.136233,1.025075,-0.064325,0.061296,-0.366234,0.121827,-0.190121,-0.192451,-0.137999,0.8641559999999999,0.723363,1.027414,-0.45069,0.298477 -33,0,1.6319780000000002,0.850497,1.6125690000000001,1.6391069999999999,0.812341,2.57468,1.616341,0.972885,1.2477040000000001,1.5712700000000002,1.460602,1.670916,1.480004,1.442275,-0.167259,1.280343,0.9654860000000001,0.696717,0.151081,-0.026595999999999998,0.5436479999999999,-1.1079350000000001,0.327681,0.612372,-0.675354,0.430453,0.102,-0.34926,-0.618989,0.038239999999999996 -251,1,-0.6832050000000001,-0.5239050000000001,-0.719066,-0.653386,-0.6167090000000001,-0.9501700000000001,-0.916991,-0.748634,-0.260067,-1.057102,-0.746188,-0.195392,-0.7698050000000001,-0.7038840000000001,-0.20711100000000002,-0.842033,-0.783673,-0.728142,0.081713,-0.49015200000000003,-0.045014,-0.67848,-0.090125,-0.293659,-0.220335,-0.828738,-0.644696,-0.425174,0.287889,-0.8828020000000001 -224,1,-0.233835,-0.33826300000000004,-0.250833,-0.30198,-0.209036,-0.783502,-0.44899300000000003,-0.271433,-0.638628,-0.42757799999999996,-0.243483,-0.528162,-0.305591,-0.308554,-0.8476,-1.030979,-0.668669,-0.628319,-1.152306,-0.8587290000000001,-0.405935,-0.591754,-0.40694400000000003,-0.344483,-0.33634000000000003,-0.806941,-0.30848400000000004,-0.44431499999999996,-0.692847,-0.49131199999999997 -186,0,1.043864,0.111186,0.951324,0.9306690000000001,-0.393146,-0.06211900000000001,0.391533,0.647036,0.49381800000000003,-0.807177,1.187949,-0.16514,1.096935,1.098139,-0.745834,-0.372605,-0.089257,0.23784299999999997,-0.6959380000000001,-1.2117129999999998,-0.532258,-1.3447069999999999,-0.519316,-0.251195,-1.391716,-0.91084,-0.589324,-0.8235610000000001,-1.1929020000000001,-1.024268 -85,0,1.379337,0.32614,1.338539,1.269946,0.325762,-0.288585,0.208748,0.7551439999999999,1.284913,0.102197,1.230551,-0.179102,1.19991,1.1948379999999998,0.169354,0.018175,0.5612159999999999,1.006761,1.169694,-0.36540300000000003,1.063013,0.46836700000000003,0.948449,0.8858530000000001,-0.190001,-0.502343,-0.127115,0.38993,0.38233,-0.015094 -422,1,-0.751542,-0.978239,-0.7545109999999999,-0.711749,0.400283,-0.237058,-0.201441,-0.062524,-0.184032,-0.536193,-0.714947,-0.7608689999999999,-0.68001,-0.701609,0.8852780000000001,0.236117,-0.223847,-0.101864,0.27156199999999997,0.057042999999999996,-0.575929,-0.876244,-0.593571,-0.537878,-0.385675,-0.30337600000000003,-0.185139,0.201767,-0.2497,-0.685733 -104,1,-0.979333,-0.385488,-0.98416,-0.839901,-0.4589,-0.672177,-0.922652,-1.257355,-0.12093800000000002,-0.466924,-1.033042,8.2e-05,-1.011592,-0.9066690000000001,0.25119400000000003,-0.351758,-0.738851,-0.952034,1.480024,0.285277,-0.181081,0.5754130000000001,-0.279227,-0.378585,0.18468099999999998,-0.183773,-0.102578,-0.516173,0.7939970000000001,-0.130082 -461,0,4.094189,0.927033,4.287337,5.930172,0.146035,1.0899299999999998,1.9723169999999999,2.251919,-0.420228,-0.536193,3.775318,1.624375,3.910226,5.250529,0.8568120000000001,1.790138,3.4488589999999997,3.094784,0.9104770000000001,-0.931027,7.730307000000001,0.161743,7.813534,11.041841999999999,0.203015,1.579542,1.613298,2.30076,-0.432529,0.288642 -142,1,-0.72255,0.176323,-0.732768,-0.663758,0.391516,-0.47751899999999997,-0.9360379999999999,-0.770256,-0.512441,-0.165462,-0.766069,-0.46067700000000006,-0.754153,-0.7306189999999999,0.913744,-0.179679,-0.8598809999999999,-0.781794,-0.6083149999999999,0.40010300000000004,-0.436253,1.253978,-0.459913,-0.417088,-0.125665,-0.45427799999999996,-0.673542,-0.369211,0.275781,-0.101713 -380,1,-0.710125,-0.8381930000000001,-0.665154,-0.711046,1.255083,-0.072298,-0.227827,0.261803,0.715454,0.45464099999999996,-0.811511,-1.472952,-0.774748,-0.763895,1.945643,0.128093,-0.12303,0.169746,0.753487,0.8991,-0.537671,-0.41921,-0.523277,-0.5345770000000001,-0.13533199999999998,-0.430804,-0.36087199999999997,-0.025327000000000002,0.065104,-0.027954000000000003 -123,1,-0.117869,-1.579133,-0.132881,-0.23746399999999998,-0.046843,-0.480063,-0.077665,0.114105,-0.019018,-0.215336,0.10585399999999999,-1.954655,0.095189,-0.040355,0.977793,0.105352,-0.004762,0.228557,0.16203399999999998,0.173287,-0.405213,-0.652898,-0.464368,-0.355263,-1.07437,-0.710811,-0.10191499999999999,0.033069,-0.17463099999999998,-0.373298 -428,1,-0.950341,-0.8772760000000001,-0.9802879999999999,-0.8077310000000001,-1.287398,-1.221866,-1.0861040000000002,-1.129299,-0.837615,-0.726824,-0.851273,-0.6212449999999999,-0.885549,-0.778684,-1.056827,-1.250816,-0.942995,-0.908442,-1.097542,-0.18678499999999998,-0.9516479999999999,-0.453139,-0.9395969999999999,-0.67398,-0.386009,-1.073925,-0.745891,-0.909532,-0.054764,-0.536324 -396,1,-0.304244,0.247975,-0.295809,-0.361046,0.45726999999999995,0.017398,0.343558,0.467362,-0.379783,-0.39266599999999996,-0.17532,-0.093001,-0.159366,-0.275278,0.678898,0.196319,-0.037656,0.126155,-0.020513,-0.2846,-0.6914239999999999,0.20891500000000002,-0.669806,-0.46307200000000004,-0.533015,-0.330203,0.038007,0.30395900000000003,-0.895048,-0.503416 -66,1,-1.213336,0.957974,-1.19832,-0.9666469999999999,0.983301,-0.558944,-0.8542879999999999,-0.752745,-0.036814,0.452425,-1.324157,0.400337,-1.31228,-1.096371,0.57215,-0.504318,-0.8421790000000001,-0.873878,-0.345447,0.8778360000000001,-0.613826,1.440855,-0.597036,-0.57506,1.159721,-0.4431,-0.48918900000000004,-0.399545,0.272148,0.167223 -170,1,-0.573451,-1.634499,-0.604391,-0.5827180000000001,0.268776,-0.812128,-0.709978,-0.315133,-0.11932100000000001,-0.8997209999999999,-0.513297,-1.605595,-0.540376,-0.542624,0.45828500000000005,-0.654413,-0.614306,-0.307442,0.538081,-0.46038199999999996,-0.610577,-1.000163,-0.592086,-0.503995,0.334687,-0.764465,-0.499468,0.099574,-0.15768,-0.585118 -481,1,0.029158999999999997,0.120957,-0.085224,-0.088042,-1.138356,-0.7173430000000001,-0.503205,-0.504095,-0.881295,-0.43866099999999997,-0.064554,-0.011554,-0.133416,-0.147862,-1.170692,-0.96806,-0.738851,-0.727884,-0.8492770000000001,-0.972138,-0.26553699999999997,-0.526982,-0.40100399999999997,-0.262416,-1.112371,-0.8192370000000001,-0.550198,-0.895582,-1.2013770000000001,-0.5968439999999999 -313,1,-0.813667,-2.0855770000000002,-0.775361,-0.725637,-1.015616,-0.583118,-0.906485,-1.0271290000000002,0.7914899999999999,-0.5323140000000001,-0.7348279999999999,-1.994215,-0.751269,-0.699049,-0.739429,-0.846202,-0.9432459999999999,-1.032073,0.078062,-0.25483,-0.988823,-1.5542639999999999,-0.8707879999999999,-0.679833,-0.972366,-0.675042,-0.7249359999999999,-1.156415,-0.026916000000000002,-0.6970810000000001 -53,0,0.896835,-0.251956,0.829202,0.7742140000000001,-0.191501,-0.15626800000000002,-0.047439999999999996,0.272461,0.194529,-0.225865,1.162387,-0.137215,1.166958,1.075386,1.312271,0.836879,1.109868,1.472343,1.023657,0.042867,1.5466469999999999,0.682459,0.9954770000000001,1.286504,-1.047369,0.23092800000000002,-0.12346700000000001,-0.41755,0.750408,0.5034890000000001 -296,1,-1.014537,-1.7680310000000001,-1.037775,-0.858535,-1.7204970000000002,-1.139994,-1.16747,-1.258725,-1.225882,-0.9706530000000001,-0.9137569999999999,-1.614903,-0.940332,-0.8281719999999999,-0.79565,-1.082717,-0.959693,-0.9087,-1.3239,-0.352645,-0.829657,-0.344459,-0.791583,-0.643486,-1.187708,-0.74155,-0.7020569999999999,-0.389812,1.073688,-0.138782 -114,1,-1.3752739999999999,-0.986381,-1.274274,-1.048038,1.7548119999999998,-0.113647,-0.12755899999999998,-0.14627,0.040839999999999994,0.9838629999999999,-1.534044,-0.805083,-1.488162,-1.205869,1.3265040000000001,-0.423205,-0.596101,-0.7655430000000001,-0.593711,1.9183569999999999,-0.861418,-1.143859,-0.748516,-0.690217,0.40669,-0.187126,0.023087,-0.226467,-0.41921,0.004197 -243,1,-0.260756,0.107929,-0.275853,-0.306902,-1.6959490000000002,-0.700167,-0.6538470000000001,-0.815327,-0.384637,-1.149092,-0.107156,1.042608,-0.141243,-0.184551,-1.133686,-0.687389,-0.525166,-0.657209,-0.140994,-1.206042,0.106573,-0.290029,-0.018345,-0.008956,-0.8966959999999999,0.066613,0.060223,-0.279996,0.658389,-0.267766 -140,1,-1.169849,-1.8852790000000001,-1.2132129999999999,-0.9452,-0.393146,-1.159206,-1.305831,-1.745063,0.330422,-0.134983,-1.246621,-1.7033310000000002,-1.265735,-1.0420479999999999,-0.274719,-1.200026,-1.114873,-1.2618200000000002,0.33362800000000004,0.201639,-0.74484,-1.307876,-0.81584,-0.617744,-0.333673,-1.10785,-1.057501,-1.9134470000000001,0.269727,-0.21745799999999998 -152,1,-1.087016,-1.007551,-1.0788790000000001,-0.879102,-0.138898,0.145898,2.635815,0.647036,0.335276,2.324925,-1.248609,-0.91911,-1.1611120000000001,-1.008772,0.771413,1.052926,4.042708999999999,0.764814,2.688487,4.275833,1.513443,2.625622,0.5974729999999999,0.20930100000000001,1.309727,3.93361,12.07268,6.649601,1.806213,9.851593 -268,1,-0.49061800000000005,-0.33174899999999996,-0.535883,-0.49763500000000005,-0.296707,-0.46734,-0.350164,-0.864965,1.1376950000000001,-0.7384609999999999,-0.357089,-0.716655,-0.394974,-0.40582199999999996,-0.150179,-0.798824,-0.625229,-0.845247,0.724279,-0.724057,-0.615991,0.0038950000000000005,-0.65347,-0.486174,-0.50768,-0.20668699999999998,-0.19906500000000002,-0.88666,0.652335,-0.6195390000000001 -91,0,0.033301,0.026507,0.007112,-0.087339,-0.292324,-0.34711,0.05954500000000001,0.502383,-0.557739,-0.868134,0.35294699999999996,0.807574,0.339035,0.208504,-0.31030100000000005,-0.014043,0.293795,0.6683439999999999,-0.345447,-0.259083,-0.333029,-0.681383,-0.391599,-0.239754,0.947045,-0.058021,0.44484399999999996,0.9463090000000001,0.5046189999999999,-0.625213 -255,0,0.025018000000000002,-0.587414,0.024984,-0.095952,0.825491,0.457607,0.23369499999999999,0.347072,0.270565,-0.24248899999999998,-0.047513,-0.521181,-0.022203,-0.149284,0.94221,0.446478,0.114133,0.091333,0.351883,-0.212302,0.071563,-0.738535,-0.150024,-0.10114400000000001,-0.230002,0.07331900000000001,-0.023333,0.263406,0.009408,-0.41603999999999997 -559,1,-0.784675,1.8698990000000002,-0.7440859999999999,-0.714386,-0.112597,-0.016316999999999998,0.43567,-0.275239,-1.2760340000000001,0.18698299999999998,-0.743348,1.079841,-0.718729,-0.7149760000000001,-0.26689,-0.04247,0.28124,-0.202977,-1.546608,0.411444,-0.600472,3.061064,-0.460408,-0.514116,0.386356,0.24266500000000002,0.8450489999999999,0.141749,-0.6855829999999999,0.356727 -14,0,-0.256615,1.031253,0.045834,-0.321493,1.4348100000000001,3.296698,2.02509,1.61697,1.1247530000000001,3.2780769999999997,-0.11283599999999999,0.772668,0.06717999999999999,-0.217827,1.191289,2.368158,1.556825,0.8081470000000001,0.939685,1.9878200000000001,-0.696838,-0.086823,-0.398529,-0.46483199999999997,-0.204001,1.8936419999999998,0.766467,0.727326,-0.112881,1.625761 -100,0,0.149267,1.562124,0.039876999999999996,0.04556,-0.257255,-0.381461,0.21450500000000003,0.057766,-0.40405,-0.552818,-0.146918,1.3241829999999999,-0.161426,-0.20531300000000002,-0.10534500000000001,-0.364456,-0.032007,-0.10392799999999999,-0.739749,-0.579461,0.185253,0.132713,-0.0025039999999999997,0.06166900000000001,-0.389675,-0.592325,-0.179834,-0.304165,-0.713431,-0.544646 -353,0,0.464033,1.228294,0.415178,0.29782,1.4742629999999999,-0.118736,0.627092,0.578516,-0.39919699999999997,0.5782189999999999,0.270583,1.50104,0.248417,0.175512,0.42981899999999995,-0.126046,0.435666,0.42846000000000006,-0.601013,0.261178,0.895907,0.5246109999999999,0.647471,0.506764,1.159721,-0.065287,0.571172,0.918733,0.07963300000000001,0.405522 -543,1,-0.393289,1.871527,-0.440271,-0.44120600000000004,-1.1032879999999998,-0.7389720000000001,-0.796334,-0.53333,-0.6920149999999999,-1.081485,-0.260524,2.0409200000000003,-0.29199899999999995,-0.331307,-0.686767,-0.6741229999999999,-0.7398560000000001,-0.417067,-0.670381,-0.707046,-0.613826,0.689717,-0.656935,-0.49475399999999997,-0.689354,-0.657157,-0.560808,-0.43458199999999997,-0.399837,-0.927436 -165,1,-0.059886,0.02325,-0.147774,-0.173125,-1.221645,-0.9816590000000001,-0.9340229999999999,-0.868924,-0.41213900000000003,-1.279874,0.239341,0.109454,0.14544100000000001,0.100428,-0.8646799999999999,-0.963133,-0.8704280000000001,-0.761674,-1.082938,-1.4371120000000002,-0.7982560000000001,-0.275514,-0.7821779999999999,-0.5213760000000001,-1.1357059999999999,-0.9777950000000001,-0.783358,-0.870115,-0.15768,-0.860485 -43,0,0.23002899999999998,0.37825,0.173913,0.04679,0.9043959999999999,0.7515029999999999,0.451982,0.526746,1.356095,1.0392780000000001,-0.240643,0.230461,-0.191495,-0.311967,0.550801,0.744016,0.12141500000000001,0.326574,0.592845,0.711976,-0.1255,-0.711138,-0.217348,-0.198171,-0.656353,-0.22401300000000002,-0.33302,-0.362723,-0.405891,-0.180768 -201,0,0.85956,0.026507,0.960259,0.630066,0.251241,0.558117,0.377141,1.207374,0.044075,-0.292364,0.969258,0.007063,0.9527690000000001,0.8438760000000001,-0.475405,0.292971,0.18582200000000001,0.6696340000000001,-1.115796,-1.1181510000000001,-0.029133999999999997,-0.705151,0.109867,0.008645,-0.317006,0.011840999999999999,-0.157951,0.268273,-0.557239,-0.43003500000000006 -108,0,2.512079,0.379878,2.964846,2.600686,1.65399,2.833589,3.3036300000000005,2.685877,1.867314,0.772728,2.3126450000000003,0.08851,2.50564,2.429179,2.579016,3.268353,4.238567,3.4404230000000005,2.717694,1.0763,2.9228389999999997,0.595371,3.5562709999999997,2.852806,-0.175333,3.420544,2.39083,2.109352,1.280732,0.469825 -328,0,0.623486,0.765818,0.671337,0.42263199999999995,1.167411,0.257223,0.419839,0.665308,0.327187,-0.107829,0.60856,0.330525,0.6150100000000001,0.45167399999999996,1.461718,0.522284,0.7407520000000001,0.927574,0.497921,-0.003914,0.116678,0.027481000000000002,0.199963,0.089611,-0.11466400000000002,-0.259782,0.019438999999999998,0.344511,-0.6274649999999999,-0.380484 -439,1,-0.281464,-1.0368629999999999,-0.31963800000000003,-0.336962,-1.2698639999999999,-0.970527,-1.005506,-0.494046,-1.237207,-0.933525,-0.030472000000000003,-0.8446440000000001,-0.097993,-0.137624,-1.188483,-0.919734,-0.852851,-0.577763,-0.812768,-0.9834780000000001,-0.689258,-1.019577,-0.623768,-0.46395200000000003,-0.734689,-0.9076540000000001,-0.752025,-0.168071,-1.069402,-0.639587 -162,0,2.166251,0.11607100000000001,2.014678,2.375673,0.501106,0.8291120000000001,1.961283,1.6763540000000001,1.200788,0.459075,1.551487,-0.265204,1.595336,1.590169,1.113008,1.1799,2.0339139999999998,2.055287,0.786345,-0.28034699999999996,1.195471,-0.306358,0.9534,1.2482209999999998,-0.9947,-0.151357,0.278396,0.20014400000000002,-0.44584700000000005,-0.18039000000000002 -325,1,-0.529963,-0.745372,-0.5528609999999999,-0.538243,0.264392,-0.8464799999999999,-0.816483,-0.892069,-0.344192,-0.834885,-0.413892,-0.463005,-0.44151899999999994,-0.469246,0.45828500000000005,-0.524975,-0.713993,-0.71834,-0.381957,-0.41927200000000003,-0.704417,-0.483257,-0.643569,-0.500035,-0.077329,-0.892228,-0.616844,-0.863302,0.0033539999999999998,-0.760627 -183,1,-0.807454,-1.299041,-0.83821,-0.726691,-0.8884920000000001,-0.593296,-0.515199,-0.786396,-1.7629860000000002,-0.536193,-0.771749,-1.0168469999999998,-0.7595069999999999,-0.7192430000000001,-0.410645,-0.43192299999999995,-0.33885,-0.652824,-2.353466,-0.08897000000000001,-0.255431,-0.197498,-0.477239,-0.386506,0.105011,0.659601,0.9037370000000001,0.863582,0.13411900000000002,0.374127 -197,0,0.7228859999999999,-0.159135,0.650487,0.610729,-1.935293,-0.368739,-0.089658,-0.347109,-0.8602639999999999,-1.017757,1.122625,0.593484,1.047507,1.0497889999999999,-1.611917,-0.339629,0.26994,0.228557,-0.151947,-1.332209,0.833828,0.159928,0.715785,0.792566,-0.50368,1.535949,0.9683930000000001,0.42886,0.103849,0.46944600000000003 -498,0,1.342063,-0.45551,1.1657819999999999,1.264672,0.387133,0.347555,0.38913400000000004,0.7871199999999999,-0.632157,0.582098,1.2390709999999998,-0.41180900000000004,1.208148,1.17493,0.344421,0.518494,0.757073,1.106842,0.074411,0.59148,1.397226,-0.311801,0.982606,1.219619,0.31101999999999996,0.372328,0.413013,0.638111,-0.529391,0.552283 -477,1,-0.233835,-0.631382,-0.180538,-0.284225,-1.6889349999999999,-0.341385,-0.641853,-0.797664,-0.358752,-0.38601599999999997,-0.064554,-0.6212449999999999,-0.12353099999999999,-0.157817,-1.998346,-0.9693870000000001,-0.83565,-0.916438,0.005043,-1.054359,-0.901119,-1.162366,-0.729704,-0.578801,-1.244377,-0.689014,-0.729114,-1.12965,-0.8938379999999999,-0.642991 -421,1,0.039513,-1.194821,0.203699,-0.12548499999999999,-0.051226,0.694887,0.238492,-0.057955999999999994,-0.11932100000000001,0.450762,0.15981700000000001,-1.2355909999999999,0.257479,0.0034439999999999996,0.47963500000000003,1.502076,0.705598,0.363201,1.001751,1.5965610000000001,0.508999,0.533683,0.954885,0.2005,0.9783799999999999,1.506886,0.692527,0.649465,0.724982,0.6237729999999999 -374,1,-0.29596,-0.8903030000000001,-0.24130100000000002,-0.369132,-0.9586290000000001,-0.284132,-0.660084,-0.68118,0.683099,-0.384354,-0.124197,-0.7492340000000001,-0.170076,-0.215552,-0.949367,-0.7694489999999999,-0.793968,-0.7379439999999999,0.22044899999999998,-0.8658170000000001,-0.846981,-1.2886440000000001,-0.7396050000000001,-0.579461,-0.937031,-0.536994,-0.669895,-0.885849,-0.134675,-0.611974 -470,1,-1.062166,-0.009318,-1.083645,-0.87084,-0.393146,-0.6365529999999999,-0.693187,-0.7461979999999999,0.44204899999999997,0.07171799999999999,-1.266786,-0.186084,-1.2554370000000001,-1.040342,-0.491062,-0.791432,-0.7447520000000001,-0.8712989999999999,1.556694,0.18888,-0.099513,0.24157399999999998,-0.147053,-0.387386,0.153347,-0.312319,-0.157619,-0.310329,-0.11409200000000001,0.04467 -382,1,-0.766038,0.49386899999999995,-0.592774,-0.6894239999999999,-1.945375,0.427072,0.091208,-0.08231799999999999,-1.1482290000000002,0.528899,-0.5899800000000001,0.798266,-0.544495,-0.588983,-1.9221990000000002,0.056077999999999996,-0.117631,-0.493675,-2.222032,0.537611,-1.031411,0.393979,-0.538623,-0.6773899999999999,-0.666353,1.124602,0.369909,0.790588,0.503408,1.3477459999999999 -510,1,-0.790887,-1.315326,-0.774766,-0.7150890000000001,-1.098904,0.15925699999999998,-0.015297,-0.13713399999999998,-0.48008599999999996,0.8226030000000001,-0.678025,-1.0703690000000001,-0.644999,-0.650984,-1.093833,-0.146514,-0.270425,-0.581116,-1.141353,0.677954,-0.767939,-1.043526,-0.7529710000000001,-0.600582,-0.01966,0.7646729999999999,0.274417,0.565116,0.24914299999999998,1.145003 -132,0,0.662832,0.9775149999999999,0.668358,0.517559,0.312612,0.32592600000000005,0.129588,0.25266700000000003,0.93709,-0.429795,0.577318,0.523672,0.586176,0.440582,0.31595500000000004,0.45595399999999997,0.19461099999999998,0.185997,1.2719209999999999,-0.551109,0.101159,0.087354,-0.010920000000000001,0.07355,-0.721356,-0.332998,-0.321746,-0.41608999999999996,-0.628676,-0.537837 -545,1,-0.190348,0.55575,-0.28836300000000004,-0.26506399999999997,-0.472051,-0.652457,-0.80257,-0.6527069999999999,-0.41861000000000004,-0.798864,-0.14407799999999998,0.9169459999999999,-0.196849,-0.232332,-0.277565,-0.69876,-0.741488,-0.6316729999999999,-0.5389470000000001,-0.6786939999999999,-0.21356399999999998,0.216173,-0.396054,-0.200151,-0.391009,-0.25084,-0.387397,-0.443179,0.039678,-0.458404 -121,0,1.238521,-0.126566,1.135996,1.175019,0.786039,-0.16008499999999998,0.263919,0.80387,-0.010929000000000001,0.034035,1.287353,-0.504892,1.212267,1.2005270000000001,0.643316,0.10724700000000001,0.714386,0.9732290000000001,0.563638,-0.094641,1.1102940000000001,0.660687,1.0043879999999998,1.10301,0.353688,-0.24972199999999997,0.050607,0.6835289999999999,-0.011175,0.29318099999999997 -45,0,1.356558,-0.709547,1.2908819999999999,1.206661,1.5575510000000001,1.62047,2.21795,1.875822,1.453162,0.438017,1.284513,-0.39319299999999996,1.307005,1.197683,0.9635600000000001,1.217803,1.363478,1.3407930000000001,0.34823200000000004,-0.32712800000000003,0.807481,-1.0043360000000001,0.706379,0.686958,-0.24900300000000003,0.808267,0.78404,0.8360059999999999,0.45255500000000004,-0.09830900000000001 -333,1,-0.726692,-0.589042,-0.7500439999999999,-0.6818649999999999,-0.69123,-0.9944459999999999,-1.279358,-1.491235,-0.138734,-0.541181,-0.817192,-1.049425,-0.848066,-0.753372,-0.94652,-1.132559,-1.1026479999999999,-1.185959,-0.140994,-0.281764,-0.6885359999999999,-0.400523,-0.6618850000000001,-0.5559189999999999,-0.474679,-1.0258040000000002,-1.025216,-1.436389,-0.428896,-0.6675770000000001 -134,0,1.294434,0.93029,1.141953,1.247093,0.619463,-0.170263,0.596387,0.354685,0.336894,-0.43478199999999995,1.227711,0.609773,1.162839,1.1948379999999998,-0.146621,-0.137417,0.332715,0.5042939999999999,-0.436721,-0.783596,0.688377,-0.026949,0.44549799999999995,0.616332,-0.346674,-0.629212,-0.11086800000000001,-0.42955299999999996,-0.68074,-0.601383 -182,0,0.795365,1.163157,0.6564439999999999,0.6828029999999999,0.3959,0.638907,0.086411,0.601356,0.867526,0.13101300000000002,0.446671,0.237443,0.380225,0.317717,-0.027775,-0.309875,-0.287124,0.07662999999999999,-0.7068909999999999,-1.03593,-0.127304,-0.121295,-0.22774299999999997,0.014145,-0.805026,-0.159181,-0.409612,-0.329146,-0.614146,-0.5162770000000001 -519,1,-0.376722,-0.641152,-0.406017,-0.450875,0.663299,-0.35856,-0.623143,-0.520083,0.275418,0.090005,-0.39116999999999996,-0.602629,-0.389619,-0.458154,1.14859,0.139464,-0.62774,-0.48928999999999995,1.125883,0.48657700000000004,-0.07858,-0.388004,-0.183686,-0.257796,0.156013,-0.5515260000000001,-0.401986,-0.421281,-0.301764,-0.062753 -207,0,0.731169,-0.102139,0.677294,0.579086,-0.932328,-0.672177,-0.378949,-0.076228,0.605445,-1.067077,0.8187300000000001,0.225807,0.730342,0.70935,-0.614889,-0.5932,-0.242302,0.128476,0.782694,-1.498069,0.651924,-0.655801,0.613809,0.6187520000000001,-0.667687,-0.583942,-0.41226499999999994,-0.091833,0.29031,-0.459161 -99,0,0.012593,0.843983,0.06668400000000001,-0.095249,0.470421,0.30747800000000003,0.226498,0.6379,-0.295659,0.531115,0.083133,0.11178099999999999,0.103427,-0.035236,0.08253200000000001,0.184948,0.063788,0.24429099999999998,0.246006,0.156275,-0.41748500000000005,1.15056,-0.242594,-0.296739,0.321353,0.19404100000000002,0.043644,0.396418,-0.717063,0.248547 -366,0,1.640261,1.3243719999999999,1.570869,1.389484,-0.200268,0.5555720000000001,0.470692,1.531701,0.598974,-0.422591,1.724735,1.75469,1.718907,1.6470509999999998,0.191415,1.185586,0.945398,2.0011189999999996,0.231402,-0.368238,2.0605990000000003,1.224948,2.109791,1.391892,0.466026,1.188316,0.90009,2.20019,2.004782,0.185758 -180,0,3.4895080000000003,1.168042,3.381848,4.105459,0.650148,0.948707,1.256047,2.3478470000000002,-0.072405,-0.17322,3.718515,0.6004649999999999,3.712513,4.536658999999999,0.927977,1.6498970000000002,2.489659,3.58229,-0.042419,-0.722639,1.555309,0.479253,1.462291,1.944135,-0.803359,-0.006044,-0.026649000000000003,0.099574,-0.580244,-0.39637100000000003 -318,1,-1.285815,-0.370832,-1.1509610000000001,-1.0257120000000002,-0.450133,0.766771,0.9053479999999999,-0.0016170000000000002,0.378956,1.194443,-1.444295,-0.090674,-1.313928,-1.167189,0.236249,1.759816,1.3647340000000001,0.004149,1.892581,3.4918980000000004,0.21701399999999998,1.259421,0.446983,-0.355043,0.9347120000000001,2.259159,2.3477259999999998,2.185591,1.7396200000000002,1.544058 -501,0,-0.053674,1.182698,-0.037566,-0.162753,2.061664,0.905449,0.316212,0.5709029999999999,1.213731,1.9037650000000002,-0.087275,1.210157,0.014868000000000001,-0.167771,1.411902,1.208328,0.5888369999999999,0.481596,1.6917790000000001,1.356986,0.25238499999999997,0.564527,0.053434,-0.028318,0.879709,0.731139,0.095037,0.707861,0.45982,1.2104409999999999 -229,0,-0.22141100000000002,0.728364,-0.058415999999999996,-0.306902,1.987143,1.7814139999999998,1.707974,1.265235,0.818992,2.2362599999999997,-0.36844899999999997,0.7075100000000001,-0.276346,-0.43141899999999994,0.8852780000000001,1.4319549999999999,1.013195,0.5079060000000001,1.136836,1.3810850000000001,-0.357572,-0.268257,-0.301503,-0.334582,-0.019327,0.7322569999999999,0.49524300000000004,0.518075,-0.453112,0.68921 -7,0,0.16376300000000002,0.401048,0.099449,0.028858999999999996,1.447961,0.7247859999999999,-0.021054,0.624196,0.47763999999999995,1.726435,-0.11851700000000001,0.35845,-0.072867,-0.21896500000000002,1.604049,1.140102,0.061026,0.28195,1.403355,1.660353,0.643623,0.290561,0.490051,0.23372199999999999,0.5880310000000001,0.268933,-0.23255399999999998,0.435349,-0.688004,0.6116689999999999 -247,1,-0.38914699999999997,-1.299041,-0.067352,-0.42450600000000005,-0.305475,2.1033,2.401216,0.6318090000000001,-0.42346300000000003,1.8760569999999999,-0.351408,-1.205339,-0.289115,-0.40582199999999996,-0.623429,0.573453,0.6101800000000001,-0.235219,-0.787211,0.18320999999999998,-0.731486,-1.409116,-0.234179,-0.5277569999999999,-0.513347,1.701382,1.6474490000000002,0.308825,-0.954377,1.422261 -494,1,-0.36636799999999997,0.453158,-0.35657300000000003,-0.408333,-0.9016430000000001,-0.570395,-0.936517,-1.106307,-0.34581,-0.535085,-0.274725,0.29096500000000003,-0.325775,-0.33045399999999997,-1.637537,-0.977725,-0.888883,-0.937847,-0.360051,-0.555362,-0.29405,0.464738,-0.26734600000000003,-0.3139,0.253684,-0.27710799999999997,-0.612865,-1.010913,0.038467,-0.413771 -174,1,-0.979333,-1.054776,-1.014542,-0.830233,-1.085753,-1.185478,-1.305831,-1.745063,-0.308601,-1.2360950000000002,-0.9847600000000001,-0.963324,-1.0082959999999999,-0.868274,-0.6006560000000001,-1.162123,-1.114873,-1.2618200000000002,0.42490200000000006,-0.43203,-0.268063,1.284821,-0.35199600000000003,-0.403887,0.557363,-0.8555649999999999,-1.057501,-1.9134470000000001,1.465982,-1.0564200000000001 -340,1,0.083001,-0.678606,0.123277,-0.032492,-0.130131,0.526946,0.49563900000000005,0.407978,0.246298,0.204716,0.083133,-0.639862,0.089834,-0.038932999999999995,0.08182,0.18115699999999998,-0.109596,-0.17254,0.366487,0.18746300000000002,-0.202376,-0.809657,-0.09358999999999999,-0.18035,-0.8213600000000001,0.282346,0.215729,0.10281900000000001,-0.21943000000000001,-0.15958599999999998 -125,1,-0.161357,-0.34152,-0.207346,-0.271919,-0.730683,-0.758692,-0.9165120000000001,-0.9678969999999999,-0.868353,-0.671962,-0.07875499999999999,-0.48394799999999993,-0.145362,-0.188249,-0.605638,-0.814553,-0.936592,-0.96751,-0.721494,-0.552527,-0.6737390000000001,-0.6545310000000001,-0.678716,-0.49343400000000004,-0.814026,-0.91151,-0.754877,-1.13257,-0.977382,-0.789374 -209,1,0.23002899999999998,-1.5889030000000002,0.19178499999999998,0.09161699999999999,-0.44574899999999995,-0.22688000000000003,0.115196,-0.16910999999999998,-0.939535,-0.5101479999999999,0.324545,-1.484587,0.255419,0.200825,-1.034766,-0.7967390000000001,-0.375134,-0.447504,-1.6524849999999998,-1.068535,-0.692146,-1.549547,-0.6638649999999999,-0.44745100000000004,-0.916697,-0.733167,-0.44708000000000003,-0.717476,-1.332263,-0.8090430000000001 -347,1,0.20725,-1.261587,0.20667800000000003,0.000381,-0.454517,-0.339476,-0.273883,0.15978499999999998,0.336894,-0.115033,0.179698,-1.058734,0.11949100000000001,0.03928,-0.541589,-0.502991,-0.5363399999999999,-0.351808,-1.061032,-0.521339,-0.22511399999999998,-1.4855,-0.162894,-0.248115,-0.769691,-0.5822649999999999,-0.442438,-0.18267,0.131697,-0.382754 -332,1,-0.888216,0.016737000000000002,-0.9040360000000001,-0.781363,0.43973599999999996,-1.002397,-1.241784,-1.437181,0.632947,-1.037706,-0.8257120000000001,0.132725,-0.825,-0.761051,0.643316,-0.692695,-1.052023,-1.066224,0.468713,-0.35689699999999996,-0.38825,1.35921,-0.44902200000000003,-0.45581099999999997,1.9497529999999998,-0.806941,-0.948182,-1.107752,2.65013,-0.69292 -361,1,-0.428493,0.573662,-0.426569,-0.45597299999999996,-0.805204,-0.557036,-0.724371,-0.890242,-0.426699,-0.962341,-0.234963,0.5306529999999999,-0.27716999999999997,-0.30940700000000004,-0.750104,-0.769638,-0.6950350000000001,-0.6365729999999999,0.012345,-0.827542,-0.5163770000000001,0.584485,-0.414865,-0.425889,-0.514347,-0.28102,-0.46233199999999997,-0.875468,-0.27270500000000003,-0.6955680000000001 -500,1,0.101638,-0.8544780000000001,0.072641,-0.041632999999999996,-0.8271219999999999,-0.23324099999999998,-0.41541000000000006,-0.194995,-1.170878,0.08557200000000001,0.25922199999999995,-0.59332,0.278486,0.098153,0.175759,0.6075659999999999,-0.145504,0.322447,-0.524343,0.8353079999999999,-0.119725,-0.679569,-0.278237,-0.12094500000000001,-0.972699,-0.407331,-0.395687,-0.284862,-1.209853,-0.21102800000000002 -300,0,2.000585,0.091645,1.9014919999999997,2.061007,0.75097,1.0008700000000001,1.630733,1.269803,0.108787,0.85031,1.534446,-0.090674,1.545908,1.598701,1.3265040000000001,1.134417,1.6434540000000002,1.477501,-0.071627,0.385927,2.5474810000000003,-0.101337,2.163749,2.038743,-0.32834,0.366181,0.8118920000000001,0.8976459999999999,-0.206112,0.375262 -283,0,0.47231599999999996,-0.095626,0.584958,0.26442,0.181104,1.376193,1.105405,0.892184,-0.21153400000000003,1.238775,0.600039,-0.120926,0.693271,0.42721499999999996,0.7287140000000001,1.437641,1.3308360000000001,1.0730520000000001,0.235053,0.573051,-0.425425,-0.543493,-0.199032,-0.269457,-0.826026,0.5215529999999999,0.22633899999999998,0.047667,-0.806661,0.108216 -107,1,-0.616938,0.295199,-0.646389,-0.591508,-0.612326,-0.368739,-0.37655,-0.459633,0.133053,-0.6703,-0.5019359999999999,-0.174448,-0.5337850000000001,-0.535229,-0.8248270000000001,-0.685873,-0.783045,-0.766317,-0.765306,-0.303028,-1.029607,-0.585041,-0.9988030000000001,-0.684475,-1.194708,-0.8387979999999999,-0.670558,-1.011886,-0.9725389999999999,-0.9225190000000001 -131,0,0.619345,0.052562,0.525386,0.484159,0.974533,-0.09456200000000001,0.512911,0.560244,-0.10314300000000001,-0.208132,0.378508,0.044295999999999995,0.40081999999999995,0.267377,0.913744,0.34035,0.7256859999999999,0.82414,0.435855,-0.685782,0.24949699999999997,-0.781898,0.11283800000000001,0.17541800000000002,-0.267004,-0.594561,-0.124794,-0.140496,-0.795764,-0.504551 -303,1,-1.078732,-0.18519000000000002,-1.0872190000000002,-0.8880680000000001,0.391516,-0.9533510000000001,-0.9017350000000001,-0.751071,-1.112638,-0.306218,-1.033042,-0.158159,-1.034246,-0.9117879999999999,0.742947,-0.7118359999999999,-0.826485,-0.802687,-1.203419,0.45397200000000004,-0.926383,0.628029,-0.90643,-0.6657069999999999,0.6113649999999999,-0.900724,-0.45802200000000004,-0.421281,-0.318715,-0.180768 -567,0,1.961239,2.237926,2.303601,1.653171,1.4304270000000001,3.9048480000000003,3.197605,2.2899849999999997,1.919083,2.219635,1.838341,2.336457,1.9825240000000002,1.735218,1.525767,3.272144,3.2969440000000003,2.658866,2.137194,1.043695,1.157935,0.686088,1.4385299999999999,1.009503,-0.173,2.017716,1.3022850000000001,0.785721,0.32663400000000004,0.904057 -507,1,-0.9482700000000001,-0.803996,-0.928759,-0.82531,1.4830299999999998,-0.325481,-0.7032619999999999,-0.296404,-0.195356,1.820641,-0.8711540000000001,-0.504892,-0.8534209999999999,-0.820208,1.639632,0.052287,-0.6047640000000001,-0.160932,0.519826,2.404595,-0.8202729999999999,-0.339016,-0.7663369999999999,-0.6168640000000001,1.859749,-0.111675,-0.460674,0.17094700000000002,0.200712,1.7089759999999998 -3,0,-0.281464,0.133984,-0.249939,-0.550021,3.394275,3.893397,1.9895880000000001,2.175786,6.046041000000001,4.93501,-0.7689090000000001,0.253732,-0.592687,-0.764464,3.283553,3.4029089999999997,1.9158970000000002,1.451707,2.8673830000000002,4.910919,0.326373,-0.110409,0.286593,-0.288378,0.689702,2.74428,0.8195180000000001,1.115007,4.73268,2.047511 -516,0,1.157759,0.085131,1.040681,1.076575,0.73782,-0.004231,0.497558,0.554153,0.280271,-0.29458,1.187949,0.300273,1.187553,1.129424,0.742947,0.387729,0.855002,1.1759700000000002,0.17663800000000002,-0.480229,0.504307,-0.534058,0.17422100000000001,0.59455,-0.288338,-0.374915,-0.091637,-0.216734,-0.599617,-0.40469299999999997 -143,1,-0.37051,-0.628125,-0.300575,-0.41624399999999995,-0.051226,0.003403,-0.30314800000000003,-0.204131,1.048717,-0.15327000000000002,-0.348568,-0.78414,-0.338956,-0.40582199999999996,-0.682497,-0.17532,-0.500433,-0.465301,-0.12274000000000002,-0.063453,-0.688897,-0.808569,-0.58268,-0.5213760000000001,-0.57235,-0.550408,-0.55683,-0.683249,0.060261,-0.7216670000000001 -398,1,-0.743259,-0.867505,-0.788467,-0.674833,-0.892875,-0.422174,-0.30842600000000003,-0.899073,-0.502735,0.423054,-0.8711540000000001,-1.03779,-0.8921399999999999,-0.786932,-1.3486049999999998,-1.073809,-0.774382,-1.074916,-1.009919,-0.093223,-0.7928430000000001,-0.959341,-0.793563,-0.601903,-0.927364,-0.6029439999999999,-0.414254,-1.237358,-1.045186,-0.097552 -530,1,-0.573451,0.37499299999999997,-0.5582229999999999,-0.5770930000000001,0.11096600000000001,-0.43807799999999997,-0.650489,-0.20717600000000003,-0.683926,-0.353321,-0.675185,-0.402501,-0.662299,-0.659801,0.77853,-0.136659,-0.451719,-0.116567,-0.77991,0.563128,0.11992699999999999,1.252163,0.14006400000000002,-0.212912,-0.151332,-0.409566,-0.481563,0.221232,-0.26544,-0.5586409999999999 -119,0,0.8926930000000001,0.350566,0.653465,0.66874,-1.1032879999999998,-0.852841,-0.226868,0.059289,3.205219,-1.265465,1.085703,0.167631,0.9156979999999999,0.9303370000000001,-0.878202,-0.703498,-0.199239,0.181612,1.158741,-1.7787540000000002,0.52488,-0.005177,0.243031,0.30148800000000003,-1.0057,-0.95326,-0.297873,-0.31357399999999996,3.5836449999999997,-0.715993 -259,0,0.459891,3.885905,0.567086,0.271451,2.451803,1.922,1.430197,1.321574,0.988859,2.020138,0.398389,3.320807,0.48320100000000005,0.256,0.707364,1.128731,1.083503,0.904617,1.020006,0.5248520000000001,-0.589283,0.11094100000000001,-0.47674399999999995,-0.38100500000000004,-0.565349,0.004575,-0.099594,-0.25566500000000003,-1.2828629999999999,-0.164881 -450,1,-0.720479,0.407562,-0.70745,-0.656375,-1.6569349999999998,0.544758,0.238012,-0.412736,-0.963802,0.8630559999999999,-0.641103,0.523672,-0.62358,-0.633919,-2.151351,0.039021,-0.012922999999999999,-0.646375,-1.6889939999999999,0.470983,-0.538393,0.6117,-0.45100200000000007,-0.44217,-0.062328999999999996,1.964621,1.151752,0.605669,0.36295700000000003,1.940466 -192,1,-1.304866,-0.7893399999999999,-1.340697,-1.0139340000000001,-2.6826950000000003,-1.443878,-1.305831,-1.745063,-1.604443,-1.017203,-1.251733,-0.248914,-1.286742,-1.0431860000000002,-1.9115240000000002,-1.533193,-1.114873,-1.2618200000000002,-0.579108,0.23707899999999998,-0.185052,6.655278999999999,-0.314869,-0.410268,-1.7760650000000002,-1.04749,-1.057501,-1.9134470000000001,2.112542,-0.7969390000000001 -514,0,0.271446,0.38802,0.19476300000000002,0.15191300000000002,-0.340543,-0.280951,0.06914,-0.039684,-1.001011,-0.7983100000000001,0.26206199999999996,-0.051114,0.21793600000000002,0.133704,-0.299627,-0.348157,-0.175008,-0.14365,-0.9149940000000001,-0.517087,-0.06919600000000001,-0.034207,-0.11685699999999999,-0.040639,-0.6963550000000001,-0.512962,-0.073732,-0.384946,-1.092406,-0.785592 -241,1,-0.635576,-0.8642479999999999,-0.6973229999999999,-0.592739,-1.256713,-1.122819,-1.006321,-1.128081,0.000395,-0.893071,-0.48489499999999996,-0.988922,-0.550261,-0.507357,-1.216949,-1.334392,-0.982669,-0.976022,-0.969759,-0.745321,-1.046209,-0.984016,-1.044049,-0.689337,-1.2587110000000001,-1.148426,-0.8422120000000001,-1.303215,-0.404681,-0.921006 -402,1,-0.442989,-0.173791,-0.326191,-0.454742,-1.7130450000000002,-0.14290899999999998,-0.536308,-0.738889,0.495436,-0.635942,-0.331527,-0.232625,-0.320832,-0.368849,-1.62615,-0.48043900000000006,-0.605518,-0.776119,0.227751,-0.539768,-0.61166,0.149042,-0.232199,-0.44283,-1.1373719999999998,0.651217,0.08707999999999999,-0.185915,0.699555,-0.034006 -148,1,-0.08680700000000001,-0.9489270000000001,0.039876999999999996,-0.199845,-0.033692,0.12236099999999998,0.182841,0.68967,-0.339339,-0.394329,0.088814,-0.956343,0.08242000000000001,-0.042062,0.237673,-0.04247,-0.049332,0.165104,-0.319891,-0.281764,-0.593975,-0.866265,-0.369322,-0.421048,-0.445011,-0.140179,-0.024991,0.508343,-0.726749,-0.47920799999999997 -395,1,-0.27939400000000003,-0.054915,-0.322915,-0.344697,-1.129589,-0.834393,-0.89996,-0.5404869999999999,-0.6111260000000001,-0.9894950000000001,-0.019112,-0.490929,-0.091402,-0.13022899999999998,-1.132262,-0.961427,-0.778274,-0.423257,-0.622919,-0.7311449999999999,-0.9195260000000001,0.8493790000000001,-0.806434,-0.6087229999999999,-0.556682,-0.712488,-0.682163,-0.428418,-0.405891,-0.8892329999999999 -299,1,-1.105653,-0.2373,-1.106878,-0.910393,-0.792053,-1.06951,-1.10635,-1.269232,-1.0899889999999999,-0.8963959999999999,-1.027362,0.884367,-1.034658,-0.9120729999999999,0.36577,-0.689284,-0.8016260000000001,-0.7781819999999999,-0.42576800000000004,0.391598,-0.42723,-0.133996,-0.28566199999999997,-0.43513,1.043049,-0.617475,-0.44044799999999995,0.114173,1.7069290000000001,-0.689894 -149,1,-0.192419,-0.5239050000000001,-0.29998,-0.271919,-1.545592,-0.457162,-0.5554979999999999,-0.828574,-0.8910020000000001,-0.765061,-0.10999600000000001,-0.32105300000000003,-0.15854200000000002,-0.198772,-1.2041389999999998,-0.76907,-0.7531640000000001,-0.919018,-1.236277,-0.991984,-0.560049,-0.833607,-0.640104,-0.415108,-1.40105,-0.5342,-0.46730600000000005,-0.96809,-0.8780969999999999,-0.6880029999999999 -60,1,-1.087016,-1.339752,-1.114026,-0.900022,-0.213419,-0.989865,-1.20182,-1.3523690000000002,1.061659,-0.207578,-1.1239270000000001,-1.026155,-1.1293950000000001,-0.9754959999999999,1.212639,-0.449737,-0.978777,-0.929077,3.4004209999999997,0.96431,0.399279,0.406679,0.220754,-0.12578599999999998,0.15768,-0.809735,-0.8033520000000001,-0.584464,2.577483,0.816303 -61,1,-1.388321,0.22191999999999998,-1.3463559999999999,-1.0664959999999999,1.382207,-0.537316,-0.874006,-1.322068,0.115258,-0.378258,-1.570397,0.393356,-1.536767,-1.23175,1.9883419999999998,-0.27879499999999996,-0.738224,-1.022993,0.059807000000000006,0.6765359999999999,-0.169532,1.5424579999999999,-0.184676,-0.48287299999999994,1.62974,0.34326599999999996,-0.06279,-0.41154799999999997,1.577375,-0.282517 -126,0,0.12855899999999998,1.622376,0.176892,-0.056048,0.645764,0.217146,0.515309,0.278552,0.9209120000000001,-0.274077,-0.146918,1.2566979999999999,-0.173371,-0.234039,-0.269025,-0.487451,-0.45134300000000005,-0.466075,-0.184806,-0.212302,-0.628624,-0.384375,-0.551493,-0.451191,-0.984366,-0.767818,-0.462001,-0.797445,-0.8853620000000001,-0.881289 -372,0,1.3296379999999999,-0.624868,1.335561,1.150408,-0.577257,0.189156,0.624693,1.248486,-0.27624499999999996,0.150409,2.057032,-0.974959,2.031952,2.079354,0.266139,0.893733,1.310748,1.975325,0.589194,-0.137169,-0.230167,0.167186,-0.227248,-0.028098,-0.8716950000000001,0.07108400000000001,0.082106,0.26502800000000004,-0.45916599999999996,0.216396 -82,0,2.843411,1.293432,3.1107970000000003,2.955784,1.09289,2.247704,1.801046,2.620403,-0.882913,1.172276,3.150487,1.3078940000000001,3.275896,3.478653,0.707364,3.073153,3.077232,3.4971699999999997,0.063458,0.711976,1.7761930000000001,0.466552,2.235529,1.75272,0.37502199999999997,1.75783,0.842397,1.379409,-1.197745,0.793608 -425,1,-1.068378,0.531323,-1.112239,-0.8864860000000001,-0.866574,-1.166203,-1.246581,-1.3523690000000002,-0.8926200000000001,-0.184857,-1.163689,0.463168,-1.185414,-0.9885790000000001,-1.081023,-1.236034,-1.083862,-1.1287479999999999,-0.663079,0.22573800000000002,-0.7942859999999999,0.22524499999999997,-0.8326709999999999,-0.632265,-0.439011,-1.105559,-0.988733,-1.341334,-0.737646,-0.523463 -337,0,1.7127400000000002,1.415565,1.603633,1.744582,0.764121,1.453165,0.917342,1.3733440000000001,1.2590290000000002,0.817615,1.3185950000000002,0.49807399999999996,1.274053,1.243188,-0.37008,0.679581,0.215954,0.30903400000000003,0.516175,-0.278929,0.855484,0.568156,0.744002,1.054166,0.169014,0.754054,0.240265,1.0322799999999999,0.137751,0.2459 -386,1,-0.650071,-1.0401200000000002,-0.584136,-0.61647,-1.304933,-0.071025,0.169888,-0.353352,-0.36198800000000003,0.23796599999999998,-0.544538,-1.209993,-0.543259,-0.548596,-1.087428,-0.494842,-0.256238,-0.6082,-0.604664,-0.178279,-0.5001359999999999,-0.7002520000000001,-0.380708,-0.44833100000000004,-0.8786950000000001,0.267256,0.38284,-0.150228,-0.161313,0.31285 -184,0,0.317004,0.383135,0.19476300000000002,0.16263699999999998,-0.099446,0.48114399999999996,0.43567,0.121718,0.44366700000000003,0.763308,0.327386,0.7261270000000001,0.286312,0.158448,-0.41206800000000005,0.016280000000000003,-0.440043,-0.420162,-0.308938,0.05279,-0.721019,-1.308602,-0.753466,-0.457791,-1.250377,-0.644302,-0.46929499999999996,-0.93873,-1.068191,-0.46143 -58,1,-0.422281,-0.5581020000000001,-0.506991,-0.450875,-1.326851,-1.223647,-1.296979,-1.575895,-0.747019,-1.166825,-0.30596599999999996,0.004736,-0.3855,-0.363161,-1.1215870000000001,-1.259344,-1.106185,-1.154336,0.026949,-1.103975,-0.004229999999999999,-0.005177,-0.134183,-0.16230799999999998,0.150013,-0.9437030000000001,-1.034556,-1.2375200000000002,0.164388,-0.304456 -88,1,-0.5051140000000001,0.785359,-0.47065200000000007,-0.537716,-0.08629500000000001,-0.050669,-0.138113,0.089742,0.115258,0.480133,-0.5019359999999999,0.584176,-0.502069,-0.536936,-0.614889,-0.187449,-0.359691,-0.295835,0.432204,0.176122,-0.38752800000000004,0.517354,-0.328235,-0.42654899999999996,0.023675,-0.030635000000000003,-0.16126600000000002,0.183924,-0.116514,0.252708 -117,0,0.526157,0.275658,0.590915,0.37692600000000004,2.429885,1.232425,0.950924,1.3413680000000001,1.106957,1.249858,0.21094000000000002,-0.60961,0.274778,0.078528,1.411902,1.147683,1.006917,1.039777,1.260968,0.69213,0.077338,-0.48616000000000004,0.060859,0.018546,-0.01866,0.008487,-0.059142999999999994,0.14823699999999998,-0.547553,0.033701 -459,1,-1.159494,1.830816,-1.168535,-0.932895,-0.9367110000000001,-0.9120020000000001,-0.960889,-1.004137,-0.937918,-0.655891,-1.241793,2.073499,-1.247611,-1.0352219999999999,-1.175673,-1.1007209999999998,-0.921401,-0.9927879999999999,-0.6959380000000001,-0.464635,-0.819551,0.853008,-0.8034640000000001,-0.639305,-0.150999,-0.714164,-0.576724,-0.92332,-0.581455,-0.577553 -221,1,-0.266969,-1.3918620000000002,-0.183517,-0.341005,0.229323,0.098824,-0.069509,-0.36096500000000004,0.26571100000000003,-0.120575,-0.161119,-1.254207,-0.139183,-0.266177,0.621966,0.2816,-0.128052,-0.113729,0.549034,0.032944,-0.535145,-1.304066,-0.42328000000000005,-0.424789,-0.39667600000000003,-0.130678,-0.213986,-0.557375,-0.256965,-0.33169 -223,0,0.681469,0.751162,0.555172,0.36462100000000003,1.0008350000000001,1.232425,0.601665,0.506951,1.7670119999999998,1.2443170000000001,0.460872,0.22348,0.437892,0.302644,0.436936,0.304342,0.325182,0.404988,0.45045799999999997,0.032944,-0.20887199999999997,-0.536961,-0.307938,-0.17925,-0.758357,-0.09714400000000001,-0.26703699999999997,-0.512443,-0.34171999999999997,-0.251123 -128,1,-0.032964999999999994,-1.19645,-0.040545,-0.207404,0.27315900000000004,0.21650999999999998,-0.365516,0.42168199999999995,-0.502735,-0.34113000000000004,0.27626300000000004,-0.674768,0.31349699999999997,0.055775,1.3265040000000001,1.447116,0.31388299999999997,0.9394389999999999,0.691421,0.265431,0.09285800000000001,-0.270071,-0.034681000000000003,-0.010937,0.655034,0.9150159999999999,0.00485,1.7492470000000002,0.361746,0.44977700000000004 -294,1,-0.573451,-1.334867,-0.5576270000000001,-0.5746319999999999,-0.112597,-0.681083,-1.054776,-0.7792399999999999,-0.8602639999999999,-0.8160430000000001,-0.39969099999999996,-1.282132,-0.419688,-0.46298900000000004,0.022040999999999998,-0.386818,-0.9531649999999999,-0.7655430000000001,-0.633872,-0.25483,-0.810167,-0.950269,-0.755446,-0.593102,-0.325673,-0.764465,-0.839858,-0.6193390000000001,-0.8236120000000001,-0.9089020000000001 -541,1,-0.010186,0.985657,0.185828,-0.12601300000000001,0.071514,1.0555780000000001,0.632369,0.089742,0.46308,1.017112,0.097334,1.32651,0.15821,0.004297,-0.568632,0.353616,0.151924,-0.258434,0.22044899999999998,0.086813,-0.54489,-0.250114,-0.12428199999999999,-0.379025,0.032342,1.176579,0.21208200000000002,-0.028571,0.016673,0.876066 -111,1,-0.6086550000000001,-0.033745,-0.543926,-0.620865,-0.16081600000000001,-0.186167,-0.24269899999999997,-0.062524,-1.091607,0.050660000000000004,-0.425252,0.342161,-0.404448,-0.49626499999999996,0.21134099999999997,0.313818,0.222232,0.291236,-0.27973000000000003,1.1202459999999999,-0.22655799999999998,1.0634709999999998,-0.07675900000000001,-0.43688999999999995,1.95642,0.835094,0.633839,1.809264,0.10869200000000001,0.792851 -266,1,-0.908925,-0.44573999999999997,-0.8632299999999999,-0.801227,-0.485202,-0.01759,-0.386625,-0.538203,0.063489,-0.447528,-1.001801,-0.079038,-0.934566,-0.877375,0.036986,0.196319,-0.312987,-0.580343,0.402996,0.299453,0.163598,-0.036021,0.27916799999999997,-0.291238,0.143013,0.577443,0.054586,0.300714,1.7553599999999998,-0.18039000000000002 -568,1,-1.410893,0.7641899999999999,-1.432735,-1.075813,-1.859019,-1.207552,-1.305831,-1.745063,-0.048138,-0.751207,-1.808401,1.221792,-1.814389,-1.347789,-3.112085,-1.150752,-1.114873,-1.2618200000000002,-0.8200700000000001,-0.561032,-0.070279,0.383092,-0.157449,-0.46615200000000007,0.049342000000000004,-1.163516,-1.057501,-1.9134470000000001,0.75283,-0.382754 -362,1,-0.52168,0.050934,-0.579073,-0.528926,-0.112597,-0.44761999999999996,-0.703741,-0.47942799999999997,-0.253596,-0.640929,-0.38833,-0.10463599999999999,-0.41598100000000005,-0.45019,0.028446,-0.470394,-0.777395,-0.802429,-0.192108,-0.137169,-0.663633,0.12364100000000001,-0.6589149999999999,-0.507735,-0.477679,-0.50402,-0.55053,-0.291837,-0.175842,-0.628617 -271,1,-0.817808,-1.546564,-0.8635280000000001,-0.743743,0.150419,-0.658818,-0.694146,-0.412736,-0.271392,-0.206469,-0.8058310000000001,-1.454335,-0.813055,-0.7590600000000001,0.140888,-0.535587,-0.7049529999999999,-0.551195,-0.15559800000000001,-0.013837,-0.775157,-1.247458,-0.842571,-0.597722,-0.189667,-0.7968810000000001,-0.632428,-0.478054,-0.438583,-0.36989299999999997 -187,1,-0.674921,-0.698148,-0.680345,-0.6312369999999999,-0.003007,-0.955896,-0.5761270000000001,-0.07166,-0.531855,-0.719066,-0.686545,-0.48860200000000004,-0.7121390000000001,-0.667195,0.098188,-0.813606,-0.6366539999999999,-0.42635200000000006,-1.079287,-0.261918,-0.577734,-0.81891,-0.5564439999999999,-0.49453400000000003,-0.045328,-0.937499,-0.40165500000000004,0.008737,-0.190372,-0.803369 -155,1,-0.554813,-0.074456,-0.615412,-0.556174,-0.467667,-0.480063,-0.373672,-0.494807,0.34336500000000003,-0.145512,-0.5331779999999999,-0.31407199999999996,-0.5642659999999999,-0.553431,-0.698865,-0.711647,-0.627112,-0.660562,0.578241,-0.073377,-0.668325,-0.42556099999999997,-0.6841619999999999,-0.5242359999999999,-0.50768,-0.550967,-0.39635,-0.6280979999999999,-0.309029,-0.49585100000000004 -5,0,-0.165498,-0.313836,-0.11500899999999999,-0.24431999999999998,2.0485130000000003,1.721616,1.2632430000000001,0.9058879999999999,1.7540689999999999,2.241802,-0.476375,-0.835335,-0.387148,-0.50565,2.237421,1.244335,0.866302,0.824656,1.0054020000000001,1.890005,-0.25506999999999996,-0.592662,-0.321304,-0.289258,0.156347,0.44554399999999994,0.160025,-0.069124,0.13411900000000002,0.486846 -511,1,-0.136507,-1.318583,-0.16564500000000001,-0.211623,-0.809587,-0.9743440000000001,-0.777623,-0.533787,-0.9168860000000001,-1.2482870000000001,0.193899,-1.068042,0.11084100000000001,0.073409,-0.8283860000000001,-1.02681,-0.685995,-0.606394,-0.5572020000000001,-1.320868,-0.674821,-1.077091,-0.588621,-0.431609,-0.110997,-0.7549640000000001,-0.566114,-0.20051300000000002,-0.57419,-0.762518 -252,0,1.8659810000000001,-0.014204,1.564912,1.850057,1.693442,2.170731,2.766787,2.072245,-0.245507,2.535505,1.591249,0.12341600000000001,1.595336,1.567416,0.700248,1.526713,1.919664,1.250514,-0.287032,0.59148,1.302665,-0.792602,0.618264,1.1544940000000001,0.48036,1.4023729999999999,1.197177,1.2837049999999999,-0.711009,1.3102989999999999 -480,1,-0.606584,0.35708,-0.5489890000000001,-0.585707,-0.50712,-0.167719,-0.528632,-0.878669,-0.8004060000000001,-0.368838,-0.558739,-0.293129,-0.563442,-0.567652,-0.390718,-0.49199899999999996,-0.7487699999999999,-0.8679450000000001,-1.269136,0.0060090000000000005,-0.6704899999999999,-0.048721,-0.588126,-0.529737,-0.710022,-0.492842,-0.5942970000000001,-1.076283,-0.726749,-0.732636 -513,1,0.101638,-1.373949,0.036898,-0.032668,-0.441366,-0.391004,-0.11028800000000001,-0.346348,-0.44449399999999994,-0.746219,0.128576,-1.310057,0.095601,0.011123000000000001,0.139464,-0.287323,-0.082603,-0.140039,-0.265127,-0.9069280000000001,0.040885000000000005,-1.076184,-0.151014,-0.071001,-0.696021,-0.411243,-0.051185,-0.510658,-0.623833,-0.5726359999999999 -86,0,-0.012256999999999999,0.581805,0.03392,-0.12601300000000001,-0.077528,-0.360469,0.30086,0.12019500000000001,0.192911,-0.858159,0.100174,0.505055,0.093953,-0.019024,-0.136658,-0.092312,0.396746,0.011887,0.96159,-0.9125979999999999,0.054961,1.820053,0.21530900000000003,-0.032277999999999994,0.776038,0.24322399999999997,0.723363,0.9430649999999999,0.44044700000000003,-0.20648899999999998 -105,0,0.008451,-0.533675,-0.025651999999999998,-0.093843,2.3597479999999997,0.9900559999999999,1.7530700000000001,1.278939,0.39836900000000003,3.133996,-0.288925,-0.867914,-0.196026,-0.35462899999999997,3.091407,1.36752,1.485262,1.214661,0.41394899999999996,2.001996,-0.051872,-0.5317,-0.22576300000000002,-0.124905,0.040342,0.203542,0.7571829999999999,0.338023,-0.614146,1.249401 -242,1,-0.7639670000000001,0.371736,-0.598731,-0.7166710000000001,0.102199,1.466524,2.2616080000000003,0.109537,0.658832,2.535505,-0.802991,-0.255896,-0.743031,-0.755078,-0.031333,0.533655,0.8286370000000001,-0.525659,0.884921,1.969391,-0.5860350000000001,0.771362,-0.24606,-0.526877,-0.125998,1.881346,1.886843,0.21798800000000002,-0.071715,1.8459029999999998 -312,1,-0.43056400000000006,-1.510738,-0.45337700000000003,-0.460192,-0.5684899999999999,-0.21288400000000002,-0.45714899999999997,-0.464354,-0.544797,-0.078459,-0.38833,-1.377542,-0.398681,-0.42886,-0.599233,-0.47115200000000007,-0.6061449999999999,-0.604589,-0.768957,-0.198126,-0.28394400000000003,-1.011412,-0.257445,-0.333482,-0.182334,0.12306099999999999,-0.017365000000000002,-0.179426,-0.391362,0.225852 -473,1,-0.583805,2.01483,-0.660686,-0.565491,-1.672278,-1.2858610000000001,-1.305831,-1.745063,-0.795553,-0.9152379999999999,-0.527497,2.485389,-0.599278,-0.538926,-1.378495,-1.333445,-1.114873,-1.2618200000000002,-0.403862,-0.453294,0.145552,4.409122,0.008881,-0.11456500000000001,0.099344,-0.9632639999999999,-1.057501,-1.9134470000000001,1.3158450000000002,-0.249231 -364,1,-0.318739,-0.647666,-0.402145,-0.38161300000000004,-0.485202,-0.551311,-0.6514479999999999,-0.68118,-0.25845,-0.450299,-0.20656100000000002,-0.544452,-0.267285,-0.29149,-1.209121,-0.89794,-0.8410489999999999,-0.8818739999999999,-0.59006,-0.8204540000000001,-0.890652,-1.096686,-0.905935,-0.596622,-0.8823620000000001,-0.725342,-0.576392,-1.02389,-0.924107,-0.650934 -565,0,1.53672,2.047399,1.42194,1.494959,-0.69123,-0.39482,0.236573,0.733827,-0.531855,-0.973978,1.704854,2.085134,1.615931,1.7238419999999999,0.102458,-0.017833,0.6930430000000001,1.2636690000000002,-0.217664,-1.058611,1.300499,2.260938,1.156857,1.291565,-0.42401000000000005,-0.069758,0.252202,0.808431,-0.189161,-0.49055600000000005 -447,1,0.033301,-0.47830900000000004,-0.040545,-0.0898,-0.42821499999999996,-0.42090200000000005,-0.31754099999999996,-0.480037,1.131224,-0.614884,0.19105899999999998,-0.37923,0.16109300000000001,0.056629,-0.325246,-0.292629,-0.604011,-0.678876,0.297119,-0.558197,-0.6668810000000001,-1.0790870000000001,-0.6851520000000001,-0.45295100000000005,-0.7483569999999999,-0.7694949999999999,-0.4746,-0.794687,0.24187899999999998,-0.689894 -488,1,-0.610726,-0.665579,-0.616305,-0.581488,0.886862,-0.6779029999999999,-0.591,-0.25057199999999996,-0.15653,-0.20536100000000002,-0.695066,-0.725963,-0.678775,-0.6666270000000001,1.16994,-0.22194,-0.577646,-0.453952,0.151081,0.171869,-0.122251,-0.11403800000000001,-0.154479,-0.280898,0.652367,-0.701869,-0.44376400000000005,-0.020461,0.118379,-0.220106 -151,1,-1.486271,0.658341,-1.464904,-1.108862,1.342755,1.1242809999999999,1.275717,-0.545359,0.681481,3.582864,-1.6780389999999998,0.328198,-1.594021,-1.282659,-0.164412,0.495752,0.543639,-0.7026060000000001,1.4982790000000001,2.808612,-0.763969,1.351952,-0.8034640000000001,-0.6628470000000001,1.796413,1.603016,1.5131629999999998,-0.25566500000000003,0.30847199999999997,3.0203729999999998 -263,0,0.339783,0.975886,0.257314,0.189884,-1.050685,-0.467976,-0.22159,-0.44044799999999995,-0.352281,-0.8675799999999999,0.42111000000000004,0.021025,0.330797,0.294964,-1.278151,-0.913101,-0.586434,-0.527465,-0.966108,-1.1861959999999998,-0.632955,-0.395624,-0.65941,-0.399487,-1.40505,-0.9150870000000001,-0.622812,-1.074175,-1.2607059999999999,-0.9266790000000001 -563,0,1.66097,0.6078600000000001,2.139779,1.6496549999999999,0.365215,1.0454,1.860055,2.125538,0.045693,0.8192780000000001,1.9292259999999999,1.349781,2.1019759999999996,1.968434,0.9635600000000001,2.260135,2.870075,2.540213,1.23176,0.849484,2.010431,-0.34627399999999997,2.916691,1.726318,-0.214002,0.984878,1.543668,2.342934,0.0033539999999999998,0.9146479999999999 -194,0,-0.039178,0.34242399999999995,0.337735,-0.168554,-0.033692,1.3392959999999998,0.895753,0.8845709999999999,0.160555,0.16980399999999998,0.2081,0.912292,0.347273,0.046959,0.57215,1.774977,1.015706,1.02817,-0.272428,0.55604,-0.453216,-0.462029,0.35886799999999996,-0.333042,0.346688,1.438701,0.783046,1.1409610000000001,0.594217,0.19559200000000002 -433,0,1.323425,0.855382,1.133017,1.269946,0.290694,0.585471,0.5709609999999999,0.85564,0.17188,-0.044655,1.3327959999999999,0.623736,1.307005,1.294382,0.38711999999999996,0.654944,0.88639,0.993606,0.479666,-0.20946700000000001,1.493953,1.295707,0.805385,1.398493,0.344354,0.8608030000000001,0.706785,1.0614780000000001,0.39807,0.272377 -244,0,1.114272,0.790245,1.121103,0.942974,0.610696,0.270582,0.353153,0.6363770000000001,0.031132999999999997,-0.432566,1.497524,0.979777,1.529432,1.422367,0.451169,0.975224,1.457641,1.030234,0.607449,-0.39659,0.429958,1.061657,0.5796520000000001,0.44163900000000006,1.189722,0.39356599999999997,0.24059699999999998,0.615401,0.159545,0.058287 -489,0,0.6027779999999999,0.143755,0.5968720000000001,0.357589,-1.3794540000000002,0.24004699999999998,-0.11748399999999999,-0.41471499999999994,2.873574,-0.42757799999999996,0.727846,0.21184499999999998,0.6232479999999999,0.576531,-1.522249,-0.629587,-0.656742,-0.666752,0.125524,-1.353473,-0.569793,-1.177425,-0.540108,-0.38254499999999997,-1.4580520000000001,-0.615799,-0.585676,-1.054223,-0.112881,-0.7924 -465,1,-0.171711,-0.028860000000000004,0.230506,-0.25855900000000004,-0.537805,1.974164,1.8394259999999998,0.321186,-0.090201,2.26951,-0.25200300000000003,0.195555,-0.21003000000000002,-0.31850900000000004,-0.9621770000000001,0.34035,0.153179,-0.531076,-0.768957,0.215815,-0.44816300000000003,-0.731822,0.248971,-0.363624,-0.704022,2.296605,1.489953,0.305581,-0.639573,1.6408909999999999 -483,1,-0.27111,-0.349662,-0.341978,-0.341181,-0.5465720000000001,-0.7612369999999999,-0.470102,-0.36294499999999996,-0.619215,-0.794985,-0.12135699999999999,-0.383884,-0.173371,-0.23830500000000002,0.223439,-0.469447,-0.5438729999999999,-0.44673,-0.29068299999999997,-0.271841,-0.584952,-0.491058,-0.644559,-0.43336899999999995,-1.2653780000000001,-0.96645,-0.49449399999999993,-0.41576599999999997,-0.931372,-0.858972 -397,1,-0.523751,-0.7518859999999999,-0.49269399999999997,-0.509062,-1.623181,-0.464796,-0.393821,-0.481864,-1.476638,-0.743448,-0.37696999999999997,-0.425771,-0.367377,-0.416914,-1.132974,-0.291682,-0.18706,-0.20865100000000003,-0.867532,-0.750991,-0.14895999999999998,0.087354,-0.09804500000000001,-0.214892,-0.5400149999999999,0.5193180000000001,0.44948599999999994,0.331534,-0.22427399999999997,0.102921 -52,1,-0.656284,-0.707918,-0.702684,-0.621217,-0.787669,-1.050935,-0.8643149999999999,-0.786396,-0.18726700000000002,-0.546722,-0.621222,-0.24425999999999998,-0.669713,-0.617993,-0.9785450000000001,-1.0770309999999998,-0.867289,-0.913859,0.20584499999999997,-0.240654,-0.6419779999999999,-1.059492,-0.666341,-0.503115,0.056343,-0.955607,-0.622812,-0.6157699999999999,-0.070504,-0.43873500000000004 -147,1,-0.003974,-0.033745,-0.004802000000000001,-0.124606,-1.432057,-0.013772999999999999,-0.10645,-0.465267,-0.078876,0.456304,0.233661,-0.120926,0.241826,0.098437,-1.066078,0.23422199999999999,0.021352,-0.343038,-0.24687199999999998,0.302288,0.060736,1.255792,0.200458,-0.019957,-0.41701000000000005,1.301771,0.700816,0.5634939999999999,1.576164,2.108031 -27,0,1.043864,0.257745,0.972174,0.9183629999999999,0.062747,-0.270773,0.347396,0.5237,-0.9055620000000001,-0.539518,1.273153,0.22348,1.241101,1.248876,-0.139504,0.042812,0.755818,0.732313,-0.418466,-0.8232889999999999,1.615944,1.146931,1.369225,1.170555,1.2363899999999999,0.097352,0.627208,1.186379,0.289099,0.159658 -436,1,-0.376722,-0.211245,-0.36104,-0.445953,-0.480818,-0.566578,-0.9639110000000001,-0.772997,0.666921,-0.36551300000000003,-0.357089,0.058258000000000004,-0.383029,-0.41435500000000003,-0.35584699999999997,-0.483471,-0.88926,-0.722725,0.180289,0.095318,-0.139576,-0.812016,-0.133193,-0.30443899999999996,-0.377008,-0.6627460000000001,-0.823213,-0.8587600000000001,0.204344,-0.535946 -434,1,0.008451,-0.836565,-0.147774,-0.18121099999999998,-0.46328400000000003,-0.631464,-0.720533,-0.531351,-0.6078899999999999,-0.8686879999999999,0.2081,-0.546779,0.120315,0.0535,-0.506718,-0.636788,-0.6947840000000001,-0.519727,-0.871183,-0.817619,-0.36948200000000003,-0.995265,-0.620798,-0.36120399999999997,-0.42834399999999995,-0.49340100000000003,-0.5730770000000001,-0.5701890000000001,-0.6093029999999999,-0.615379 -190,0,-0.10958599999999999,1.873156,-0.025651999999999998,-0.207756,0.9175469999999999,4.315794,2.766307,0.95309,3.664668,3.3612010000000003,0.026330000000000003,0.891349,0.098896,-0.12795399999999998,0.792763,2.595576,1.372267,0.445226,2.089732,1.789355,-0.430117,1.620475,-0.373282,-0.18959,0.309686,6.143482,2.808608,0.788966,3.7035129999999996,2.997678 -407,1,-0.387077,0.217034,-0.46558900000000003,-0.412728,-1.6810450000000001,-0.385914,-0.424046,-0.8922209999999999,-0.667748,-0.134983,-0.362769,0.484112,-0.384677,-0.399281,-1.483819,-0.401411,-0.34575500000000003,-0.780246,-0.845627,-0.234983,0.33972800000000003,1.0544,-0.155469,0.019865999999999998,-0.34334000000000003,1.07989,0.658375,0.261784,0.744354,1.488834 -202,0,1.832848,1.1403590000000001,2.077228,1.943226,0.9306979999999999,1.033313,2.480375,2.416366,0.480876,0.20360799999999998,2.6023389999999997,1.717457,2.7569,2.929741,1.2624549999999999,1.972072,3.308243,2.9168060000000002,1.417958,-0.07195900000000001,0.53679,0.622586,0.8915209999999999,0.942177,0.762038,1.438142,1.9127049999999999,2.085021,0.146227,0.5295880000000001 -455,1,-0.252473,2.594554,-0.314872,-0.307605,-0.664929,-0.713526,-0.96986,-0.563022,-1.14014,-0.398762,-0.21224099999999999,2.65992,-0.231861,-0.27783800000000003,-0.278277,-0.570079,-0.760948,-0.419904,-1.59407,-0.373909,-0.232332,1.283007,-0.286652,-0.250975,-0.400009,-0.727578,-0.794367,-0.43279799999999996,-0.5935630000000001,-0.306347 -452,1,-0.658355,1.9871459999999999,-0.660984,-0.627369,-0.50712,-0.436806,-0.43699899999999997,-0.661538,-0.7340770000000001,-0.111154,-0.6041810000000001,2.08048,-0.626051,-0.604056,-0.853294,-0.755046,-0.605769,-0.760127,-0.717843,-0.24915900000000002,-0.77227,0.885666,-0.668321,-0.582541,0.097678,0.023019,-0.082353,-0.426634,-0.374411,0.19180999999999998 -385,0,-0.099232,0.9824,-0.150752,-0.215139,-0.051226,-0.611744,-0.022493,0.324232,-0.6855439999999999,-0.863701,0.134256,0.930909,0.08242000000000001,0.027902999999999997,-0.678938,-0.719796,-0.061511,0.09778099999999999,-0.674032,-1.2244709999999999,0.037998000000000004,0.744147,0.023732,-0.161208,0.423691,-0.450366,0.066191,0.642977,-0.380465,-0.349468 -188,1,-0.766038,0.130727,-0.8248059999999999,-0.6874899999999999,0.141652,-0.981341,-0.8835049999999999,-1.0894059999999999,0.484112,-1.007782,-0.658144,-0.442061,-0.687837,-0.642736,0.308838,-0.923335,-0.8194540000000001,-0.861239,-0.341796,-0.7084630000000001,-0.791399,1.2866360000000001,-0.9183110000000001,-0.56912,0.263351,-0.933475,-0.5415770000000001,-0.901259,1.313423,-0.6834640000000001 -159,1,-0.809525,-1.217619,-0.8694850000000001,-0.721769,-0.669312,-1.089867,-1.216885,-1.143155,-0.263303,-0.393221,-0.9165969999999999,-1.472952,-0.958868,-0.8193549999999999,-1.509439,-1.2728,-1.076078,-1.0918889999999999,-1.349457,-0.760915,-0.445276,-0.826349,-0.523772,-0.47957299999999997,-0.299672,-1.081135,-0.996658,-1.333386,-0.5051760000000001,-0.427388 -32,0,0.9548180000000001,1.044281,0.8589870000000001,0.814646,1.3602889999999999,0.64654,1.375025,1.06729,1.0179790000000002,0.048444,0.82157,1.0914760000000001,0.858032,0.69513,1.660981,0.857725,1.919664,1.8411959999999998,1.593204,0.14493399999999998,0.7064229999999999,0.328662,0.56084,0.603791,0.409024,0.298554,0.614276,-0.109676,0.057839,0.022353 -264,0,1.099776,0.5948319999999999,0.9900450000000001,0.976374,1.0271370000000002,0.01549,0.559926,1.275894,0.509996,-0.456949,0.8698530000000001,0.6470060000000001,0.808603,0.777609,0.064029,-0.27273000000000003,0.022733,0.421754,0.202194,-0.991984,0.0546,-0.86826,-0.023296,0.11183299999999999,-0.849361,-0.749934,-0.378445,-0.311627,-1.102093,-0.8396819999999999 -245,1,-0.991758,0.616002,-1.000245,-0.839901,0.838642,-0.964802,-0.739243,-0.719399,-0.028725,-0.358309,-1.0358829999999999,0.132725,-1.0400129999999999,-0.902118,0.75718,-0.845823,-0.508342,-0.469944,-0.272428,0.22715500000000002,-0.12008599999999998,2.5312759999999996,-0.172795,-0.376605,2.999795,-0.649332,-0.439122,-0.075612,1.7214580000000002,-0.088853 -535,0,1.66304,-0.032117,1.576826,1.632076,-0.244104,0.376817,0.8209120000000001,1.5256100000000001,0.285125,-0.457503,1.82414,0.365431,1.8877869999999999,1.8575139999999999,0.586383,1.3182459999999998,1.502839,2.148145,1.15144,-0.040772,1.059043,-0.411409,0.910827,1.043825,-0.821026,0.038109,0.270438,0.391552,-0.12862200000000001,-0.41831 -430,0,0.016734,0.308227,0.540279,-0.084174,0.417818,2.89275,3.021056,2.02352,-0.056227,1.7486009999999998,0.21946,0.7540520000000001,0.41729700000000003,0.085638,0.22130500000000003,2.239288,2.316401,1.243034,0.837458,0.876418,-0.549221,-0.620421,0.296989,-0.355263,-0.025327000000000002,2.0484560000000003,1.570525,1.710316,-0.6722640000000001,0.752378 -443,1,-1.103582,-0.385488,-1.129217,-0.904065,-1.5092079999999999,-1.201318,-1.114601,-1.406728,-0.326396,-0.919117,-1.0103209999999998,-0.225644,-1.035894,-0.8930170000000001,-1.063232,-1.131801,-0.8646520000000001,-0.975248,2.045921,-0.725474,-0.806197,2.404272,-0.786633,-0.5988220000000001,1.2263899999999999,-0.680072,-0.396681,-0.11129800000000001,-0.40831300000000004,0.263678 -154,1,-0.310456,-0.8430790000000001,-0.285682,-0.357354,0.676449,-0.18235,0.137744,-0.264733,1.534051,0.13212100000000002,-0.277565,-0.91911,-0.274287,-0.329885,-0.179357,-0.366919,0.051861000000000004,-0.363415,0.037902,-0.103146,-0.484255,-0.76956,-0.5183260000000001,-0.38606599999999996,0.514361,-0.296669,-0.047206,-0.366616,0.865433,-0.11949100000000001 -9,0,-0.24419000000000002,2.443109,-0.28627800000000003,-0.29740900000000003,2.320295,5.112877,3.9954330000000002,1.620015,2.370444,6.846856,-0.47353500000000004,1.1054389999999998,-0.329482,-0.509063,1.582699,2.563358,1.7388720000000002,0.94176,0.797298,2.783096,-0.38825,0.693345,-0.40942,-0.360764,0.036008,2.609587,1.5098479999999999,0.409395,-0.32113600000000003,2.3773459999999997 -63,1,-1.2961690000000001,-1.04989,-1.241212,-1.00286,-1.490797,-0.550038,-0.635617,-0.9704860000000001,0.6167699999999999,0.052877,-1.407089,-1.263516,-1.3497629999999998,-1.1205450000000001,-1.362838,-0.318972,-0.363081,-0.699511,1.932741,0.968562,0.016703,1.9016979999999999,-0.127747,-0.370004,0.565697,0.7769689999999999,0.37223,0.617023,2.58838,0.766752 -329,0,0.302508,-0.076084,0.19178499999999998,0.166328,0.448503,-0.271409,0.29846100000000003,-0.150838,-0.26653899999999997,-0.244706,0.605719,0.602792,0.639724,0.488932,1.433252,0.454059,1.143766,0.796798,0.20949600000000002,0.357575,0.597064,0.435708,0.046999,0.382454,1.173055,0.67525,0.8785379999999999,0.010359,2.382547,0.9048139999999999 -536,0,-0.202773,1.3992799999999999,-0.088202,-0.2677,0.246858,0.12108900000000002,0.725441,0.3288,-0.328014,-0.024151,0.040531,0.758706,0.07418200000000001,-0.07135599999999999,0.529451,0.209585,0.721919,0.321673,0.4176,-0.42210699999999995,-0.7307640000000001,1.15056,-0.480704,-0.47957299999999997,-0.309339,0.019665000000000002,0.48264300000000004,0.15634800000000001,-0.730382,-0.014715 -65,0,0.215534,1.255978,0.218592,0.07825700000000001,1.4216600000000001,0.5555720000000001,0.14494,0.7125100000000001,0.679863,0.286177,0.185378,1.082168,0.223703,0.038142,1.483068,0.825508,0.475842,1.0671190000000002,0.516175,0.530523,-0.171337,0.11638399999999999,-0.20596199999999998,-0.11214400000000001,-0.11266400000000001,-0.132913,-0.289916,0.007115000000000001,-0.18673900000000002,-0.21594499999999997 -438,1,-0.132365,0.379878,-0.189474,-0.231136,-0.9016430000000001,-0.8916459999999999,-1.077804,-0.848216,-0.627304,-0.822139,-0.07875499999999999,0.07222100000000001,-0.13547599999999999,-0.17715699999999998,-0.677515,-0.777787,-0.946385,-0.6703640000000001,-0.9369,-0.8601469999999999,-0.22836199999999998,0.836678,-0.264871,-0.23557399999999998,-0.40167600000000003,-0.811971,-0.864792,-0.771654,-0.04871,-0.555615 -269,1,-0.9482700000000001,-0.076084,-0.9159510000000001,-0.826541,0.049596,0.004675,-0.090138,-0.435575,-0.478468,0.16980399999999998,-0.9705590000000001,0.25605900000000004,-0.9255040000000001,-0.881641,0.8425790000000001,0.46543,-0.054229,-0.522306,-0.524343,0.825384,-0.308126,0.493767,-0.314869,-0.43116899999999997,0.620366,1.250353,1.375893,0.917111,0.8157909999999999,1.443065 -36,0,-0.078524,0.7625609999999999,0.26624899999999996,-0.142361,0.536175,1.0784790000000002,1.182165,0.458226,1.1166639999999999,0.9672379999999999,0.034851,0.565559,0.06841599999999999,-0.062255,0.13305899999999998,0.10345599999999999,0.5411279999999999,0.18212799999999998,0.267911,-0.21939,-0.430117,-0.35897399999999996,-0.10349100000000001,-0.339422,-0.387675,0.24993,0.53901,-0.030193,-0.031758999999999996,0.085899 -98,1,-0.664567,-1.3869770000000001,-0.7238319999999999,-0.647058,0.470421,-0.439986,-0.383747,-0.45856800000000003,-0.208298,0.200283,-0.717787,-1.500877,-0.726143,-0.689095,-0.46473000000000003,-0.5513170000000001,-0.588066,-0.397721,-0.699589,0.42845500000000003,-0.626819,-1.229678,-0.688617,-0.5409579999999999,-0.296005,-0.680631,-0.49615200000000004,-0.796796,-0.488225,-0.470508 -392,0,1.021085,0.6078600000000001,1.0377020000000001,0.841015,1.566318,0.871734,1.358234,1.4844979999999999,0.46308,0.9949459999999999,0.38702800000000004,0.158322,0.429654,0.255431,1.397669,0.9828049999999999,1.259272,1.088786,0.42855299999999996,0.658107,0.872808,0.20710100000000004,0.895481,0.58465,0.07601000000000001,0.212484,0.591066,0.745169,-0.24485700000000002,0.165332 -69,1,-0.581734,-0.963583,-0.643112,-0.572523,-0.12136400000000001,-1.168303,-0.807368,-0.849434,-0.837615,-1.099772,-0.38265,-0.651497,-0.436576,-0.43341,0.138753,-0.9854959999999999,-0.6562399999999999,-0.52308,-0.809117,-0.8884989999999999,-0.60769,-0.623505,-0.690597,-0.48419300000000004,0.307019,-1.110309,-0.5319619999999999,-0.508387,0.24187899999999998,-0.71448 -517,0,1.5450030000000001,-0.072828,1.5857620000000001,1.345536,0.40905,0.487505,0.701933,0.710987,-0.569064,0.41086300000000003,1.636691,0.225807,1.587098,1.590169,0.522334,0.505228,0.656634,1.170811,-0.035117,-0.130081,0.370767,-0.622598,0.39005500000000004,0.42601700000000003,-0.650686,-0.136825,-0.045548000000000005,-0.002618,-1.207431,-0.152777 -92,1,0.018805000000000002,-0.541818,-0.082245,-0.087866,-1.3926049999999999,-0.82994,-0.658165,-0.22088000000000002,-1.413545,-1.21282,-0.243483,-1.05408,-0.297765,-0.29348,-1.623304,-1.019419,-0.705455,-0.578795,-1.5539100000000001,-1.3633959999999998,0.001905,-0.115852,-0.081709,-0.087723,-0.853361,-0.843828,-0.607228,-0.158339,-1.1929020000000001,-0.8926370000000001 -552,1,-0.49683,1.681,-0.570733,-0.502558,-0.393146,-0.940628,-0.8907010000000001,-0.7556390000000001,-0.798788,-1.058764,-0.38549,2.359728,-0.4374,-0.41805200000000003,-0.9678700000000001,-1.17501,-0.86415,-0.8751680000000001,-0.995315,-0.9111809999999999,-0.592892,0.272418,-0.687627,-0.474733,0.5980310000000001,-0.734843,-0.617175,-0.404087,-0.190372,-0.782566 -505,1,-1.1739899999999999,-1.243674,-1.125643,-0.971217,2.990984,0.712699,0.091688,-0.108204,-0.085347,2.906791,-1.26423,-1.431064,-1.1471069999999999,-1.0875540000000001,2.073741,2.19949,0.376658,0.553561,0.8958729999999999,4.671344,-0.471984,0.31414699999999995,-0.5341670000000001,-0.49871400000000005,4.909873,1.307919,0.663017,0.438593,0.699555,2.906898 -566,0,0.561361,1.374854,0.579001,0.42790600000000006,-0.809587,0.35073499999999996,0.32676700000000003,0.41406899999999996,-1.104549,-0.318409,0.702284,2.0455740000000002,0.672676,0.5779529999999999,-0.840484,-0.03868,0.046588,0.105777,-0.809117,-0.8955870000000001,0.184892,-0.257371,0.27669299999999997,0.180698,-0.379342,0.661277,0.510827,0.612157,-0.891416,0.036726999999999996 -163,1,-0.556884,0.48898400000000003,-0.592774,-0.575863,0.562476,-0.130186,-0.49600900000000003,-0.497395,-1.02366,0.380938,-0.507616,0.681912,-0.49918500000000005,-0.5414859999999999,0.344421,-0.053841,-0.440671,-0.533913,-0.9515040000000001,0.6822060000000001,-0.397995,0.796763,-0.45100200000000007,-0.41334799999999994,1.433065,0.350531,-0.021344,-0.072368,-0.212166,0.587461 -358,1,-1.302174,-1.299041,-1.250743,-1.017099,-1.3531520000000001,-0.8235790000000001,-0.8528959999999999,-1.0199719999999999,-0.755108,-0.533977,-1.490873,-0.8842040000000001,-1.451091,-1.177143,-0.955772,-0.5185310000000001,-0.522153,-0.6476649999999999,0.432204,0.483742,0.47976499999999994,-0.030577999999999998,0.698459,-0.223473,1.2963930000000001,0.196276,0.008166,0.5294300000000001,0.947766,0.143394 -442,1,-0.20691500000000002,-1.33161,-0.278832,-0.305847,-1.1032879999999998,-0.936175,-1.137102,-1.240758,-1.685332,-0.878109,-0.098636,-0.814392,-0.14824500000000002,-0.196212,-0.582865,-0.704256,-0.982418,-1.0055049999999999,-1.484542,-0.612066,-0.17639000000000002,-1.330918,-0.312394,-0.24195500000000003,-0.203001,-0.777878,-0.800832,-0.995827,-1.001598,-0.465591 -15,0,0.246596,1.865014,0.5015569999999999,0.110075,1.553167,2.56641,2.064909,0.8617309999999999,2.1310119999999997,2.7793349999999997,0.11721500000000001,1.9199119999999998,0.196105,0.011123000000000001,1.248222,1.045345,0.942887,0.637649,1.7940060000000002,1.130169,-0.126943,-0.333573,0.006406,-0.171329,-0.478012,0.9457549999999999,0.514474,-0.145362,-0.23880300000000002,0.6320939999999999 -503,0,3.007006,-0.294295,3.1048400000000003,3.342525,-0.5465720000000001,0.688526,0.51435,1.7022389999999998,0.01172,-0.619317,2.5455360000000002,0.125743,2.476807,2.921209,-0.20924600000000002,0.43889799999999995,0.98934,1.325317,-1.1194469999999999,-1.128074,3.197139,-0.855742,3.350833,3.077223,-0.42934399999999995,0.451691,0.260823,1.583793,0.177707,-0.339633 -176,1,-1.037316,-0.209616,-1.018414,-0.862051,-0.099446,0.259131,0.366586,-0.236107,-0.463908,1.7873919999999999,-1.199475,-0.286147,-1.1273360000000001,-1.002515,0.044814,0.474905,0.526062,-0.303315,-0.5206930000000001,2.6030599999999997,0.093579,1.894441,0.131649,-0.282878,1.9397520000000001,3.498789,2.911395,2.07691,2.210615,5.342837 -327,1,-0.662496,-0.5581020000000001,-0.7303850000000001,-0.627897,-1.36192,-1.147374,-1.268736,-1.3193270000000001,-1.180585,-0.752315,-0.595661,-0.316399,-0.654061,-0.594102,-1.389881,-1.239824,-1.0954629999999999,-1.11758,-1.5685129999999998,-0.297358,-0.6196010000000001,-0.5572819999999999,-0.693072,-0.514116,-0.770691,-1.03905,-1.018243,-1.272557,-0.71222,-0.771596 -1,0,1.805927,-0.369203,1.535126,1.8904889999999999,-0.375612,-0.43044399999999994,-0.146749,1.087084,-0.24389,0.28119,1.829821,-0.353632,1.6859549999999999,1.908708,-0.8269620000000001,-0.487072,-0.023846,0.548144,0.001392,-0.8686520000000001,0.499255,-0.876244,0.263327,0.742402,-0.605351,-0.692926,-0.44078,0.260162,-0.80545,-0.099444 -404,1,-0.639717,-1.437458,-0.689578,-0.610845,-1.208494,-1.188468,-1.0697450000000002,-1.015252,-0.9751270000000001,-1.341385,-0.507616,-1.009865,-0.563442,-0.528403,-0.678938,-1.1111440000000001,-0.850089,-0.732011,-0.8784850000000001,-0.810531,-0.078941,-0.560729,-0.130717,-0.224133,0.220349,-0.9494030000000001,-0.624138,-0.24268800000000001,1.108801,-0.8937719999999999 -24,0,2.1103389999999997,0.957974,2.077228,2.345788,2.109883,0.6586270000000001,0.946607,1.444909,1.152255,0.648043,0.716485,0.486439,0.742699,0.710203,1.120124,0.7838149999999999,0.79976,1.103489,0.669515,0.071219,1.44956,-0.571797,1.2816049999999999,1.369891,-0.33100599999999997,-0.37212,-0.148667,-0.080478,-0.7097979999999999,-0.375945 -139,1,-0.900641,-1.61333,-0.9153549999999999,-0.785054,0.189871,-0.458434,-0.889933,-0.43390100000000004,-1.2922120000000001,-0.892517,-0.808671,-1.3728879999999999,-0.781338,-0.768161,1.426135,0.175472,-0.53295,-0.02474,-0.148296,-0.294523,-0.240994,0.228873,-0.5024850000000001,-0.30818,1.4097309999999998,0.531055,-0.332357,1.2739719999999999,-0.57419,-0.133486 -101,1,-1.7269009999999998,-0.999409,-1.6933610000000001,-1.222423,1.14111,-0.852841,-1.305831,-1.745063,0.050546,0.547186,-2.029648,-1.36358,-1.984504,-1.454443,1.4688350000000001,-0.543168,-1.114873,-1.2618200000000002,0.432204,2.180614,-0.6535270000000001,0.5282399999999999,-0.650005,-0.671142,1.049716,-0.8181189999999999,-1.057501,-1.9134470000000001,0.732247,0.11540299999999999 -350,1,-0.619009,-0.9668399999999999,-0.704471,-0.594321,-1.437317,-1.205517,-1.159698,-1.0961049999999999,-0.274627,-0.8697969999999999,-0.700746,-0.5165270000000001,-0.752505,-0.665205,-1.476703,-1.289477,-1.010591,-0.962094,-0.5133909999999999,-0.777926,-0.186856,-0.987826,-0.317344,-0.31478,-0.152666,-1.0332379999999999,-0.86025,-0.892175,0.19586900000000002,-0.426253 -90,1,-0.032964999999999994,0.559006,-0.129902,-0.135154,-0.9147930000000001,-0.494058,-0.864986,-0.687423,-0.612744,-0.636496,0.13993599999999998,1.1007850000000001,0.107134,0.022215000000000002,-0.471135,-0.346452,-0.725418,-0.499092,-0.46227700000000005,-0.586549,-0.11936400000000001,-0.192055,-0.290612,-0.144707,-0.724356,-0.40789000000000003,-0.68581,-0.517471,0.037256,-0.341146 -191,1,-0.52168,-0.354547,-0.542734,-0.529278,-1.687182,-1.046355,-1.057031,-1.018297,-1.167642,-0.844305,-0.38549,0.49341999999999997,-0.409803,-0.41947399999999996,-0.631257,-0.726429,-0.724162,-0.52308,-0.42941899999999994,0.010262,1.176342,0.963683,1.1147799999999999,0.29290700000000003,-0.8233600000000001,-0.42353900000000005,-0.33600399999999997,0.9365760000000001,1.596747,0.786799 -383,1,-0.43263500000000005,-0.414799,-0.35836,-0.49236199999999997,0.45288599999999996,0.668169,0.23225500000000002,-0.252247,-0.13226300000000002,1.543562,-0.49341599999999997,-0.421117,-0.466645,-0.546037,0.557917,0.480591,-0.375134,-0.518953,-0.11908900000000001,0.436961,-0.521069,-0.623868,-0.370807,-0.465052,-0.10866400000000001,0.646746,0.519779,-0.20862399999999998,-0.261808,1.168455 -11,0,0.85956,0.261002,0.8709020000000001,0.73554,0.31699499999999997,1.950627,0.596387,1.010951,1.441838,1.1556520000000001,0.469393,-0.325708,0.47908199999999995,0.358672,0.052641999999999994,0.47111499999999995,0.134849,0.44213100000000005,0.110921,-0.28034699999999996,0.363187,-0.420843,0.345502,0.304128,-0.423343,0.8457129999999999,-0.132088,0.16608,-0.055974,0.132046 -314,1,-1.515262,-0.527162,-1.5074969999999999,-1.125913,0.102199,-1.123391,-1.305831,-1.745063,0.39028,-0.154378,-1.570681,-0.160486,-1.560245,-1.2334559999999999,0.785646,-0.869323,-1.114873,-1.2618200000000002,1.282873,1.529934,-0.24676900000000002,2.8306419999999997,-0.318829,-0.495634,4.569858999999999,-0.639831,-1.057501,-1.9134470000000001,4.954254,1.144247 -352,0,3.491579,-0.34152,3.6350279999999997,4.137101,0.9043959999999999,2.159281,1.789052,2.451387,1.276824,0.23297800000000002,3.2953339999999995,-0.425771,3.38711,3.8540739999999998,1.319387,2.500819,3.113641,3.672569,0.5271279999999999,-0.22505999999999998,2.128091,-0.6639649999999999,2.156324,2.480977,-0.224002,0.9474319999999999,0.356978,0.532674,0.339952,-0.155047 -235,1,-0.19449,0.749534,-0.267811,-0.290202,-0.16081600000000001,-0.6550020000000001,-1.006897,-0.532569,-1.091607,-0.430903,-0.027632,0.456187,-0.089755,-0.14644000000000001,-0.402816,-0.6612359999999999,-0.931319,-0.772766,-1.075636,-0.630495,-0.527927,0.5191680000000001,-0.593571,-0.401907,0.11601199999999999,-0.6510090000000001,-0.815388,-0.284862,-0.958009,-0.329042 -305,1,-0.7929579999999999,0.967744,-0.7705960000000001,-0.7103430000000001,-1.618359,-0.751695,-0.9585379999999999,-1.011902,0.555294,-0.9141290000000001,-0.717787,1.210157,-0.7306739999999999,-0.676012,-1.538617,-0.8994559999999999,-0.867038,-0.9231450000000001,0.45045799999999997,-0.569538,-0.555717,1.032628,-0.448032,-0.486834,-0.306339,-0.117823,-0.528315,-0.778304,1.380017,-0.599114 -375,1,0.145126,-1.064546,0.173913,-0.033546,-0.388763,0.004675,-0.291634,0.15978499999999998,0.408076,0.31333099999999997,0.580158,-0.7492340000000001,0.590295,0.38000300000000004,0.173624,0.747807,-0.279842,0.13028199999999998,0.6512600000000001,0.414279,-0.8325440000000001,-1.320576,-0.750991,-0.559439,-0.843694,-0.411243,-0.41060699999999994,0.02658,-0.145573,-0.037411 -215,0,-0.10751500000000001,0.204007,-0.085224,-0.22937800000000003,0.597545,1.16245,0.918301,0.773416,1.179757,1.216609,-0.075915,-0.5491060000000001,-0.041562,-0.216121,0.44405200000000006,0.897523,0.128195,0.183159,1.07477,0.901935,-0.537311,-0.041464,-0.46189300000000005,-0.388266,-0.36034099999999997,0.497521,0.238607,0.414261,-0.139519,0.28939899999999996 -466,1,-0.304244,-0.035373,-0.189474,-0.336611,0.119734,0.6401789999999999,0.854974,0.053198,-0.546415,-0.12223699999999998,-0.280405,0.337507,-0.246689,-0.335573,-0.68392,0.0864,0.247342,-0.356451,-0.911343,-0.368238,-0.324728,-0.777362,-0.274276,-0.284638,0.084677,0.352767,0.472696,0.12066199999999999,-0.597195,-0.21367600000000003 -486,1,0.039513,-0.03863,-0.037566,-0.087163,-0.796437,-0.300672,-0.136674,-0.553125,-0.7211350000000001,-0.996699,0.145616,-0.567723,0.092306,0.031601,-0.708116,-0.708046,-0.46301899999999996,-0.54191,-1.469938,-1.310945,-0.6668810000000001,-0.38256100000000004,-0.690597,-0.447891,-1.168707,-0.64542,-0.460674,-0.916831,-0.9689059999999999,-0.974339 -250,0,1.928106,0.215406,1.728734,1.985416,-0.49396899999999994,0.400354,2.048118,1.460136,0.364396,-0.30233899999999997,1.9349060000000002,0.9937389999999999,1.933096,2.016784,0.308838,1.066192,2.290035,2.1171919999999997,1.436213,-0.541186,2.161296,-0.718577,1.7355479999999999,2.1465509999999997,-0.586017,0.760202,2.098385,1.110141,0.419864,0.456586 -339,0,2.9821560000000003,0.8228129999999999,2.833789,3.5605059999999997,0.838642,0.086101,1.0060959999999999,1.435773,-0.497881,-0.36385,2.664822,1.1589610000000001,2.600377,3.106075,0.7500640000000001,0.454059,1.782814,2.375132,-0.053372,-1.0968870000000002,2.1793419999999997,-0.5304300000000001,1.780101,2.7229959999999997,-0.24966999999999998,-0.32237899999999997,0.130184,0.1969,-0.69648,-0.25604 -103,1,-1.140857,0.187723,-1.0437319999999999,-0.9130299999999999,1.03152,-0.153087,-0.037364999999999995,-0.26062199999999996,-0.450966,0.052877,-1.207427,0.02568,-1.15411,-1.014176,0.294605,-0.139691,-0.34224,-0.48052,0.486968,0.059878,-0.81161,0.009337999999999999,-0.66238,-0.628525,0.6723680000000001,-0.196627,-0.053174,-0.109676,-0.539078,-0.08507 -206,1,-1.211265,-0.400144,-1.196831,-0.9650639999999999,0.400283,-0.8242149999999999,-1.0077610000000001,-0.894201,0.14276,-0.562239,-1.207427,-0.46998599999999996,-1.196536,-1.022424,0.892395,-0.606845,-0.894408,-0.758321,0.44680699999999995,0.007427,-0.691063,0.227059,-0.667826,-0.616204,0.89271,-0.725901,-0.7930729999999999,-0.692495,0.188604,-0.50039 -345,1,-1.116007,-1.009179,-1.0833469999999998,-0.9202379999999999,0.159186,-0.576756,-0.962232,-1.124731,-0.755108,0.051769,-1.098366,-1.065715,-1.061432,-0.947908,0.175047,-0.24165,-0.665279,-0.736396,-0.652127,1.028102,-0.24243800000000001,2.344398,-0.233684,-0.462192,3.4398129999999996,1.186639,-0.191771,0.18879,1.962404,1.120795 -346,1,-0.544459,0.225177,-0.6171989999999999,-0.558987,-0.152049,-0.75742,-1.0896540000000001,-0.969572,-0.033578,-0.17266600000000001,-0.58714,-0.090674,-0.630582,-0.596093,-0.8895879999999999,-0.879367,-1.020585,-1.04288,-0.9369,-0.328545,-0.585313,-0.117666,-0.647034,-0.491014,0.046342,-0.811412,-0.864129,-1.0221049999999998,-0.08745499999999999,-0.393723 -317,0,1.1536170000000001,-0.11028199999999999,1.001959,1.0625120000000001,0.483572,0.140173,0.542655,0.959181,-0.143587,-0.10893699999999999,1.162387,-0.09765499999999999,1.101054,1.058322,0.078262,0.139464,0.30383899999999997,0.788802,-0.016862000000000002,-0.8729049999999999,-0.0038689999999999996,-1.209357,-0.157944,0.188399,-0.740023,-0.496754,-0.25908000000000003,-0.059391,-0.9434799999999999,-0.508333 -499,0,1.5719239999999999,0.827699,1.666184,1.5459379999999998,0.615079,0.670714,1.178807,1.472317,-0.68069,0.334943,1.8355009999999998,0.45386000000000004,1.8877869999999999,1.8916439999999999,0.863928,1.138207,1.632155,1.629686,0.132826,-0.08188200000000001,0.6685270000000001,-0.0015480000000000001,0.663312,0.764624,-0.124998,0.135916,0.289338,0.485634,-1.134784,-0.025685000000000003 -478,1,-0.801242,-0.615097,-0.751235,-0.725988,0.124117,-0.33884000000000003,-0.060394,-0.613574,0.065106,0.435246,-0.749028,-1.09364,-0.74056,-0.7109949999999999,0.586383,-0.418088,-0.448455,-0.7539359999999999,-0.11908900000000001,0.41711400000000004,-0.728238,-0.092266,-0.6430739999999999,-0.57198,-0.694688,-0.242457,0.32050500000000004,-0.6096060000000001,-0.255754,-0.068427 -47,0,-0.124082,0.370108,-0.132881,-0.213029,2.026595,1.032677,1.09581,1.434251,1.616558,1.8815990000000002,-0.271884,-0.14652300000000001,-0.246689,-0.342115,1.383436,0.355511,0.424367,0.631459,1.15509,0.704888,-0.426147,-0.5863109999999999,-0.47971400000000003,-0.353943,-0.169667,-0.11838199999999999,-0.094289,0.0574,-0.376832,-0.057458 -456,1,-0.652142,2.138591,-0.632092,-0.620162,0.360831,-0.325481,0.096485,-0.704325,-0.027107,-0.650904,-0.709266,2.3271490000000004,-0.704313,-0.6819850000000001,-0.198572,-0.352516,-0.215937,-0.741555,-0.04607,-0.161268,-0.330864,2.193808,-0.354471,-0.378585,0.940045,-0.07255199999999999,0.359962,-0.41868500000000003,0.509462,-0.630509 -211,1,-0.614867,-0.11191,-0.656516,-0.587641,-0.191501,-0.421538,-0.600115,-0.692448,-0.591712,-0.22254000000000002,-0.649623,-0.081366,-0.677951,-0.645296,-0.5444359999999999,-0.669764,-0.7797810000000001,-0.902509,-1.017221,-0.315787,-0.660385,-0.63802,-0.703963,-0.510815,-0.5080140000000001,-0.45874899999999996,-0.37944,-0.8190189999999999,-0.530602,-0.330177 -490,1,-0.43470600000000004,1.0279969999999998,-0.432526,-0.452984,-0.296707,-0.46988500000000005,-0.715735,-0.780458,0.322334,-0.10616700000000001,-0.5331779999999999,0.733108,-0.567973,-0.535798,-1.027649,-0.9919389999999999,-0.899681,-0.936557,-0.97706,-0.430613,-0.654249,-0.141253,-0.638124,-0.49057399999999995,-0.648353,-0.750492,-0.7454930000000001,-1.175231,-0.540288,-0.528003 -113,1,-1.058024,-0.47668,-1.0318180000000001,-0.889826,-0.10382899999999999,-0.314031,-0.684551,-0.810759,-0.837615,0.349906,-1.027362,0.20951799999999998,-0.9609280000000001,-0.9120729999999999,1.127241,0.491962,-0.301813,-0.47046000000000004,0.402996,2.12958,-0.258319,1.1668889999999998,-0.40843,-0.449431,1.6130719999999998,0.6702199999999999,0.464738,0.59107,0.281835,1.133656 -359,1,-0.879933,-0.107025,-0.9373959999999999,-0.7752100000000001,0.040829000000000004,-0.9501700000000001,-0.756994,-0.975815,-0.722753,-0.143295,-1.332393,-0.225644,-1.324225,-1.0702049999999999,0.323071,-0.848666,-0.774633,-0.899156,-1.115796,0.962892,0.370767,0.054696,0.198478,-0.216873,-0.068329,-0.9219620000000001,-0.278974,-0.848541,-0.13588599999999998,-0.40923200000000004 -344,1,-0.664567,-1.224133,-0.6880890000000001,-0.640202,0.597545,-0.908185,-0.7843399999999999,-0.547643,-0.219623,-0.326167,-0.686545,-0.8935120000000001,-0.6977220000000001,-0.667195,1.3265040000000001,-0.5975590000000001,-0.611921,-0.423515,0.7206279999999999,0.32071700000000003,-0.218617,-0.866083,-0.25299,-0.347783,0.831707,-0.8108529999999999,-0.5100779999999999,-0.095077,-0.12256800000000001,-0.262848 -532,1,-0.08680700000000001,-0.891932,-0.168624,-0.188419,-0.261639,-0.622558,-0.7272489999999999,-0.41973999999999995,-0.153294,-0.339467,-0.12703699999999998,-0.6887300000000001,-0.173371,-0.225791,-0.255504,-0.602486,-0.89491,-0.7768930000000001,-0.659428,-0.176862,-0.723546,-1.336724,-0.73911,-0.507955,-1.071036,-1.019992,-0.822815,-1.089909,-1.211063,-0.793535 -261,0,0.7415229999999999,0.943318,0.623679,0.5931489999999999,-0.366845,-0.672177,-0.724851,-0.49115299999999995,-0.725988,-1.0415860000000001,0.9152950000000001,0.8773860000000001,0.7838890000000001,0.7912600000000001,-0.693171,-0.7853680000000001,-0.751909,-0.530044,-0.904042,-1.37899,-0.016141,0.1817,-0.143093,0.089611,-0.43834399999999996,-0.805823,-0.644364,-0.669137,-0.778813,-0.8385469999999999 -4,0,1.298575,-1.4667700000000001,1.338539,1.220724,0.220556,-0.313395,0.613179,0.729259,-0.868353,-0.3971,1.7502970000000002,-1.151816,1.776573,1.826229,0.28037199999999995,0.5393399999999999,1.371011,1.428493,-0.009559999999999999,-0.56245,1.270543,-0.790244,1.273189,1.1903569999999999,1.483067,-0.04852,0.828471,1.144205,-0.361092,0.499328 -226,1,-0.983474,-0.9570690000000001,-1.0065,-0.8529100000000001,0.07589800000000001,-0.8840120000000001,-1.179224,-1.065348,-0.46229,-0.069592,-1.047243,-0.891185,-1.044132,-0.925156,0.636199,-0.513983,-1.03147,-0.9481649999999999,-0.08623,0.24133200000000002,-0.7719090000000001,-0.569982,-0.8207899999999999,-0.626544,-0.176,-0.9734360000000001,-0.964097,-1.106941,-0.668632,-0.36838000000000004 -464,1,-0.283535,-0.291038,-0.362232,-0.33924699999999997,-0.18273399999999998,-0.36746599999999996,-0.40581500000000004,-0.153883,-1.0770469999999999,-0.814381,-0.271884,-0.248914,-0.316713,-0.334435,-1.54431,-0.841464,-0.504827,-0.521532,-1.3056450000000002,-1.03593,-0.732208,-0.964965,-0.806929,-0.515876,-0.35734099999999996,-0.589531,-0.538925,-0.540667,-1.163843,-0.802991 -277,0,0.764302,-0.22427199999999997,0.647508,0.624792,-0.353694,-0.879559,-0.24557800000000002,0.22525900000000001,-0.539944,-1.472721,1.329956,0.16064900000000001,1.1916719999999998,1.2716290000000001,-0.50743,-0.862311,-0.107964,0.245323,-0.955155,-1.819865,-0.277447,-0.7055140000000001,-0.24903000000000003,-0.07914199999999999,0.176681,-0.8013520000000001,-0.18779200000000001,0.45968000000000003,-0.15041600000000002,-0.801478 -257,0,0.302508,-0.491336,0.37347800000000003,0.084761,1.9345400000000001,1.247056,0.8189930000000001,1.648946,0.577943,1.9480979999999999,0.338746,-0.46998599999999996,0.462606,0.166127,2.643065,2.351102,1.958584,1.941793,2.140845,1.865906,0.91684,-0.2864,0.591532,0.42073699999999997,1.036382,1.1402510000000001,0.5947140000000001,1.5367520000000001,-0.206112,1.8402290000000001 -260,0,1.6692529999999999,2.195586,1.639376,1.693603,0.8693270000000001,0.25595100000000004,0.511471,0.8388909999999999,0.40484000000000003,-0.21921500000000002,1.7559770000000001,1.808213,1.6859549999999999,1.800632,0.259022,0.084505,0.792227,1.145533,0.008694,-1.003325,-0.026968000000000002,-0.333573,-0.13814300000000002,0.264085,-0.66602,-0.542025,-0.355567,-0.585761,-0.9350040000000001,-0.719776 -301,1,-0.581734,-0.42457,-0.569839,-0.578851,-1.199727,-0.244691,-0.392382,-0.5840350000000001,-0.349045,-0.34944200000000003,-0.47353500000000004,0.139706,-0.47529499999999997,-0.522146,-0.8433299999999999,-0.055736,-0.257368,-0.46246400000000004,-0.11178699999999998,-0.043607,-0.147877,-0.320873,-0.142103,-0.264396,-0.17033299999999998,0.45895699999999995,0.504858,0.362354,0.8303200000000001,0.32382 -537,1,-0.681134,1.060565,-0.629709,-0.690654,1.9476900000000001,0.45060900000000004,-0.636576,0.24657600000000002,-0.158147,0.873031,-0.692225,1.198521,-0.6425270000000001,-0.706729,1.9385270000000001,0.9638540000000001,-0.548016,-0.09309400000000001,1.166043,1.595143,-0.395108,1.380982,-0.350511,-0.42654899999999996,1.946419,0.529378,-0.439122,0.951175,-0.598406,0.767508 -403,1,-0.49890100000000004,-0.432712,-0.523373,-0.526817,-0.664929,-0.371919,-0.437479,-0.46785600000000005,0.6410359999999999,-0.310651,-0.337208,-0.725963,-0.36202199999999995,-0.418905,0.17291199999999998,-0.302863,-0.7010609999999999,-0.645343,-0.27973000000000003,-0.113069,-0.9361280000000001,-0.565809,-0.9249940000000001,-0.637545,-1.384716,-0.705781,-0.522678,-0.728019,-0.223063,-0.689516 -233,0,1.698245,1.905725,1.651291,1.7428240000000002,-0.441366,0.138901,0.6832229999999999,0.634854,-0.750255,-0.036897000000000006,1.8127799999999998,1.982743,1.74774,1.8888,-0.339479,0.057973000000000004,0.8361700000000001,0.8893989999999999,-1.327551,-0.974973,0.428875,-0.050536000000000005,0.44599300000000003,0.652855,-0.673687,-0.271519,0.088738,-0.14374,-0.915632,-0.343416 -487,0,1.592632,0.767446,1.389175,1.51078,0.834259,0.7521399999999999,1.541979,1.391616,0.590885,0.340485,1.508885,-0.10929000000000001,1.488242,1.456496,0.892395,0.7667579999999999,1.717529,1.817982,0.041553,-0.233566,0.580101,0.346806,0.378669,0.602911,-0.58435,0.159389,0.353662,-0.005862,-0.40831300000000004,-0.220862 -141,0,0.756019,-0.066314,0.647508,0.619518,-0.042459,-0.195073,0.038435000000000004,0.106491,-0.17594300000000002,-0.131104,0.563117,-0.288475,0.540867,0.44968400000000003,0.060471000000000004,0.177367,0.07119500000000001,0.271117,0.180289,-0.045025,1.0817809999999999,0.20891500000000002,0.8251870000000001,0.742402,-0.09033,-0.34082199999999996,-0.040575,-0.020461,-0.217009,-0.136512 -68,1,-1.2340440000000001,-0.492965,-1.2438930000000001,-0.977194,0.693984,1.159269,4.700669,0.919592,2.14719,1.859432,-1.4479870000000001,-0.45602299999999996,-1.366651,-1.150124,0.7287140000000001,0.7004279999999999,2.814833,-0.13333299999999998,1.093024,2.503828,-0.280696,-0.041464,-0.485654,-0.49871400000000005,0.836041,3.3858919999999997,9.015603,3.475158,2.594434,2.1802770000000002 -363,1,0.385341,-0.037002,0.296035,0.225746,0.062747,-0.5494020000000001,-0.5080020000000001,-0.35594000000000003,-0.8198200000000001,-1.067077,0.673883,-0.232625,0.6026520000000001,0.521071,0.035563,-0.372605,-0.3789,-0.014681,-1.155957,-0.973555,-0.23919,0.40305,-0.25843499999999997,-0.148667,0.07201,-0.415155,-0.450064,-0.23782199999999998,-0.43616099999999997,-0.678547 -379,0,-0.627292,1.163157,-0.461717,-0.6547930000000001,3.771263,4.348873,2.7250490000000003,2.09813,2.027474,3.122913,-0.865474,-0.10696300000000002,-0.768981,-0.834144,1.796196,2.104733,1.005662,0.380483,1.403355,2.367738,-0.699364,-0.344459,-0.567829,-0.579681,0.12134500000000001,1.118454,0.46374399999999993,0.25854,-0.382886,0.242117 -51,1,-0.331164,-0.405029,-0.333042,-0.393567,-1.028767,-0.611108,-0.802091,-0.43770699999999996,-0.897473,-0.20480700000000002,-0.138398,-0.686403,-0.196026,-0.236314,-1.388458,-0.8291459999999999,-0.881727,-0.817389,-1.674391,-0.463218,-0.7867069999999999,-0.532425,-0.701488,-0.5673600000000001,-0.8546940000000001,-0.766142,-0.6997359999999999,-0.622907,-0.882941,-0.470508 -195,1,-0.494759,-0.598813,-0.49001300000000003,-0.49218599999999996,-0.993698,-0.659455,-0.45954799999999996,-0.49115299999999995,0.199382,-0.801081,-0.34572800000000004,-0.6887300000000001,-0.38879600000000003,-0.393877,-1.206274,-0.9604799999999999,-0.6286189999999999,-0.6486970000000001,0.063458,-0.8686520000000001,-0.761442,-0.5592779999999999,-0.679706,-0.5409579999999999,-0.581017,-0.536994,-0.287926,-0.547642,-0.243646,-0.6214310000000001 -239,0,1.292363,3.125425,1.010895,0.9271530000000001,0.181104,0.758501,0.249047,1.400752,-0.07725800000000001,0.056201999999999995,0.946536,4.651889,0.8827459999999999,0.755709,0.12523099999999998,0.48248599999999997,0.664167,1.010888,-0.009559999999999999,-0.44478900000000005,0.474351,-0.6545310000000001,0.06729500000000001,0.190599,-0.7270220000000001,0.13256199999999999,-0.19475499999999998,0.315313,-1.002808,-0.391832 -161,0,1.192963,-1.281128,1.1717389999999999,1.080091,-0.875341,-0.335023,-0.219671,0.9607040000000001,-0.740548,-1.187883,1.437881,-0.779486,1.414099,1.4280549999999999,-0.6703979999999999,0.268334,0.38293499999999997,1.2316850000000001,-0.257825,-1.5646959999999999,2.146859,-1.058222,2.0320709999999997,1.7373189999999998,0.7883720000000001,0.283464,0.38284,2.618691,1.334007,-0.163746 -169,1,-0.032964999999999994,-0.43596899999999994,-0.079266,-0.15273299999999998,-0.472051,-0.57612,-0.986891,-0.453086,-0.803642,-1.089797,0.239341,-0.544452,0.175098,0.088198,0.155832,-0.483092,-0.788192,-0.286549,-0.11543800000000001,-0.892752,-0.48317200000000005,0.000266,-0.48169399999999996,-0.353283,-0.653686,-0.658275,-0.81625,-0.441719,-0.852671,-0.790131 -212,0,2.452025,-1.1736520000000001,2.419765,2.845036,-0.796437,-0.653093,0.229857,0.683579,-2.026684,-1.5902020000000001,3.971288,-0.19073800000000002,3.9761300000000004,5.244841,1.269571,0.8956280000000001,2.903973,2.852321,-0.597362,-1.069952,8.906908999999999,0.470181,9.461986,10.676614,2.136427,0.125297,1.060902,0.368843,3.303954,0.257626 -120,1,-0.714267,-1.580761,-0.700599,-0.650574,0.983301,-0.097107,-0.29739099999999996,-0.381064,0.18644000000000002,0.071164,-0.771749,-1.9709439999999998,-0.767334,-0.715545,-0.187185,-0.7105100000000001,-0.673942,-0.5852430000000001,-0.527994,-0.23640100000000003,-0.954174,-1.3719219999999999,-0.872768,-0.656467,-0.333673,-0.5694100000000001,-0.555503,-0.8655729999999999,-0.859936,-0.601005 -367,1,-0.40985600000000005,-0.266612,-0.399464,-0.449996,0.19425499999999998,-0.237058,-0.14818800000000001,-0.40070700000000004,0.513232,-0.512365,-0.544538,-0.295456,-0.5626180000000001,-0.5588350000000001,-0.28824,-0.6176470000000001,-0.563459,-0.7389760000000001,-0.42576800000000004,-0.5156689999999999,-0.550304,-0.795142,-0.4911,-0.47891300000000003,-0.40267600000000003,-0.648215,-0.39436,-0.763868,-0.140729,-0.694055 -171,0,0.354279,0.682768,0.27816399999999997,0.198674,0.338913,-0.634009,-0.037364999999999995,0.021222,-0.027107,-0.567226,-0.198041,0.079202,-0.252456,-0.25451599999999996,-0.418473,-0.785747,-0.37940300000000005,-0.37502199999999997,-0.77991,-0.8629819999999999,0.232895,-0.12673800000000002,0.136599,0.067389,-0.346007,-0.8298559999999999,-0.34429299999999996,-0.381702,-0.646838,-0.728854 -348,1,-0.778463,-0.795854,-0.8218270000000001,-0.711573,0.9087799999999999,-0.9050040000000001,-0.83457,-0.748025,-0.08049400000000001,0.20416199999999998,-0.754709,-0.758542,-0.780514,-0.717252,-0.398546,-0.8619319999999999,-0.790076,-0.662883,-0.648476,0.13075799999999999,-0.846259,-0.826168,-0.8792040000000001,-0.6179640000000001,0.716703,-0.946218,-0.745825,-0.889093,-0.36230300000000004,-0.297269 -409,1,-0.449201,0.521552,-0.543926,-0.47531,-0.366845,-0.47561000000000003,-0.645212,-0.293664,0.8966459999999999,-0.830451,-0.527497,-0.318726,-0.558499,-0.536936,-0.6768029999999999,-0.740642,-0.711733,-0.577505,0.563638,-0.967885,-0.256153,1.023556,-0.38961799999999996,-0.32006100000000004,-0.384342,-0.132913,-0.374798,-0.169694,0.634173,-0.577932 -112,1,-0.200702,-0.31709299999999996,-0.00778,-0.301628,-1.879621,1.0498530000000001,1.94833,0.5465399999999999,-0.813348,1.344065,0.037691,0.083856,0.241414,-0.071072,-1.280286,2.254449,2.655385,0.749595,-0.39291,2.111151,-0.15292999999999998,0.49558199999999997,0.263822,-0.24393499999999999,-0.581017,2.737574,3.700532,1.804398,0.619644,3.474275 -524,1,-1.041457,-0.43759799999999993,-0.981182,-0.886134,0.417818,-0.19062,-0.901208,-0.751071,-0.645099,0.451317,-1.2156639999999999,-0.839989,-1.1932399999999999,-1.028681,-0.102498,-0.381891,-0.822342,-0.638637,-1.550259,0.8664950000000001,-0.56077,-0.0015480000000000001,-0.44060699999999997,-0.552179,0.563697,-0.282697,-0.7053729999999999,-0.8102600000000001,-0.27875900000000003,-0.113817 -62,0,0.290083,0.6241439999999999,0.352628,0.138729,1.386591,2.356484,2.015015,0.972885,-0.091818,1.6211440000000001,0.034851,0.665623,0.183336,-0.026135000000000002,0.607733,1.828041,1.565613,0.970134,0.501572,1.434954,1.077089,0.092797,1.241012,0.449779,0.788706,2.519604,1.230002,1.084187,-0.428896,0.8768229999999999 -393,0,2.06271,0.498754,1.9282990000000002,2.1102279999999998,0.781656,2.01933,2.0778630000000002,1.9428189999999999,1.500078,0.9284469999999999,2.125195,0.695875,2.159642,2.139081,1.4474850000000001,1.9777580000000001,2.413074,2.7672,1.279223,0.462478,0.790518,-0.536961,0.63955,0.8944340000000001,-0.608684,0.658483,0.506848,0.175813,-0.011175,0.08816900000000001 -75,0,0.724957,-0.18193299999999998,0.641551,0.601939,0.772888,-0.316575,0.051389,0.56938,-0.405668,-1.1125180000000001,0.5517569999999999,0.083856,0.49967700000000004,0.46305100000000005,-0.333074,-0.380943,0.111622,0.45038500000000004,-0.049721,-1.259911,1.2351729999999999,-0.364417,1.070722,0.856151,1.259725,-0.192715,0.102995,1.020925,-0.610514,-0.6985939999999999 -124,1,-0.41606800000000005,-0.47668,-0.454866,-0.436812,-1.309316,-0.007411,0.28119,-0.378019,-1.379572,-0.424808,-0.21508200000000002,-0.674768,-0.241747,-0.28836100000000003,-1.794101,-0.58922,-0.098925,-0.539588,-1.422476,-0.647506,-0.8708020000000001,-0.139439,-0.813365,-0.56494,-0.374008,0.40362600000000004,0.586093,-0.22971100000000003,-1.024602,0.106325 -231,1,-0.867508,1.314602,-0.8173600000000001,-0.752884,-1.7682779999999998,-0.706529,-0.78338,-1.427893,-0.083729,-0.724607,-0.797311,1.8128669999999998,-0.832414,-0.73716,-1.9592049999999999,-1.254796,-0.90985,-1.1812129999999998,0.20949600000000002,-0.923939,-1.0256370000000001,-0.588126,-0.89455,-0.69816,-1.129372,-0.5034609999999999,-0.516046,-1.406542,-0.626254,-0.6592560000000001 -285,1,-0.573451,-0.422942,-0.646389,-0.55635,-1.25233,-1.196102,-1.279065,-1.6114950000000001,-0.640246,-1.088135,-0.439453,-0.207027,-0.500009,-0.47180600000000006,-0.884606,-1.1784219999999999,-1.091521,-1.186398,-0.418466,-0.602143,-0.48100699999999996,0.24157399999999998,-0.566839,-0.393546,-0.219335,-0.976398,-0.9958290000000001,-1.4391459999999998,0.625698,-0.673251 -448,1,0.006379999999999999,0.441759,0.024984,-0.088042,-1.028767,0.06765299999999999,0.507153,-0.284223,-0.69525,-0.5162439999999999,0.114375,0.011717,0.093953,0.013683,-0.888165,-0.499201,-0.007901,-0.507346,-1.236277,-0.756662,-0.547416,0.248831,-0.431696,-0.380565,-0.964699,-0.279344,0.062875,-0.478216,-0.799396,-0.555237 -460,0,1.38555,1.435106,1.335561,1.3490520000000001,1.211247,-0.06275499999999999,-0.039764,0.6226729999999999,0.176733,0.368747,0.8386110000000001,1.829157,0.792127,0.785003,0.186433,0.126198,0.14941300000000002,0.396991,-0.067976,0.001756,1.890966,-0.117666,1.576643,1.647112,0.5663630000000001,-0.183773,-0.155298,0.45156899999999994,-0.011175,0.23530900000000002 -218,0,1.959169,0.48246999999999995,1.877663,1.9836580000000001,0.128501,0.440431,0.42943400000000004,1.026178,0.2738,-0.07735,1.61113,0.5283260000000001,1.5541459999999998,1.635674,-0.180069,0.497647,0.48211899999999996,0.979935,1.030958,-0.990566,1.985527,-0.055978999999999994,1.792477,1.8495279999999998,-0.078996,0.34717800000000004,0.08509,0.863582,-0.190372,0.094599 -467,1,-1.060095,-0.17216199999999998,-1.076794,-0.8796299999999999,0.281926,-0.819126,-0.9983569999999999,-1.364398,0.252769,-0.28793,-1.266502,-0.276839,-1.273149,-1.0483049999999998,-0.9429620000000001,-0.94873,-0.929185,-1.113014,-0.48053199999999996,0.18746300000000002,-0.22944499999999998,0.172629,-0.292592,-0.425889,1.3130600000000001,-0.72143,-0.7156520000000001,-1.275315,0.773413,-0.30861700000000003 -525,1,-1.4073719999999998,-1.176908,-1.3094219999999999,-1.063508,1.390974,-0.195709,-0.464345,-0.448975,0.133053,1.161193,-1.578066,-1.440373,-1.5421209999999999,-1.233172,0.5152180000000001,-0.531039,-0.792838,-0.8723299999999999,-0.487834,1.199632,-1.005064,-0.9753069999999999,-0.889599,-0.727884,0.285352,-0.439188,-0.460343,-0.726073,-0.559661,0.049208999999999996 -80,1,-0.654213,1.05568,-0.677068,-0.624908,1.022753,-0.551311,-0.46386499999999997,-0.812129,-0.224476,0.25292800000000004,-0.760389,0.391029,-0.747974,-0.720665,0.9849100000000001,-0.203179,-0.538474,-0.68584,0.110921,1.028102,-0.288997,1.736593,-0.390608,-0.345803,1.1097190000000001,-0.47048599999999996,-0.20006,-0.695902,-0.288445,0.0685 -472,1,0.188613,-1.2143620000000002,0.141149,0.045735000000000005,-1.133973,0.157985,0.205869,0.001428,-0.344192,-0.067376,0.22514,-1.014519,0.184572,0.091042,-1.094545,-0.357254,-0.41945299999999996,-0.43099499999999996,-0.45497600000000005,-0.8658170000000001,-0.579538,-1.421454,-0.514861,-0.374625,-1.2567110000000001,-0.434717,-0.291573,-0.551048,-1.09725,-0.535568 -549,1,-0.67078,0.940061,-0.695833,-0.659188,-0.524654,-0.578665,-1.008672,-1.248067,0.25600500000000004,-0.42591599999999996,-0.939318,1.1449989999999999,-0.95063,-0.834144,-1.027649,-0.7262390000000001,-0.920522,-1.051341,0.600147,0.068384,0.41299399999999997,1.2721209999999998,0.345502,-0.161428,0.407357,-0.378827,-0.6340859999999999,-0.9536530000000001,0.498565,-0.309373 -527,1,-0.550672,-1.043377,-0.5969439999999999,-0.5549430000000001,-0.138898,-0.298127,-0.44659399999999994,-0.115817,0.338511,-0.44475699999999996,-0.507616,-1.633519,-0.536668,-0.5301100000000001,-0.45049700000000004,-0.782146,-0.7434970000000001,-0.5790529999999999,-0.44767399999999996,-0.66877,-1.041517,-1.3084200000000001,-1.03692,-0.690459,-1.1200379999999999,-0.91151,-0.7679739999999999,-0.9823639999999999,-1.120254,-0.9187360000000001 -168,0,1.422825,1.0833629999999999,1.430876,1.370147,0.229323,0.818934,1.040158,0.8754350000000001,-1.19838,0.501745,0.949377,1.254371,0.993959,0.937732,0.607733,1.060506,1.595745,1.428493,-0.998966,0.12083499999999998,2.464469,0.350434,2.213253,1.8033240000000001,-0.289005,0.607064,0.482311,0.631622,-1.100882,0.501219 -351,0,0.225888,-0.245442,0.361564,0.06102899999999999,0.9920680000000001,1.59248,1.9910279999999998,1.505816,2.174692,1.166735,0.460872,-0.016208,0.6232479999999999,0.294964,1.9883419999999998,2.502714,2.543646,1.941793,2.056873,1.8758290000000002,0.415882,0.194401,0.30243400000000004,0.239443,0.762705,2.241833,2.2426180000000002,1.7897990000000001,4.224151,1.3371549999999999 -57,0,0.3315,0.817928,0.25135599999999997,0.184435,0.19425499999999998,1.111558,0.415041,1.047495,1.289766,1.410564,0.165497,0.535307,0.147501,0.005719,1.233989,0.6094609999999999,0.5084850000000001,0.833426,0.786345,0.677954,0.062901,-0.121295,-0.06487799999999999,-0.005436,-1.127372,0.171685,-0.204702,0.149859,-0.287234,0.11842899999999999 -560,1,-0.200702,1.2201520000000001,-0.21032399999999998,-0.305671,-0.36246100000000003,-0.177261,-0.669679,-0.149315,-1.0527799999999998,-0.040776,-0.021952,1.829157,-0.024262,-0.154973,0.20849499999999999,0.156521,-0.55467,-0.151647,-1.002617,-0.15417999999999998,-0.146794,0.49921099999999996,0.010860999999999999,-0.230954,0.071677,0.07276,-0.370819,0.724082,0.031201999999999997,0.570818 -424,1,-1.04767,-0.40828600000000004,-1.05654,-0.8783989999999999,0.325762,-0.75742,-1.205802,-1.046314,0.47763999999999995,-0.213673,-1.245485,-0.039478,-1.237314,-1.038635,0.792763,-0.39818899999999996,-1.002707,-0.754452,2.651977,1.062124,1.051464,0.9618680000000001,0.861819,0.07003,2.0097549999999997,-0.370444,-0.8578620000000001,-0.207002,1.266203,0.162684 -355,1,-0.600372,-0.52879,-0.54333,-0.585707,-0.998082,-0.343929,-0.160182,-0.33431900000000003,-1.261474,-0.668637,-0.445133,-0.051114,-0.413922,-0.480907,-0.623429,-0.010251999999999999,0.178289,-0.129206,-1.017221,-0.135751,-0.162314,0.47381,0.171251,-0.282658,0.937378,0.942961,1.021776,1.275595,0.7080310000000001,0.155119 -42,0,1.619553,1.2201520000000001,2.089143,1.354326,-0.33616,3.117943,2.168536,2.050928,2.8622490000000003,1.100236,1.403799,1.2846229999999998,1.49648,1.277317,-0.394988,2.1729580000000004,1.53046,1.307519,1.819562,0.089648,2.078645,0.814906,2.9523330000000003,1.420495,-0.16433299999999998,4.198526,2.166357,2.365644,3.9698860000000002,1.456682 -217,1,-0.991758,-0.196589,-0.9490129999999999,-0.838319,-1.62625,-0.7287939999999999,-0.382307,-1.2013209999999999,-0.052992,-0.32450500000000004,-1.115407,-0.421117,-1.1088,-0.949046,-1.1258569999999999,-0.857952,-0.389949,-0.9855659999999999,0.5563359999999999,0.049955,-0.174946,1.2793780000000001,-0.058938,-0.38606599999999996,-0.787025,0.319792,0.856986,-0.176182,0.612379,0.309446 -286,1,-0.627292,0.262631,-0.448611,-0.587992,-0.91041,0.17198,-0.171216,0.013609,-0.704957,0.879127,-0.621222,0.342161,-0.580742,-0.608323,-0.7337359999999999,-0.061421,-0.289509,-0.28397,-0.816419,0.689294,-0.472706,0.31414699999999995,0.164321,-0.405427,-0.107331,1.457703,0.397098,0.735436,-0.22064099999999998,1.596257 -135,0,-0.36843899999999996,1.252721,-0.45337700000000003,-0.399017,0.417818,-0.64864,-0.26141,-0.32426900000000003,-0.116085,-0.181532,-0.38549,0.740089,-0.42216000000000004,-0.42260299999999995,-0.413491,-0.8856209999999999,-0.523408,-0.56435,-0.8273719999999999,-0.304446,-0.608051,0.296004,-0.697527,-0.45031099999999996,0.15267999999999998,-0.752169,-0.284279,-0.466537,-0.493068,-0.44100500000000004 -521,0,2.826844,0.204007,2.932082,3.0964169999999998,0.08028099999999999,1.0466719999999998,0.9288559999999999,2.02352,0.414547,0.707338,2.982918,0.5376340000000001,3.028755,3.373421,0.47251899999999997,2.013766,1.785325,2.532475,0.654911,0.651019,2.116181,-0.574155,2.071179,2.190554,-0.684021,0.371211,0.12653699999999998,0.677041,-0.212166,0.36504899999999996 -278,1,-0.15928599999999998,0.068847,-0.248748,-0.24871500000000002,-1.199727,-1.132615,-0.797293,-0.955564,-0.915269,-1.181233,-0.152598,-0.337343,-0.23598000000000002,-0.23489200000000002,-1.201293,-1.209502,-0.86415,-0.9424899999999999,-0.871183,-1.07704,-0.531175,-0.092266,-0.585651,-0.398607,-1.1000379999999998,-1.1292,-0.704378,-1.0952620000000002,-0.859936,-1.009516 -504,1,-1.240257,-1.513995,-1.138153,-1.020263,2.535091,0.571476,-0.29883000000000004,-0.184337,0.222031,2.286134,-1.380108,-1.493895,-1.2554370000000001,-1.155243,4.770911,2.26582,0.106726,0.09287999999999999,2.0678259999999997,4.5678589999999994,0.008763,-0.22471300000000002,0.07323500000000001,-0.44656999999999997,0.9140440000000001,1.114541,0.097358,1.6357,0.590585,2.517299 -97,1,-1.107724,0.099787,-1.145302,-0.904416,-0.033692,-1.014866,-1.207529,-1.3825180000000001,-1.563999,0.32884800000000003,-1.232704,0.151341,-1.2299,-1.0249840000000001,0.42981899999999995,-0.9727979999999999,-1.0291350000000001,-1.057093,-1.685343,0.8650770000000001,-0.253266,1.4989139999999999,-0.36338200000000004,-0.44635,1.363062,-0.606297,-0.881503,-1.061847,-0.306607,0.708123 -84,1,-0.538247,-0.126566,-0.580264,-0.54967,0.23370700000000003,-0.34329299999999996,-0.218232,-0.582969,0.773694,-0.260777,-0.6041810000000001,-0.8469709999999999,-0.618637,-0.601781,0.061894000000000005,-0.6195430000000001,-0.593716,-0.781278,0.976194,-0.441954,-0.642699,0.069211,-0.705448,-0.531937,-0.35734099999999996,-0.411243,-0.39203899999999997,-0.7736,-0.099563,-0.449326 -553,1,-1.330337,-0.102139,-1.322527,-1.027998,-0.9673959999999999,-1.089612,-0.922365,-1.354653,-0.753491,-0.5550350000000001,-1.3616469999999998,0.616755,-1.357589,-1.1117290000000002,-0.281835,-0.915186,-0.613176,-0.931141,-0.436721,0.41995,-0.374896,1.201362,-0.368827,-0.49453400000000003,1.299726,-0.398947,0.26745399999999997,0.16608,2.06411,0.313228 -469,1,-0.602442,-0.045144,-0.569541,-0.619635,2.000293,0.213329,0.22266,0.41102299999999997,-0.38949,0.48512,-0.712106,-0.258223,-0.642115,-0.6999029999999999,1.5044170000000001,0.8330879999999999,0.165734,0.17335799999999998,0.530779,1.382503,0.017786,0.949168,0.079671,-0.274737,2.5164419999999996,0.367857,0.5874189999999999,1.072832,-0.299342,0.53791 -87,0,1.716882,0.770703,1.3593899999999999,1.305104,-0.327392,0.421983,1.455144,1.23326,1.707154,0.495095,1.389599,1.233427,1.236982,1.197683,-0.43199399999999993,0.308132,0.728197,0.8716010000000001,0.516175,-0.9225209999999999,0.52091,-1.003792,0.09353099999999999,0.380914,-1.0563690000000001,-0.39447600000000005,0.172625,0.033069,-0.109249,-0.173203 -22,0,0.37291599999999997,-1.074317,0.531343,0.176348,0.290694,2.170095,1.719008,1.8986619999999998,2.857396,0.8597309999999999,0.344426,-1.170433,0.43377299999999996,0.14081400000000002,0.77853,2.068725,1.492795,1.254641,2.589911,1.0663770000000001,0.12137,-0.9203319999999999,0.256397,0.10061200000000001,-0.083996,1.553833,1.079801,1.739514,1.9587720000000002,0.226609 -19,1,-0.240048,-1.045005,-0.225217,-0.297761,0.5098729999999999,-0.489605,-0.159223,0.216123,0.123347,-0.629292,-0.166799,-1.147162,-0.185728,-0.25195700000000004,0.101747,-0.43685,-0.27821,-0.028609,0.267911,-0.72831,-0.488225,-0.776999,-0.400014,-0.36912399999999995,0.47369300000000003,-0.607974,-0.266043,0.21961,-0.089876,-0.565449 -265,0,3.3590459999999998,3.4983370000000003,3.179304,4.485168,0.338913,0.064472,0.345477,0.78103,-0.052992,-0.097854,1.8752630000000001,2.753002,1.8012869999999999,2.1732099999999996,-0.11886600000000001,0.18873800000000002,0.601392,0.9683280000000001,-0.15559800000000001,-0.8587290000000001,2.767643,0.726003,2.417205,3.506257,-0.8300270000000001,-0.5979140000000001,-0.346946,-0.408142,-0.8320879999999999,-0.565828 -38,0,-0.264898,-0.077713,-0.34912600000000005,-0.31955900000000004,-1.6876200000000001,-1.2910780000000002,-1.1907860000000001,-1.303644,-2.1609599999999998,-1.601839,0.24502100000000002,1.375378,0.147089,0.12488699999999998,-0.177222,-1.005016,-0.813805,-0.5140520000000001,-0.9003909999999999,-1.0997219999999999,2.9192299999999998,1.7619939999999998,2.579575,1.444697,-0.052662,-0.81253,-0.45470600000000005,1.196112,-1.5328899999999999,-0.771975 -479,0,0.2321,-0.42782700000000007,0.441986,0.103922,0.23370700000000003,1.220974,1.523269,0.957658,0.67501,0.41086300000000003,0.6028789999999999,0.051276999999999996,0.7344609999999999,0.457647,0.44405200000000006,1.610099,1.692419,1.109679,1.239062,0.422785,-0.326533,-0.419392,0.100957,-0.158788,0.718703,1.6331959999999999,1.621255,1.679497,0.870276,0.96836 -17,0,0.971385,0.944946,0.8798379999999999,0.763667,2.039746,1.0752979999999999,0.9893049999999999,1.411411,1.3027090000000001,1.6765599999999998,0.5687979999999999,0.323544,0.664438,0.40929699999999997,1.4688350000000001,1.854573,1.047093,1.389802,1.286524,1.5256809999999998,0.592011,-0.261,0.48906099999999997,0.304568,-0.004993,-0.026164,-0.000455,0.190412,-0.44221499999999997,0.13129000000000002 -189,1,-0.604513,-0.991267,-0.6139220000000001,-0.586937,-0.998082,-0.56785,-0.623143,-1.011902,-0.677455,-1.157405,-0.518977,-0.788794,-0.541199,-0.5437609999999999,-1.1073540000000002,-0.602865,-0.63226,-0.8351870000000001,-0.527994,-1.14225,-0.602637,-0.691906,-0.58367,-0.48441300000000004,-0.34834,-0.187685,-0.355567,-0.8699530000000001,-0.035391000000000006,-0.782944 -178,1,-0.46991000000000005,0.54435,-0.56835,-0.477771,-2.240829,-1.399158,-1.2675610000000002,-1.60408,-0.97998,-1.41398,-0.317327,0.681912,-0.41021499999999994,-0.36543600000000004,-2.40897,-1.610136,-1.094848,-1.214049,-1.521051,-1.482475,-0.8375969999999999,-0.13581,-0.873758,-0.57198,-1.207709,-1.298098,-1.004615,-1.613035,-0.534235,-1.069167 -164,0,2.431317,0.414075,2.291686,2.6762759999999997,-0.41944799999999993,0.661808,0.588232,1.8270970000000002,1.113428,0.439125,2.596659,0.640025,2.476807,2.932585,-0.8518700000000001,0.192528,0.547405,1.240713,-0.038768000000000004,-1.030259,0.9348860000000001,-0.6545310000000001,0.8598389999999999,1.265383,-0.7103550000000001,-0.002132,-0.12180899999999999,0.7192149999999999,-0.11893499999999999,-0.020766999999999997 -451,0,1.070784,0.860267,0.969195,0.9500059999999999,0.8956290000000001,-0.44380299999999995,0.602144,0.48715600000000003,-0.983215,-1.276549,1.551487,1.328837,1.471766,1.5247540000000002,0.486752,-0.106715,0.962975,1.0758889999999999,-0.5425979999999999,-1.259911,0.22459400000000002,0.286932,0.024722,0.348571,1.619739,-0.345852,0.5695140000000001,0.518075,-0.500333,-0.751927 -290,1,-0.103373,-0.577643,-0.16564500000000001,-0.199142,-1.426358,-0.044944,-0.24078000000000002,-0.19042699999999999,-1.017189,0.22411199999999998,0.08029299999999999,0.10247300000000001,0.167272,-0.011061,-0.625564,1.198852,0.595114,0.44109899999999996,-0.3564,1.293194,1.7177240000000003,1.003598,0.739546,0.809067,0.24035,4.522686,2.244939,2.581382,2.4491400000000003,7.211398 -67,1,-0.8157369999999999,-0.29918,-0.8715700000000001,-0.7275699999999999,-0.14766500000000002,-1.03554,-0.6130680000000001,-0.685139,-0.810113,-0.9717620000000001,-0.8001510000000001,-0.05809500000000001,-0.830767,-0.741711,-1.065367,-1.086507,-0.649209,-0.6866140000000001,-1.079287,-0.8686520000000001,-0.47811899999999996,-0.497046,-0.512386,-0.488154,0.747037,-0.908883,-0.373471,-0.45923800000000004,0.15591300000000002,-0.6237 -110,1,-1.080803,-0.68512,-1.059816,-0.902834,0.6282300000000001,-0.49469399999999997,-0.682153,-0.932876,-0.5949479999999999,0.040685,-1.2355450000000001,-0.535144,-1.213835,-1.037213,0.522334,-0.384734,-0.5707399999999999,-0.803203,-0.831023,1.113158,-0.007839,0.375835,-0.058938,-0.384306,2.269766,0.21472,-0.154967,-0.25404299999999996,1.485355,0.318902 -556,1,-1.1636360000000001,-0.45551,-1.173002,-0.937465,-0.257255,-0.8541129999999999,-1.257616,-1.405205,-1.033367,-0.915792,-1.126767,0.06989400000000001,-1.121981,-0.976065,0.28037199999999995,-0.555297,-1.051784,-0.973959,-0.075277,0.07263700000000001,-0.581343,1.584188,-0.6029770000000001,-0.5178560000000001,1.95642,-0.182096,-0.9191030000000001,-0.764679,0.626908,-0.573771 -150,1,-0.436776,-0.255213,-0.48971499999999996,-0.463884,-0.11698,-0.9145469999999999,-0.9166559999999999,-0.786396,0.47763999999999995,-1.085918,-0.32016700000000003,0.346815,-0.348429,-0.385345,1.2197559999999998,-0.539188,-0.7211489999999999,-0.579569,2.659279,-0.27325900000000003,0.054238999999999996,0.190772,0.003436,-0.12226500000000001,-0.007993,-0.7857029999999999,-0.41126999999999997,-0.04317,1.085796,-0.855568 -321,0,1.406258,-0.431084,1.278968,1.364873,-1.182192,-0.639734,-0.055596000000000007,0.424727,0.249533,-1.364106,1.713375,0.08618300000000001,1.611812,1.7608150000000002,-1.1500540000000001,-0.35441100000000003,0.335226,0.731023,0.42490200000000006,-1.678104,0.676106,-0.962606,0.49599099999999996,0.759343,-0.8350270000000001,-0.654921,-0.180497,0.10930699999999999,0.168021,-0.8343860000000001 -526,1,-0.190348,-0.084227,-0.15968800000000002,-0.282643,1.316453,0.36982,-0.032568,0.42777299999999996,0.998566,0.149854,-0.18952,-0.12558,-0.186552,-0.295187,0.792763,0.17926199999999998,-0.587439,-0.44879300000000005,-0.323542,0.05279,-0.7412310000000001,-1.106847,-0.704458,-0.533917,-0.876028,-0.617475,-0.557161,-0.718124,-0.847828,-0.7594920000000001 -18,0,2.28843,0.84724,2.369129,2.6674860000000002,0.825491,0.386359,1.271399,1.8910490000000002,-0.21477,-0.432012,1.6139700000000001,0.665623,1.566503,1.720997,0.138753,-0.031099,0.742007,1.188093,-0.838325,-1.254241,1.274152,-0.362603,1.484567,1.585507,-0.182334,-0.36597199999999996,0.066854,0.553762,-0.845406,-0.68006 -446,0,1.089422,2.094623,1.135996,0.978132,0.338913,0.7756770000000001,1.764105,1.254577,0.115258,0.37705900000000003,1.028901,2.033938,1.043388,0.929199,0.256887,0.512808,1.016961,0.877275,-0.360051,-0.5156689999999999,-0.055841999999999996,-0.253742,0.003436,0.07948999999999999,-0.775691,-0.29778699999999997,0.16831400000000002,-0.11292,-0.98949,-0.468617 -432,0,1.192963,-0.098883,1.153867,1.051965,1.496181,0.254042,1.240695,1.563677,0.21232399999999998,-0.177099,1.719055,0.058258000000000004,1.723026,1.6925560000000002,1.205523,0.844459,1.563102,1.985643,-0.319891,-0.321457,0.100798,-0.39163200000000004,0.070265,0.267385,0.6820350000000001,0.09344,0.781388,1.184757,0.48040299999999997,0.07946900000000001 -523,1,-0.240048,-0.007690000000000001,-0.23325900000000002,-0.314109,0.444119,0.014854,-0.37751,0.21003200000000002,-0.083729,0.35267600000000005,-0.11851700000000001,-0.141869,-0.133416,-0.238589,0.199243,0.050392,-0.438788,-0.286033,-0.3564,0.79845,-0.310652,0.058325,-0.288137,-0.305539,-0.100664,-0.165888,-0.365845,0.279627,-0.223063,-0.018119999999999997 -418,1,-0.542388,-1.426059,-0.570137,-0.551428,-0.042459,-0.595204,-0.855583,-0.49282700000000007,-0.20344500000000001,0.592627,-0.405371,-1.6567900000000002,-0.45676000000000005,-0.454741,-0.605638,-0.879367,-0.818576,-0.6422479999999999,-0.834674,-0.006749,-0.649196,-1.036269,-0.662875,-0.505315,-0.30333899999999997,-0.7180770000000001,-0.756601,-0.5783,-0.413156,0.23417399999999997 -272,0,2.4685919999999997,0.407562,2.640181,2.642876,-0.22657,1.388279,1.480091,1.058154,-0.109614,0.256807,2.164957,0.395683,2.2790939999999997,2.377985,-0.167259,1.7389689999999998,1.640943,1.5445659999999999,-0.330843,-0.121575,2.749597,0.245202,2.9706490000000003,2.562383,-0.451345,1.34816,1.041007,0.618645,-0.15768,0.309824 -153,1,-0.8861450000000001,-1.527023,-0.923695,-0.7731,0.07589800000000001,-1.0468,-0.9644389999999999,-0.9066860000000001,-0.067552,-0.899167,-0.845593,-1.445027,-0.869073,-0.776409,0.083955,-1.008427,-0.8660329999999999,-0.801139,0.067109,-0.24774200000000002,-0.649918,-0.789881,-0.711388,-0.546898,0.659367,-0.921794,-0.6609430000000001,-0.578137,0.404124,-0.823039 -509,0,0.174117,1.734739,0.31092800000000004,0.050658,1.789881,1.542225,1.529986,1.54845,0.181587,1.260942,0.378508,1.084495,0.48732,0.21731999999999999,1.56135,1.566511,1.433786,0.9358280000000001,-0.016862000000000002,1.138675,-0.260123,1.3501379999999998,0.035118,-0.171989,0.832374,1.336982,0.9382209999999999,1.397252,0.061472000000000006,0.834081 -288,1,-0.9130659999999999,-0.545075,-0.8635280000000001,-0.778726,-1.296166,-0.445075,-0.564134,-0.326858,0.087755,-0.767831,-0.8143520000000001,0.155995,-0.751681,-0.741711,-1.1500540000000001,0.260754,0.049475,0.179548,2.860081,-0.066289,0.29389099999999996,1.248535,0.005416,-0.124465,2.899791,3.193632,1.627887,3.742804,1.651232,1.0296370000000001 -258,0,0.7415229999999999,0.971001,1.08536,0.607213,0.790423,1.672634,1.659519,2.003725,0.608681,0.9949459999999999,0.43531099999999995,0.9099649999999999,0.750937,0.33734200000000003,1.034726,3.924075,2.872586,2.290012,2.494987,2.586049,3.200748,2.244609,3.5909230000000005,2.159752,1.773079,1.9266159999999999,1.6719849999999998,2.991773,0.34237399999999996,0.849967 -228,1,-0.428493,0.9172629999999999,-0.494183,-0.45105100000000004,-0.423831,0.579746,0.570481,0.051676,-0.12093800000000002,0.65968,-0.42809200000000003,1.089149,-0.4374,-0.450759,-1.233317,-0.550559,-0.432134,-0.736654,-1.086589,-0.369656,-0.5784560000000001,-0.2737,-0.703468,-0.48023299999999997,-0.6240180000000001,-0.141855,-0.057485,-0.503683,-0.8345090000000001,-0.146725 -476,1,0.037443000000000004,0.257745,0.144127,-0.091558,-0.748217,0.563842,-0.10069299999999999,0.293779,-0.59333,-0.29735100000000003,0.02065,0.288638,0.018164,-0.103779,-0.501736,0.12240799999999999,-0.47921499999999995,-0.47303999999999996,-1.115796,-0.383832,-0.20706799999999997,-0.36078899999999997,-0.057948,-0.20521199999999998,-0.978033,0.41368699999999997,-0.12214100000000001,0.276383,-0.537867,-0.39750599999999997 -423,1,-0.233835,-0.028860000000000004,-0.174581,-0.30198,-0.774519,0.39717399999999997,0.450063,0.397319,-0.253596,0.24627800000000002,-0.132717,-0.037150999999999997,-0.103348,-0.22635999999999998,-0.41206800000000005,0.196319,0.097561,-0.020613,0.132826,-0.140004,-0.6524439999999999,-0.5839529999999999,-0.525752,-0.461532,-1.020368,0.145976,0.15903,0.153104,-0.57298,0.060935 -64,0,0.169975,1.269005,0.135192,0.013566,2.311528,0.965882,0.624693,0.867821,0.780165,1.061445,-0.411051,1.058897,-0.382205,-0.44336499999999995,1.127241,0.414261,0.30132800000000004,0.511001,0.34093,0.439796,0.07336799999999999,-0.070493,0.030168,-0.085302,0.24668400000000001,0.055994,-0.071742,0.179057,-0.5075970000000001,-0.073344 -137,1,-0.817808,-0.5955560000000001,-0.814083,-0.735833,-0.586024,-0.5691229999999999,-0.6346569999999999,-0.454456,-0.363605,-0.9030459999999999,-0.766069,-0.907474,-0.778867,-0.7255,0.0021149999999999997,-0.671849,-0.675072,-0.520243,-0.283381,-0.587967,-0.827491,-0.404696,-0.852967,-0.6087229999999999,-0.636019,-0.573882,-0.582029,-0.51812,-0.669842,-0.834764 -343,0,1.342063,1.462789,1.499383,1.159197,-0.46328400000000003,0.58229,0.965317,1.688535,1.8511360000000001,-0.264102,1.577048,0.556251,1.562384,1.533287,0.11455599999999999,0.560187,1.224118,1.583257,0.9871469999999999,-0.800607,0.7847430000000001,1.9361709999999999,1.1420059999999999,0.601151,-0.761691,0.45839799999999997,0.38317199999999996,1.016059,2.060478,-0.191737 -302,0,1.534649,0.611116,1.535126,1.433432,0.102199,0.539669,1.060308,1.1830120000000002,0.6361829999999999,0.595398,1.6934939999999998,1.063551,1.760097,1.6840240000000002,0.8283459999999999,1.505866,1.751427,2.0398099999999997,1.596855,1.68587,2.4067220000000002,0.9546110000000001,2.444432,1.9903389999999999,0.307686,1.220732,1.47868,1.226932,0.825477,0.806846 -561,1,-0.900641,2.055541,-0.955268,-0.7752100000000001,-1.7402229999999999,-1.267986,-1.305831,-1.745063,-2.1593419999999997,-1.379622,-0.831392,2.345765,-0.8773110000000001,-0.764748,-1.556408,-1.303122,-1.114873,-1.2618200000000002,-2.7441169999999997,-1.102557,-0.328698,4.860893,-0.40843,-0.385626,0.184348,-0.927774,-1.057501,-1.9134470000000001,-0.078979,-0.764788 -8,0,-0.161357,0.8228129999999999,-0.031609,-0.248363,1.662757,1.8183099999999999,1.280035,1.391616,2.389857,1.28865,-0.32016700000000003,0.58883,-0.18408,-0.384207,2.201839,1.6840099999999998,1.219096,1.150692,1.9656,1.572462,-0.35685,-0.389818,-0.22774299999999997,-0.352403,-0.43667700000000004,0.5332899999999999,0.12056800000000001,0.07524299999999999,0.107482,-0.017363 -502,1,-0.558955,-0.696519,-0.6133270000000001,-0.57762,1.1235760000000001,-0.5036,-0.39957800000000004,-0.464354,0.411311,-0.474682,-0.450813,-0.6910569999999999,-0.44151899999999994,-0.507926,1.383436,0.07882,-0.370614,-0.41603500000000004,0.479666,0.470983,-0.532258,-0.221084,-0.643569,-0.48067299999999996,0.887043,-0.548172,-0.203707,-0.030193,0.904178,-0.49736400000000003 -28,0,0.828498,1.796619,1.252161,0.6828029999999999,1.390974,2.269333,1.7334009999999997,1.3368,1.822016,0.82094,0.333066,1.3916680000000001,0.429654,0.22044899999999998,0.8425790000000001,1.23865,0.998129,0.9954120000000001,0.4176,0.36891599999999997,0.122092,-0.371675,0.31283,0.06959,-0.602684,0.284582,0.128194,-0.156717,-0.346563,-0.313156 -457,1,-0.397431,1.392767,-0.47571599999999997,-0.43540500000000004,-0.152049,-0.941264,-0.638975,-0.8307059999999999,-0.738931,-0.8903,-0.260524,1.387014,-0.324127,-0.332729,-0.601368,-0.990992,-0.766849,-0.7284,-0.70324,-0.9863139999999999,-0.710192,0.24157399999999998,-0.768317,-0.500695,-0.42434399999999994,-0.9722620000000001,-0.55683,-0.8670329999999999,-0.856303,-0.743984 -136,1,-0.6086550000000001,-0.032117,-0.628517,-0.586937,-0.230954,-0.9635290000000001,-0.80401,-0.684074,-1.923146,-0.582743,-0.686545,-0.60961,-0.710491,-0.65781,0.621966,-0.8223229999999999,-0.663898,-0.591176,-1.7255040000000001,-0.474559,0.157823,2.342584,0.194023,-0.13128599999999999,-0.154333,-0.652686,-0.174861,0.20663299999999998,-0.841774,-0.03325 -365,0,1.665111,0.11281400000000001,1.6066120000000002,1.5810959999999998,0.014527000000000002,-0.10601300000000001,-0.00954,0.9424319999999999,-0.471997,-0.9196709999999999,1.7928990000000002,0.579522,1.723026,1.8148529999999998,-0.345884,0.165996,0.115389,0.746242,-0.7068909999999999,-1.024589,0.6241329999999999,-0.5444,0.6692520000000001,0.7063189999999999,-0.277671,-0.358707,-0.270021,0.45643599999999995,-0.737646,-0.71448 -384,1,-0.42021000000000003,-1.3527799999999999,-0.317851,-0.451578,-0.69123,0.090554,0.06914,-0.348327,-0.26653899999999997,-0.595488,-0.240643,-1.296095,-0.25451599999999996,-0.321637,-0.905956,-0.352327,-0.47745699999999996,-0.52308,-0.710542,-0.972138,-0.800783,-1.244737,-0.630698,-0.551739,-0.923364,-0.265371,-0.11982000000000001,-0.539856,-0.718274,-0.447057 -81,1,-0.15307300000000001,-0.405029,-0.315766,-0.46704799999999996,0.9306979999999999,1.430264,1.0248059999999999,0.85564,1.013126,0.9783209999999999,-0.223602,-0.7981020000000001,-0.22568200000000002,-0.38363800000000003,0.814113,0.931636,0.35280300000000003,0.540406,0.47601499999999997,0.8820889999999999,-0.430117,-0.364417,-0.6589149999999999,-0.6023430000000001,-0.08233,0.57409,0.26214899999999997,0.329912,0.096584,0.305663 -306,1,-0.385006,-0.8512209999999999,-0.454568,-0.428374,-0.857807,-0.7612369999999999,-1.252098,-1.364398,-0.40405,-0.0053100000000000005,-0.263364,-0.8074100000000001,-0.325363,-0.334435,-0.8006310000000001,-0.982274,-1.09653,-1.177705,-0.655777,-0.546856,-0.775518,-1.1672639999999999,-0.8227700000000001,-0.546458,-1.136372,-0.984614,-1.0201,-1.5264149999999999,-0.859936,-0.45764799999999994 -282,0,1.557428,0.484098,1.3444969999999998,1.3138940000000001,0.851793,0.767407,0.764781,1.6839669999999998,1.115046,-0.336696,1.497524,-0.258223,1.451171,1.393926,0.522334,0.755387,0.926565,1.179323,0.297119,-0.549692,0.23722600000000002,-0.402337,0.018287,0.28212600000000004,-0.462345,-0.19495,-0.043227,0.518075,-0.522127,-0.692164 -475,1,-0.451272,-1.030349,-0.418229,-0.483045,0.010144,0.042844,0.36178899999999997,-0.255445,0.170262,-0.328384,-0.36844899999999997,-0.828354,-0.373967,-0.420896,-0.424166,-0.410318,-0.38229,-0.46788100000000005,-0.389259,-0.519922,-0.92133,-1.3232979999999999,-0.827225,-0.6313850000000001,-0.722689,-0.42018500000000003,0.042649,-0.557375,-0.548764,-0.5696100000000001 -118,0,0.8119310000000001,0.785359,0.68623,0.688077,2.329063,1.515507,2.2232279999999998,1.352027,0.603827,2.286134,0.469393,0.8424799999999999,0.565581,0.363223,1.362087,1.342883,1.563102,1.183192,1.03826,1.490241,0.529933,-0.262814,0.362333,0.402476,0.552696,0.798207,0.915342,0.341267,-0.677107,0.8280290000000001 -55,1,-0.710125,-0.5222760000000001,-0.7580859999999999,-0.658133,-0.327392,-1.062767,-0.8704069999999999,-0.783351,0.655596,-0.752869,-0.7405079999999999,-0.12558,-0.767334,-0.699334,-0.079725,-0.9402020000000001,-0.733704,-0.674233,0.395694,-0.5284270000000001,-0.289719,-0.467653,-0.33813499999999996,-0.371105,0.42902399999999996,-0.936493,-0.610212,-0.5070899999999999,1.409075,-0.53292 -557,1,-1.196769,1.394395,-1.214107,-0.966822,-1.098904,-1.162132,-1.305831,-1.745063,-0.688779,-0.789998,-1.336086,1.999032,-1.347292,-1.090967,-1.076753,-1.0353379999999999,-1.114873,-1.2618200000000002,-0.25417399999999996,-0.312952,0.477599,3.102794,0.372234,-0.247015,1.516402,-0.795763,-1.057501,-1.9134470000000001,1.149967,-0.17812 -284,1,-0.49061800000000005,-0.974982,-0.45099399999999995,-0.500975,-1.451345,-0.143545,0.29846100000000003,-0.196518,-1.4588430000000001,-0.702441,-0.351408,-0.835335,-0.32495100000000005,-0.393308,-1.293808,-0.161864,0.285006,-0.38740399999999997,-1.385966,-0.48873500000000003,-0.410988,0.31233299999999997,-0.25695,-0.375065,-0.207668,0.7898229999999999,1.570857,0.964152,-0.213376,-0.037411 -256,0,1.818352,1.724968,2.124886,1.837752,-0.187118,1.7725080000000002,0.733597,1.21042,-0.133881,0.917364,1.540126,2.206141,1.7147880000000002,1.57026,-0.267602,1.9322740000000003,1.124934,1.689012,0.297119,-0.06770599999999999,1.578769,-0.032392000000000004,2.1246419999999997,1.453497,-0.228336,1.239175,0.22335500000000003,0.550517,-0.143151,0.551149 -426,1,-0.8571540000000001,-0.668836,-0.77,-0.773804,0.014527000000000002,0.288394,0.10416199999999999,-0.327467,0.192911,0.693484,-1.0358829999999999,-1.002884,-1.0082959999999999,-0.9137790000000001,0.128078,-0.057630999999999995,-0.319515,-0.689709,0.41394899999999996,0.900517,-0.279974,-0.163025,-0.149529,-0.43050900000000003,0.107678,0.737287,0.687553,0.136883,0.129276,0.391527 -387,1,-0.15721500000000002,-0.929386,-0.226408,-0.23781599999999997,-2.0834580000000003,-0.8331209999999999,-0.782421,-1.054232,-0.580388,-0.981736,-0.070234,-0.72829,-0.14824500000000002,-0.16577999999999998,-1.8574380000000001,-1.06187,-0.858124,-1.04239,-0.747051,-1.14225,-0.545251,-1.079632,-0.5727800000000001,-0.378805,-1.104371,-0.633124,-0.398339,-0.775384,-0.493068,-0.690272 -497,1,-0.45748500000000003,-0.21775799999999998,-0.43014399999999997,-0.480408,-0.209036,-0.023315,-0.332893,-0.141702,0.217178,-0.40652,-0.47069399999999995,-0.46067700000000006,-0.474471,-0.497118,-0.503871,-0.5314180000000001,-0.661764,-0.65076,-1.042777,-0.331381,-0.9094200000000001,-0.790788,-0.7985140000000001,-0.625444,-1.081703,-0.657716,-0.611207,-0.762408,-0.6274649999999999,-0.852542 -298,1,-0.010186,-0.067942,-0.043523,-0.107027,-1.662195,-0.23896599999999998,-0.555018,-0.5985,-0.428317,-0.398208,0.037691,-0.26055,-0.030853,-0.061970000000000004,-2.177683,-0.9881489999999999,-0.804137,-0.90741,-0.644825,-0.9834780000000001,-0.632233,-0.9939950000000001,-0.596541,-0.43513,-1.290712,-0.654362,-0.6997359999999999,-1.0629819999999999,-1.151735,-0.69519 -157,1,0.403979,0.38964899999999997,0.388371,0.266178,-1.956334,-0.529682,-0.402937,-0.46054700000000004,-0.604655,-1.342493,0.7704479999999999,0.039642000000000004,0.676795,0.640807,-1.5592549999999998,-0.6085510000000001,-0.468292,-0.547068,0.11822300000000001,-1.434276,0.2661,1.529758,0.30342399999999997,0.138015,-1.199375,0.06325900000000001,-0.044222000000000004,-0.11292,-0.646838,-0.860864 -249,1,-0.749471,-0.730716,-0.785787,-0.683447,0.28631,-0.611108,-0.440357,-0.282091,-0.383019,-0.32450500000000004,-0.7405079999999999,-1.014519,-0.745503,-0.707013,0.351537,-0.49768500000000004,-0.571494,-0.506314,0.26060900000000004,-0.158433,-0.537671,-0.324502,-0.5841649999999999,-0.477813,-0.126331,-0.737638,-0.359545,-0.281618,-0.456744,-0.38237600000000005 -342,1,-0.900641,-0.940785,-0.8191470000000001,-0.774507,0.413434,-0.211612,-0.20288,-0.108204,0.6475069999999999,0.37983,-0.8711540000000001,-1.0075379999999998,-0.843535,-0.799162,0.49386800000000003,-0.2534,-0.43728100000000003,-0.400043,-0.13004200000000002,0.889177,-0.884517,-0.715493,-0.748021,-0.6498659999999999,0.12501199999999998,-0.374915,-0.14303,-0.27513000000000004,0.355692,-0.332068 -508,1,0.217604,-1.2892709999999998,0.07562,0.08370599999999999,0.132884,-0.751695,-0.371753,0.321186,-0.971891,-0.645362,0.6170800000000001,-0.835335,0.5243909999999999,0.469024,-0.148756,-0.705393,-0.42108500000000004,-0.08484,-0.36735300000000004,-0.8828280000000001,-0.716327,-1.35396,-0.851482,-0.432709,0.11767799999999999,-0.75161,-0.23852199999999998,0.407773,-0.859936,-0.4637 -267,1,-0.304244,0.710451,-0.28598,-0.385129,-1.396988,-0.5169590000000001,-0.60875,-0.802689,-0.735695,-0.7595189999999999,-0.152598,0.593484,-0.19808499999999998,-0.26703000000000005,-1.195599,-0.412213,-0.603634,-0.7093119999999999,-0.644825,-0.596472,-0.24243800000000001,1.268492,-0.136163,-0.298719,-0.535015,-0.07925900000000001,-0.029965,-0.30449,-0.029337000000000002,-0.297269 -338,1,-1.058024,0.189351,-1.05088,-0.87295,0.34329699999999996,-0.7256130000000001,-0.799692,-0.755486,-0.010929000000000001,-0.404858,-1.158009,-0.409482,-1.135162,-0.978625,0.308838,-0.5890310000000001,-0.799618,-0.803976,0.286166,0.07263700000000001,-0.517099,1.448112,-0.538623,-0.516756,0.254017,-0.6141220000000001,-0.497147,-0.608795,0.055417999999999995,-0.38464499999999996 -360,1,-0.5278930000000001,-0.764914,-0.608859,-0.518379,-1.7288259999999998,-1.342223,-1.288651,-1.496108,-1.080282,-1.592419,-0.450813,-0.28381999999999996,-0.516897,-0.46355799999999997,-1.56566,-1.4752020000000001,-1.099882,-1.121268,-1.035476,-1.5519379999999998,-0.195157,-0.47926499999999994,-0.265861,-0.264836,-0.419343,-1.161672,-1.03121,-1.326735,-0.013597,-1.0352379999999999 -12,0,0.971385,0.694167,1.323647,0.793551,-1.256713,0.8653719999999999,0.439988,0.945477,0.44528500000000004,1.017112,1.432201,1.2822959999999999,1.66536,1.3313549999999998,0.073992,2.680858,1.477729,1.621948,2.137194,2.155097,1.9862490000000002,4.265788,4.061202,1.669114,-1.300712,3.2131939999999997,1.890159,4.720928,2.941929,3.4213199999999997 -210,0,1.443533,0.352195,1.520233,1.3631149999999999,-0.6386270000000001,0.24004699999999998,0.546493,1.178444,0.013337,-1.4017879999999998,1.832661,0.663296,1.760097,1.8063209999999998,-0.388583,0.5772430000000001,0.944142,1.204343,-0.170202,-1.780172,1.64193,0.47743800000000003,2.0607830000000003,1.570106,0.36102199999999995,0.59421,0.762488,2.57165,1.358222,-0.540863 -533,0,1.441462,0.239833,1.332582,1.3437780000000001,-0.993698,-0.005503,0.177564,0.710987,0.516467,-1.1130719999999998,1.8014189999999999,0.32121700000000003,1.760097,1.8319169999999998,-0.341614,0.5109130000000001,0.797249,1.35627,1.293826,-1.220218,1.546286,0.94191,1.1395309999999998,1.321487,-0.701021,0.302466,0.299616,0.8424950000000001,0.9223399999999999,-0.406962 -167,0,0.78294,0.10141499999999999,0.698144,0.666982,-0.682463,-0.26950100000000005,-0.19376500000000002,0.499338,-0.146823,-0.646471,0.7534069999999999,-0.113944,0.713866,0.6581560000000001,-0.548706,-0.23729099999999997,-0.057492999999999995,0.43439300000000003,0.297119,-1.057194,0.699566,0.31596199999999997,0.6251939999999999,0.59411,-0.306005,-0.04349,-0.186797,0.688396,0.044521,-0.114196 -79,1,-0.42021000000000003,-0.139593,-0.458142,-0.454391,-0.152049,-0.255506,-0.475379,-0.538203,-0.196974,-0.264102,-0.359929,-0.30011,-0.36161,-0.42260299999999995,0.21205300000000002,-0.16830799999999999,-0.62661,-0.6646890000000001,-0.341796,-0.400843,-0.504106,-0.221084,-0.538623,-0.43975,-0.582683,-0.49563599999999997,-0.370819,-0.586735,-0.37077899999999997,-0.358168 -369,0,2.358838,0.019993,2.613373,2.366883,-0.130131,0.853922,0.975872,1.958046,-0.25845,0.099426,2.238801,0.607446,2.274975,2.352388,0.707364,1.725703,1.958584,2.609857,0.045204,-0.198126,2.175733,-0.937931,2.3241389999999997,1.977138,-1.021034,0.15268199999999998,0.128194,0.472657,-0.649259,0.00041500000000000006 -437,1,-0.126153,-0.667207,-0.180538,-0.229554,-0.564106,-0.821034,-0.769947,-0.610225,-0.28433400000000003,-0.643146,-0.024791999999999998,-0.770177,-0.090167,-0.12425599999999999,-0.838349,-0.860226,-0.67118,-0.502445,-0.3564,-0.541186,-0.057647000000000004,-0.309987,-0.109926,-0.167149,0.311686,-0.700192,-0.5243359999999999,-0.44609899999999997,-0.05960700000000001,-0.364976 -485,1,-0.515468,-0.756771,-0.28121399999999996,-0.5273439999999999,-0.651778,0.965882,1.043037,0.298347,0.534263,1.078069,-0.476375,-0.670114,-0.37561500000000003,-0.506788,-0.086842,0.8861530000000001,0.823615,-0.011843000000000001,0.9871469999999999,1.481735,-0.04718,-0.017878,1.058346,-0.223253,0.064343,2.751546,2.636192,2.500278,1.426026,2.205998 -408,0,0.996235,-0.043516,0.918559,0.8234360000000001,0.693984,0.758501,0.277832,1.2606680000000001,0.25762199999999996,0.060080999999999996,1.097064,0.31889,1.063983,0.957925,0.5152180000000001,0.493857,0.392979,1.014241,0.658562,-0.29877600000000004,0.175148,-0.623324,0.096502,0.20842,0.063343,0.125297,-0.22559099999999999,0.487256,-0.775181,-0.17358099999999999 -308,1,-0.26904,-1.422803,-0.35091300000000003,-0.319735,-1.8471830000000002,-1.24623,-1.239673,-1.408555,-1.0252780000000001,-1.220579,-0.17815999999999999,-1.531128,-0.258635,-0.252241,-1.608359,-1.29251,-1.0802459999999998,-1.147836,-1.630579,-1.339297,-0.6524439999999999,-0.962425,-0.671786,-0.43887,-1.234376,-1.214599,-0.990192,-1.387564,-0.6952689999999999,-0.8430860000000001 -414,0,0.205179,1.829188,0.08455599999999999,0.08933200000000001,-0.770135,-0.989865,-0.563654,-0.7439140000000001,0.5374979999999999,-1.235541,0.284783,2.448156,0.19528099999999998,0.18375999999999998,-0.936557,-1.1047,-0.526547,-0.555322,0.14743,-1.397419,0.22712,0.744147,0.087591,0.110953,-0.069996,-0.626418,-0.23222199999999998,-0.43944799999999995,1.3279530000000002,-0.7734880000000001 -41,0,-0.710125,1.573523,-0.5969439999999999,-0.644421,2.565776,0.098824,0.624213,0.423205,0.102315,0.671317,-0.9023959999999999,0.479458,-0.8266479999999999,-0.807125,1.8744779999999999,0.330874,0.195866,0.200441,0.304421,0.836725,-0.608412,0.383092,-0.516841,-0.514116,0.341021,-0.43806999999999996,-0.197076,-0.23133299999999998,-0.844196,-0.285543 -276,1,-0.842658,-1.088973,-0.890335,-0.7428640000000001,-0.283557,-1.150045,-1.282059,-1.575895,-0.23094699999999999,-1.1130719999999998,-0.794471,-1.193704,-0.8311780000000001,-0.7346010000000001,-0.18291500000000002,-1.243615,-1.0962040000000002,-1.175848,0.519826,-0.650341,-0.605164,0.11457,-0.6440640000000001,-0.511475,0.461692,-0.921067,-1.0081959999999999,-1.372803,0.3678,-0.8200129999999999 -146,0,-0.523751,0.114443,-0.456653,-0.507831,0.268776,0.9856030000000001,0.854974,1.094698,4.6482790000000005,1.055903,-0.660984,-0.6305529999999999,-0.534609,-0.633919,0.9066280000000001,1.244335,0.9679969999999999,0.650804,3.163109,1.546945,-0.308487,0.37946399999999997,-0.289622,-0.343602,-0.5380149999999999,0.606506,0.483969,1.076077,4.3270669999999996,0.317767 -405,1,-0.801242,-0.015832,-0.729789,-0.71755,0.172337,-0.571667,-0.62842,-0.544141,-1.0511620000000002,-0.367175,-0.905236,-0.162813,-0.8888450000000001,-0.810254,0.287488,-0.5636359999999999,-0.494155,-0.50554,-1.188815,0.475236,-0.092295,0.9546110000000001,0.075215,-0.320281,0.826041,-0.23127899999999998,-0.39767600000000003,-0.039925999999999996,0.029991000000000004,-0.413771 -35,0,0.774656,0.54435,0.7815439999999999,0.612486,1.0490549999999998,0.8221149999999999,1.28915,1.015519,3.174481,0.13212100000000002,0.742046,0.535307,0.746818,0.610375,-0.018522999999999998,0.554502,0.577537,0.290462,0.30807199999999996,-0.8842459999999999,0.203299,-0.539139,0.070265,0.106773,-0.421677,-0.027282,0.167651,0.024958,0.889649,-0.427388 -13,0,0.11820499999999999,0.322883,0.141149,-0.007178,-0.844656,-0.393548,-0.19184600000000002,-0.041207,-0.14844100000000002,-1.167934,0.489274,1.084495,0.48320100000000005,0.363507,-0.8789129999999999,-0.07847799999999999,0.13284,0.12177,0.129175,-1.3350440000000001,-0.006756999999999999,-0.25192800000000004,0.018287,-0.082662,0.909377,0.323146,0.617261,1.317769,1.122119,-0.299917 -166,1,-0.966908,-2.223994,-1.00084,-0.820212,0.492339,-0.8178529999999999,-0.80353,-1.044183,-1.311625,-0.38546199999999997,-0.944998,-2.2292490000000003,-0.955573,-0.845521,-0.029908999999999998,-0.890359,-0.797107,-0.823838,-1.572164,0.170451,-0.83868,-1.4704409999999999,-0.861383,-0.634905,0.256017,-0.87535,-0.693105,-1.046599,-0.9689059999999999,-0.6335350000000001 -173,1,-1.018678,-1.442344,-1.049987,-0.8509760000000001,-0.472051,-1.093302,-1.1169040000000001,-1.0894059999999999,-1.615768,-0.599367,-0.865474,-1.065715,-0.896259,-0.802575,0.301721,-0.889033,-0.818199,-0.595561,-0.89674,0.551787,-0.714162,1.0671,-0.7371300000000001,-0.467692,2.639781,-0.238544,-0.575729,0.654332,1.244409,0.37450500000000003 -531,1,-0.604513,0.510153,-0.6034970000000001,-0.580082,0.9920680000000001,0.26803699999999997,0.017325999999999998,-0.508663,0.49381800000000003,0.30779,-0.697906,0.169958,-0.690308,-0.678856,0.37288699999999997,-0.185933,-0.5875640000000001,-0.705443,0.172987,0.256925,-0.716327,-0.621147,-0.729209,-0.549978,-0.596684,-0.45874899999999996,-0.44741099999999995,-1.0540610000000001,-0.732803,-0.42511800000000005 -388,1,-0.875791,-1.098743,-0.82004,-0.7569279999999999,-0.97178,0.169436,0.143501,-0.485519,-1.203234,1.127944,-0.811511,-0.881877,-0.765686,-0.747684,-0.904533,0.133779,0.14941300000000002,-0.550679,-0.00591,1.37825,-0.269507,-0.271886,-0.147053,-0.38210500000000003,1.1130520000000002,2.303312,2.083133,1.406984,-1.011284,3.228412 -205,0,0.310792,-0.885418,0.31092800000000004,0.191466,0.733436,0.5015,0.29030500000000004,0.161307,0.8319340000000001,0.745575,0.281943,-0.607283,0.280545,0.175512,-0.540877,-0.160348,-0.166973,-0.20968299999999998,-0.794513,-0.416437,-0.48389399999999994,-1.5508170000000001,-0.441597,-0.305759,-0.523014,-0.351441,-0.381429,-0.573596,-0.643205,-0.34568499999999996 -6,0,1.368983,0.322883,1.368325,1.27522,0.51864,0.021215,0.509552,1.196716,0.26247600000000004,-0.014730000000000002,1.170908,0.16064900000000001,1.138125,1.0952950000000001,-0.123136,0.088295,0.300072,0.646935,-0.064325,-0.762332,0.149883,-0.80494,0.15541,0.298627,-0.9090299999999999,-0.651568,-0.310141,-0.228089,-0.829666,-0.611218 -144,1,-0.8944290000000001,-0.807253,-0.877825,-0.772397,-1.085753,-0.839482,-0.837832,-1.225379,-0.971891,-0.9008290000000001,-0.959199,-1.005211,-0.9765799999999999,-0.8520620000000001,-1.311599,-1.003499,-0.83226,-1.058692,-1.5064469999999999,-0.838883,-0.551026,0.040181,-0.524762,-0.497174,-0.164666,-0.428569,-0.388392,-1.003127,-0.464009,-0.542754 -540,1,-0.830233,-0.976611,-0.8483370000000001,-0.743216,0.093432,-0.270137,-0.443716,-0.6916869999999999,-0.924975,-0.144403,-0.7348279999999999,-1.1285459999999998,-0.713374,-0.716683,0.24763600000000002,0.14515,-0.269044,-0.592724,0.023298,0.711976,-0.45754700000000004,0.999969,-0.612877,-0.428529,1.7030759999999998,0.874216,0.783709,0.509965,-0.259386,0.6494939999999999 -30,0,1.424896,1.356941,1.5857620000000001,1.387726,0.733436,1.090566,1.636491,1.068812,0.87885,0.768849,1.2788329999999999,1.354435,1.352314,1.2318120000000001,0.714481,1.5987280000000001,1.796625,1.946952,1.355892,-0.11732200000000001,1.53582,0.452038,1.340514,1.422695,-0.264337,0.46175200000000005,0.665338,-0.03506,-0.057185,0.28939899999999996 -435,0,0.159621,0.8342120000000001,0.197742,-0.019835,1.268234,0.652266,0.646282,1.036837,0.45013800000000004,1.194443,-0.041832999999999995,0.076875,-0.034972,-0.157532,0.6860149999999999,0.169787,0.298817,0.40524499999999997,-0.5206930000000001,0.37458600000000003,-0.665437,-0.478177,-0.625748,-0.47275200000000006,-0.575683,-0.42298,-0.33302,-0.361588,-1.006441,-0.35892399999999997 -31,0,0.11406300000000001,0.397791,0.361564,0.014269,1.37344,2.0562259999999997,2.031327,0.608969,3.009467,3.117372,-0.649623,-0.137215,-0.578271,-0.60946,1.034726,0.8956280000000001,0.41432299999999994,0.074825,1.786704,2.153679,0.279093,-0.339016,0.301444,0.014584999999999999,-0.49668,0.48410699999999995,0.336752,-0.219979,0.264884,0.708123 -203,0,0.60692,2.633636,0.6326149999999999,0.47888500000000006,3.955374,1.6961709999999999,0.923099,1.320051,2.477217,1.366232,-0.090115,1.037954,-0.016848,-0.162367,2.5576659999999998,1.373206,0.841192,1.1050360000000001,1.604157,1.617825,0.5761310000000001,1.293893,0.516288,0.272445,0.594364,0.313085,-0.025654000000000003,0.180679,-0.068082,0.268973 -59,1,-1.400331,-1.673582,-1.410693,-1.064738,1.7942650000000002,-0.829304,-0.74548,-1.072199,0.516467,0.349906,-1.564717,-1.7452189999999999,-1.549947,-1.224071,0.08253200000000001,-0.978294,-0.856115,-1.0606520000000002,-0.46957899999999997,1.286106,-0.899675,-1.156197,-0.900985,-0.704386,1.0230489999999999,-0.834327,-0.40066,-0.9820399999999999,0.04331,-0.380863 -95,0,1.6464740000000002,0.962859,1.454704,1.528359,-0.586024,0.633818,0.660194,0.650081,1.2752059999999998,-0.014730000000000002,1.7417759999999998,0.8704049999999999,1.66536,1.732374,-0.397123,0.5109130000000001,0.72443,0.977872,1.0346090000000001,-0.8941690000000001,1.271986,0.530054,0.8355819999999999,1.045806,-0.341673,0.522112,0.34570500000000004,0.144993,0.729825,0.233039 -37,1,-0.614867,-0.466909,-0.679153,-0.588344,-1.549975,-1.323648,-1.073966,-0.9817530000000001,-1.478256,-1.233324,-0.311646,-0.202373,-0.3855,-0.37283099999999997,-0.46473000000000003,-1.263703,-0.7932140000000001,-0.507861,-1.258183,-0.590802,-0.798617,2.041403,-0.839601,-0.57594,-0.8963629999999999,-1.150159,-0.612202,-0.025327000000000002,0.746776,-0.763275 -331,1,-0.38293499999999997,-0.606955,-0.239812,-0.43294399999999994,-0.156432,0.45188199999999995,0.344038,-0.244025,1.1247530000000001,0.42748800000000003,-0.325847,0.014044,-0.306827,-0.40070300000000003,-0.040583999999999995,0.154625,-0.22259099999999998,-0.500897,-0.184806,0.36891599999999997,-0.49363900000000005,-1.180146,-0.198537,-0.43314899999999995,-0.438011,0.395243,0.39908699999999997,-0.322008,0.8436389999999999,0.29393800000000003 -237,0,1.6464740000000002,0.080246,1.621505,1.528359,-0.41944799999999993,-0.147362,0.209227,0.45518100000000006,-1.072193,-0.702441,1.804259,0.505055,1.6694790000000002,1.8518259999999998,-0.9116489999999999,-0.395347,0.020347999999999998,0.29149400000000003,-1.258183,-1.5632780000000002,1.01862,-0.31905900000000004,1.12765,0.949658,0.306019,0.327058,0.35399400000000003,0.797076,-0.863568,0.052235000000000004 -10,0,0.604849,1.3357709999999998,0.492622,0.47361099999999995,-0.6254770000000001,-0.6308279999999999,-0.6058720000000001,-0.22621,0.076431,0.031819,0.537556,0.919273,0.442011,0.406453,-1.017686,-0.713542,-0.700684,-0.404686,-1.035476,-0.826124,-0.09265599999999999,-0.054164,-0.198042,0.003805,-1.004034,-0.9059209999999999,-0.692442,-0.682114,-0.719485,-0.284787 -389,0,0.942393,0.7755890000000001,1.034724,0.760151,-0.318625,-0.08184,0.531141,1.033791,-0.525383,-0.439216,1.540126,0.912292,1.5211940000000002,1.476405,0.33018800000000004,0.520389,1.21533,1.371746,0.647609,-0.561032,0.741794,2.9376889999999998,1.245963,0.6548350000000001,1.39973,0.865833,1.4189969999999998,3.668187,0.8642219999999999,0.924861 -324,1,-0.52168,-0.6997760000000001,-0.48107700000000003,-0.522949,-0.296707,-0.391004,-0.7459600000000001,-0.899073,-0.387872,-0.24027300000000001,-0.5473779999999999,-0.949362,-0.574975,-0.560257,-0.685343,-0.737042,-0.864527,-0.825385,-0.633872,-0.213719,-0.532979,-0.743071,-0.44902200000000003,-0.469232,-0.5460149999999999,-0.631448,-0.70902,-1.079365,-0.8732540000000001,-0.654338 -179,1,-0.5465300000000001,-1.551449,-0.612433,-0.544747,-0.708765,-1.271103,-1.1735149999999999,-1.137522,-1.808284,-0.596042,-0.37413,-1.449681,-0.43987200000000004,-0.41549200000000003,-0.638374,-1.2621870000000002,-0.999455,-0.91876,-1.2618340000000001,-0.20804899999999998,-0.41965,-0.411772,-0.538623,-0.408067,0.49769399999999997,-1.06828,-0.85259,-0.7117979999999999,-1.197745,-0.167907 -220,1,-0.192419,-1.51888,-0.224919,-0.30655,-0.05561,-0.043670999999999995,-0.461946,-0.518408,-0.842468,0.179225,-0.13555799999999998,-1.42641,-0.168428,-0.244562,0.007097,-0.32655300000000004,-0.626736,-0.60072,-1.648834,0.091065,-0.703695,-1.4210909999999999,-0.7301989999999999,-0.504655,-0.969366,-0.476634,-0.5097470000000001,-0.8332930000000001,-0.8272450000000001,-0.40091 -50,1,-0.681134,0.006966,-0.723236,-0.640026,-1.046301,-1.069447,-1.0408629999999999,-1.179395,-0.756726,-1.0149860000000002,-0.6723439999999999,0.5376340000000001,-0.710491,-0.64558,-0.710963,-1.036286,-0.9068370000000001,-0.974217,-1.155957,-0.555362,0.00371,-0.012434,-0.114381,-0.261096,-0.394676,-0.878591,-0.670227,-0.705796,0.425918,-0.766301 -401,1,-0.5113260000000001,-0.901702,-0.584434,-0.511699,0.220556,-0.6155609999999999,-0.579486,-0.698082,-0.713046,-0.627629,-0.624062,-1.95,-0.652001,-0.603488,-0.543724,-0.983979,-0.78769,-0.79856,-0.768957,-1.0472709999999998,-0.552108,-0.311801,-0.602482,-0.470552,-0.288671,-0.751051,-0.60126,-0.967441,-1.16021,-0.733015 -368,0,2.998723,0.12421300000000002,2.74741,3.977131,0.172337,-0.581845,0.066741,1.026178,-0.632157,-1.053223,2.153597,-0.47463999999999995,2.015476,2.534411,-0.179357,-0.35479,0.351548,0.9216409999999999,-0.345447,-1.737644,2.893965,-0.300915,2.4092849999999997,4.0430980000000005,-0.49101300000000003,-0.80247,-0.362529,0.028201999999999998,-0.958009,-0.75344 -29,0,0.774656,-1.002666,0.8232440000000001,0.608971,-0.301091,0.171344,-0.11172699999999999,0.47193,-0.234183,-0.263547,0.977778,-0.9865950000000001,0.94865,0.8538309999999999,0.15013900000000002,0.21527,0.124931,0.7895760000000001,-0.265127,-0.185367,0.7042579999999999,-0.715493,0.8855799999999999,0.45682,-0.47134499999999996,0.27116799999999996,0.072159,0.282871,-0.15647,-0.020011 -78,0,1.4704540000000001,0.984029,1.877663,1.305104,1.382207,2.303684,2.379147,2.073768,4.10794,0.869706,1.719055,1.089149,2.130809,1.678336,2.2943540000000002,4.568425,3.5982629999999998,2.8755349999999997,3.9955239999999996,2.639918,1.90035,1.212248,2.862733,1.673514,1.1130520000000002,2.396089,2.559931,2.2926490000000004,7.071917,0.8291629999999999 -204,1,-0.26904,-0.168905,-0.333935,-0.356299,0.448503,-0.10474100000000001,-0.024412,-0.19956300000000002,0.183204,0.196958,-0.47069399999999995,-0.160486,-0.44811000000000006,-0.49199899999999996,0.234114,0.027651,-0.109847,-0.27623200000000003,0.41394899999999996,0.132176,-0.032743,-0.313616,-0.182696,-0.22105300000000003,-0.029327,-0.355912,-0.161929,-0.23133299999999998,-0.329612,-0.07901799999999999 -214,0,0.12234600000000001,1.49373,0.230506,-0.121794,1.03152,0.9646100000000001,0.49036199999999996,0.95309,2.949609,1.0337370000000001,0.01781,1.051916,0.037111,-0.12567799999999998,-0.123136,0.497647,0.285006,0.404988,1.545741,0.217232,0.056044000000000004,1.139674,0.33065100000000003,-0.205432,1.279726,0.64954,0.16533,0.725704,2.960091,0.368075 -127,0,1.253017,0.008594,1.219396,1.155681,-1.326851,-0.177261,0.232735,0.109537,-0.09667200000000001,-1.027177,1.383918,-0.08834700000000001,1.294648,1.374017,-1.009858,-0.455991,0.049099000000000004,0.189608,0.49061899999999997,-1.7518200000000002,1.0265600000000001,0.227059,1.1632930000000001,0.8997139999999999,-0.871028,0.102382,0.282043,0.29422600000000004,-0.029337000000000002,-0.419444 -46,1,-1.512777,-0.6053270000000001,-1.489328,-1.1222219999999998,-0.11698,-0.754239,-0.9757610000000001,-1.354653,0.330422,-0.546168,-1.6845709999999998,-0.57005,-1.658278,-1.288347,-0.737294,-0.8511299999999999,-0.9155,-1.109197,-0.15559800000000001,0.316465,-0.8982319999999999,-0.472008,-0.877224,-0.706961,0.642366,-0.50402,-0.530967,-0.9536530000000001,0.6293300000000001,-0.458783 -554,1,-0.49268900000000004,1.6386610000000001,-0.548691,-0.5008,-0.423831,-0.586935,-0.135715,-0.7564,-0.855411,-0.638713,-0.354249,2.241047,-0.390031,-0.39985,-1.076753,-0.873682,-0.337092,-0.657467,-0.89674,-0.810531,-0.698642,0.25971700000000003,-0.675251,-0.517196,0.45702600000000004,-0.22066,0.23496,-0.6774100000000001,-0.43495,-0.375945 -449,0,1.948814,1.041024,1.8151130000000002,2.006511,0.19425499999999998,0.355188,0.8046,1.7266009999999998,-1.02366,-0.5373020000000001,1.980348,0.28631100000000004,1.9001439999999998,2.073666,0.034138999999999996,0.249383,0.858769,1.717385,-0.940551,-0.877158,0.9352469999999999,0.261531,0.829642,0.914235,-0.5246810000000001,-0.264254,-0.00145,0.464546,-1.241333,-0.601383 -564,0,1.901185,0.1177,1.752563,2.015301,0.378365,-0.273318,0.664512,1.629151,-1.360158,-0.709091,2.110995,0.7214729999999999,2.0607860000000002,2.343856,1.041842,0.21906,1.947285,2.320965,-0.312589,-0.931027,2.78208,0.071025,2.3795830000000002,2.604187,1.086384,0.191805,0.6660010000000001,2.0671779999999997,-1.138416,0.16798 -297,0,-0.602442,-0.37246,-0.66009,-0.574808,-0.818354,-1.110223,-1.012222,-0.654838,-1.492816,-0.8199219999999999,-0.6723439999999999,-0.267531,-0.698958,-0.636479,0.236249,-0.856626,-0.777772,-0.355161,-0.70324,0.010262,0.8655889999999999,1.6114030000000001,0.62965,0.193019,-0.481679,-0.8622719999999999,-0.6357430000000001,0.409395,-0.580244,-0.39221 -491,1,0.7353109999999999,-1.181794,0.590915,0.579086,-1.4794,-0.9828680000000001,-0.80305,-0.475012,-1.808284,-1.398463,1.057302,-1.410121,0.932174,0.9590620000000001,-1.279575,-0.799203,-0.556804,-0.184147,-2.159966,-1.469717,0.282341,-0.309987,0.14699500000000001,0.233502,-0.8906959999999999,-0.9615309999999999,-0.6752,-0.707094,-0.910789,-0.9402959999999999 -492,0,1.089422,0.062333000000000006,1.076424,0.9587950000000001,-0.064377,-0.137184,-0.085341,0.5221779999999999,0.5666180000000001,-0.42646999999999996,1.102744,0.295619,1.088697,1.0014399999999999,0.266139,0.46543,0.354059,0.740309,1.1112790000000001,-0.287435,1.261881,0.129084,1.231112,1.086949,0.318687,0.08505599999999999,0.181577,0.759768,1.020413,0.076065 -181,0,2.155897,1.270634,2.062335,2.124291,0.733436,3.207003,1.9468900000000002,2.675218,1.936879,2.4634650000000002,1.9775080000000003,1.6941869999999999,2.089619,1.866047,1.2624549999999999,3.389643,2.007548,2.5969599999999997,2.129892,1.58522,0.810729,-0.823628,0.766278,0.904775,-0.929364,1.235822,0.22633899999999998,0.628378,-0.310239,0.5674130000000001 -427,1,-0.726692,1.036139,-0.702088,-0.6874899999999999,-0.090679,-0.538588,-0.381348,-0.605352,0.103933,-0.405966,-0.944998,0.626063,-0.954749,-0.8389790000000001,-0.594251,-0.889033,-0.661136,-0.8996719999999999,0.746185,-0.429195,-0.35179699999999997,0.7332609999999999,-0.30991799999999997,-0.44305,-0.166,-0.22345399999999999,-0.065774,-0.218356,-0.25454299999999996,-0.41793100000000005 -471,1,-0.552743,1.246207,-0.596349,-0.550197,-1.239179,-0.998771,-1.040815,-0.900443,-0.803642,-0.9728700000000001,-0.59282,2.059536,-0.6227560000000001,-0.58301,-0.629123,-0.840327,-0.817697,-0.6486970000000001,0.15473199999999998,-0.824707,0.725191,2.58752,0.6103430000000001,0.101712,0.15868,-0.554879,-0.57175,0.005493,-0.008754000000000001,0.033322000000000004 -515,1,-0.7867460000000001,-0.431084,-0.837316,-0.706651,0.698367,-0.616197,-0.526713,-0.444407,0.25762199999999996,-0.893071,-0.7916300000000001,-0.158159,-0.7912239999999999,-0.749959,0.607733,-0.36673,-0.574758,-0.592724,0.42125100000000004,-0.097476,-0.585313,-0.375303,-0.680696,-0.487274,0.5120279999999999,-0.506814,-0.361535,-0.11778599999999999,0.45982,-0.975096 -445,1,-0.681134,0.7625609999999999,-0.678558,-0.644597,-0.05561,-0.458434,-0.533909,0.085174,-0.488175,-0.079567,-0.607021,1.303239,-0.5914510000000001,-0.6074689999999999,0.47251899999999997,-0.230469,-0.43175699999999995,-0.159385,0.0306,0.808373,-0.515655,-0.023321,-0.495555,-0.460872,0.426358,-0.292198,-0.283616,0.787343,0.048152999999999994,-0.045732 -506,1,-0.643859,-0.245442,-0.659197,-0.6421359999999999,0.34329699999999996,-0.144817,0.390094,-0.513536,-0.310219,0.24627800000000002,-0.5416979999999999,0.174612,-0.514838,-0.573909,0.94221,0.205794,-0.088504,-0.703122,1.140487,0.870748,-0.808723,-0.763754,-0.9308350000000001,-0.610703,-0.256337,-0.195509,0.25750700000000004,-0.30854499999999996,-0.8889950000000001,0.006845 -93,1,-0.242119,0.042792000000000004,-0.288065,-0.318504,0.067131,-0.5036,-0.643293,-0.5404869999999999,-0.36036999999999997,-0.99282,-0.19236,-0.230298,-0.221152,-0.28381,0.41558599999999996,-0.43002799999999997,-0.615938,-0.544747,-0.633872,-0.8076949999999999,-0.397634,0.283304,-0.379718,-0.332602,-0.385675,-0.590648,-0.436801,-0.394192,-0.206112,-0.748145 -453,1,-0.097161,-1.424431,-0.123945,-0.22973000000000002,0.102199,-0.677266,-0.647131,-0.11734000000000001,-0.47685,-0.32395100000000004,0.114375,-1.2355909999999999,0.077889,-0.030400999999999997,0.9635600000000001,-0.22591999999999998,-0.24920799999999999,0.4135,-0.59006,-0.22505999999999998,-0.357933,-0.8991040000000001,-0.35793600000000003,-0.32204099999999997,-0.302672,-0.724783,-0.522015,-0.070746,0.184972,-0.08771799999999999 -357,1,-0.240048,-0.015832,-0.31338299999999997,-0.32729400000000003,-0.748217,-0.976252,-1.052282,-0.899073,-0.871589,-0.710199,-0.073075,-0.716655,-0.142066,-0.174028,-0.6355270000000001,-0.936601,-0.9262969999999999,-0.723241,-1.415174,-0.56245,-0.544529,0.26516,-0.5589189999999999,-0.43116899999999997,-0.467679,-0.9802540000000001,-0.883294,-0.933377,-0.617779,-0.646017 -544,1,-0.252473,-0.15099300000000002,-0.241004,-0.33749,-0.261639,-0.321664,-0.645212,-0.702802,-1.054398,0.053985000000000005,-0.073075,0.328198,-0.09057899999999999,-0.19934100000000002,-0.041296,-0.048155,-0.651846,-0.65076,-0.699589,0.578721,-0.480646,-0.308173,-0.391104,-0.378805,-0.24766999999999997,-0.21004099999999998,-0.190445,-0.443666,-0.683161,-0.074101 -26,0,0.279729,1.226666,0.450921,0.028683999999999998,0.8824780000000001,2.6083950000000002,1.351518,2.367641,2.20543,2.413591,0.128576,0.521345,0.224115,-0.028694,0.643316,1.56272,0.674211,1.003666,1.607807,0.913276,-0.5438069999999999,-0.42392799999999997,-0.374272,-0.424349,-0.863028,0.283464,-0.168561,0.279627,-0.726749,-0.031737 -356,1,-0.43056400000000006,-0.134708,-0.388443,-0.50871,0.084665,0.073378,-0.07142799999999999,0.170443,0.34336500000000003,-0.042993,-0.30596599999999996,-0.162813,-0.28334899999999996,-0.406391,0.8425790000000001,0.493857,0.090781,0.183417,0.815553,0.313629,-0.34133,0.5318689999999999,-0.136658,-0.412908,0.255351,0.7736149999999999,0.637486,1.1296059999999999,0.975614,0.726279 -227,1,0.029158999999999997,-1.0368629999999999,0.20667800000000003,-0.127243,-0.822738,0.6897979999999999,0.326287,0.354685,0.086138,-0.018055,0.247862,-0.87955,0.225762,0.084216,-0.9002629999999999,0.09966599999999999,-0.298172,-0.28680700000000003,0.253307,-0.5284270000000001,-0.625736,-1.3067870000000001,-0.292097,-0.45009099999999996,-0.974032,0.36841599999999997,0.150741,-0.039925999999999996,-0.803029,-0.22351 -377,1,-0.327023,1.6207479999999999,-0.30236199999999996,-0.351553,-0.9454790000000001,-0.690625,-0.9251950000000001,-0.8648129999999999,-0.334485,-0.739015,-0.18952,2.0758259999999997,-0.250397,-0.263902,-1.508016,-1.081769,-0.955299,-0.973701,-1.4261270000000001,-0.7325619999999999,-0.8527549999999999,-0.121295,-0.725744,-0.559439,-0.699688,-0.75161,-0.808558,-1.073364,-0.741279,-0.798452 -234,1,-1.192628,-1.061289,-1.236744,-0.9575049999999999,0.790423,-1.012194,-0.9622799999999999,-0.6453979999999999,-0.23256500000000002,-0.120021,-1.295188,-0.786467,-1.3081610000000001,-1.067361,-0.8340790000000001,-1.202869,-0.907465,-0.831834,-0.9515040000000001,0.174704,-0.685649,-0.701704,-0.817325,-0.609383,1.533069,-0.8427100000000001,-0.664258,-0.352504,0.39807,-0.09641799999999999 -341,1,-1.1429280000000002,-0.42457,-1.072624,-0.9263899999999999,-0.39753,0.5555720000000001,0.776774,-0.508663,0.131435,0.792678,-1.284111,-0.57005,-1.2492590000000001,-1.064801,-0.821981,-0.228573,-0.057492999999999995,-0.670622,0.819204,1.198214,-0.796813,-0.497046,-0.711388,-0.621924,-0.36234099999999997,0.515964,0.609634,-0.533043,-0.368357,0.089304 -129,0,1.317213,1.286918,1.234289,1.245335,-0.213419,0.8386549999999999,1.415804,0.892184,0.653979,0.039023,1.60829,1.356762,1.5829790000000001,1.527599,0.36577,1.033974,2.080367,1.7019090000000001,1.42526,-0.23640100000000003,0.325291,-0.032392000000000004,-0.050026999999999995,0.505884,-0.669353,0.352767,0.519116,-0.22160100000000002,-0.576612,-0.21594499999999997 -289,1,-0.809525,0.07536,-0.8331459999999999,-0.740579,-0.9016430000000001,-0.999916,-0.944625,-0.800557,0.592503,-0.776144,-0.7831100000000001,-0.093001,-0.815526,-0.7363069999999999,-0.656877,-1.028326,-0.813679,-0.7013159999999999,0.735232,-0.46038199999999996,-0.503745,1.373724,-0.45149700000000004,-0.502675,-0.167666,-0.644302,-0.60126,-0.303679,1.648811,-0.327908 -539,1,-1.572003,1.0117120000000002,-1.571835,-1.154919,1.193713,0.33165100000000003,0.321969,-0.983733,-0.179178,1.2554,-1.827998,1.4312280000000002,-1.797089,-1.3779370000000002,-0.688901,0.29486599999999996,0.046713,-0.90999,0.8228549999999999,2.085634,-0.669769,0.475624,-0.703468,-0.629405,2.809788,2.1848259999999997,2.0101869999999997,0.299092,0.061472000000000006,1.420748 -145,1,-0.64593,-1.492825,-0.6255390000000001,-0.6521560000000001,0.43973599999999996,-0.016316999999999998,-0.8539040000000001,-0.8250719999999999,-0.281099,1.089153,-0.632582,-1.079677,-0.570856,-0.631644,1.340737,0.478696,-0.649083,-0.48722600000000005,0.669515,2.210384,-0.032382,-1.021573,0.0767,-0.336782,1.043049,1.225762,-0.132751,-0.11292,1.298894,2.128835 -138,0,0.47231599999999996,-0.691634,0.42113599999999995,0.159648,0.382749,-0.240875,0.303738,0.793211,0.830317,-0.691358,0.233661,-0.400174,0.201048,0.066014,1.4474850000000001,0.495752,0.817337,0.9626530000000001,0.530779,-0.090388,3.215185,0.426637,2.7488759999999997,1.35449,0.9863799999999999,0.520995,1.123237,2.630045,3.7701059999999997,-0.343416 -410,1,-0.6666380000000001,1.73311,-0.660984,-0.631588,0.56686,-0.585662,-0.43699899999999997,-0.420653,0.116875,-0.359971,-0.78595,-0.400174,-0.802345,-0.7255,-0.553687,-0.9705239999999999,-0.765468,-0.720146,-0.768957,-0.519922,-0.770826,0.613514,-0.7460399999999999,-0.5869409999999999,-0.5500149999999999,-0.868028,-0.6718850000000001,-0.960952,-0.863568,-0.807909 -56,0,2.0440720000000003,0.401048,1.871706,2.222734,1.316453,0.616006,0.555129,1.438819,1.029304,-0.055738,1.443561,-0.167467,1.381147,1.413834,0.636199,0.42373599999999995,0.54615,1.058091,0.384741,-0.45187700000000003,1.163349,-0.043278,0.975676,1.367691,-0.19433399999999998,-0.135149,-0.081026,0.581337,-0.24485700000000002,-0.44894799999999996 -216,1,-0.625221,0.23169,-0.627326,-0.614185,0.35644699999999996,0.320201,0.040354,-0.012276,0.802814,-0.016393,-0.635423,-0.21866300000000002,-0.6033970000000001,-0.633351,-0.194302,0.209585,-0.281725,-0.45137299999999997,0.567289,0.048537000000000004,-0.392942,0.628029,-0.385658,-0.415328,0.6103649999999999,0.9189280000000001,0.9140159999999999,0.9819950000000001,0.894492,0.370723 -281,1,-0.612797,-1.207849,-0.672005,-0.6097899999999999,-1.261097,-1.076762,-0.9827170000000001,-0.482778,0.32395100000000004,-0.945716,-0.678025,-1.226283,-0.730262,-0.647287,-1.297366,-1.154922,-0.8330139999999999,-0.549132,1.0565149999999999,-0.23640100000000003,0.5656640000000001,0.092797,0.421242,-0.05516,0.33102,-0.618034,-0.555503,1.080943,1.049472,-0.677034 -16,0,0.5799989999999999,0.84724,0.480707,0.452516,0.615079,-0.42726400000000003,0.092168,0.704897,0.20747100000000002,-0.098963,0.156977,0.195555,0.11413699999999999,0.084216,0.164372,-0.612909,-0.186433,0.094686,-0.8237209999999999,-0.507163,0.243723,0.041996,0.16283599999999998,0.111393,-0.441011,-0.774525,-0.395023,-0.114542,-0.7800239999999999,-0.6467729999999999 -444,0,0.8512770000000001,-0.5955560000000001,0.775587,0.7232350000000001,-0.266022,0.078468,0.752307,0.59222,-0.095054,-0.093975,1.108425,-0.567723,1.051626,0.95309,-0.49035,0.357406,0.253619,0.35133600000000004,-0.334494,-0.7084630000000001,-0.384641,-1.136239,-0.467833,-0.100484,-0.974699,-0.552085,-0.071079,-0.331417,-0.919264,-0.514007 -547,1,-1.1263610000000002,-0.592299,-1.077688,-0.91971,0.601928,-0.188711,-0.45043199999999994,-0.47623000000000004,-0.339339,0.600939,-1.098366,-0.6305529999999999,-1.075848,-0.950184,-0.540166,-0.44878999999999997,-0.5677270000000001,-0.632962,-0.5206930000000001,0.615579,-1.0494569999999999,-0.351717,-0.92935,-0.7263,1.076384,0.299672,-0.191108,-0.134007,0.269727,0.792473 -270,1,-0.281464,-0.818652,-0.381891,-0.344521,-2.0470740000000003,-1.297121,-1.120358,-1.23756,-0.716282,-1.260478,0.046211,-0.574704,-0.068748,-0.06339199999999999,-2.282296,-1.470464,-1.023849,-1.100607,-1.108494,-1.281175,-0.992432,-0.901826,-1.001031,-0.650526,-1.183041,-1.216611,-0.8974850000000001,-1.328195,-0.6274649999999999,-0.913062 -44,0,-0.008116,0.686025,-0.052459000000000006,-0.245902,0.786039,0.866009,0.48268599999999995,0.701851,1.281677,0.677967,-0.271884,0.586503,-0.269756,-0.350931,0.055489,0.006804000000000001,-0.077958,0.09287999999999999,-0.23956999999999998,-0.145674,-0.762886,-1.096868,-0.758416,-0.56868,-1.230376,-0.65045,-0.5760609999999999,-0.801825,-1.139627,-0.7848350000000001 -34,0,0.8160729999999999,0.257745,0.757716,0.66874,0.536175,2.0746740000000004,1.224383,1.093175,2.215137,2.180844,0.5687979999999999,-0.32803499999999997,0.619129,0.433188,0.5436840000000001,0.9771200000000001,0.58507,0.73773,0.6804680000000001,0.333476,-0.256875,-0.9636950000000001,-0.33813499999999996,-0.11676500000000001,-0.952032,0.178951,-0.174198,-0.44269200000000003,-0.42526400000000003,0.008357999999999999 -371,1,-0.014328,-1.619844,-0.082245,-0.108082,-0.866574,-0.512506,-0.652408,-0.499832,-0.669366,-0.902492,0.301824,-1.414775,0.234,0.16186,-1.190618,-0.66332,-0.688883,-0.5764729999999999,-0.330843,-1.043018,-0.818829,-1.459374,-0.756436,-0.49761400000000006,-0.676354,-0.594002,-0.543235,-0.428418,-0.493068,-0.767057 -310,1,-0.757754,0.142126,-0.7845949999999999,-0.698741,-0.441366,-0.925997,-0.926107,-0.8709040000000001,0.948415,-0.796094,-0.689385,-0.041805,-0.7265550000000001,-0.671746,-0.585,-0.9818950000000001,-0.916128,-0.9657049999999999,0.45410900000000004,-0.215137,-0.884517,0.386721,-0.869798,-0.639305,-0.325673,-0.914808,-0.712005,-0.6744899999999999,0.356903,-0.8071520000000001 -538,1,-1.489377,0.8537540000000001,-1.492009,-1.112026,-0.296707,-1.08694,-1.305831,-1.745063,0.254387,0.855298,-1.817206,1.442863,-1.8119169999999998,-1.3540459999999999,-1.094545,-1.0529629999999999,-1.114873,-1.2618200000000002,0.21314699999999998,1.425031,-0.099152,0.44478,-0.185171,-0.466372,1.8730830000000003,-0.88228,-1.057501,-1.9134470000000001,1.0022520000000001,1.163916 -72,0,1.4601,1.326001,1.320668,1.407063,1.145493,3.086136,1.844223,1.146468,0.666921,2.768252,0.872693,1.2171379999999998,0.9156979999999999,0.780737,0.764297,1.490705,1.009428,0.787254,0.42125100000000004,0.293783,0.669609,-0.31905900000000004,0.41530100000000003,0.6409739999999999,-0.407009,1.714795,0.35233600000000004,-0.085345,-0.638362,0.9471780000000001 -73,0,0.062292999999999994,-0.784455,0.090513,-0.11986,0.382749,0.635726,0.027401,0.36077600000000004,-0.504352,1.055903,-0.092956,-0.814392,-0.06339299999999999,-0.20133099999999998,0.308838,0.44837299999999997,-0.136966,0.045676999999999995,-0.546249,0.40577399999999997,-0.45646400000000004,-1.08199,-0.45001199999999997,-0.373745,-0.774691,-0.269842,-0.47426899999999994,-0.420145,-1.010073,-0.25150100000000003 -279,1,-0.266969,-0.641152,-0.264832,-0.370187,-0.607942,-0.520776,-0.6073109999999999,-0.223469,0.087755,-0.821585,-0.07875499999999999,-0.956343,-0.122707,-0.191946,-0.08541900000000001,-0.520426,-0.5525359999999999,-0.304605,1.089373,-0.604978,-0.567628,-0.539501,-0.512881,-0.46043199999999995,-0.935364,-0.562704,-0.574403,-0.22160100000000002,-0.637151,-0.832873 -196,0,0.025018000000000002,1.356941,0.129234,-0.129529,1.8117990000000002,0.36854699999999996,0.521546,0.802347,0.289978,0.5200319999999999,-0.101476,0.698202,-0.055154999999999996,-0.18767999999999999,1.682331,0.42373599999999995,0.623991,0.421496,0.081713,0.846648,0.772111,1.624104,1.009833,0.206,2.253098,0.44722,0.489274,1.428072,0.76857,0.193323 -74,1,-0.44713,-0.40177199999999996,-0.522778,-0.473728,-0.6473939999999999,-0.445075,-0.48881199999999997,-0.426439,-0.457437,-0.435337,-0.516137,-0.644516,-0.526371,-0.523284,-0.330228,-0.6832189999999999,-0.691519,-0.67578,-0.334494,-0.518504,-0.558244,-0.348088,-0.557434,-0.454491,-0.729023,-0.407331,-0.452053,-0.621447,-0.809083,-0.563936 -23,0,2.671532,1.614234,2.404872,3.048953,0.338913,0.036482,0.207788,1.3139610000000002,-0.127409,-0.481332,1.9973889999999999,0.872732,1.863073,2.130548,-0.148044,-0.040575,0.262407,0.964717,-0.15559800000000001,-1.4201,1.03414,-0.163025,0.71133,1.180456,-0.7710239999999999,-0.720312,-0.488858,-0.22971100000000003,-1.175951,-0.6838420000000001 -334,1,-0.604513,0.453158,-0.677068,-0.591156,-0.44574899999999995,-1.041647,-1.132208,-1.138587,-0.560975,-0.658108,-0.518977,-0.062749,-0.58033,-0.541771,-0.9415389999999999,-1.181075,-1.017496,-1.041668,-0.995315,-0.474559,-0.7982560000000001,0.5717840000000001,-0.8252450000000001,-0.596182,0.280018,-0.952701,-0.825269,-0.8555159999999999,-0.139519,-0.594953 -400,0,0.9382520000000001,0.34242399999999995,1.261096,0.74433,2.4079669999999997,2.146558,3.028252,1.245441,0.556912,1.986889,1.074343,0.40266399999999997,1.3358379999999999,0.9644659999999999,1.8958270000000002,2.9044849999999998,2.888907,1.8282990000000001,1.100326,1.184038,-0.007839,-0.802218,0.127194,0.025806,0.039342,0.654012,0.9866299999999999,-0.20862399999999998,-0.560872,0.49327600000000005 -431,1,-0.701842,-0.450625,-0.525756,-0.641257,0.553709,0.05492999999999999,-0.15298599999999998,-0.622863,-0.557739,0.5344399999999999,-0.490575,-0.374576,-0.432457,-0.532101,0.643316,0.516599,-0.142993,-0.539846,-0.002259,1.1656090000000001,-0.8246040000000001,0.44115200000000004,-0.32774000000000003,-0.547998,0.9863799999999999,0.417599,0.554262,-0.020461,0.16075599999999998,0.8359719999999999 -254,0,1.9529560000000001,-0.180304,1.6632049999999998,1.9186159999999999,0.759738,0.393357,0.76526,1.298734,0.773694,0.30779,1.511725,0.00939,1.4223370000000002,1.462184,0.508101,0.27402,0.616458,0.954141,-0.13004200000000002,-0.8970049999999999,0.688377,-1.057134,0.460844,0.674637,-0.797359,-0.41795,-0.146014,0.141749,-0.8345090000000001,-0.470887 -48,1,-0.519609,-0.8105100000000001,-0.517714,-0.523828,0.746587,-0.245964,0.157414,-0.748025,-0.248743,-0.051859,-0.5899800000000001,-1.084331,-0.5737399999999999,-0.584717,0.47963500000000003,-0.254348,-0.287249,-0.552743,-0.498787,-0.335633,-0.5109630000000001,-0.884408,-0.50397,-0.45031099999999996,-0.5176810000000001,-0.626418,-0.287595,-0.995341,-0.7582300000000001,-0.519303 -130,1,-0.606584,-1.281128,-0.47303500000000004,-0.5895739999999999,0.45288599999999996,0.02694,-0.8301559999999999,-0.498461,0.9192950000000001,0.46905,-0.550218,-1.396158,-0.5309020000000001,-0.56623,0.7287140000000001,-0.17532,-0.756428,-0.518437,0.24965700000000002,0.27110100000000004,-0.738704,-0.726742,-0.442092,-0.552179,-0.08932999999999999,-0.051314,-0.719962,-0.600522,0.735879,0.131668 -555,1,-1.12429,1.5035,-1.1226639999999999,-0.919359,0.264392,-0.529682,-0.346326,-0.355331,-1.091607,-0.061834,-1.0898450000000002,1.936202,-1.0832620000000002,-0.948477,-0.43128299999999997,-0.526112,-0.3617,-0.55558,-0.798164,-0.21655500000000003,-0.668686,1.854525,-0.707428,-0.56934,1.669741,0.105176,0.535363,0.878181,-0.255754,0.432378 -520,1,-1.1802030000000001,-1.276243,-1.174194,-0.9738540000000001,2.3071450000000002,-0.283496,-0.8262700000000001,-0.639307,1.262264,0.325523,-1.372439,-1.254207,-1.318458,-1.129362,2.89926,0.34414,-0.696541,-0.637347,1.407006,2.007666,-0.18541300000000002,-0.157582,-0.236654,-0.45559099999999997,2.806454,-0.004367,-0.329041,0.649465,2.352277,0.040131 -122,0,2.019222,-0.274754,2.193393,2.096165,1.632072,1.082296,1.478172,1.677876,0.519703,-0.213673,2.874993,0.21184499999999998,3.057588,3.145893,3.440117,3.455973,4.243589,3.9279300000000004,3.079138,0.846648,3.983947,3.4529620000000003,3.435978,4.238914,5.429894,4.056567,3.179967,1.0420129999999999,3.018209,2.299805 -496,1,-0.391218,-0.574386,-0.35657300000000003,-0.43399899999999997,0.9175469999999999,0.8265680000000001,0.412642,0.397319,0.532645,1.0725280000000001,-0.419572,-0.26055,-0.382205,-0.481476,0.79988,0.550711,-0.108341,0.046967,-0.622919,0.814044,-0.623571,-1.058948,-0.5792149999999999,-0.482653,-0.44567799999999996,-0.025605000000000003,-0.183482,-0.23944400000000002,-0.35746,-0.08771799999999999 -225,1,0.103709,-1.429316,0.09349099999999999,-0.012979,-0.11698,-0.647368,-0.522875,-0.08993200000000001,0.260858,-1.2870780000000002,0.060412,-1.354271,0.022283,-0.038932999999999995,0.192127,-0.532555,-0.396226,-0.074523,0.96159,-1.179108,0.42165699999999995,-0.734362,0.444013,0.174978,0.016008,-0.625859,-0.274995,0.172569,0.25640799999999997,-0.8820459999999999 -326,1,-0.15307300000000001,-1.2501879999999999,-0.263939,-0.22973000000000002,-0.187118,-0.9120020000000001,-1.051226,-0.848216,-1.295447,-0.726824,-0.004911,-1.491568,-0.07986900000000001,-0.10918299999999999,-0.232731,-0.971851,-0.893278,-0.55687,-1.601372,-0.8218719999999999,-0.534423,-0.24648499999999998,-0.647529,-0.36120399999999997,-0.11633099999999999,-0.791292,-0.867942,-0.838484,-0.772759,-0.49887700000000007 -106,1,-0.648001,0.583433,-0.647878,-0.630885,1.5970030000000002,0.074651,0.072498,0.109537,-0.153294,0.389251,-0.706426,-0.22331700000000002,-0.691956,-0.689379,1.269571,-0.050051,-0.22723600000000002,-0.36289899999999997,-0.038768000000000004,0.340564,-0.357933,0.7985770000000001,-0.35199600000000003,-0.433809,0.49969399999999997,-0.132913,-0.081026,0.354244,-0.592352,0.017058 -381,1,-0.8654370000000001,-0.7893399999999999,-0.82004,-0.762026,-1.002465,-0.356652,-0.560775,-0.716658,0.48734700000000003,-0.613775,-0.876835,-1.014519,-0.8773110000000001,-0.802575,-1.173538,-0.635841,-0.669674,-0.726852,0.698723,-0.04786,-0.869719,-0.337202,-0.7846529999999999,-0.630505,-0.581683,-0.36038400000000004,-0.486205,-0.784468,-0.21095500000000003,-0.701242 -415,1,-0.6666380000000001,0.24960300000000002,-0.660388,-0.628776,0.448503,-0.22624299999999997,-0.7473989999999999,-0.486889,0.281889,-0.578309,-0.635423,0.43756999999999996,-0.6417039999999999,-0.6288,0.097477,-0.43855600000000006,-0.794093,-0.699769,0.757138,0.014515,-0.470901,-0.025135,-0.46337799999999996,-0.457791,0.9513790000000001,0.282346,-0.517041,-0.40879099999999996,0.24672199999999997,-0.57604 -71,1,-1.353531,-1.629614,-1.3314629999999998,-1.048038,-0.5115029999999999,-0.067845,-0.617866,-1.016318,-1.046309,1.355149,-1.488033,-1.0820040000000002,-1.366651,-1.168611,0.104593,0.924055,-0.034392,-0.521016,0.329977,3.82787,0.43681499999999995,-0.661607,0.14947,-0.327761,3.389811,3.811771,0.8228340000000001,0.951175,0.589374,6.859624 -558,1,-0.163427,0.259374,-0.040545,-0.25855900000000004,-1.304933,0.39971799999999996,0.45102200000000003,-0.062524,-1.0398379999999998,-0.216444,0.131416,0.788958,0.1821,0.006288,-0.827674,0.543131,0.177034,-0.298156,-1.3056450000000002,-0.188203,-0.648835,-0.197498,-0.317839,-0.457571,-0.9330309999999999,1.168754,1.1235680000000001,0.6916399999999999,-0.503965,0.231148 -21,1,-1.2506110000000001,-1.631243,-1.254913,-0.9944219999999999,0.001377,-0.887193,-0.880434,-0.796903,-0.729224,-0.344455,-1.31308,-1.593959,-1.302806,-1.083572,0.42981899999999995,-0.7470859999999999,-0.743748,-0.726337,0.012345,0.8863409999999999,-0.461517,-0.435539,-0.473774,-0.542058,0.855042,-0.6236229999999999,-0.399334,0.391552,-0.03297,-0.31277699999999997 -316,1,-0.708054,-1.499339,-0.7643409999999999,-0.646003,-1.414523,-1.278291,-1.108365,-1.463066,-0.983215,-1.306473,-0.5530579999999999,-1.21232,-0.60628,-0.550303,-1.353587,-1.368694,-0.973881,-1.131534,-0.506089,-0.8941690000000001,-0.699725,-1.11991,-0.706933,-0.5394180000000001,-0.566016,-1.101646,-0.6851470000000001,-1.094126,-0.09350900000000001,-1.0760129999999999 -463,1,-0.724621,-0.269869,-0.732172,-0.677646,0.08028099999999999,-0.46734,-0.413491,-0.48308199999999996,0.500289,-0.29458,-0.717787,-0.216335,-0.745091,-0.688811,-0.802766,-0.867807,-0.6921470000000001,-0.803461,-1.079287,-0.596472,-0.8069189999999999,-0.818729,-0.773762,-0.603883,-0.11066400000000001,-0.47328000000000003,-0.36783499999999997,-0.697362,0.084477,-0.388428 -454,1,-0.399502,-0.574386,-0.465887,-0.43435100000000004,-0.43259899999999996,-0.652457,-0.400538,-0.24509099999999998,0.597356,-0.589947,-0.42809200000000003,-0.497911,-0.46746899999999997,-0.460714,-0.7493920000000001,-0.948351,-0.742492,-0.67578,-0.04607,-0.643253,-0.8516729999999999,-0.9968969999999999,-0.866333,-0.594422,-1.051036,-0.9467209999999999,-0.641048,-0.796147,-0.540288,-0.815852 -413,1,0.101638,0.9563450000000001,0.087534,-0.023702,-1.0813700000000002,0.510406,0.188118,0.24657600000000002,0.424254,0.474591,0.24502100000000002,0.656315,0.229057,0.110382,-0.7977850000000001,-0.034888999999999996,-0.253727,-0.262045,0.483317,-0.519922,-0.31245700000000004,0.216173,-0.275266,-0.260216,-0.864028,0.14541700000000002,0.04066,0.02658,-0.179475,0.08325199999999999 -330,0,0.515803,-0.60207,0.5075149999999999,0.332978,0.487955,1.2311530000000002,1.0718219999999998,1.271326,0.19129300000000002,0.404213,0.540396,-0.87955,0.5697,0.39337,-0.10321,0.6208319999999999,0.396746,0.554335,-0.108136,-0.430613,-0.24568600000000002,-0.8513870000000001,-0.117352,-0.15548800000000002,-0.40067600000000003,0.389654,0.174283,0.453192,-0.711009,-0.254905 -468,0,1.097705,0.5199239999999999,1.082381,0.978132,-0.5115029999999999,1.426448,1.1720899999999999,1.294166,-0.9702729999999999,2.13097,0.986298,0.940217,1.1134110000000002,0.9260709999999999,-0.246964,1.8204599999999997,1.566869,1.322737,-0.422117,1.54411,1.8902439999999998,0.45022299999999993,1.4528860000000001,1.420495,-0.091663,2.502279,1.127879,1.835218,-0.46158699999999997,2.838813 -322,1,-0.46162600000000004,-0.748629,-0.43073900000000004,-0.49412,0.978917,-0.19825299999999998,-0.44659399999999994,0.013609,-0.839233,0.08778899999999999,-0.359929,-1.389177,-0.37685100000000005,-0.42686899999999994,1.212639,-0.303242,-0.637784,-0.384824,-0.9807110000000001,0.27818899999999996,-0.663994,-0.317244,-0.619808,-0.5229159999999999,-0.377008,-0.29722800000000005,-0.426854,-0.273508,-1.031867,-0.260201 -307,1,-1.360572,-0.913101,-1.380908,-1.0461040000000001,-1.479838,-1.284653,-1.235211,-1.533565,0.145995,-0.327276,-1.456224,-1.1378540000000001,-1.4667430000000001,-1.162069,-1.8723830000000001,-1.386888,-1.0686579999999999,-1.1722629999999998,-0.08623,0.784274,-0.832183,0.159928,-0.8524719999999999,-0.67211,0.11601199999999999,-1.151053,-0.93545,-1.350255,0.7831,-0.621052 -373,0,1.8846189999999998,-0.40828600000000004,1.7734130000000001,1.8729099999999999,1.044671,0.32592600000000005,0.689459,1.470795,-0.342574,-0.74234,1.849702,-0.45136899999999996,1.764216,1.934305,-0.135234,0.061763,0.802271,1.0444200000000001,-0.8784850000000001,-1.13658,0.752621,-1.014859,0.6202439999999999,0.807087,-0.276671,-0.36485500000000004,-0.168561,0.084976,-0.942269,-0.788239 -304,1,-0.743259,-0.6623220000000001,-0.731874,-0.686963,-0.787669,-0.47942700000000005,-0.717654,-0.90623,-1.120727,-0.41926599999999997,-0.757549,-0.26287699999999997,-0.7570359999999999,-0.716114,-0.557245,-0.519289,-0.6950350000000001,-0.874394,-1.462636,-0.052113,-0.279252,-0.2864,-0.19358599999999998,-0.38298499999999996,-0.129665,0.058228999999999996,-0.321083,-0.647077,-0.194004,-0.14521199999999998 -529,1,-0.583805,-1.61333,-0.60588,-0.5813119999999999,0.864944,-0.5793010000000001,-0.527672,-0.61936,-0.19373800000000002,-0.189844,-0.5843,-1.3612520000000001,-0.58239,-0.596377,0.970677,-0.270077,-0.640169,-0.540104,-0.564504,0.46531300000000003,-0.555357,-1.293361,-0.5703050000000001,-0.47957299999999997,0.09534400000000001,-0.779555,-0.461337,-0.618041,-0.111671,-0.590414 -40,0,-0.07024,0.744648,-0.141817,-0.162929,-1.0068489999999999,-0.317847,-0.30554699999999996,-0.051864999999999994,0.150849,-0.691912,-0.19520099999999999,0.53298,-0.23845100000000002,-0.261342,-1.048999,-0.8344520000000001,-0.724413,-0.7379439999999999,-0.10083400000000001,-0.9820610000000001,-0.601554,-0.7082350000000001,-0.640599,-0.43578999999999996,-1.25371,-0.808059,-0.596618,-0.797283,-0.816347,-0.948996 -115,1,-0.538247,0.076989,-0.587413,-0.523125,0.772888,-0.091382,-0.584763,-0.641591,-0.748637,0.081139,-0.624062,0.521345,-0.635937,-0.6151479999999999,0.093918,-0.489914,-0.6970430000000001,-0.743876,-0.451325,-0.121575,-0.336999,-0.533695,-0.428726,-0.342062,0.254017,-0.022811,-0.44906899999999994,-0.6626489999999999,-0.9398479999999999,0.02311 -2,0,1.51187,-0.023974000000000002,1.347475,1.456285,0.527407,1.082932,0.854974,1.955,1.152255,0.20139100000000001,1.579888,0.456187,1.566503,1.558884,0.94221,1.052926,1.363478,2.0372310000000002,0.939685,-0.398008,1.228676,-0.780083,0.850928,1.1813360000000002,-0.297005,0.814974,0.21307600000000002,1.424827,0.23703600000000002,0.293559 -39,0,-0.15307300000000001,0.05581900000000001,0.001155,-0.24643,1.255083,1.070209,1.107324,1.6931029999999998,-0.151676,1.283108,-0.18384,0.356123,-0.147009,-0.27215,0.37288699999999997,0.400995,0.219721,0.14111500000000002,-0.334494,0.19738599999999998,-0.693589,-1.134788,-0.653965,-0.480013,-0.558016,-0.172595,-0.046543,0.133639,-0.8199799999999999,-0.22994 diff --git a/fml_manager/Examples/Hetero_LR/data/breast_a.csv b/fml_manager/Examples/Hetero_LR/data/breast_a.csv deleted file mode 100644 index 7f744c76e..000000000 --- a/fml_manager/Examples/Hetero_LR/data/breast_a.csv +++ /dev/null @@ -1,570 +0,0 @@ -id,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11,x12,x13,x14,x15,x16,x17,x18,x19 -133,0.449512,-1.247226,0.413178,0.303781,-0.123848,-0.184227,-0.219076,0.268537,0.015996,-0.789267,-0.337360,-0.728193,-0.442587,-0.272757,-0.608018,-0.577235,-0.501126,0.143371,-0.466431,-0.554102 -273,-1.245485,-0.842317,-1.255026,-1.038066,-0.426301,-1.088781,-0.976392,-0.898898,0.983496,0.045702,-0.493639,0.348620,-0.552483,-0.526877,2.253098,-0.827620,-0.780739,-0.376997,-0.310239,0.176301 -175,-1.549664,-1.126219,-1.546652,-1.216392,-0.354424,-1.167051,-1.114873,-1.261820,-0.327193,0.629755,-0.666881,-0.779358,-0.708418,-0.637545,0.710369,-0.976454,-1.057501,-1.913447,0.795207,-0.149751 -551,-0.851273,0.733108,-0.843535,-0.786363,-0.049836,-0.424532,-0.509221,-0.679649,0.797298,0.385927,-0.451772,0.453852,-0.431696,-0.494754,-1.182041,0.281228,0.084759,-0.252420,1.038575,0.351054 -199,0.091654,0.216499,0.103839,-0.034667,0.167930,0.308132,0.366614,0.280661,0.505223,0.264013,-0.707304,-1.026834,-0.702973,-0.460212,-0.999033,-0.531406,-0.394360,-0.728830,-0.644416,-0.688003 -274,1.080023,1.207830,0.956888,0.978402,-0.555822,-0.645696,-0.399365,-0.038153,-0.998966,-1.091216,0.057848,0.392164,-0.050027,0.120414,-0.532348,-0.770613,-0.519694,-0.531097,-0.769127,-0.394858 -420,-0.726307,-0.058095,-0.731910,-0.697343,-0.775723,-0.513983,-0.426233,-0.893482,0.800949,-0.018090,-0.428673,0.404865,-0.326750,-0.440850,0.079010,-0.279903,0.416992,-0.486165,-0.225484,-0.172446 -76,-0.169639,-1.943019,-0.167192,-0.272150,2.329937,0.006804,-0.251467,0.429234,2.159100,0.512094,0.017786,-0.368046,-0.105966,-0.169129,2.119760,0.162743,-0.672216,-0.577002,0.626908,0.896114 -315,-0.465014,-0.567723,-0.526371,-0.492852,-0.800631,-1.250816,-1.058714,-1.096145,-2.178221,-0.860147,-0.843011,-0.910353,-0.900490,-0.608283,-0.704355,-1.255622,-0.970629,-1.363557,-0.800607,-0.927058 -399,-0.660984,-0.472313,-0.688248,-0.634204,-0.390718,-0.796360,-0.756680,-0.839314,0.129175,-0.369656,-0.221505,-0.139439,-0.317344,-0.336122,-0.526014,-0.326291,-0.368166,-1.037840,-0.698901,-0.273818 -238,0.026330,1.992051,0.023930,-0.088136,-1.005588,-0.008357,0.269940,-0.124821,-1.714551,-0.213719,-0.251822,2.008745,-0.376748,-0.228313,-0.244670,0.166096,0.219045,-0.273508,-1.052451,-0.077883 -246,-0.263364,-0.432753,-0.322891,-0.322206,-1.722935,-1.120051,-0.570489,-0.976796,-1.185164,-0.914016,-0.874050,0.696974,-0.986625,-0.589142,-0.260004,-0.547055,-0.036596,-1.040273,-0.111671,-0.584362 -253,0.901094,-0.514200,0.866270,0.777324,0.315955,-0.004567,0.474586,0.892752,0.005043,-0.945203,-0.346022,-0.653261,-0.333185,-0.147567,-0.761357,-0.583942,-0.284279,0.133639,-0.799396,-0.541998 -550,-0.927957,0.509709,-0.966282,-0.837273,-1.569218,-1.176337,-1.114873,-1.261820,-0.549900,-0.470306,-0.320758,0.158114,-0.371797,-0.432709,0.846041,-0.806941,-1.057501,-1.913447,1.149967,-0.592683 -208,-0.288925,0.756379,-0.203852,-0.356904,0.273255,0.833088,-0.021963,0.054189,0.140128,1.460471,-0.765413,-0.534421,-0.680696,-0.555479,-0.596684,0.275639,-0.219954,-0.566458,0.556683,0.152472 -185,-1.149488,-0.972632,-1.161936,-0.959569,-0.262620,-1.087644,-1.094823,-1.199811,-0.396560,-0.328545,0.069759,0.092797,-0.092105,-0.305980,2.449773,-0.753287,-1.004549,-1.523495,0.585742,-0.122895 -156,1.009020,0.337507,1.047507,0.878290,1.077425,1.178005,1.214074,1.456866,0.581892,-0.161268,1.465801,0.332291,1.323683,1.178696,0.665367,1.344806,0.668654,1.072832,-0.334455,0.443725 -0,1.097064,-2.073335,1.269934,0.984375,1.568466,3.283515,2.652874,2.532475,2.217515,2.255747,2.489734,-0.565265,2.833031,2.487578,-0.214002,1.316862,0.724026,0.660820,1.148757,0.907083 -70,1.366877,0.470149,1.302886,1.351264,-0.446227,-0.027309,0.241064,0.789060,-0.838325,-1.160679,1.384594,-0.760851,1.296951,1.225780,-0.865695,-0.500666,-0.305168,0.308825,-0.809083,-0.793157 -293,-0.646783,-0.425771,-0.676715,-0.631929,-0.899551,-0.908174,-0.777395,-0.673717,0.231402,-0.800607,-0.715245,0.038367,-0.807919,-0.582101,0.184681,-0.585618,-0.589324,-0.522013,-0.317504,-0.760627 -287,-0.351408,-1.435719,-0.415157,-0.395299,-1.907966,-1.270715,-0.831130,-0.959772,-1.732806,-0.990566,-0.909420,-1.356863,-0.866828,-0.608503,-0.770024,-0.672247,-0.509747,-0.955599,-0.526970,-0.648286 -222,-1.121087,-0.409482,-1.105917,-0.972083,0.693131,-0.366161,-0.892901,-0.767865,0.359185,0.890594,-0.571959,0.000266,-0.606442,-0.556359,0.286019,-0.641508,-0.774539,-0.676761,0.705609,-0.012824 -262,0.898254,0.660969,0.923936,0.833069,-0.453344,0.435107,0.102583,0.674535,1.082072,-1.156426,1.550617,0.755033,1.623671,1.113351,-0.107997,1.918792,0.480322,1.572438,0.838796,0.772804 -309,-0.305966,-1.268170,-0.381381,-0.353491,-0.913784,-1.269578,-1.057635,-1.034085,-1.309296,-1.079875,-0.027690,-0.704607,-0.148043,-0.161208,-0.964365,-1.160666,-0.963633,-1.131435,-0.765494,-0.575662 -534,-0.899556,-0.388538,-0.872369,-0.822768,0.036274,-0.129268,-0.454105,-0.542683,-0.703240,0.181792,-0.918443,0.664316,-0.842076,-0.665487,0.820040,0.463987,0.401740,0.269895,-0.886573,-0.098687 -54,0.276263,0.635371,0.217936,0.164705,-0.412780,-0.635462,-0.455360,-0.401848,-0.714193,-0.844553,-0.341691,-0.694809,-0.380708,-0.229414,-0.788692,-0.848299,-0.525994,-0.417550,-1.161421,-0.819256 -172,0.378508,-1.721948,0.433773,0.233247,2.087974,0.969539,1.436297,1.567780,0.563638,1.118828,0.056765,-1.013408,-0.030226,0.094672,-0.549349,-0.126765,0.369246,0.227720,-0.317504,0.141124 -484,0.455192,-1.863899,0.446130,0.262542,0.565034,0.484381,0.380424,0.340245,-0.100834,-0.029431,-0.874050,-1.505458,-0.852967,-0.582321,-0.335673,-0.406772,0.048618,-0.182670,-1.064558,-0.582092 -102,-0.553058,0.286311,-0.607516,-0.557982,-1.155035,-1.212155,-0.815688,-0.805266,-0.265127,-0.854476,-0.767939,0.642544,-0.833166,-0.564500,-0.653686,-1.083147,-0.703052,-0.810908,-0.735225,-0.855946 -458,-0.320167,1.359089,-0.385500,-0.383070,-0.901686,-1.016007,-0.963460,-0.807330,-0.527994,-1.177690,-0.516377,0.027481,-0.598521,-0.421268,-0.329006,-0.922409,-0.869136,-0.885687,-1.016127,-0.862755 -495,0.210940,0.214172,0.170979,0.073978,-0.034891,-0.395915,-0.258122,0.015240,-1.185164,-0.753827,-0.623932,0.760476,-0.628718,-0.406967,-0.542015,-0.657716,-0.343630,0.005493,-0.115303,-0.750036 -158,-0.587140,-1.524147,-0.623168,-0.586707,-0.231308,-0.984169,-0.867289,-0.755484,-0.809117,-0.528427,-0.804753,-0.886041,-0.839106,-0.595962,-0.504347,-0.876859,-0.780905,-0.895906,-0.713431,-0.478452 -160,-0.675185,0.207191,-0.653649,-0.668618,0.892395,0.184948,-0.255736,-0.297641,0.662213,0.245584,0.348751,0.863894,0.524703,-0.043939,0.797373,-0.079818,0.324152,-0.044792,1.625806,0.478903 -292,-0.334368,-0.760869,-0.363670,-0.401556,0.294605,-0.472100,-0.342114,-0.392562,-0.297985,0.269684,-0.706582,-0.822358,-0.809404,-0.498714,0.561363,-0.304494,-0.283284,-0.077234,0.691080,0.352188 -493,-0.473535,-1.503204,-0.541199,-0.505082,-1.611206,-1.211208,-1.024816,-0.965447,-0.725145,-0.378161,-0.279974,0.488324,-0.375263,-0.346243,1.116386,-0.863389,-0.844765,-0.632802,0.986511,0.388122 -394,-0.575780,-0.365268,-0.572504,-0.593533,0.465402,-0.128131,-0.514369,-0.403912,0.457760,-0.168356,-0.436974,0.789506,-0.493575,-0.398607,0.368355,-0.512403,-0.446417,-0.694766,-0.047499,-0.755332 -528,-0.053194,-1.424083,-0.068336,-0.172606,2.023925,-0.128699,0.153179,0.444452,0.600147,0.251255,0.508639,2.573006,0.606383,0.096872,0.999714,0.390772,0.521768,2.714394,-0.410734,0.655546 -335,0.832931,0.398010,0.816841,0.750021,1.105891,0.023860,0.778417,1.300555,-0.308938,-0.295940,1.483125,1.654948,1.589019,1.030404,-0.195334,-0.419626,0.435228,0.915489,-0.272705,-0.023415 -311,0.137096,-0.837662,0.029285,0.028472,-1.436139,-1.311272,-0.933202,-0.777667,-0.655777,-1.452705,-0.321841,-0.554016,-0.451497,-0.251635,-0.670020,-1.087451,-0.880940,-0.887634,-0.678318,-1.096968 -25,0.855652,-0.672441,0.989840,0.733241,1.582699,2.335941,1.683630,2.351917,4.484751,1.606484,2.312883,-0.436991,2.183056,1.563506,0.329354,0.699282,0.179919,1.974718,0.307261,1.380276 -522,-0.814352,0.125743,-0.851362,-0.758776,-0.800631,-1.141087,-1.051257,-1.115722,-0.637523,0.089648,-0.977273,-0.242856,-0.933063,-0.682165,-0.947031,-1.094213,-0.929979,-1.254065,-0.686793,-0.567341 -512,-0.206561,0.286311,-0.137124,-0.279260,1.013376,0.806556,0.699320,0.846065,1.111279,1.481735,-0.052594,-0.519362,0.112343,-0.146687,-0.542348,-0.158063,0.087080,0.250429,-0.422842,0.079469 -230,0.830091,-0.048787,0.882746,0.682900,1.262455,1.001757,1.283127,1.549724,1.166043,0.064131,-0.394386,-0.975851,-0.352986,-0.183870,-0.503013,-0.301699,-0.044554,0.331534,-1.062137,-0.551832 -548,-1.262242,0.011717,-1.273561,-1.050012,-0.814864,-1.024157,-0.821463,-1.013810,-0.845627,-0.063453,-0.395108,0.265160,-0.401994,-0.486174,0.133012,-0.796322,-0.282621,-0.353802,0.180129,0.135829 -213,0.935176,1.459153,0.928055,0.833637,0.301721,0.194424,0.996873,0.439809,-1.838683,-0.586549,0.449087,0.816721,0.445993,0.400275,8.029999,3.357389,3.710479,4.456526,0.146227,3.315409 -390,-1.098366,-1.645155,-1.079967,-0.947908,0.256176,-0.548095,-0.873441,-0.754194,-0.042419,0.410026,-0.772631,-1.214074,-0.751486,-0.626104,-0.453011,-0.661069,-0.775799,-0.789497,-0.140729,-0.538594 -275,-0.635423,-0.449042,-0.649530,-0.623681,1.860245,-0.611014,-0.370489,0.647967,0.742534,-0.573791,0.851874,1.952500,0.571731,0.187079,2.379770,-0.591766,-0.637401,1.184757,0.755252,-0.299917 -219,1.534446,3.067156,1.484123,1.615766,-0.865392,0.164101,0.322671,0.450127,-1.400570,-1.370484,1.205577,0.188958,0.918747,1.530503,-0.500680,0.053758,-0.174198,-0.164827,-0.874465,-0.582092 -236,2.579618,1.787269,2.534473,2.887080,-0.090400,1.210223,1.333347,1.928896,0.355534,0.041449,2.356193,-0.459670,2.168700,2.540382,-0.204335,0.176156,0.433571,0.870070,-0.562083,-0.280626 -336,-0.323007,-1.177414,-0.324951,-0.399850,-0.123848,-0.088901,-0.645568,-0.720662,-0.582759,1.358404,-0.807640,-1.044433,-0.962418,-0.570440,-0.603351,-0.135708,-0.025323,-0.726884,-0.502754,0.720983 -20,-0.297446,-0.833008,-0.261106,-0.383638,0.792763,0.429422,-0.541362,-0.459627,0.567289,0.753087,-0.793925,-0.851206,-0.734160,-0.564720,-0.981366,-0.363178,-0.494494,-0.860707,-0.455534,-0.518168 -193,-0.507616,1.761672,-0.445638,-0.504797,0.500985,0.586719,0.247342,-0.085098,0.479666,0.931705,0.000462,1.074358,-0.110916,-0.129746,0.685701,0.724991,0.190198,0.229342,-0.213376,0.710014 -200,-0.538858,0.062913,-0.553145,-0.551441,-0.035603,-0.444810,-0.589196,-0.202461,0.611100,-0.378161,-0.186856,0.198029,-0.276256,-0.288158,0.157680,-0.429687,-0.592971,-0.064257,-0.667421,-0.172825 -83,1.412320,1.629029,1.529432,1.356952,1.789079,1.416794,1.317025,2.527316,-0.648476,1.338557,0.410829,3.071950,1.452886,0.588830,0.168014,1.957356,-0.349930,1.076077,1.212928,2.494604 -376,-1.010321,0.216499,-0.898730,-0.900412,-0.400681,1.168530,1.747661,0.270601,1.374147,3.076540,-1.059924,0.025667,-0.249030,-0.728456,0.486027,2.847676,4.032102,2.821453,-0.529391,3.179239 -295,-0.101476,-1.400813,-0.161014,-0.205313,-0.311725,-0.798444,-0.981414,-0.767349,-0.801815,-0.521339,-0.671573,-0.947547,-0.686637,-0.497174,-0.897696,-0.968294,-0.915854,-0.805718,0.120800,-0.753819 -77,1.114105,-0.730617,1.162839,0.998595,0.721598,2.089571,0.999384,1.523931,1.242713,0.557457,2.076840,-1.208994,1.705351,2.078346,0.299686,1.839429,0.486953,1.444293,0.649913,1.232757 -441,0.892574,1.426574,0.841556,0.779031,-0.928729,0.124303,0.396746,0.217723,-1.258183,-1.237230,0.378346,0.838493,0.206398,0.396975,0.356021,0.983760,0.581119,0.912245,-0.557239,-0.021146 -109,-0.791630,0.458514,-0.802757,-0.734885,-0.624141,-0.731356,-0.470426,-0.771992,-1.185164,0.353322,-0.616352,-0.418666,-0.628223,-0.526437,0.690702,-0.553761,-0.247475,-0.869628,-0.588720,-0.498499 -319,-0.482055,-0.532817,-0.550673,-0.505082,-1.479550,-1.322832,-0.946385,-0.823580,-1.239928,-1.018918,-0.098791,1.783766,-0.187646,-0.201911,0.105345,-0.820913,-0.727290,-0.096699,1.669394,-0.315425 -349,-0.618382,-1.007538,-0.607104,-0.648993,1.383436,0.308132,-0.967854,-0.800881,2.363553,0.427038,-0.159426,-0.302729,-0.203487,-0.301139,-0.413676,-0.073111,-0.798345,-0.207002,0.823055,-0.257553 -89,0.145616,-0.942380,0.156563,-0.008501,1.198406,0.560187,0.136356,0.560267,1.111279,0.093901,0.383760,-0.870256,0.469260,0.053308,-0.511014,1.041885,0.413345,0.719215,0.451344,0.395687 -419,-0.842753,0.493420,-0.865778,-0.780960,0.387120,-0.844497,-1.002443,-0.984276,-0.717843,-0.192455,-0.428312,0.836678,-0.444567,-0.469672,-0.044328,-0.896252,-0.826330,-0.914398,0.774624,-0.656230 -562,0.310345,2.636649,0.470844,0.176365,0.600616,1.977758,2.086645,1.170295,1.155090,1.236490,-0.523235,-0.021506,-0.249525,-0.389146,-0.805359,1.283328,1.382525,0.694884,0.100217,0.887792 -370,0.631281,0.930909,0.701509,0.527612,0.075415,0.859620,1.158832,1.001086,1.326685,-0.087553,0.093940,-0.353531,0.052444,0.113593,-0.468679,0.765232,0.955794,0.772745,1.383649,0.109729 -546,-1.081325,-0.684076,-1.098091,-0.938523,-0.143774,-1.030979,-0.987817,-1.120082,0.267911,-0.111652,-0.702973,-0.453320,-0.747526,-0.602123,0.015008,-1.018930,-0.721952,-1.022105,-0.598406,-0.449704 -49,-0.181000,0.700529,-0.208383,-0.267030,-0.629123,-0.518531,-0.518386,-0.388951,-0.009560,-0.796355,-0.618518,0.247017,-0.559909,-0.443050,-0.862028,-0.651568,-0.362861,0.007115,-0.500333,-0.695568 -412,-1.343470,0.556251,-1.327108,-1.098077,-1.186348,-0.830283,-0.645945,-1.129548,-1.962815,0.629755,-1.034299,-0.063236,-0.837621,-0.737829,-0.508680,0.070525,0.180914,-1.081636,-0.124989,0.298098 -417,0.389869,0.416627,0.450249,0.421527,1.113008,0.999861,0.795994,0.925768,0.998100,0.828220,3.482267,-0.006991,3.246381,2.995817,0.385689,0.749583,0.432244,1.556217,-0.043867,0.764104 -440,-0.896716,-0.486275,-0.833650,-0.805988,-0.513123,0.131884,0.072451,-0.329883,-1.177862,0.510676,-0.533340,0.288747,-0.029731,-0.488154,0.508028,1.168195,1.074496,0.954419,-0.651681,0.446751 -482,-0.186680,-1.216974,-0.191495,-0.308839,0.764297,0.211480,-0.388442,0.096492,-0.119089,0.509259,-0.889208,-1.167627,-0.873263,-0.604983,-0.863695,-0.612445,-0.615186,-0.487462,-0.431318,-0.381241 -291,0.236501,-0.044132,0.208462,0.092180,-0.458325,-0.115812,-0.369108,-0.018808,0.246006,-0.606396,-0.423981,-0.487793,-0.344076,-0.340302,-0.569683,-0.241898,-0.548209,0.011981,-0.644416,-0.370650 -198,1.435041,0.744743,1.463528,1.402458,-0.792091,0.728855,0.283751,0.484949,-0.162900,-1.064282,0.110182,-0.261000,0.478665,0.305448,-0.505680,0.642834,-0.160272,0.067133,-0.773970,-0.150508 -542,0.174018,1.426574,0.112489,0.038995,-0.968582,-0.610256,-0.599491,-0.481036,0.103619,-0.850224,-0.368399,0.305076,-0.341105,-0.284418,-0.755357,-0.768936,-0.411933,0.144993,-0.223063,-0.442139 -320,-1.101206,-0.723636,-1.048251,-0.940514,0.693131,0.128093,-0.270425,-0.239088,-0.250523,1.416525,-0.135245,0.461109,-0.628223,-0.388486,1.149720,0.959727,0.270107,0.591070,0.804894,1.437770 -94,0.264902,0.125743,0.343154,0.144227,0.536567,0.965749,1.019472,1.011920,0.158383,0.006009,0.258520,-0.458037,0.415796,0.149676,0.736370,0.652335,0.556252,1.089053,-0.673475,-0.103983 -116,-1.470424,-0.821373,-1.368711,-1.165198,-0.123848,0.378253,0.048094,-0.666494,-1.849636,1.252083,-0.331946,-0.431729,0.204913,-0.514776,3.769827,2.354171,2.013834,1.830352,0.399281,0.683158 -429,-0.399691,-0.376903,-0.452640,-0.436823,-1.238299,-1.120430,-0.938852,-0.788500,-1.287391,-1.043018,-0.396190,-0.604636,-0.374767,-0.376165,0.098678,-0.767818,-0.879016,-1.001343,-0.138308,-0.989091 -518,-0.354249,-0.248914,-0.309710,-0.460145,1.810429,1.170425,-0.509095,0.106035,-0.374655,1.379668,0.135085,-0.086823,0.153430,-0.131286,-0.589350,-0.122294,-0.591645,0.104441,-0.288445,-0.187576 -323,1.764497,0.516691,1.809525,1.732374,1.468835,1.575986,2.105477,2.617595,2.765157,0.553204,0.595621,-0.351717,0.567276,0.631953,-0.518681,-0.065287,0.000208,0.307203,0.864222,-0.170177 -248,-0.987600,1.380033,-0.986877,-0.875668,0.014925,-0.606466,-0.816190,-0.845247,0.311723,0.069801,-0.561131,0.501025,-0.677726,-0.521376,0.049342,-0.845505,-0.699073,-0.900448,0.125643,-0.444787 -280,1.429361,1.701168,1.409980,1.374017,0.401353,0.776234,1.296937,1.230911,0.329977,-0.084717,0.833467,-0.391632,0.720240,0.644934,0.117012,-0.055227,0.264801,0.183924,-0.749754,-0.131973 -391,-1.531771,-0.570050,-1.511641,-1.196199,0.536567,-0.569700,-1.114873,-1.261820,0.633006,1.159939,0.403250,1.564230,0.148975,-0.252735,2.926459,-0.325173,-1.057501,-1.913447,-0.229117,1.112474 -462,0.077453,1.791923,0.011573,-0.024997,-1.879499,-0.987580,-0.678462,-0.813778,-0.381957,-1.200372,-0.626819,-0.554560,-0.563869,-0.436010,-0.561682,-0.485576,-0.403976,-0.880172,-0.775181,-0.719776 -406,0.571638,-1.030809,0.507915,0.412710,-0.100363,-0.366351,-0.424349,-0.093868,-0.279730,-0.573791,-0.600833,-1.051691,-0.562879,-0.407187,-1.027701,-0.727578,-0.450396,-0.494600,-0.671053,-0.822282 -416,-1.341198,0.560905,-1.333287,-1.091251,0.572150,-0.810194,-0.857873,-0.937589,0.779043,0.455390,0.090331,3.013891,-0.052998,-0.333702,2.566444,-0.488371,-0.604244,-0.506441,1.198399,0.300746 -411,-0.876835,-0.572377,-0.867014,-0.801153,0.806996,-0.498443,-0.732448,-0.622129,-0.356400,0.085395,-0.752419,0.308704,-0.754456,-0.589582,-0.627685,-0.901115,-0.707363,-0.699633,-0.407102,-0.604031 -474,-0.922277,-0.853952,-0.888021,-0.841823,0.308838,0.048497,-0.472686,-0.856596,0.180289,0.789944,-0.927466,-1.231674,-0.774752,-0.676334,-0.855694,0.304702,-0.143362,-0.828103,-1.019759,0.331763 -96,-0.553058,-0.337343,-0.584037,-0.579597,0.579267,-0.640010,-0.802254,-0.503219,0.322675,0.503588,-0.141019,0.533683,-0.225763,-0.349763,-0.536015,-0.765024,-0.682495,0.550517,0.200712,-0.146347 -232,-0.825712,3.378983,-0.872369,-0.762473,-1.320851,-1.300090,-1.052512,-1.095861,0.121873,-0.640418,-0.654249,0.780434,-0.681686,-0.547338,-0.894029,-1.043186,-0.950636,-1.358853,-0.167367,-0.476939 -354,-0.848433,-1.214647,-0.853833,-0.768730,-1.680948,-0.828198,-0.549271,-0.882390,-0.444023,-0.278929,0.061457,-0.739624,0.229665,-0.252955,-0.500013,0.469017,0.436223,0.472657,1.268624,0.391905 -240,-0.138398,-0.858606,-0.189023,-0.226360,-0.151602,-0.720933,-0.524161,-0.299446,-0.345447,-0.878576,-0.292245,-1.008146,-0.430706,-0.289258,-0.190334,-0.726460,-0.457359,-0.124275,-0.189161,-0.756845 -378,-0.132717,-0.963324,-0.152364,-0.211286,-0.973563,-0.546958,-0.581412,-0.624450,-0.071627,-0.542604,-0.956340,-1.224961,-0.873758,-0.637765,-0.609684,0.243782,-0.247475,-0.558023,-0.286024,0.405900 -177,0.662522,0.190901,0.713866,0.506281,0.138753,0.971434,1.136233,1.025075,-0.064325,0.061296,-0.366234,0.121827,-0.190121,-0.192451,-0.137999,0.864156,0.723363,1.027414,-0.450690,0.298477 -33,1.460602,1.670916,1.480004,1.442275,-0.167259,1.280343,0.965486,0.696717,0.151081,-0.026596,0.543648,-1.107935,0.327681,0.612372,-0.675354,0.430453,0.102000,-0.349260,-0.618989,0.038240 -251,-0.746188,-0.195392,-0.769805,-0.703884,-0.207111,-0.842033,-0.783673,-0.728142,0.081713,-0.490152,-0.045014,-0.678480,-0.090125,-0.293659,-0.220335,-0.828738,-0.644696,-0.425174,0.287889,-0.882802 -224,-0.243483,-0.528162,-0.305591,-0.308554,-0.847600,-1.030979,-0.668669,-0.628319,-1.152306,-0.858729,-0.405935,-0.591754,-0.406944,-0.344483,-0.336340,-0.806941,-0.308484,-0.444315,-0.692847,-0.491312 -186,1.187949,-0.165140,1.096935,1.098139,-0.745834,-0.372605,-0.089257,0.237843,-0.695938,-1.211713,-0.532258,-1.344707,-0.519316,-0.251195,-1.391716,-0.910840,-0.589324,-0.823561,-1.192902,-1.024268 -85,1.230551,-0.179102,1.199910,1.194838,0.169354,0.018175,0.561216,1.006761,1.169694,-0.365403,1.063013,0.468367,0.948449,0.885853,-0.190001,-0.502343,-0.127115,0.389930,0.382330,-0.015094 -422,-0.714947,-0.760869,-0.680010,-0.701609,0.885278,0.236117,-0.223847,-0.101864,0.271562,0.057043,-0.575929,-0.876244,-0.593571,-0.537878,-0.385675,-0.303376,-0.185139,0.201767,-0.249700,-0.685733 -104,-1.033042,0.000082,-1.011592,-0.906669,0.251194,-0.351758,-0.738851,-0.952034,1.480024,0.285277,-0.181081,0.575413,-0.279227,-0.378585,0.184681,-0.183773,-0.102578,-0.516173,0.793997,-0.130082 -461,3.775318,1.624375,3.910226,5.250529,0.856812,1.790138,3.448859,3.094784,0.910477,-0.931027,7.730307,0.161743,7.813534,11.041842,0.203015,1.579542,1.613298,2.300760,-0.432529,0.288642 -142,-0.766069,-0.460677,-0.754153,-0.730619,0.913744,-0.179679,-0.859881,-0.781794,-0.608315,0.400103,-0.436253,1.253978,-0.459913,-0.417088,-0.125665,-0.454278,-0.673542,-0.369211,0.275781,-0.101713 -380,-0.811511,-1.472952,-0.774748,-0.763895,1.945643,0.128093,-0.123030,0.169746,0.753487,0.899100,-0.537671,-0.419210,-0.523277,-0.534577,-0.135332,-0.430804,-0.360872,-0.025327,0.065104,-0.027954 -123,0.105854,-1.954655,0.095189,-0.040355,0.977793,0.105352,-0.004762,0.228557,0.162034,0.173287,-0.405213,-0.652898,-0.464368,-0.355263,-1.074370,-0.710811,-0.101915,0.033069,-0.174631,-0.373298 -428,-0.851273,-0.621245,-0.885549,-0.778684,-1.056827,-1.250816,-0.942995,-0.908442,-1.097542,-0.186785,-0.951648,-0.453139,-0.939597,-0.673980,-0.386009,-1.073925,-0.745891,-0.909532,-0.054764,-0.536324 -396,-0.175320,-0.093001,-0.159366,-0.275278,0.678898,0.196319,-0.037656,0.126155,-0.020513,-0.284600,-0.691424,0.208915,-0.669806,-0.463072,-0.533015,-0.330203,0.038007,0.303959,-0.895048,-0.503416 -66,-1.324157,0.400337,-1.312280,-1.096371,0.572150,-0.504318,-0.842179,-0.873878,-0.345447,0.877836,-0.613826,1.440855,-0.597036,-0.575060,1.159721,-0.443100,-0.489189,-0.399545,0.272148,0.167223 -170,-0.513297,-1.605595,-0.540376,-0.542624,0.458285,-0.654413,-0.614306,-0.307442,0.538081,-0.460382,-0.610577,-1.000163,-0.592086,-0.503995,0.334687,-0.764465,-0.499468,0.099574,-0.157680,-0.585118 -481,-0.064554,-0.011554,-0.133416,-0.147862,-1.170692,-0.968060,-0.738851,-0.727884,-0.849277,-0.972138,-0.265537,-0.526982,-0.401004,-0.262416,-1.112371,-0.819237,-0.550198,-0.895582,-1.201377,-0.596844 -313,-0.734828,-1.994215,-0.751269,-0.699049,-0.739429,-0.846202,-0.943246,-1.032073,0.078062,-0.254830,-0.988823,-1.554264,-0.870788,-0.679833,-0.972366,-0.675042,-0.724936,-1.156415,-0.026916,-0.697081 -53,1.162387,-0.137215,1.166958,1.075386,1.312271,0.836879,1.109868,1.472343,1.023657,0.042867,1.546647,0.682459,0.995477,1.286504,-1.047369,0.230928,-0.123467,-0.417550,0.750408,0.503489 -296,-0.913757,-1.614903,-0.940332,-0.828172,-0.795650,-1.082717,-0.959693,-0.908700,-1.323900,-0.352645,-0.829657,-0.344459,-0.791583,-0.643486,-1.187708,-0.741550,-0.702057,-0.389812,1.073688,-0.138782 -114,-1.534044,-0.805083,-1.488162,-1.205869,1.326504,-0.423205,-0.596101,-0.765543,-0.593711,1.918357,-0.861418,-1.143859,-0.748516,-0.690217,0.406690,-0.187126,0.023087,-0.226467,-0.419210,0.004197 -243,-0.107156,1.042608,-0.141243,-0.184551,-1.133686,-0.687389,-0.525166,-0.657209,-0.140994,-1.206042,0.106573,-0.290029,-0.018345,-0.008956,-0.896696,0.066613,0.060223,-0.279996,0.658389,-0.267766 -140,-1.246621,-1.703331,-1.265735,-1.042048,-0.274719,-1.200026,-1.114873,-1.261820,0.333628,0.201639,-0.744840,-1.307876,-0.815840,-0.617744,-0.333673,-1.107850,-1.057501,-1.913447,0.269727,-0.217458 -152,-1.248609,-0.919110,-1.161112,-1.008772,0.771413,1.052926,4.042709,0.764814,2.688487,4.275833,1.513443,2.625622,0.597473,0.209301,1.309727,3.933610,12.072680,6.649601,1.806213,9.851593 -268,-0.357089,-0.716655,-0.394974,-0.405822,-0.150179,-0.798824,-0.625229,-0.845247,0.724279,-0.724057,-0.615991,0.003895,-0.653470,-0.486174,-0.507680,-0.206687,-0.199065,-0.886660,0.652335,-0.619539 -91,0.352947,0.807574,0.339035,0.208504,-0.310301,-0.014043,0.293795,0.668344,-0.345447,-0.259083,-0.333029,-0.681383,-0.391599,-0.239754,0.947045,-0.058021,0.444844,0.946309,0.504619,-0.625213 -255,-0.047513,-0.521181,-0.022203,-0.149284,0.942210,0.446478,0.114133,0.091333,0.351883,-0.212302,0.071563,-0.738535,-0.150024,-0.101144,-0.230002,0.073319,-0.023333,0.263406,0.009408,-0.416040 -559,-0.743348,1.079841,-0.718729,-0.714976,-0.266890,-0.042470,0.281240,-0.202977,-1.546608,0.411444,-0.600472,3.061064,-0.460408,-0.514116,0.386356,0.242665,0.845049,0.141749,-0.685583,0.356727 -14,-0.112836,0.772668,0.067180,-0.217827,1.191289,2.368158,1.556825,0.808147,0.939685,1.987820,-0.696838,-0.086823,-0.398529,-0.464832,-0.204001,1.893642,0.766467,0.727326,-0.112881,1.625761 -100,-0.146918,1.324183,-0.161426,-0.205313,-0.105345,-0.364456,-0.032007,-0.103928,-0.739749,-0.579461,0.185253,0.132713,-0.002504,0.061669,-0.389675,-0.592325,-0.179834,-0.304165,-0.713431,-0.544646 -353,0.270583,1.501040,0.248417,0.175512,0.429819,-0.126046,0.435666,0.428460,-0.601013,0.261178,0.895907,0.524611,0.647471,0.506764,1.159721,-0.065287,0.571172,0.918733,0.079633,0.405522 -543,-0.260524,2.040920,-0.291999,-0.331307,-0.686767,-0.674123,-0.739856,-0.417067,-0.670381,-0.707046,-0.613826,0.689717,-0.656935,-0.494754,-0.689354,-0.657157,-0.560808,-0.434582,-0.399837,-0.927436 -165,0.239341,0.109454,0.145441,0.100428,-0.864680,-0.963133,-0.870428,-0.761674,-1.082938,-1.437112,-0.798256,-0.275514,-0.782178,-0.521376,-1.135706,-0.977795,-0.783358,-0.870115,-0.157680,-0.860485 -43,-0.240643,0.230461,-0.191495,-0.311967,0.550801,0.744016,0.121415,0.326574,0.592845,0.711976,-0.125500,-0.711138,-0.217348,-0.198171,-0.656353,-0.224013,-0.333020,-0.362723,-0.405891,-0.180768 -201,0.969258,0.007063,0.952769,0.843876,-0.475405,0.292971,0.185822,0.669634,-1.115796,-1.118151,-0.029134,-0.705151,0.109867,0.008645,-0.317006,0.011841,-0.157951,0.268273,-0.557239,-0.430035 -108,2.312645,0.088510,2.505640,2.429179,2.579016,3.268353,4.238567,3.440423,2.717694,1.076300,2.922839,0.595371,3.556271,2.852806,-0.175333,3.420544,2.390830,2.109352,1.280732,0.469825 -328,0.608560,0.330525,0.615010,0.451674,1.461718,0.522284,0.740752,0.927574,0.497921,-0.003914,0.116678,0.027481,0.199963,0.089611,-0.114664,-0.259782,0.019439,0.344511,-0.627465,-0.380484 -439,-0.030472,-0.844644,-0.097993,-0.137624,-1.188483,-0.919734,-0.852851,-0.577763,-0.812768,-0.983478,-0.689258,-1.019577,-0.623768,-0.463952,-0.734689,-0.907654,-0.752025,-0.168071,-1.069402,-0.639587 -162,1.551487,-0.265204,1.595336,1.590169,1.113008,1.179900,2.033914,2.055287,0.786345,-0.280347,1.195471,-0.306358,0.953400,1.248221,-0.994700,-0.151357,0.278396,0.200144,-0.445847,-0.180390 -325,-0.413892,-0.463005,-0.441519,-0.469246,0.458285,-0.524975,-0.713993,-0.718340,-0.381957,-0.419272,-0.704417,-0.483257,-0.643569,-0.500035,-0.077329,-0.892228,-0.616844,-0.863302,0.003354,-0.760627 -183,-0.771749,-1.016847,-0.759507,-0.719243,-0.410645,-0.431923,-0.338850,-0.652824,-2.353466,-0.088970,-0.255431,-0.197498,-0.477239,-0.386506,0.105011,0.659601,0.903737,0.863582,0.134119,0.374127 -197,1.122625,0.593484,1.047507,1.049789,-1.611917,-0.339629,0.269940,0.228557,-0.151947,-1.332209,0.833828,0.159928,0.715785,0.792566,-0.503680,1.535949,0.968393,0.428860,0.103849,0.469446 -498,1.239071,-0.411809,1.208148,1.174930,0.344421,0.518494,0.757073,1.106842,0.074411,0.591480,1.397226,-0.311801,0.982606,1.219619,0.311020,0.372328,0.413013,0.638111,-0.529391,0.552283 -477,-0.064554,-0.621245,-0.123531,-0.157817,-1.998346,-0.969387,-0.835650,-0.916438,0.005043,-1.054359,-0.901119,-1.162366,-0.729704,-0.578801,-1.244377,-0.689014,-0.729114,-1.129650,-0.893838,-0.642991 -421,0.159817,-1.235591,0.257479,0.003444,0.479635,1.502076,0.705598,0.363201,1.001751,1.596561,0.508999,0.533683,0.954885,0.200500,0.978380,1.506886,0.692527,0.649465,0.724982,0.623773 -374,-0.124197,-0.749234,-0.170076,-0.215552,-0.949367,-0.769449,-0.793968,-0.737944,0.220449,-0.865817,-0.846981,-1.288644,-0.739605,-0.579461,-0.937031,-0.536994,-0.669895,-0.885849,-0.134675,-0.611974 -470,-1.266786,-0.186084,-1.255437,-1.040342,-0.491062,-0.791432,-0.744752,-0.871299,1.556694,0.188880,-0.099513,0.241574,-0.147053,-0.387386,0.153347,-0.312319,-0.157619,-0.310329,-0.114092,0.044670 -382,-0.589980,0.798266,-0.544495,-0.588983,-1.922199,0.056078,-0.117631,-0.493675,-2.222032,0.537611,-1.031411,0.393979,-0.538623,-0.677390,-0.666353,1.124602,0.369909,0.790588,0.503408,1.347746 -510,-0.678025,-1.070369,-0.644999,-0.650984,-1.093833,-0.146514,-0.270425,-0.581116,-1.141353,0.677954,-0.767939,-1.043526,-0.752971,-0.600582,-0.019660,0.764673,0.274417,0.565116,0.249143,1.145003 -132,0.577318,0.523672,0.586176,0.440582,0.315955,0.455954,0.194611,0.185997,1.271921,-0.551109,0.101159,0.087354,-0.010920,0.073550,-0.721356,-0.332998,-0.321746,-0.416090,-0.628676,-0.537837 -545,-0.144078,0.916946,-0.196849,-0.232332,-0.277565,-0.698760,-0.741488,-0.631673,-0.538947,-0.678694,-0.213564,0.216173,-0.396054,-0.200151,-0.391009,-0.250840,-0.387397,-0.443179,0.039678,-0.458404 -121,1.287353,-0.504892,1.212267,1.200527,0.643316,0.107247,0.714386,0.973229,0.563638,-0.094641,1.110294,0.660687,1.004388,1.103010,0.353688,-0.249722,0.050607,0.683529,-0.011175,0.293181 -45,1.284513,-0.393193,1.307005,1.197683,0.963560,1.217803,1.363478,1.340793,0.348232,-0.327128,0.807481,-1.004336,0.706379,0.686958,-0.249003,0.808267,0.784040,0.836006,0.452555,-0.098309 -333,-0.817192,-1.049425,-0.848066,-0.753372,-0.946520,-1.132559,-1.102648,-1.185959,-0.140994,-0.281764,-0.688536,-0.400523,-0.661885,-0.555919,-0.474679,-1.025804,-1.025216,-1.436389,-0.428896,-0.667577 -134,1.227711,0.609773,1.162839,1.194838,-0.146621,-0.137417,0.332715,0.504294,-0.436721,-0.783596,0.688377,-0.026949,0.445498,0.616332,-0.346674,-0.629212,-0.110868,-0.429553,-0.680740,-0.601383 -182,0.446671,0.237443,0.380225,0.317717,-0.027775,-0.309875,-0.287124,0.076630,-0.706891,-1.035930,-0.127304,-0.121295,-0.227743,0.014145,-0.805026,-0.159181,-0.409612,-0.329146,-0.614146,-0.516277 -519,-0.391170,-0.602629,-0.389619,-0.458154,1.148590,0.139464,-0.627740,-0.489290,1.125883,0.486577,-0.078580,-0.388004,-0.183686,-0.257796,0.156013,-0.551526,-0.401986,-0.421281,-0.301764,-0.062753 -207,0.818730,0.225807,0.730342,0.709350,-0.614889,-0.593200,-0.242302,0.128476,0.782694,-1.498069,0.651924,-0.655801,0.613809,0.618752,-0.667687,-0.583942,-0.412265,-0.091833,0.290310,-0.459161 -99,0.083133,0.111781,0.103427,-0.035236,0.082532,0.184948,0.063788,0.244291,0.246006,0.156275,-0.417485,1.150560,-0.242594,-0.296739,0.321353,0.194041,0.043644,0.396418,-0.717063,0.248547 -366,1.724735,1.754690,1.718907,1.647051,0.191415,1.185586,0.945398,2.001119,0.231402,-0.368238,2.060599,1.224948,2.109791,1.391892,0.466026,1.188316,0.900090,2.200190,2.004782,0.185758 -180,3.718515,0.600465,3.712513,4.536659,0.927977,1.649897,2.489659,3.582290,-0.042419,-0.722639,1.555309,0.479253,1.462291,1.944135,-0.803359,-0.006044,-0.026649,0.099574,-0.580244,-0.396371 -318,-1.444295,-0.090674,-1.313928,-1.167189,0.236249,1.759816,1.364734,0.004149,1.892581,3.491898,0.217014,1.259421,0.446983,-0.355043,0.934712,2.259159,2.347726,2.185591,1.739620,1.544058 -501,-0.087275,1.210157,0.014868,-0.167771,1.411902,1.208328,0.588837,0.481596,1.691779,1.356986,0.252385,0.564527,0.053434,-0.028318,0.879709,0.731139,0.095037,0.707861,0.459820,1.210441 -229,-0.368449,0.707510,-0.276346,-0.431419,0.885278,1.431955,1.013195,0.507906,1.136836,1.381085,-0.357572,-0.268257,-0.301503,-0.334582,-0.019327,0.732257,0.495243,0.518075,-0.453112,0.689210 -7,-0.118517,0.358450,-0.072867,-0.218965,1.604049,1.140102,0.061026,0.281950,1.403355,1.660353,0.643623,0.290561,0.490051,0.233722,0.588031,0.268933,-0.232554,0.435349,-0.688004,0.611669 -247,-0.351408,-1.205339,-0.289115,-0.405822,-0.623429,0.573453,0.610180,-0.235219,-0.787211,0.183210,-0.731486,-1.409116,-0.234179,-0.527757,-0.513347,1.701382,1.647449,0.308825,-0.954377,1.422261 -494,-0.274725,0.290965,-0.325775,-0.330454,-1.637537,-0.977725,-0.888883,-0.937847,-0.360051,-0.555362,-0.294050,0.464738,-0.267346,-0.313900,0.253684,-0.277108,-0.612865,-1.010913,0.038467,-0.413771 -174,-0.984760,-0.963324,-1.008296,-0.868274,-0.600656,-1.162123,-1.114873,-1.261820,0.424902,-0.432030,-0.268063,1.284821,-0.351996,-0.403887,0.557363,-0.855565,-1.057501,-1.913447,1.465982,-1.056420 -340,0.083133,-0.639862,0.089834,-0.038933,0.081820,0.181157,-0.109596,-0.172540,0.366487,0.187463,-0.202376,-0.809657,-0.093590,-0.180350,-0.821360,0.282346,0.215729,0.102819,-0.219430,-0.159586 -125,-0.078755,-0.483948,-0.145362,-0.188249,-0.605638,-0.814553,-0.936592,-0.967510,-0.721494,-0.552527,-0.673739,-0.654531,-0.678716,-0.493434,-0.814026,-0.911510,-0.754877,-1.132570,-0.977382,-0.789374 -209,0.324545,-1.484587,0.255419,0.200825,-1.034766,-0.796739,-0.375134,-0.447504,-1.652485,-1.068535,-0.692146,-1.549547,-0.663865,-0.447451,-0.916697,-0.733167,-0.447080,-0.717476,-1.332263,-0.809043 -347,0.179698,-1.058734,0.119491,0.039280,-0.541589,-0.502991,-0.536340,-0.351808,-1.061032,-0.521339,-0.225114,-1.485500,-0.162894,-0.248115,-0.769691,-0.582265,-0.442438,-0.182670,0.131697,-0.382754 -332,-0.825712,0.132725,-0.825000,-0.761051,0.643316,-0.692695,-1.052023,-1.066224,0.468713,-0.356897,-0.388250,1.359210,-0.449022,-0.455811,1.949753,-0.806941,-0.948182,-1.107752,2.650130,-0.692920 -361,-0.234963,0.530653,-0.277170,-0.309407,-0.750104,-0.769638,-0.695035,-0.636573,0.012345,-0.827542,-0.516377,0.584485,-0.414865,-0.425889,-0.514347,-0.281020,-0.462332,-0.875468,-0.272705,-0.695568 -500,0.259222,-0.593320,0.278486,0.098153,0.175759,0.607566,-0.145504,0.322447,-0.524343,0.835308,-0.119725,-0.679569,-0.278237,-0.120945,-0.972699,-0.407331,-0.395687,-0.284862,-1.209853,-0.211028 -300,1.534446,-0.090674,1.545908,1.598701,1.326504,1.134417,1.643454,1.477501,-0.071627,0.385927,2.547481,-0.101337,2.163749,2.038743,-0.328340,0.366181,0.811892,0.897646,-0.206112,0.375262 -283,0.600039,-0.120926,0.693271,0.427215,0.728714,1.437641,1.330836,1.073052,0.235053,0.573051,-0.425425,-0.543493,-0.199032,-0.269457,-0.826026,0.521553,0.226339,0.047667,-0.806661,0.108216 -107,-0.501936,-0.174448,-0.533785,-0.535229,-0.824827,-0.685873,-0.783045,-0.766317,-0.765306,-0.303028,-1.029607,-0.585041,-0.998803,-0.684475,-1.194708,-0.838798,-0.670558,-1.011886,-0.972539,-0.922519 -131,0.378508,0.044296,0.400820,0.267377,0.913744,0.340350,0.725686,0.824140,0.435855,-0.685782,0.249497,-0.781898,0.112838,0.175418,-0.267004,-0.594561,-0.124794,-0.140496,-0.795764,-0.504551 -303,-1.033042,-0.158159,-1.034246,-0.911788,0.742947,-0.711836,-0.826485,-0.802687,-1.203419,0.453972,-0.926383,0.628029,-0.906430,-0.665707,0.611365,-0.900724,-0.458022,-0.421281,-0.318715,-0.180768 -567,1.838341,2.336457,1.982524,1.735218,1.525767,3.272144,3.296944,2.658866,2.137194,1.043695,1.157935,0.686088,1.438530,1.009503,-0.173000,2.017716,1.302285,0.785721,0.326634,0.904057 -507,-0.871154,-0.504892,-0.853421,-0.820208,1.639632,0.052287,-0.604764,-0.160932,0.519826,2.404595,-0.820273,-0.339016,-0.766337,-0.616864,1.859749,-0.111675,-0.460674,0.170947,0.200712,1.708976 -3,-0.768909,0.253732,-0.592687,-0.764464,3.283553,3.402909,1.915897,1.451707,2.867383,4.910919,0.326373,-0.110409,0.286593,-0.288378,0.689702,2.744280,0.819518,1.115007,4.732680,2.047511 -516,1.187949,0.300273,1.187553,1.129424,0.742947,0.387729,0.855002,1.175970,0.176638,-0.480229,0.504307,-0.534058,0.174221,0.594550,-0.288338,-0.374915,-0.091637,-0.216734,-0.599617,-0.404693 -143,-0.348568,-0.784140,-0.338956,-0.405822,-0.682497,-0.175320,-0.500433,-0.465301,-0.122740,-0.063453,-0.688897,-0.808569,-0.582680,-0.521376,-0.572350,-0.550408,-0.556830,-0.683249,0.060261,-0.721667 -398,-0.871154,-1.037790,-0.892140,-0.786932,-1.348605,-1.073809,-0.774382,-1.074916,-1.009919,-0.093223,-0.792843,-0.959341,-0.793563,-0.601903,-0.927364,-0.602944,-0.414254,-1.237358,-1.045186,-0.097552 -530,-0.675185,-0.402501,-0.662299,-0.659801,0.778530,-0.136659,-0.451719,-0.116567,-0.779910,0.563128,0.119927,1.252163,0.140064,-0.212912,-0.151332,-0.409566,-0.481563,0.221232,-0.265440,-0.558641 -119,1.085703,0.167631,0.915698,0.930337,-0.878202,-0.703498,-0.199239,0.181612,1.158741,-1.778754,0.524880,-0.005177,0.243031,0.301488,-1.005700,-0.953260,-0.297873,-0.313574,3.583645,-0.715993 -259,0.398389,3.320807,0.483201,0.256000,0.707364,1.128731,1.083503,0.904617,1.020006,0.524852,-0.589283,0.110941,-0.476744,-0.381005,-0.565349,0.004575,-0.099594,-0.255665,-1.282863,-0.164881 -450,-0.641103,0.523672,-0.623580,-0.633919,-2.151351,0.039021,-0.012923,-0.646375,-1.688994,0.470983,-0.538393,0.611700,-0.451002,-0.442170,-0.062329,1.964621,1.151752,0.605669,0.362957,1.940466 -192,-1.251733,-0.248914,-1.286742,-1.043186,-1.911524,-1.533193,-1.114873,-1.261820,-0.579108,0.237079,-0.185052,6.655279,-0.314869,-0.410268,-1.776065,-1.047490,-1.057501,-1.913447,2.112542,-0.796939 -514,0.262062,-0.051114,0.217936,0.133704,-0.299627,-0.348157,-0.175008,-0.143650,-0.914994,-0.517087,-0.069196,-0.034207,-0.116857,-0.040639,-0.696355,-0.512962,-0.073732,-0.384946,-1.092406,-0.785592 -241,-0.484895,-0.988922,-0.550261,-0.507357,-1.216949,-1.334392,-0.982669,-0.976022,-0.969759,-0.745321,-1.046209,-0.984016,-1.044049,-0.689337,-1.258711,-1.148426,-0.842212,-1.303215,-0.404681,-0.921006 -402,-0.331527,-0.232625,-0.320832,-0.368849,-1.626150,-0.480439,-0.605518,-0.776119,0.227751,-0.539768,-0.611660,0.149042,-0.232199,-0.442830,-1.137372,0.651217,0.087080,-0.185915,0.699555,-0.034006 -148,0.088814,-0.956343,0.082420,-0.042062,0.237673,-0.042470,-0.049332,0.165104,-0.319891,-0.281764,-0.593975,-0.866265,-0.369322,-0.421048,-0.445011,-0.140179,-0.024991,0.508343,-0.726749,-0.479208 -395,-0.019112,-0.490929,-0.091402,-0.130229,-1.132262,-0.961427,-0.778274,-0.423257,-0.622919,-0.731145,-0.919526,0.849379,-0.806434,-0.608723,-0.556682,-0.712488,-0.682163,-0.428418,-0.405891,-0.889233 -299,-1.027362,0.884367,-1.034658,-0.912073,0.365770,-0.689284,-0.801626,-0.778182,-0.425768,0.391598,-0.427230,-0.133996,-0.285662,-0.435130,1.043049,-0.617475,-0.440448,0.114173,1.706929,-0.689894 -149,-0.109996,-0.321053,-0.158542,-0.198772,-1.204139,-0.769070,-0.753164,-0.919018,-1.236277,-0.991984,-0.560049,-0.833607,-0.640104,-0.415108,-1.401050,-0.534200,-0.467306,-0.968090,-0.878097,-0.688003 -60,-1.123927,-1.026155,-1.129395,-0.975496,1.212639,-0.449737,-0.978777,-0.929077,3.400421,0.964310,0.399279,0.406679,0.220754,-0.125786,0.157680,-0.809735,-0.803352,-0.584464,2.577483,0.816303 -61,-1.570397,0.393356,-1.536767,-1.231750,1.988342,-0.278795,-0.738224,-1.022993,0.059807,0.676536,-0.169532,1.542458,-0.184676,-0.482873,1.629740,0.343266,-0.062790,-0.411548,1.577375,-0.282517 -126,-0.146918,1.256698,-0.173371,-0.234039,-0.269025,-0.487451,-0.451343,-0.466075,-0.184806,-0.212302,-0.628624,-0.384375,-0.551493,-0.451191,-0.984366,-0.767818,-0.462001,-0.797445,-0.885362,-0.881289 -372,2.057032,-0.974959,2.031952,2.079354,0.266139,0.893733,1.310748,1.975325,0.589194,-0.137169,-0.230167,0.167186,-0.227248,-0.028098,-0.871695,0.071084,0.082106,0.265028,-0.459166,0.216396 -82,3.150487,1.307894,3.275896,3.478653,0.707364,3.073153,3.077232,3.497170,0.063458,0.711976,1.776193,0.466552,2.235529,1.752720,0.375022,1.757830,0.842397,1.379409,-1.197745,0.793608 -425,-1.163689,0.463168,-1.185414,-0.988579,-1.081023,-1.236034,-1.083862,-1.128748,-0.663079,0.225738,-0.794286,0.225245,-0.832671,-0.632265,-0.439011,-1.105559,-0.988733,-1.341334,-0.737646,-0.523463 -337,1.318595,0.498074,1.274053,1.243188,-0.370080,0.679581,0.215954,0.309034,0.516175,-0.278929,0.855484,0.568156,0.744002,1.054166,0.169014,0.754054,0.240265,1.032280,0.137751,0.245900 -386,-0.544538,-1.209993,-0.543259,-0.548596,-1.087428,-0.494842,-0.256238,-0.608200,-0.604664,-0.178279,-0.500136,-0.700252,-0.380708,-0.448331,-0.878695,0.267256,0.382840,-0.150228,-0.161313,0.312850 -184,0.327386,0.726127,0.286312,0.158448,-0.412068,0.016280,-0.440043,-0.420162,-0.308938,0.052790,-0.721019,-1.308602,-0.753466,-0.457791,-1.250377,-0.644302,-0.469295,-0.938730,-1.068191,-0.461430 -58,-0.305966,0.004736,-0.385500,-0.363161,-1.121587,-1.259344,-1.106185,-1.154336,0.026949,-1.103975,-0.004230,-0.005177,-0.134183,-0.162308,0.150013,-0.943703,-1.034556,-1.237520,0.164388,-0.304456 -88,-0.501936,0.584176,-0.502069,-0.536936,-0.614889,-0.187449,-0.359691,-0.295835,0.432204,0.176122,-0.387528,0.517354,-0.328235,-0.426549,0.023675,-0.030635,-0.161266,0.183924,-0.116514,0.252708 -117,0.210940,-0.609610,0.274778,0.078528,1.411902,1.147683,1.006917,1.039777,1.260968,0.692130,0.077338,-0.486160,0.060859,0.018546,-0.018660,0.008487,-0.059143,0.148237,-0.547553,0.033701 -459,-1.241793,2.073499,-1.247611,-1.035222,-1.175673,-1.100721,-0.921401,-0.992788,-0.695938,-0.464635,-0.819551,0.853008,-0.803464,-0.639305,-0.150999,-0.714164,-0.576724,-0.923320,-0.581455,-0.577553 -221,-0.161119,-1.254207,-0.139183,-0.266177,0.621966,0.281600,-0.128052,-0.113729,0.549034,0.032944,-0.535145,-1.304066,-0.423280,-0.424789,-0.396676,-0.130678,-0.213986,-0.557375,-0.256965,-0.331690 -223,0.460872,0.223480,0.437892,0.302644,0.436936,0.304342,0.325182,0.404988,0.450458,0.032944,-0.208872,-0.536961,-0.307938,-0.179250,-0.758357,-0.097144,-0.267037,-0.512443,-0.341720,-0.251123 -128,0.276263,-0.674768,0.313497,0.055775,1.326504,1.447116,0.313883,0.939439,0.691421,0.265431,0.092858,-0.270071,-0.034681,-0.010937,0.655034,0.915016,0.004850,1.749247,0.361746,0.449777 -294,-0.399691,-1.282132,-0.419688,-0.462989,0.022041,-0.386818,-0.953165,-0.765543,-0.633872,-0.254830,-0.810167,-0.950269,-0.755446,-0.593102,-0.325673,-0.764465,-0.839858,-0.619339,-0.823612,-0.908902 -541,0.097334,1.326510,0.158210,0.004297,-0.568632,0.353616,0.151924,-0.258434,0.220449,0.086813,-0.544890,-0.250114,-0.124282,-0.379025,0.032342,1.176579,0.212082,-0.028571,0.016673,0.876066 -111,-0.425252,0.342161,-0.404448,-0.496265,0.211341,0.313818,0.222232,0.291236,-0.279730,1.120246,-0.226558,1.063471,-0.076759,-0.436890,1.956420,0.835094,0.633839,1.809264,0.108692,0.792851 -266,-1.001801,-0.079038,-0.934566,-0.877375,0.036986,0.196319,-0.312987,-0.580343,0.402996,0.299453,0.163598,-0.036021,0.279168,-0.291238,0.143013,0.577443,0.054586,0.300714,1.755360,-0.180390 -568,-1.808401,1.221792,-1.814389,-1.347789,-3.112085,-1.150752,-1.114873,-1.261820,-0.820070,-0.561032,-0.070279,0.383092,-0.157449,-0.466152,0.049342,-1.163516,-1.057501,-1.913447,0.752830,-0.382754 -362,-0.388330,-0.104636,-0.415981,-0.450190,0.028446,-0.470394,-0.777395,-0.802429,-0.192108,-0.137169,-0.663633,0.123641,-0.658915,-0.507735,-0.477679,-0.504020,-0.550530,-0.291837,-0.175842,-0.628617 -271,-0.805831,-1.454335,-0.813055,-0.759060,0.140888,-0.535587,-0.704953,-0.551195,-0.155598,-0.013837,-0.775157,-1.247458,-0.842571,-0.597722,-0.189667,-0.796881,-0.632428,-0.478054,-0.438583,-0.369893 -187,-0.686545,-0.488602,-0.712139,-0.667195,0.098188,-0.813606,-0.636654,-0.426352,-1.079287,-0.261918,-0.577734,-0.818910,-0.556444,-0.494534,-0.045328,-0.937499,-0.401655,0.008737,-0.190372,-0.803369 -155,-0.533178,-0.314072,-0.564266,-0.553431,-0.698865,-0.711647,-0.627112,-0.660562,0.578241,-0.073377,-0.668325,-0.425561,-0.684162,-0.524236,-0.507680,-0.550967,-0.396350,-0.628098,-0.309029,-0.495851 -5,-0.476375,-0.835335,-0.387148,-0.505650,2.237421,1.244335,0.866302,0.824656,1.005402,1.890005,-0.255070,-0.592662,-0.321304,-0.289258,0.156347,0.445544,0.160025,-0.069124,0.134119,0.486846 -511,0.193899,-1.068042,0.110841,0.073409,-0.828386,-1.026810,-0.685995,-0.606394,-0.557202,-1.320868,-0.674821,-1.077091,-0.588621,-0.431609,-0.110997,-0.754964,-0.566114,-0.200513,-0.574190,-0.762518 -252,1.591249,0.123416,1.595336,1.567416,0.700248,1.526713,1.919664,1.250514,-0.287032,0.591480,1.302665,-0.792602,0.618264,1.154494,0.480360,1.402373,1.197177,1.283705,-0.711009,1.310299 -480,-0.558739,-0.293129,-0.563442,-0.567652,-0.390718,-0.491999,-0.748770,-0.867945,-1.269136,0.006009,-0.670490,-0.048721,-0.588126,-0.529737,-0.710022,-0.492842,-0.594297,-1.076283,-0.726749,-0.732636 -513,0.128576,-1.310057,0.095601,0.011123,0.139464,-0.287323,-0.082603,-0.140039,-0.265127,-0.906928,0.040885,-1.076184,-0.151014,-0.071001,-0.696021,-0.411243,-0.051185,-0.510658,-0.623833,-0.572636 -86,0.100174,0.505055,0.093953,-0.019024,-0.136658,-0.092312,0.396746,0.011887,0.961590,-0.912598,0.054961,1.820053,0.215309,-0.032278,0.776038,0.243224,0.723363,0.943065,0.440447,-0.206489 -105,-0.288925,-0.867914,-0.196026,-0.354629,3.091407,1.367520,1.485262,1.214661,0.413949,2.001996,-0.051872,-0.531700,-0.225763,-0.124905,0.040342,0.203542,0.757183,0.338023,-0.614146,1.249401 -242,-0.802991,-0.255896,-0.743031,-0.755078,-0.031333,0.533655,0.828637,-0.525659,0.884921,1.969391,-0.586035,0.771362,-0.246060,-0.526877,-0.125998,1.881346,1.886843,0.217988,-0.071715,1.845903 -312,-0.388330,-1.377542,-0.398681,-0.428860,-0.599233,-0.471152,-0.606145,-0.604589,-0.768957,-0.198126,-0.283944,-1.011412,-0.257445,-0.333482,-0.182334,0.123061,-0.017365,-0.179426,-0.391362,0.225852 -473,-0.527497,2.485389,-0.599278,-0.538926,-1.378495,-1.333445,-1.114873,-1.261820,-0.403862,-0.453294,0.145552,4.409122,0.008881,-0.114565,0.099344,-0.963264,-1.057501,-1.913447,1.315845,-0.249231 -364,-0.206561,-0.544452,-0.267285,-0.291490,-1.209121,-0.897940,-0.841049,-0.881874,-0.590060,-0.820454,-0.890652,-1.096686,-0.905935,-0.596622,-0.882362,-0.725342,-0.576392,-1.023890,-0.924107,-0.650934 -565,1.704854,2.085134,1.615931,1.723842,0.102458,-0.017833,0.693043,1.263669,-0.217664,-1.058611,1.300499,2.260938,1.156857,1.291565,-0.424010,-0.069758,0.252202,0.808431,-0.189161,-0.490556 -447,0.191059,-0.379230,0.161093,0.056629,-0.325246,-0.292629,-0.604011,-0.678876,0.297119,-0.558197,-0.666881,-1.079087,-0.685152,-0.452951,-0.748357,-0.769495,-0.474600,-0.794687,0.241879,-0.689894 -488,-0.695066,-0.725963,-0.678775,-0.666627,1.169940,-0.221940,-0.577646,-0.453952,0.151081,0.171869,-0.122251,-0.114038,-0.154479,-0.280898,0.652367,-0.701869,-0.443764,-0.020461,0.118379,-0.220106 -151,-1.678039,0.328198,-1.594021,-1.282659,-0.164412,0.495752,0.543639,-0.702606,1.498279,2.808612,-0.763969,1.351952,-0.803464,-0.662847,1.796413,1.603016,1.513163,-0.255665,0.308472,3.020373 -263,0.421110,0.021025,0.330797,0.294964,-1.278151,-0.913101,-0.586434,-0.527465,-0.966108,-1.186196,-0.632955,-0.395624,-0.659410,-0.399487,-1.405050,-0.915087,-0.622812,-1.074175,-1.260706,-0.926679 -563,1.929226,1.349781,2.101976,1.968434,0.963560,2.260135,2.870075,2.540213,1.231760,0.849484,2.010431,-0.346274,2.916691,1.726318,-0.214002,0.984878,1.543668,2.342934,0.003354,0.914648 -194,0.208100,0.912292,0.347273,0.046959,0.572150,1.774977,1.015706,1.028170,-0.272428,0.556040,-0.453216,-0.462029,0.358868,-0.333042,0.346688,1.438701,0.783046,1.140961,0.594217,0.195592 -433,1.332796,0.623736,1.307005,1.294382,0.387120,0.654944,0.886390,0.993606,0.479666,-0.209467,1.493953,1.295707,0.805385,1.398493,0.344354,0.860803,0.706785,1.061478,0.398070,0.272377 -244,1.497524,0.979777,1.529432,1.422367,0.451169,0.975224,1.457641,1.030234,0.607449,-0.396590,0.429958,1.061657,0.579652,0.441639,1.189722,0.393566,0.240597,0.615401,0.159545,0.058287 -489,0.727846,0.211845,0.623248,0.576531,-1.522249,-0.629587,-0.656742,-0.666752,0.125524,-1.353473,-0.569793,-1.177425,-0.540108,-0.382545,-1.458052,-0.615799,-0.585676,-1.054223,-0.112881,-0.792400 -465,-0.252003,0.195555,-0.210030,-0.318509,-0.962177,0.340350,0.153179,-0.531076,-0.768957,0.215815,-0.448163,-0.731822,0.248971,-0.363624,-0.704022,2.296605,1.489953,0.305581,-0.639573,1.640891 -483,-0.121357,-0.383884,-0.173371,-0.238305,0.223439,-0.469447,-0.543873,-0.446730,-0.290683,-0.271841,-0.584952,-0.491058,-0.644559,-0.433369,-1.265378,-0.966450,-0.494494,-0.415766,-0.931372,-0.858972 -397,-0.376970,-0.425771,-0.367377,-0.416914,-1.132974,-0.291682,-0.187060,-0.208651,-0.867532,-0.750991,-0.148960,0.087354,-0.098045,-0.214892,-0.540015,0.519318,0.449486,0.331534,-0.224274,0.102921 -52,-0.621222,-0.244260,-0.669713,-0.617993,-0.978545,-1.077031,-0.867289,-0.913859,0.205845,-0.240654,-0.641978,-1.059492,-0.666341,-0.503115,0.056343,-0.955607,-0.622812,-0.615770,-0.070504,-0.438735 -147,0.233661,-0.120926,0.241826,0.098437,-1.066078,0.234222,0.021352,-0.343038,-0.246872,0.302288,0.060736,1.255792,0.200458,-0.019957,-0.417010,1.301771,0.700816,0.563494,1.576164,2.108031 -27,1.273153,0.223480,1.241101,1.248876,-0.139504,0.042812,0.755818,0.732313,-0.418466,-0.823289,1.615944,1.146931,1.369225,1.170555,1.236390,0.097352,0.627208,1.186379,0.289099,0.159658 -436,-0.357089,0.058258,-0.383029,-0.414355,-0.355847,-0.483471,-0.889260,-0.722725,0.180289,0.095318,-0.139576,-0.812016,-0.133193,-0.304439,-0.377008,-0.662746,-0.823213,-0.858760,0.204344,-0.535946 -434,0.208100,-0.546779,0.120315,0.053500,-0.506718,-0.636788,-0.694784,-0.519727,-0.871183,-0.817619,-0.369482,-0.995265,-0.620798,-0.361204,-0.428344,-0.493401,-0.573077,-0.570189,-0.609303,-0.615379 -190,0.026330,0.891349,0.098896,-0.127954,0.792763,2.595576,1.372267,0.445226,2.089732,1.789355,-0.430117,1.620475,-0.373282,-0.189590,0.309686,6.143482,2.808608,0.788966,3.703513,2.997678 -407,-0.362769,0.484112,-0.384677,-0.399281,-1.483819,-0.401411,-0.345755,-0.780246,-0.845627,-0.234983,0.339728,1.054400,-0.155469,0.019866,-0.343340,1.079890,0.658375,0.261784,0.744354,1.488834 -202,2.602339,1.717457,2.756900,2.929741,1.262455,1.972072,3.308243,2.916806,1.417958,-0.071959,0.536790,0.622586,0.891521,0.942177,0.762038,1.438142,1.912705,2.085021,0.146227,0.529588 -455,-0.212241,2.659920,-0.231861,-0.277838,-0.278277,-0.570079,-0.760948,-0.419904,-1.594070,-0.373909,-0.232332,1.283007,-0.286652,-0.250975,-0.400009,-0.727578,-0.794367,-0.432798,-0.593563,-0.306347 -452,-0.604181,2.080480,-0.626051,-0.604056,-0.853294,-0.755046,-0.605769,-0.760127,-0.717843,-0.249159,-0.772270,0.885666,-0.668321,-0.582541,0.097678,0.023019,-0.082353,-0.426634,-0.374411,0.191810 -385,0.134256,0.930909,0.082420,0.027903,-0.678938,-0.719796,-0.061511,0.097781,-0.674032,-1.224471,0.037998,0.744147,0.023732,-0.161208,0.423691,-0.450366,0.066191,0.642977,-0.380465,-0.349468 -188,-0.658144,-0.442061,-0.687837,-0.642736,0.308838,-0.923335,-0.819454,-0.861239,-0.341796,-0.708463,-0.791399,1.286636,-0.918311,-0.569120,0.263351,-0.933475,-0.541577,-0.901259,1.313423,-0.683464 -159,-0.916597,-1.472952,-0.958868,-0.819355,-1.509439,-1.272800,-1.076078,-1.091889,-1.349457,-0.760915,-0.445276,-0.826349,-0.523772,-0.479573,-0.299672,-1.081135,-0.996658,-1.333386,-0.505176,-0.427388 -32,0.821570,1.091476,0.858032,0.695130,1.660981,0.857725,1.919664,1.841196,1.593204,0.144934,0.706423,0.328662,0.560840,0.603791,0.409024,0.298554,0.614276,-0.109676,0.057839,0.022353 -264,0.869853,0.647006,0.808603,0.777609,0.064029,-0.272730,0.022733,0.421754,0.202194,-0.991984,0.054600,-0.868260,-0.023296,0.111833,-0.849361,-0.749934,-0.378445,-0.311627,-1.102093,-0.839682 -245,-1.035883,0.132725,-1.040013,-0.902118,0.757180,-0.845823,-0.508342,-0.469944,-0.272428,0.227155,-0.120086,2.531276,-0.172795,-0.376605,2.999795,-0.649332,-0.439122,-0.075612,1.721458,-0.088853 -535,1.824140,0.365431,1.887787,1.857514,0.586383,1.318246,1.502839,2.148145,1.151440,-0.040772,1.059043,-0.411409,0.910827,1.043825,-0.821026,0.038109,0.270438,0.391552,-0.128622,-0.418310 -430,0.219460,0.754052,0.417297,0.085638,0.221305,2.239288,2.316401,1.243034,0.837458,0.876418,-0.549221,-0.620421,0.296989,-0.355263,-0.025327,2.048456,1.570525,1.710316,-0.672264,0.752378 -443,-1.010321,-0.225644,-1.035894,-0.893017,-1.063232,-1.131801,-0.864652,-0.975248,2.045921,-0.725474,-0.806197,2.404272,-0.786633,-0.598822,1.226390,-0.680072,-0.396681,-0.111298,-0.408313,0.263678 -154,-0.277565,-0.919110,-0.274287,-0.329885,-0.179357,-0.366919,0.051861,-0.363415,0.037902,-0.103146,-0.484255,-0.769560,-0.518326,-0.386066,0.514361,-0.296669,-0.047206,-0.366616,0.865433,-0.119491 -9,-0.473535,1.105439,-0.329482,-0.509063,1.582699,2.563358,1.738872,0.941760,0.797298,2.783096,-0.388250,0.693345,-0.409420,-0.360764,0.036008,2.609587,1.509848,0.409395,-0.321136,2.377346 -63,-1.407089,-1.263516,-1.349763,-1.120545,-1.362838,-0.318972,-0.363081,-0.699511,1.932741,0.968562,0.016703,1.901698,-0.127747,-0.370004,0.565697,0.776969,0.372230,0.617023,2.588380,0.766752 -329,0.605719,0.602792,0.639724,0.488932,1.433252,0.454059,1.143766,0.796798,0.209496,0.357575,0.597064,0.435708,0.046999,0.382454,1.173055,0.675250,0.878538,0.010359,2.382547,0.904814 -536,0.040531,0.758706,0.074182,-0.071356,0.529451,0.209585,0.721919,0.321673,0.417600,-0.422107,-0.730764,1.150560,-0.480704,-0.479573,-0.309339,0.019665,0.482643,0.156348,-0.730382,-0.014715 -65,0.185378,1.082168,0.223703,0.038142,1.483068,0.825508,0.475842,1.067119,0.516175,0.530523,-0.171337,0.116384,-0.205962,-0.112144,-0.112664,-0.132913,-0.289916,0.007115,-0.186739,-0.215945 -438,-0.078755,0.072221,-0.135476,-0.177157,-0.677515,-0.777787,-0.946385,-0.670364,-0.936900,-0.860147,-0.228362,0.836678,-0.264871,-0.235574,-0.401676,-0.811971,-0.864792,-0.771654,-0.048710,-0.555615 -269,-0.970559,0.256059,-0.925504,-0.881641,0.842579,0.465430,-0.054229,-0.522306,-0.524343,0.825384,-0.308126,0.493767,-0.314869,-0.431169,0.620366,1.250353,1.375893,0.917111,0.815791,1.443065 -36,0.034851,0.565559,0.068416,-0.062255,0.133059,0.103456,0.541128,0.182128,0.267911,-0.219390,-0.430117,-0.358974,-0.103491,-0.339422,-0.387675,0.249930,0.539010,-0.030193,-0.031759,0.085899 -98,-0.717787,-1.500877,-0.726143,-0.689095,-0.464730,-0.551317,-0.588066,-0.397721,-0.699589,0.428455,-0.626819,-1.229678,-0.688617,-0.540958,-0.296005,-0.680631,-0.496152,-0.796796,-0.488225,-0.470508 -392,0.387028,0.158322,0.429654,0.255431,1.397669,0.982805,1.259272,1.088786,0.428553,0.658107,0.872808,0.207101,0.895481,0.584650,0.076010,0.212484,0.591066,0.745169,-0.244857,0.165332 -69,-0.382650,-0.651497,-0.436576,-0.433410,0.138753,-0.985496,-0.656240,-0.523080,-0.809117,-0.888499,-0.607690,-0.623505,-0.690597,-0.484193,0.307019,-1.110309,-0.531962,-0.508387,0.241879,-0.714480 -517,1.636691,0.225807,1.587098,1.590169,0.522334,0.505228,0.656634,1.170811,-0.035117,-0.130081,0.370767,-0.622598,0.390055,0.426017,-0.650686,-0.136825,-0.045548,-0.002618,-1.207431,-0.152777 -92,-0.243483,-1.054080,-0.297765,-0.293480,-1.623304,-1.019419,-0.705455,-0.578795,-1.553910,-1.363396,0.001905,-0.115852,-0.081709,-0.087723,-0.853361,-0.843828,-0.607228,-0.158339,-1.192902,-0.892637 -552,-0.385490,2.359728,-0.437400,-0.418052,-0.967870,-1.175010,-0.864150,-0.875168,-0.995315,-0.911181,-0.592892,0.272418,-0.687627,-0.474733,0.598031,-0.734843,-0.617175,-0.404087,-0.190372,-0.782566 -505,-1.264230,-1.431064,-1.147107,-1.087554,2.073741,2.199490,0.376658,0.553561,0.895873,4.671344,-0.471984,0.314147,-0.534167,-0.498714,4.909873,1.307919,0.663017,0.438593,0.699555,2.906898 -566,0.702284,2.045574,0.672676,0.577953,-0.840484,-0.038680,0.046588,0.105777,-0.809117,-0.895587,0.184892,-0.257371,0.276693,0.180698,-0.379342,0.661277,0.510827,0.612157,-0.891416,0.036727 -163,-0.507616,0.681912,-0.499185,-0.541486,0.344421,-0.053841,-0.440671,-0.533913,-0.951504,0.682206,-0.397995,0.796763,-0.451002,-0.413348,1.433065,0.350531,-0.021344,-0.072368,-0.212166,0.587461 -358,-1.490873,-0.884204,-1.451091,-1.177143,-0.955772,-0.518531,-0.522153,-0.647665,0.432204,0.483742,0.479765,-0.030578,0.698459,-0.223473,1.296393,0.196276,0.008166,0.529430,0.947766,0.143394 -442,-0.098636,-0.814392,-0.148245,-0.196212,-0.582865,-0.704256,-0.982418,-1.005505,-1.484542,-0.612066,-0.176390,-1.330918,-0.312394,-0.241955,-0.203001,-0.777878,-0.800832,-0.995827,-1.001598,-0.465591 -15,0.117215,1.919912,0.196105,0.011123,1.248222,1.045345,0.942887,0.637649,1.794006,1.130169,-0.126943,-0.333573,0.006406,-0.171329,-0.478012,0.945755,0.514474,-0.145362,-0.238803,0.632094 -503,2.545536,0.125743,2.476807,2.921209,-0.209246,0.438898,0.989340,1.325317,-1.119447,-1.128074,3.197139,-0.855742,3.350833,3.077223,-0.429344,0.451691,0.260823,1.583793,0.177707,-0.339633 -176,-1.199475,-0.286147,-1.127336,-1.002515,0.044814,0.474905,0.526062,-0.303315,-0.520693,2.603060,0.093579,1.894441,0.131649,-0.282878,1.939752,3.498789,2.911395,2.076910,2.210615,5.342837 -327,-0.595661,-0.316399,-0.654061,-0.594102,-1.389881,-1.239824,-1.095463,-1.117580,-1.568513,-0.297358,-0.619601,-0.557282,-0.693072,-0.514116,-0.770691,-1.039050,-1.018243,-1.272557,-0.712220,-0.771596 -1,1.829821,-0.353632,1.685955,1.908708,-0.826962,-0.487072,-0.023846,0.548144,0.001392,-0.868652,0.499255,-0.876244,0.263327,0.742402,-0.605351,-0.692926,-0.440780,0.260162,-0.805450,-0.099444 -404,-0.507616,-1.009865,-0.563442,-0.528403,-0.678938,-1.111144,-0.850089,-0.732011,-0.878485,-0.810531,-0.078941,-0.560729,-0.130717,-0.224133,0.220349,-0.949403,-0.624138,-0.242688,1.108801,-0.893772 -24,0.716485,0.486439,0.742699,0.710203,1.120124,0.783815,0.799760,1.103489,0.669515,0.071219,1.449560,-0.571797,1.281605,1.369891,-0.331006,-0.372120,-0.148667,-0.080478,-0.709798,-0.375945 -139,-0.808671,-1.372888,-0.781338,-0.768161,1.426135,0.175472,-0.532950,-0.024740,-0.148296,-0.294523,-0.240994,0.228873,-0.502485,-0.308180,1.409731,0.531055,-0.332357,1.273972,-0.574190,-0.133486 -101,-2.029648,-1.363580,-1.984504,-1.454443,1.468835,-0.543168,-1.114873,-1.261820,0.432204,2.180614,-0.653527,0.528240,-0.650005,-0.671142,1.049716,-0.818119,-1.057501,-1.913447,0.732247,0.115403 -350,-0.700746,-0.516527,-0.752505,-0.665205,-1.476703,-1.289477,-1.010591,-0.962094,-0.513391,-0.777926,-0.186856,-0.987826,-0.317344,-0.314780,-0.152666,-1.033238,-0.860250,-0.892175,0.195869,-0.426253 -90,0.139936,1.100785,0.107134,0.022215,-0.471135,-0.346452,-0.725418,-0.499092,-0.462277,-0.586549,-0.119364,-0.192055,-0.290612,-0.144707,-0.724356,-0.407890,-0.685810,-0.517471,0.037256,-0.341146 -191,-0.385490,0.493420,-0.409803,-0.419474,-0.631257,-0.726429,-0.724162,-0.523080,-0.429419,0.010262,1.176342,0.963683,1.114780,0.292907,-0.823360,-0.423539,-0.336004,0.936576,1.596747,0.786799 -383,-0.493416,-0.421117,-0.466645,-0.546037,0.557917,0.480591,-0.375134,-0.518953,-0.119089,0.436961,-0.521069,-0.623868,-0.370807,-0.465052,-0.108664,0.646746,0.519779,-0.208624,-0.261808,1.168455 -11,0.469393,-0.325708,0.479082,0.358672,0.052642,0.471115,0.134849,0.442131,0.110921,-0.280347,0.363187,-0.420843,0.345502,0.304128,-0.423343,0.845713,-0.132088,0.166080,-0.055974,0.132046 -314,-1.570681,-0.160486,-1.560245,-1.233456,0.785646,-0.869323,-1.114873,-1.261820,1.282873,1.529934,-0.246769,2.830642,-0.318829,-0.495634,4.569859,-0.639831,-1.057501,-1.913447,4.954254,1.144247 -352,3.295334,-0.425771,3.387110,3.854074,1.319387,2.500819,3.113641,3.672569,0.527128,-0.225060,2.128091,-0.663965,2.156324,2.480977,-0.224002,0.947432,0.356978,0.532674,0.339952,-0.155047 -235,-0.027632,0.456187,-0.089755,-0.146440,-0.402816,-0.661236,-0.931319,-0.772766,-1.075636,-0.630495,-0.527927,0.519168,-0.593571,-0.401907,0.116012,-0.651009,-0.815388,-0.284862,-0.958009,-0.329042 -305,-0.717787,1.210157,-0.730674,-0.676012,-1.538617,-0.899456,-0.867038,-0.923145,0.450458,-0.569538,-0.555717,1.032628,-0.448032,-0.486834,-0.306339,-0.117823,-0.528315,-0.778304,1.380017,-0.599114 -375,0.580158,-0.749234,0.590295,0.380003,0.173624,0.747807,-0.279842,0.130282,0.651260,0.414279,-0.832544,-1.320576,-0.750991,-0.559439,-0.843694,-0.411243,-0.410607,0.026580,-0.145573,-0.037411 -215,-0.075915,-0.549106,-0.041562,-0.216121,0.444052,0.897523,0.128195,0.183159,1.074770,0.901935,-0.537311,-0.041464,-0.461893,-0.388266,-0.360341,0.497521,0.238607,0.414261,-0.139519,0.289399 -466,-0.280405,0.337507,-0.246689,-0.335573,-0.683920,0.086400,0.247342,-0.356451,-0.911343,-0.368238,-0.324728,-0.777362,-0.274276,-0.284638,0.084677,0.352767,0.472696,0.120662,-0.597195,-0.213676 -486,0.145616,-0.567723,0.092306,0.031601,-0.708116,-0.708046,-0.463019,-0.541910,-1.469938,-1.310945,-0.666881,-0.382561,-0.690597,-0.447891,-1.168707,-0.645420,-0.460674,-0.916831,-0.968906,-0.974339 -250,1.934906,0.993739,1.933096,2.016784,0.308838,1.066192,2.290035,2.117192,1.436213,-0.541186,2.161296,-0.718577,1.735548,2.146551,-0.586017,0.760202,2.098385,1.110141,0.419864,0.456586 -339,2.664822,1.158961,2.600377,3.106075,0.750064,0.454059,1.782814,2.375132,-0.053372,-1.096887,2.179342,-0.530430,1.780101,2.722996,-0.249670,-0.322379,0.130184,0.196900,-0.696480,-0.256040 -103,-1.207427,0.025680,-1.154110,-1.014176,0.294605,-0.139691,-0.342240,-0.480520,0.486968,0.059878,-0.811610,0.009338,-0.662380,-0.628525,0.672368,-0.196627,-0.053174,-0.109676,-0.539078,-0.085070 -206,-1.207427,-0.469986,-1.196536,-1.022424,0.892395,-0.606845,-0.894408,-0.758321,0.446807,0.007427,-0.691063,0.227059,-0.667826,-0.616204,0.892710,-0.725901,-0.793073,-0.692495,0.188604,-0.500390 -345,-1.098366,-1.065715,-1.061432,-0.947908,0.175047,-0.241650,-0.665279,-0.736396,-0.652127,1.028102,-0.242438,2.344398,-0.233684,-0.462192,3.439813,1.186639,-0.191771,0.188790,1.962404,1.120795 -346,-0.587140,-0.090674,-0.630582,-0.596093,-0.889588,-0.879367,-1.020585,-1.042880,-0.936900,-0.328545,-0.585313,-0.117666,-0.647034,-0.491014,0.046342,-0.811412,-0.864129,-1.022105,-0.087455,-0.393723 -317,1.162387,-0.097655,1.101054,1.058322,0.078262,0.139464,0.303839,0.788802,-0.016862,-0.872905,-0.003869,-1.209357,-0.157944,0.188399,-0.740023,-0.496754,-0.259080,-0.059391,-0.943480,-0.508333 -499,1.835501,0.453860,1.887787,1.891644,0.863928,1.138207,1.632155,1.629686,0.132826,-0.081882,0.668527,-0.001548,0.663312,0.764624,-0.124998,0.135916,0.289338,0.485634,-1.134784,-0.025685 -478,-0.749028,-1.093640,-0.740560,-0.710995,0.586383,-0.418088,-0.448455,-0.753936,-0.119089,0.417114,-0.728238,-0.092266,-0.643074,-0.571980,-0.694688,-0.242457,0.320505,-0.609606,-0.255754,-0.068427 -47,-0.271884,-0.146523,-0.246689,-0.342115,1.383436,0.355511,0.424367,0.631459,1.155090,0.704888,-0.426147,-0.586311,-0.479714,-0.353943,-0.169667,-0.118382,-0.094289,0.057400,-0.376832,-0.057458 -456,-0.709266,2.327149,-0.704313,-0.681985,-0.198572,-0.352516,-0.215937,-0.741555,-0.046070,-0.161268,-0.330864,2.193808,-0.354471,-0.378585,0.940045,-0.072552,0.359962,-0.418685,0.509462,-0.630509 -211,-0.649623,-0.081366,-0.677951,-0.645296,-0.544436,-0.669764,-0.779781,-0.902509,-1.017221,-0.315787,-0.660385,-0.638020,-0.703963,-0.510815,-0.508014,-0.458749,-0.379440,-0.819019,-0.530602,-0.330177 -490,-0.533178,0.733108,-0.567973,-0.535798,-1.027649,-0.991939,-0.899681,-0.936557,-0.977060,-0.430613,-0.654249,-0.141253,-0.638124,-0.490574,-0.648353,-0.750492,-0.745493,-1.175231,-0.540288,-0.528003 -113,-1.027362,0.209518,-0.960928,-0.912073,1.127241,0.491962,-0.301813,-0.470460,0.402996,2.129580,-0.258319,1.166889,-0.408430,-0.449431,1.613072,0.670220,0.464738,0.591070,0.281835,1.133656 -359,-1.332393,-0.225644,-1.324225,-1.070205,0.323071,-0.848666,-0.774633,-0.899156,-1.115796,0.962892,0.370767,0.054696,0.198478,-0.216873,-0.068329,-0.921962,-0.278974,-0.848541,-0.135886,-0.409232 -344,-0.686545,-0.893512,-0.697722,-0.667195,1.326504,-0.597559,-0.611921,-0.423515,0.720628,0.320717,-0.218617,-0.866083,-0.252990,-0.347783,0.831707,-0.810853,-0.510078,-0.095077,-0.122568,-0.262848 -532,-0.127037,-0.688730,-0.173371,-0.225791,-0.255504,-0.602486,-0.894910,-0.776893,-0.659428,-0.176862,-0.723546,-1.336724,-0.739110,-0.507955,-1.071036,-1.019992,-0.822815,-1.089909,-1.211063,-0.793535 -261,0.915295,0.877386,0.783889,0.791260,-0.693171,-0.785368,-0.751909,-0.530044,-0.904042,-1.378990,-0.016141,0.181700,-0.143093,0.089611,-0.438344,-0.805823,-0.644364,-0.669137,-0.778813,-0.838547 -4,1.750297,-1.151816,1.776573,1.826229,0.280372,0.539340,1.371011,1.428493,-0.009560,-0.562450,1.270543,-0.790244,1.273189,1.190357,1.483067,-0.048520,0.828471,1.144205,-0.361092,0.499328 -226,-1.047243,-0.891185,-1.044132,-0.925156,0.636199,-0.513983,-1.031470,-0.948165,-0.086230,0.241332,-0.771909,-0.569982,-0.820790,-0.626544,-0.176000,-0.973436,-0.964097,-1.106941,-0.668632,-0.368380 -464,-0.271884,-0.248914,-0.316713,-0.334435,-1.544310,-0.841464,-0.504827,-0.521532,-1.305645,-1.035930,-0.732208,-0.964965,-0.806929,-0.515876,-0.357341,-0.589531,-0.538925,-0.540667,-1.163843,-0.802991 -277,1.329956,0.160649,1.191672,1.271629,-0.507430,-0.862311,-0.107964,0.245323,-0.955155,-1.819865,-0.277447,-0.705514,-0.249030,-0.079142,0.176681,-0.801352,-0.187792,0.459680,-0.150416,-0.801478 -257,0.338746,-0.469986,0.462606,0.166127,2.643065,2.351102,1.958584,1.941793,2.140845,1.865906,0.916840,-0.286400,0.591532,0.420737,1.036382,1.140251,0.594714,1.536752,-0.206112,1.840229 -260,1.755977,1.808213,1.685955,1.800632,0.259022,0.084505,0.792227,1.145533,0.008694,-1.003325,-0.026968,-0.333573,-0.138143,0.264085,-0.666020,-0.542025,-0.355567,-0.585761,-0.935004,-0.719776 -301,-0.473535,0.139706,-0.475295,-0.522146,-0.843330,-0.055736,-0.257368,-0.462464,-0.111787,-0.043607,-0.147877,-0.320873,-0.142103,-0.264396,-0.170333,0.458957,0.504858,0.362354,0.830320,0.323820 -537,-0.692225,1.198521,-0.642527,-0.706729,1.938527,0.963854,-0.548016,-0.093094,1.166043,1.595143,-0.395108,1.380982,-0.350511,-0.426549,1.946419,0.529378,-0.439122,0.951175,-0.598406,0.767508 -403,-0.337208,-0.725963,-0.362022,-0.418905,0.172912,-0.302863,-0.701061,-0.645343,-0.279730,-0.113069,-0.936128,-0.565809,-0.924994,-0.637545,-1.384716,-0.705781,-0.522678,-0.728019,-0.223063,-0.689516 -233,1.812780,1.982743,1.747740,1.888800,-0.339479,0.057973,0.836170,0.889399,-1.327551,-0.974973,0.428875,-0.050536,0.445993,0.652855,-0.673687,-0.271519,0.088738,-0.143740,-0.915632,-0.343416 -487,1.508885,-0.109290,1.488242,1.456496,0.892395,0.766758,1.717529,1.817982,0.041553,-0.233566,0.580101,0.346806,0.378669,0.602911,-0.584350,0.159389,0.353662,-0.005862,-0.408313,-0.220862 -141,0.563117,-0.288475,0.540867,0.449684,0.060471,0.177367,0.071195,0.271117,0.180289,-0.045025,1.081781,0.208915,0.825187,0.742402,-0.090330,-0.340822,-0.040575,-0.020461,-0.217009,-0.136512 -68,-1.447987,-0.456023,-1.366651,-1.150124,0.728714,0.700428,2.814833,-0.133333,1.093024,2.503828,-0.280696,-0.041464,-0.485654,-0.498714,0.836041,3.385892,9.015603,3.475158,2.594434,2.180277 -363,0.673883,-0.232625,0.602652,0.521071,0.035563,-0.372605,-0.378900,-0.014681,-1.155957,-0.973555,-0.239190,0.403050,-0.258435,-0.148667,0.072010,-0.415155,-0.450064,-0.237822,-0.436161,-0.678547 -379,-0.865474,-0.106963,-0.768981,-0.834144,1.796196,2.104733,1.005662,0.380483,1.403355,2.367738,-0.699364,-0.344459,-0.567829,-0.579681,0.121345,1.118454,0.463744,0.258540,-0.382886,0.242117 -51,-0.138398,-0.686403,-0.196026,-0.236314,-1.388458,-0.829146,-0.881727,-0.817389,-1.674391,-0.463218,-0.786707,-0.532425,-0.701488,-0.567360,-0.854694,-0.766142,-0.699736,-0.622907,-0.882941,-0.470508 -195,-0.345728,-0.688730,-0.388796,-0.393877,-1.206274,-0.960480,-0.628619,-0.648697,0.063458,-0.868652,-0.761442,-0.559278,-0.679706,-0.540958,-0.581017,-0.536994,-0.287926,-0.547642,-0.243646,-0.621431 -239,0.946536,4.651889,0.882746,0.755709,0.125231,0.482486,0.664167,1.010888,-0.009560,-0.444789,0.474351,-0.654531,0.067295,0.190599,-0.727022,0.132562,-0.194755,0.315313,-1.002808,-0.391832 -161,1.437881,-0.779486,1.414099,1.428055,-0.670398,0.268334,0.382935,1.231685,-0.257825,-1.564696,2.146859,-1.058222,2.032071,1.737319,0.788372,0.283464,0.382840,2.618691,1.334007,-0.163746 -169,0.239341,-0.544452,0.175098,0.088198,0.155832,-0.483092,-0.788192,-0.286549,-0.115438,-0.892752,-0.483172,0.000266,-0.481694,-0.353283,-0.653686,-0.658275,-0.816250,-0.441719,-0.852671,-0.790131 -212,3.971288,-0.190738,3.976130,5.244841,1.269571,0.895628,2.903973,2.852321,-0.597362,-1.069952,8.906909,0.470181,9.461986,10.676614,2.136427,0.125297,1.060902,0.368843,3.303954,0.257626 -120,-0.771749,-1.970944,-0.767334,-0.715545,-0.187185,-0.710510,-0.673942,-0.585243,-0.527994,-0.236401,-0.954174,-1.371922,-0.872768,-0.656467,-0.333673,-0.569410,-0.555503,-0.865573,-0.859936,-0.601005 -367,-0.544538,-0.295456,-0.562618,-0.558835,-0.288240,-0.617647,-0.563459,-0.738976,-0.425768,-0.515669,-0.550304,-0.795142,-0.491100,-0.478913,-0.402676,-0.648215,-0.394360,-0.763868,-0.140729,-0.694055 -171,-0.198041,0.079202,-0.252456,-0.254516,-0.418473,-0.785747,-0.379403,-0.375022,-0.779910,-0.862982,0.232895,-0.126738,0.136599,0.067389,-0.346007,-0.829856,-0.344293,-0.381702,-0.646838,-0.728854 -348,-0.754709,-0.758542,-0.780514,-0.717252,-0.398546,-0.861932,-0.790076,-0.662883,-0.648476,0.130758,-0.846259,-0.826168,-0.879204,-0.617964,0.716703,-0.946218,-0.745825,-0.889093,-0.362303,-0.297269 -409,-0.527497,-0.318726,-0.558499,-0.536936,-0.676803,-0.740642,-0.711733,-0.577505,0.563638,-0.967885,-0.256153,1.023556,-0.389618,-0.320061,-0.384342,-0.132913,-0.374798,-0.169694,0.634173,-0.577932 -112,0.037691,0.083856,0.241414,-0.071072,-1.280286,2.254449,2.655385,0.749595,-0.392910,2.111151,-0.152930,0.495582,0.263822,-0.243935,-0.581017,2.737574,3.700532,1.804398,0.619644,3.474275 -524,-1.215664,-0.839989,-1.193240,-1.028681,-0.102498,-0.381891,-0.822342,-0.638637,-1.550259,0.866495,-0.560770,-0.001548,-0.440607,-0.552179,0.563697,-0.282697,-0.705373,-0.810260,-0.278759,-0.113817 -62,0.034851,0.665623,0.183336,-0.026135,0.607733,1.828041,1.565613,0.970134,0.501572,1.434954,1.077089,0.092797,1.241012,0.449779,0.788706,2.519604,1.230002,1.084187,-0.428896,0.876823 -393,2.125195,0.695875,2.159642,2.139081,1.447485,1.977758,2.413074,2.767200,1.279223,0.462478,0.790518,-0.536961,0.639550,0.894434,-0.608684,0.658483,0.506848,0.175813,-0.011175,0.088169 -75,0.551757,0.083856,0.499677,0.463051,-0.333074,-0.380943,0.111622,0.450385,-0.049721,-1.259911,1.235173,-0.364417,1.070722,0.856151,1.259725,-0.192715,0.102995,1.020925,-0.610514,-0.698594 -124,-0.215082,-0.674768,-0.241747,-0.288361,-1.794101,-0.589220,-0.098925,-0.539588,-1.422476,-0.647506,-0.870802,-0.139439,-0.813365,-0.564940,-0.374008,0.403626,0.586093,-0.229711,-1.024602,0.106325 -231,-0.797311,1.812867,-0.832414,-0.737160,-1.959205,-1.254796,-0.909850,-1.181213,0.209496,-0.923939,-1.025637,-0.588126,-0.894550,-0.698160,-1.129372,-0.503461,-0.516046,-1.406542,-0.626254,-0.659256 -285,-0.439453,-0.207027,-0.500009,-0.471806,-0.884606,-1.178422,-1.091521,-1.186398,-0.418466,-0.602143,-0.481007,0.241574,-0.566839,-0.393546,-0.219335,-0.976398,-0.995829,-1.439146,0.625698,-0.673251 -448,0.114375,0.011717,0.093953,0.013683,-0.888165,-0.499201,-0.007901,-0.507346,-1.236277,-0.756662,-0.547416,0.248831,-0.431696,-0.380565,-0.964699,-0.279344,0.062875,-0.478216,-0.799396,-0.555237 -460,0.838611,1.829157,0.792127,0.785003,0.186433,0.126198,0.149413,0.396991,-0.067976,0.001756,1.890966,-0.117666,1.576643,1.647112,0.566363,-0.183773,-0.155298,0.451569,-0.011175,0.235309 -218,1.611130,0.528326,1.554146,1.635674,-0.180069,0.497647,0.482119,0.979935,1.030958,-0.990566,1.985527,-0.055979,1.792477,1.849528,-0.078996,0.347178,0.085090,0.863582,-0.190372,0.094599 -467,-1.266502,-0.276839,-1.273149,-1.048305,-0.942962,-0.948730,-0.929185,-1.113014,-0.480532,0.187463,-0.229445,0.172629,-0.292592,-0.425889,1.313060,-0.721430,-0.715652,-1.275315,0.773413,-0.308617 -525,-1.578066,-1.440373,-1.542121,-1.233172,0.515218,-0.531039,-0.792838,-0.872330,-0.487834,1.199632,-1.005064,-0.975307,-0.889599,-0.727884,0.285352,-0.439188,-0.460343,-0.726073,-0.559661,0.049209 -80,-0.760389,0.391029,-0.747974,-0.720665,0.984910,-0.203179,-0.538474,-0.685840,0.110921,1.028102,-0.288997,1.736593,-0.390608,-0.345803,1.109719,-0.470486,-0.200060,-0.695902,-0.288445,0.068500 -472,0.225140,-1.014519,0.184572,0.091042,-1.094545,-0.357254,-0.419453,-0.430995,-0.454976,-0.865817,-0.579538,-1.421454,-0.514861,-0.374625,-1.256711,-0.434717,-0.291573,-0.551048,-1.097250,-0.535568 -549,-0.939318,1.144999,-0.950630,-0.834144,-1.027649,-0.726239,-0.920522,-1.051341,0.600147,0.068384,0.412994,1.272121,0.345502,-0.161428,0.407357,-0.378827,-0.634086,-0.953653,0.498565,-0.309373 -527,-0.507616,-1.633519,-0.536668,-0.530110,-0.450497,-0.782146,-0.743497,-0.579053,-0.447674,-0.668770,-1.041517,-1.308420,-1.036920,-0.690459,-1.120038,-0.911510,-0.767974,-0.982364,-1.120254,-0.918736 -168,0.949377,1.254371,0.993959,0.937732,0.607733,1.060506,1.595745,1.428493,-0.998966,0.120835,2.464469,0.350434,2.213253,1.803324,-0.289005,0.607064,0.482311,0.631622,-1.100882,0.501219 -351,0.460872,-0.016208,0.623248,0.294964,1.988342,2.502714,2.543646,1.941793,2.056873,1.875829,0.415882,0.194401,0.302434,0.239443,0.762705,2.241833,2.242618,1.789799,4.224151,1.337155 -57,0.165497,0.535307,0.147501,0.005719,1.233989,0.609461,0.508485,0.833426,0.786345,0.677954,0.062901,-0.121295,-0.064878,-0.005436,-1.127372,0.171685,-0.204702,0.149859,-0.287234,0.118429 -560,-0.021952,1.829157,-0.024262,-0.154973,0.208495,0.156521,-0.554670,-0.151647,-1.002617,-0.154180,-0.146794,0.499211,0.010861,-0.230954,0.071677,0.072760,-0.370819,0.724082,0.031202,0.570818 -424,-1.245485,-0.039478,-1.237314,-1.038635,0.792763,-0.398189,-1.002707,-0.754452,2.651977,1.062124,1.051464,0.961868,0.861819,0.070030,2.009755,-0.370444,-0.857862,-0.207002,1.266203,0.162684 -355,-0.445133,-0.051114,-0.413922,-0.480907,-0.623429,-0.010252,0.178289,-0.129206,-1.017221,-0.135751,-0.162314,0.473810,0.171251,-0.282658,0.937378,0.942961,1.021776,1.275595,0.708031,0.155119 -42,1.403799,1.284623,1.496480,1.277317,-0.394988,2.172958,1.530460,1.307519,1.819562,0.089648,2.078645,0.814906,2.952333,1.420495,-0.164333,4.198526,2.166357,2.365644,3.969886,1.456682 -217,-1.115407,-0.421117,-1.108800,-0.949046,-1.125857,-0.857952,-0.389949,-0.985566,0.556336,0.049955,-0.174946,1.279378,-0.058938,-0.386066,-0.787025,0.319792,0.856986,-0.176182,0.612379,0.309446 -286,-0.621222,0.342161,-0.580742,-0.608323,-0.733736,-0.061421,-0.289509,-0.283970,-0.816419,0.689294,-0.472706,0.314147,0.164321,-0.405427,-0.107331,1.457703,0.397098,0.735436,-0.220641,1.596257 -135,-0.385490,0.740089,-0.422160,-0.422603,-0.413491,-0.885621,-0.523408,-0.564350,-0.827372,-0.304446,-0.608051,0.296004,-0.697527,-0.450311,0.152680,-0.752169,-0.284279,-0.466537,-0.493068,-0.441005 -521,2.982918,0.537634,3.028755,3.373421,0.472519,2.013766,1.785325,2.532475,0.654911,0.651019,2.116181,-0.574155,2.071179,2.190554,-0.684021,0.371211,0.126537,0.677041,-0.212166,0.365049 -278,-0.152598,-0.337343,-0.235980,-0.234892,-1.201293,-1.209502,-0.864150,-0.942490,-0.871183,-1.077040,-0.531175,-0.092266,-0.585651,-0.398607,-1.100038,-1.129200,-0.704378,-1.095262,-0.859936,-1.009516 -504,-1.380108,-1.493895,-1.255437,-1.155243,4.770911,2.265820,0.106726,0.092880,2.067826,4.567859,0.008763,-0.224713,0.073235,-0.446570,0.914044,1.114541,0.097358,1.635700,0.590585,2.517299 -97,-1.232704,0.151341,-1.229900,-1.024984,0.429819,-0.972798,-1.029135,-1.057093,-1.685343,0.865077,-0.253266,1.498914,-0.363382,-0.446350,1.363062,-0.606297,-0.881503,-1.061847,-0.306607,0.708123 -84,-0.604181,-0.846971,-0.618637,-0.601781,0.061894,-0.619543,-0.593716,-0.781278,0.976194,-0.441954,-0.642699,0.069211,-0.705448,-0.531937,-0.357341,-0.411243,-0.392039,-0.773600,-0.099563,-0.449326 -553,-1.361647,0.616755,-1.357589,-1.111729,-0.281835,-0.915186,-0.613176,-0.931141,-0.436721,0.419950,-0.374896,1.201362,-0.368827,-0.494534,1.299726,-0.398947,0.267454,0.166080,2.064110,0.313228 -469,-0.712106,-0.258223,-0.642115,-0.699903,1.504417,0.833088,0.165734,0.173358,0.530779,1.382503,0.017786,0.949168,0.079671,-0.274737,2.516442,0.367857,0.587419,1.072832,-0.299342,0.537910 -87,1.389599,1.233427,1.236982,1.197683,-0.431994,0.308132,0.728197,0.871601,0.516175,-0.922521,0.520910,-1.003792,0.093531,0.380914,-1.056369,-0.394476,0.172625,0.033069,-0.109249,-0.173203 -22,0.344426,-1.170433,0.433773,0.140814,0.778530,2.068725,1.492795,1.254641,2.589911,1.066377,0.121370,-0.920332,0.256397,0.100612,-0.083996,1.553833,1.079801,1.739514,1.958772,0.226609 -19,-0.166799,-1.147162,-0.185728,-0.251957,0.101747,-0.436850,-0.278210,-0.028609,0.267911,-0.728310,-0.488225,-0.776999,-0.400014,-0.369124,0.473693,-0.607974,-0.266043,0.219610,-0.089876,-0.565449 -265,1.875263,2.753002,1.801287,2.173210,-0.118866,0.188738,0.601392,0.968328,-0.155598,-0.858729,2.767643,0.726003,2.417205,3.506257,-0.830027,-0.597914,-0.346946,-0.408142,-0.832088,-0.565828 -38,0.245021,1.375378,0.147089,0.124887,-0.177222,-1.005016,-0.813805,-0.514052,-0.900391,-1.099722,2.919230,1.761994,2.579575,1.444697,-0.052662,-0.812530,-0.454706,1.196112,-1.532890,-0.771975 -479,0.602879,0.051277,0.734461,0.457647,0.444052,1.610099,1.692419,1.109679,1.239062,0.422785,-0.326533,-0.419392,0.100957,-0.158788,0.718703,1.633196,1.621255,1.679497,0.870276,0.968360 -17,0.568798,0.323544,0.664438,0.409297,1.468835,1.854573,1.047093,1.389802,1.286524,1.525681,0.592011,-0.261000,0.489061,0.304568,-0.004993,-0.026164,-0.000455,0.190412,-0.442215,0.131290 -189,-0.518977,-0.788794,-0.541199,-0.543761,-1.107354,-0.602865,-0.632260,-0.835187,-0.527994,-1.142250,-0.602637,-0.691906,-0.583670,-0.484413,-0.348340,-0.187685,-0.355567,-0.869953,-0.035391,-0.782944 -178,-0.317327,0.681912,-0.410215,-0.365436,-2.408970,-1.610136,-1.094848,-1.214049,-1.521051,-1.482475,-0.837597,-0.135810,-0.873758,-0.571980,-1.207709,-1.298098,-1.004615,-1.613035,-0.534235,-1.069167 -164,2.596659,0.640025,2.476807,2.932585,-0.851870,0.192528,0.547405,1.240713,-0.038768,-1.030259,0.934886,-0.654531,0.859839,1.265383,-0.710355,-0.002132,-0.121809,0.719215,-0.118935,-0.020767 -451,1.551487,1.328837,1.471766,1.524754,0.486752,-0.106715,0.962975,1.075889,-0.542598,-1.259911,0.224594,0.286932,0.024722,0.348571,1.619739,-0.345852,0.569514,0.518075,-0.500333,-0.751927 -290,0.080293,0.102473,0.167272,-0.011061,-0.625564,1.198852,0.595114,0.441099,-0.356400,1.293194,1.717724,1.003598,0.739546,0.809067,0.240350,4.522686,2.244939,2.581382,2.449140,7.211398 -67,-0.800151,-0.058095,-0.830767,-0.741711,-1.065367,-1.086507,-0.649209,-0.686614,-1.079287,-0.868652,-0.478119,-0.497046,-0.512386,-0.488154,0.747037,-0.908883,-0.373471,-0.459238,0.155913,-0.623700 -110,-1.235545,-0.535144,-1.213835,-1.037213,0.522334,-0.384734,-0.570740,-0.803203,-0.831023,1.113158,-0.007839,0.375835,-0.058938,-0.384306,2.269766,0.214720,-0.154967,-0.254043,1.485355,0.318902 -556,-1.126767,0.069894,-1.121981,-0.976065,0.280372,-0.555297,-1.051784,-0.973959,-0.075277,0.072637,-0.581343,1.584188,-0.602977,-0.517856,1.956420,-0.182096,-0.919103,-0.764679,0.626908,-0.573771 -150,-0.320167,0.346815,-0.348429,-0.385345,1.219756,-0.539188,-0.721149,-0.579569,2.659279,-0.273259,0.054239,0.190772,0.003436,-0.122265,-0.007993,-0.785703,-0.411270,-0.043170,1.085796,-0.855568 -321,1.713375,0.086183,1.611812,1.760815,-1.150054,-0.354411,0.335226,0.731023,0.424902,-1.678104,0.676106,-0.962606,0.495991,0.759343,-0.835027,-0.654921,-0.180497,0.109307,0.168021,-0.834386 -526,-0.189520,-0.125580,-0.186552,-0.295187,0.792763,0.179262,-0.587439,-0.448793,-0.323542,0.052790,-0.741231,-1.106847,-0.704458,-0.533917,-0.876028,-0.617475,-0.557161,-0.718124,-0.847828,-0.759492 -18,1.613970,0.665623,1.566503,1.720997,0.138753,-0.031099,0.742007,1.188093,-0.838325,-1.254241,1.274152,-0.362603,1.484567,1.585507,-0.182334,-0.365972,0.066854,0.553762,-0.845406,-0.680060 -446,1.028901,2.033938,1.043388,0.929199,0.256887,0.512808,1.016961,0.877275,-0.360051,-0.515669,-0.055842,-0.253742,0.003436,0.079490,-0.775691,-0.297787,0.168314,-0.112920,-0.989490,-0.468617 -432,1.719055,0.058258,1.723026,1.692556,1.205523,0.844459,1.563102,1.985643,-0.319891,-0.321457,0.100798,-0.391632,0.070265,0.267385,0.682035,0.093440,0.781388,1.184757,0.480403,0.079469 -523,-0.118517,-0.141869,-0.133416,-0.238589,0.199243,0.050392,-0.438788,-0.286033,-0.356400,0.798450,-0.310652,0.058325,-0.288137,-0.305539,-0.100664,-0.165888,-0.365845,0.279627,-0.223063,-0.018120 -418,-0.405371,-1.656790,-0.456760,-0.454741,-0.605638,-0.879367,-0.818576,-0.642248,-0.834674,-0.006749,-0.649196,-1.036269,-0.662875,-0.505315,-0.303339,-0.718077,-0.756601,-0.578300,-0.413156,0.234174 -272,2.164957,0.395683,2.279094,2.377985,-0.167259,1.738969,1.640943,1.544566,-0.330843,-0.121575,2.749597,0.245202,2.970649,2.562383,-0.451345,1.348160,1.041007,0.618645,-0.157680,0.309824 -153,-0.845593,-1.445027,-0.869073,-0.776409,0.083955,-1.008427,-0.866033,-0.801139,0.067109,-0.247742,-0.649918,-0.789881,-0.711388,-0.546898,0.659367,-0.921794,-0.660943,-0.578137,0.404124,-0.823039 -509,0.378508,1.084495,0.487320,0.217320,1.561350,1.566511,1.433786,0.935828,-0.016862,1.138675,-0.260123,1.350138,0.035118,-0.171989,0.832374,1.336982,0.938221,1.397252,0.061472,0.834081 -288,-0.814352,0.155995,-0.751681,-0.741711,-1.150054,0.260754,0.049475,0.179548,2.860081,-0.066289,0.293891,1.248535,0.005416,-0.124465,2.899791,3.193632,1.627887,3.742804,1.651232,1.029637 -258,0.435311,0.909965,0.750937,0.337342,1.034726,3.924075,2.872586,2.290012,2.494987,2.586049,3.200748,2.244609,3.590923,2.159752,1.773079,1.926616,1.671985,2.991773,0.342374,0.849967 -228,-0.428092,1.089149,-0.437400,-0.450759,-1.233317,-0.550559,-0.432134,-0.736654,-1.086589,-0.369656,-0.578456,-0.273700,-0.703468,-0.480233,-0.624018,-0.141855,-0.057485,-0.503683,-0.834509,-0.146725 -476,0.020650,0.288638,0.018164,-0.103779,-0.501736,0.122408,-0.479215,-0.473040,-1.115796,-0.383832,-0.207068,-0.360789,-0.057948,-0.205212,-0.978033,0.413687,-0.122141,0.276383,-0.537867,-0.397506 -423,-0.132717,-0.037151,-0.103348,-0.226360,-0.412068,0.196319,0.097561,-0.020613,0.132826,-0.140004,-0.652444,-0.583953,-0.525752,-0.461532,-1.020368,0.145976,0.159030,0.153104,-0.572980,0.060935 -64,-0.411051,1.058897,-0.382205,-0.443365,1.127241,0.414261,0.301328,0.511001,0.340930,0.439796,0.073368,-0.070493,0.030168,-0.085302,0.246684,0.055994,-0.071742,0.179057,-0.507597,-0.073344 -137,-0.766069,-0.907474,-0.778867,-0.725500,0.002115,-0.671849,-0.675072,-0.520243,-0.283381,-0.587967,-0.827491,-0.404696,-0.852967,-0.608723,-0.636019,-0.573882,-0.582029,-0.518120,-0.669842,-0.834764 -343,1.577048,0.556251,1.562384,1.533287,0.114556,0.560187,1.224118,1.583257,0.987147,-0.800607,0.784743,1.936171,1.142006,0.601151,-0.761691,0.458398,0.383172,1.016059,2.060478,-0.191737 -302,1.693494,1.063551,1.760097,1.684024,0.828346,1.505866,1.751427,2.039810,1.596855,1.685870,2.406722,0.954611,2.444432,1.990339,0.307686,1.220732,1.478680,1.226932,0.825477,0.806846 -561,-0.831392,2.345765,-0.877311,-0.764748,-1.556408,-1.303122,-1.114873,-1.261820,-2.744117,-1.102557,-0.328698,4.860893,-0.408430,-0.385626,0.184348,-0.927774,-1.057501,-1.913447,-0.078979,-0.764788 -8,-0.320167,0.588830,-0.184080,-0.384207,2.201839,1.684010,1.219096,1.150692,1.965600,1.572462,-0.356850,-0.389818,-0.227743,-0.352403,-0.436677,0.533290,0.120568,0.075243,0.107482,-0.017363 -502,-0.450813,-0.691057,-0.441519,-0.507926,1.383436,0.078820,-0.370614,-0.416035,0.479666,0.470983,-0.532258,-0.221084,-0.643569,-0.480673,0.887043,-0.548172,-0.203707,-0.030193,0.904178,-0.497364 -28,0.333066,1.391668,0.429654,0.220449,0.842579,1.238650,0.998129,0.995412,0.417600,0.368916,0.122092,-0.371675,0.312830,0.069590,-0.602684,0.284582,0.128194,-0.156717,-0.346563,-0.313156 -457,-0.260524,1.387014,-0.324127,-0.332729,-0.601368,-0.990992,-0.766849,-0.728400,-0.703240,-0.986314,-0.710192,0.241574,-0.768317,-0.500695,-0.424344,-0.972262,-0.556830,-0.867033,-0.856303,-0.743984 -136,-0.686545,-0.609610,-0.710491,-0.657810,0.621966,-0.822323,-0.663898,-0.591176,-1.725504,-0.474559,0.157823,2.342584,0.194023,-0.131286,-0.154333,-0.652686,-0.174861,0.206633,-0.841774,-0.033250 -365,1.792899,0.579522,1.723026,1.814853,-0.345884,0.165996,0.115389,0.746242,-0.706891,-1.024589,0.624133,-0.544400,0.669252,0.706319,-0.277671,-0.358707,-0.270021,0.456436,-0.737646,-0.714480 -384,-0.240643,-1.296095,-0.254516,-0.321637,-0.905956,-0.352327,-0.477457,-0.523080,-0.710542,-0.972138,-0.800783,-1.244737,-0.630698,-0.551739,-0.923364,-0.265371,-0.119820,-0.539856,-0.718274,-0.447057 -81,-0.223602,-0.798102,-0.225682,-0.383638,0.814113,0.931636,0.352803,0.540406,0.476015,0.882089,-0.430117,-0.364417,-0.658915,-0.602343,-0.082330,0.574090,0.262149,0.329912,0.096584,0.305663 -306,-0.263364,-0.807410,-0.325363,-0.334435,-0.800631,-0.982274,-1.096530,-1.177705,-0.655777,-0.546856,-0.775518,-1.167264,-0.822770,-0.546458,-1.136372,-0.984614,-1.020100,-1.526415,-0.859936,-0.457648 -282,1.497524,-0.258223,1.451171,1.393926,0.522334,0.755387,0.926565,1.179323,0.297119,-0.549692,0.237226,-0.402337,0.018287,0.282126,-0.462345,-0.194950,-0.043227,0.518075,-0.522127,-0.692164 -475,-0.368449,-0.828354,-0.373967,-0.420896,-0.424166,-0.410318,-0.382290,-0.467881,-0.389259,-0.519922,-0.921330,-1.323298,-0.827225,-0.631385,-0.722689,-0.420185,0.042649,-0.557375,-0.548764,-0.569610 -118,0.469393,0.842480,0.565581,0.363223,1.362087,1.342883,1.563102,1.183192,1.038260,1.490241,0.529933,-0.262814,0.362333,0.402476,0.552696,0.798207,0.915342,0.341267,-0.677107,0.828029 -55,-0.740508,-0.125580,-0.767334,-0.699334,-0.079725,-0.940202,-0.733704,-0.674233,0.395694,-0.528427,-0.289719,-0.467653,-0.338135,-0.371105,0.429024,-0.936493,-0.610212,-0.507090,1.409075,-0.532920 -557,-1.336086,1.999032,-1.347292,-1.090967,-1.076753,-1.035338,-1.114873,-1.261820,-0.254174,-0.312952,0.477599,3.102794,0.372234,-0.247015,1.516402,-0.795763,-1.057501,-1.913447,1.149967,-0.178120 -284,-0.351408,-0.835335,-0.324951,-0.393308,-1.293808,-0.161864,0.285006,-0.387404,-1.385966,-0.488735,-0.410988,0.312333,-0.256950,-0.375065,-0.207668,0.789823,1.570857,0.964152,-0.213376,-0.037411 -256,1.540126,2.206141,1.714788,1.570260,-0.267602,1.932274,1.124934,1.689012,0.297119,-0.067706,1.578769,-0.032392,2.124642,1.453497,-0.228336,1.239175,0.223355,0.550517,-0.143151,0.551149 -426,-1.035883,-1.002884,-1.008296,-0.913779,0.128078,-0.057631,-0.319515,-0.689709,0.413949,0.900517,-0.279974,-0.163025,-0.149529,-0.430509,0.107678,0.737287,0.687553,0.136883,0.129276,0.391527 -387,-0.070234,-0.728290,-0.148245,-0.165780,-1.857438,-1.061870,-0.858124,-1.042390,-0.747051,-1.142250,-0.545251,-1.079632,-0.572780,-0.378805,-1.104371,-0.633124,-0.398339,-0.775384,-0.493068,-0.690272 -497,-0.470694,-0.460677,-0.474471,-0.497118,-0.503871,-0.531418,-0.661764,-0.650760,-1.042777,-0.331381,-0.909420,-0.790788,-0.798514,-0.625444,-1.081703,-0.657716,-0.611207,-0.762408,-0.627465,-0.852542 -298,0.037691,-0.260550,-0.030853,-0.061970,-2.177683,-0.988149,-0.804137,-0.907410,-0.644825,-0.983478,-0.632233,-0.993995,-0.596541,-0.435130,-1.290712,-0.654362,-0.699736,-1.062982,-1.151735,-0.695190 -157,0.770448,0.039642,0.676795,0.640807,-1.559255,-0.608551,-0.468292,-0.547068,0.118223,-1.434276,0.266100,1.529758,0.303424,0.138015,-1.199375,0.063259,-0.044222,-0.112920,-0.646838,-0.860864 -249,-0.740508,-1.014519,-0.745503,-0.707013,0.351537,-0.497685,-0.571494,-0.506314,0.260609,-0.158433,-0.537671,-0.324502,-0.584165,-0.477813,-0.126331,-0.737638,-0.359545,-0.281618,-0.456744,-0.382376 -342,-0.871154,-1.007538,-0.843535,-0.799162,0.493868,-0.253400,-0.437281,-0.400043,-0.130042,0.889177,-0.884517,-0.715493,-0.748021,-0.649866,0.125012,-0.374915,-0.143030,-0.275130,0.355692,-0.332068 -508,0.617080,-0.835335,0.524391,0.469024,-0.148756,-0.705393,-0.421085,-0.084840,-0.367353,-0.882828,-0.716327,-1.353960,-0.851482,-0.432709,0.117678,-0.751610,-0.238522,0.407773,-0.859936,-0.463700 -267,-0.152598,0.593484,-0.198085,-0.267030,-1.195599,-0.412213,-0.603634,-0.709312,-0.644825,-0.596472,-0.242438,1.268492,-0.136163,-0.298719,-0.535015,-0.079259,-0.029965,-0.304490,-0.029337,-0.297269 -338,-1.158009,-0.409482,-1.135162,-0.978625,0.308838,-0.589031,-0.799618,-0.803976,0.286166,0.072637,-0.517099,1.448112,-0.538623,-0.516756,0.254017,-0.614122,-0.497147,-0.608795,0.055418,-0.384645 -360,-0.450813,-0.283820,-0.516897,-0.463558,-1.565660,-1.475202,-1.099882,-1.121268,-1.035476,-1.551938,-0.195157,-0.479265,-0.265861,-0.264836,-0.419343,-1.161672,-1.031210,-1.326735,-0.013597,-1.035238 -12,1.432201,1.282296,1.665360,1.331355,0.073992,2.680858,1.477729,1.621948,2.137194,2.155097,1.986249,4.265788,4.061202,1.669114,-1.300712,3.213194,1.890159,4.720928,2.941929,3.421320 -210,1.832661,0.663296,1.760097,1.806321,-0.388583,0.577243,0.944142,1.204343,-0.170202,-1.780172,1.641930,0.477438,2.060783,1.570106,0.361022,0.594210,0.762488,2.571650,1.358222,-0.540863 -533,1.801419,0.321217,1.760097,1.831917,-0.341614,0.510913,0.797249,1.356270,1.293826,-1.220218,1.546286,0.941910,1.139531,1.321487,-0.701021,0.302466,0.299616,0.842495,0.922340,-0.406962 -167,0.753407,-0.113944,0.713866,0.658156,-0.548706,-0.237291,-0.057493,0.434393,0.297119,-1.057194,0.699566,0.315962,0.625194,0.594110,-0.306005,-0.043490,-0.186797,0.688396,0.044521,-0.114196 -79,-0.359929,-0.300110,-0.361610,-0.422603,0.212053,-0.168308,-0.626610,-0.664689,-0.341796,-0.400843,-0.504106,-0.221084,-0.538623,-0.439750,-0.582683,-0.495636,-0.370819,-0.586735,-0.370779,-0.358168 -369,2.238801,0.607446,2.274975,2.352388,0.707364,1.725703,1.958584,2.609857,0.045204,-0.198126,2.175733,-0.937931,2.324139,1.977138,-1.021034,0.152682,0.128194,0.472657,-0.649259,0.000415 -437,-0.024792,-0.770177,-0.090167,-0.124256,-0.838349,-0.860226,-0.671180,-0.502445,-0.356400,-0.541186,-0.057647,-0.309987,-0.109926,-0.167149,0.311686,-0.700192,-0.524336,-0.446099,-0.059607,-0.364976 -485,-0.476375,-0.670114,-0.375615,-0.506788,-0.086842,0.886153,0.823615,-0.011843,0.987147,1.481735,-0.047180,-0.017878,1.058346,-0.223253,0.064343,2.751546,2.636192,2.500278,1.426026,2.205998 -408,1.097064,0.318890,1.063983,0.957925,0.515218,0.493857,0.392979,1.014241,0.658562,-0.298776,0.175148,-0.623324,0.096502,0.208420,0.063343,0.125297,-0.225591,0.487256,-0.775181,-0.173581 -308,-0.178160,-1.531128,-0.258635,-0.252241,-1.608359,-1.292510,-1.080246,-1.147836,-1.630579,-1.339297,-0.652444,-0.962425,-0.671786,-0.438870,-1.234376,-1.214599,-0.990192,-1.387564,-0.695269,-0.843086 -414,0.284783,2.448156,0.195281,0.183760,-0.936557,-1.104700,-0.526547,-0.555322,0.147430,-1.397419,0.227120,0.744147,0.087591,0.110953,-0.069996,-0.626418,-0.232222,-0.439448,1.327953,-0.773488 -41,-0.902396,0.479458,-0.826648,-0.807125,1.874478,0.330874,0.195866,0.200441,0.304421,0.836725,-0.608412,0.383092,-0.516841,-0.514116,0.341021,-0.438070,-0.197076,-0.231333,-0.844196,-0.285543 -276,-0.794471,-1.193704,-0.831178,-0.734601,-0.182915,-1.243615,-1.096204,-1.175848,0.519826,-0.650341,-0.605164,0.114570,-0.644064,-0.511475,0.461692,-0.921067,-1.008196,-1.372803,0.367800,-0.820013 -146,-0.660984,-0.630553,-0.534609,-0.633919,0.906628,1.244335,0.967997,0.650804,3.163109,1.546945,-0.308487,0.379464,-0.289622,-0.343602,-0.538015,0.606506,0.483969,1.076077,4.327067,0.317767 -405,-0.905236,-0.162813,-0.888845,-0.810254,0.287488,-0.563636,-0.494155,-0.505540,-1.188815,0.475236,-0.092295,0.954611,0.075215,-0.320281,0.826041,-0.231279,-0.397676,-0.039926,0.029991,-0.413771 -35,0.742046,0.535307,0.746818,0.610375,-0.018523,0.554502,0.577537,0.290462,0.308072,-0.884246,0.203299,-0.539139,0.070265,0.106773,-0.421677,-0.027282,0.167651,0.024958,0.889649,-0.427388 -13,0.489274,1.084495,0.483201,0.363507,-0.878913,-0.078478,0.132840,0.121770,0.129175,-1.335044,-0.006757,-0.251928,0.018287,-0.082662,0.909377,0.323146,0.617261,1.317769,1.122119,-0.299917 -166,-0.944998,-2.229249,-0.955573,-0.845521,-0.029909,-0.890359,-0.797107,-0.823838,-1.572164,0.170451,-0.838680,-1.470441,-0.861383,-0.634905,0.256017,-0.875350,-0.693105,-1.046599,-0.968906,-0.633535 -173,-0.865474,-1.065715,-0.896259,-0.802575,0.301721,-0.889033,-0.818199,-0.595561,-0.896740,0.551787,-0.714162,1.067100,-0.737130,-0.467692,2.639781,-0.238544,-0.575729,0.654332,1.244409,0.374505 -531,-0.697906,0.169958,-0.690308,-0.678856,0.372887,-0.185933,-0.587564,-0.705443,0.172987,0.256925,-0.716327,-0.621147,-0.729209,-0.549978,-0.596684,-0.458749,-0.447411,-1.054061,-0.732803,-0.425118 -388,-0.811511,-0.881877,-0.765686,-0.747684,-0.904533,0.133779,0.149413,-0.550679,-0.005910,1.378250,-0.269507,-0.271886,-0.147053,-0.382105,1.113052,2.303312,2.083133,1.406984,-1.011284,3.228412 -205,0.281943,-0.607283,0.280545,0.175512,-0.540877,-0.160348,-0.166973,-0.209683,-0.794513,-0.416437,-0.483894,-1.550817,-0.441597,-0.305759,-0.523014,-0.351441,-0.381429,-0.573596,-0.643205,-0.345685 -6,1.170908,0.160649,1.138125,1.095295,-0.123136,0.088295,0.300072,0.646935,-0.064325,-0.762332,0.149883,-0.804940,0.155410,0.298627,-0.909030,-0.651568,-0.310141,-0.228089,-0.829666,-0.611218 -144,-0.959199,-1.005211,-0.976580,-0.852062,-1.311599,-1.003499,-0.832260,-1.058692,-1.506447,-0.838883,-0.551026,0.040181,-0.524762,-0.497174,-0.164666,-0.428569,-0.388392,-1.003127,-0.464009,-0.542754 -540,-0.734828,-1.128546,-0.713374,-0.716683,0.247636,0.145150,-0.269044,-0.592724,0.023298,0.711976,-0.457547,0.999969,-0.612877,-0.428529,1.703076,0.874216,0.783709,0.509965,-0.259386,0.649494 -30,1.278833,1.354435,1.352314,1.231812,0.714481,1.598728,1.796625,1.946952,1.355892,-0.117322,1.535820,0.452038,1.340514,1.422695,-0.264337,0.461752,0.665338,-0.035060,-0.057185,0.289399 -435,-0.041833,0.076875,-0.034972,-0.157532,0.686015,0.169787,0.298817,0.405245,-0.520693,0.374586,-0.665437,-0.478177,-0.625748,-0.472752,-0.575683,-0.422980,-0.333020,-0.361588,-1.006441,-0.358924 -31,-0.649623,-0.137215,-0.578271,-0.609460,1.034726,0.895628,0.414323,0.074825,1.786704,2.153679,0.279093,-0.339016,0.301444,0.014585,-0.496680,0.484107,0.336752,-0.219979,0.264884,0.708123 -203,-0.090115,1.037954,-0.016848,-0.162367,2.557666,1.373206,0.841192,1.105036,1.604157,1.617825,0.576131,1.293893,0.516288,0.272445,0.594364,0.313085,-0.025654,0.180679,-0.068082,0.268973 -59,-1.564717,-1.745219,-1.549947,-1.224071,0.082532,-0.978294,-0.856115,-1.060652,-0.469579,1.286106,-0.899675,-1.156197,-0.900985,-0.704386,1.023049,-0.834327,-0.400660,-0.982040,0.043310,-0.380863 -95,1.741776,0.870405,1.665360,1.732374,-0.397123,0.510913,0.724430,0.977872,1.034609,-0.894169,1.271986,0.530054,0.835582,1.045806,-0.341673,0.522112,0.345705,0.144993,0.729825,0.233039 -37,-0.311646,-0.202373,-0.385500,-0.372831,-0.464730,-1.263703,-0.793214,-0.507861,-1.258183,-0.590802,-0.798617,2.041403,-0.839601,-0.575940,-0.896363,-1.150159,-0.612202,-0.025327,0.746776,-0.763275 -331,-0.325847,0.014044,-0.306827,-0.400703,-0.040584,0.154625,-0.222591,-0.500897,-0.184806,0.368916,-0.493639,-1.180146,-0.198537,-0.433149,-0.438011,0.395243,0.399087,-0.322008,0.843639,0.293938 -237,1.804259,0.505055,1.669479,1.851826,-0.911649,-0.395347,0.020348,0.291494,-1.258183,-1.563278,1.018620,-0.319059,1.127650,0.949658,0.306019,0.327058,0.353994,0.797076,-0.863568,0.052235 -10,0.537556,0.919273,0.442011,0.406453,-1.017686,-0.713542,-0.700684,-0.404686,-1.035476,-0.826124,-0.092656,-0.054164,-0.198042,0.003805,-1.004034,-0.905921,-0.692442,-0.682114,-0.719485,-0.284787 -389,1.540126,0.912292,1.521194,1.476405,0.330188,0.520389,1.215330,1.371746,0.647609,-0.561032,0.741794,2.937689,1.245963,0.654835,1.399730,0.865833,1.418997,3.668187,0.864222,0.924861 -324,-0.547378,-0.949362,-0.574975,-0.560257,-0.685343,-0.737042,-0.864527,-0.825385,-0.633872,-0.213719,-0.532979,-0.743071,-0.449022,-0.469232,-0.546015,-0.631448,-0.709020,-1.079365,-0.873254,-0.654338 -179,-0.374130,-1.449681,-0.439872,-0.415492,-0.638374,-1.262187,-0.999455,-0.918760,-1.261834,-0.208049,-0.419650,-0.411772,-0.538623,-0.408067,0.497694,-1.068280,-0.852590,-0.711798,-1.197745,-0.167907 -220,-0.135558,-1.426410,-0.168428,-0.244562,0.007097,-0.326553,-0.626736,-0.600720,-1.648834,0.091065,-0.703695,-1.421091,-0.730199,-0.504655,-0.969366,-0.476634,-0.509747,-0.833293,-0.827245,-0.400910 -50,-0.672344,0.537634,-0.710491,-0.645580,-0.710963,-1.036286,-0.906837,-0.974217,-1.155957,-0.555362,0.003710,-0.012434,-0.114381,-0.261096,-0.394676,-0.878591,-0.670227,-0.705796,0.425918,-0.766301 -401,-0.624062,-1.950000,-0.652001,-0.603488,-0.543724,-0.983979,-0.787690,-0.798560,-0.768957,-1.047271,-0.552108,-0.311801,-0.602482,-0.470552,-0.288671,-0.751051,-0.601260,-0.967441,-1.160210,-0.733015 -368,2.153597,-0.474640,2.015476,2.534411,-0.179357,-0.354790,0.351548,0.921641,-0.345447,-1.737644,2.893965,-0.300915,2.409285,4.043098,-0.491013,-0.802470,-0.362529,0.028202,-0.958009,-0.753440 -29,0.977778,-0.986595,0.948650,0.853831,0.150139,0.215270,0.124931,0.789576,-0.265127,-0.185367,0.704258,-0.715493,0.885580,0.456820,-0.471345,0.271168,0.072159,0.282871,-0.156470,-0.020011 -78,1.719055,1.089149,2.130809,1.678336,2.294354,4.568425,3.598263,2.875535,3.995524,2.639918,1.900350,1.212248,2.862733,1.673514,1.113052,2.396089,2.559931,2.292649,7.071917,0.829163 -204,-0.470694,-0.160486,-0.448110,-0.491999,0.234114,0.027651,-0.109847,-0.276232,0.413949,0.132176,-0.032743,-0.313616,-0.182696,-0.221053,-0.029327,-0.355912,-0.161929,-0.231333,-0.329612,-0.079018 -214,0.017810,1.051916,0.037111,-0.125678,-0.123136,0.497647,0.285006,0.404988,1.545741,0.217232,0.056044,1.139674,0.330651,-0.205432,1.279726,0.649540,0.165330,0.725704,2.960091,0.368075 -127,1.383918,-0.088347,1.294648,1.374017,-1.009858,-0.455991,0.049099,0.189608,0.490619,-1.751820,1.026560,0.227059,1.163293,0.899714,-0.871028,0.102382,0.282043,0.294226,-0.029337,-0.419444 -46,-1.684571,-0.570050,-1.658278,-1.288347,-0.737294,-0.851130,-0.915500,-1.109197,-0.155598,0.316465,-0.898232,-0.472008,-0.877224,-0.706961,0.642366,-0.504020,-0.530967,-0.953653,0.629330,-0.458783 -554,-0.354249,2.241047,-0.390031,-0.399850,-1.076753,-0.873682,-0.337092,-0.657467,-0.896740,-0.810531,-0.698642,0.259717,-0.675251,-0.517196,0.457026,-0.220660,0.234960,-0.677410,-0.434950,-0.375945 -449,1.980348,0.286311,1.900144,2.073666,0.034139,0.249383,0.858769,1.717385,-0.940551,-0.877158,0.935247,0.261531,0.829642,0.914235,-0.524681,-0.264254,-0.001450,0.464546,-1.241333,-0.601383 -564,2.110995,0.721473,2.060786,2.343856,1.041842,0.219060,1.947285,2.320965,-0.312589,-0.931027,2.782080,0.071025,2.379583,2.604187,1.086384,0.191805,0.666001,2.067178,-1.138416,0.167980 -297,-0.672344,-0.267531,-0.698958,-0.636479,0.236249,-0.856626,-0.777772,-0.355161,-0.703240,0.010262,0.865589,1.611403,0.629650,0.193019,-0.481679,-0.862272,-0.635743,0.409395,-0.580244,-0.392210 -491,1.057302,-1.410121,0.932174,0.959062,-1.279575,-0.799203,-0.556804,-0.184147,-2.159966,-1.469717,0.282341,-0.309987,0.146995,0.233502,-0.890696,-0.961531,-0.675200,-0.707094,-0.910789,-0.940296 -492,1.102744,0.295619,1.088697,1.001440,0.266139,0.465430,0.354059,0.740309,1.111279,-0.287435,1.261881,0.129084,1.231112,1.086949,0.318687,0.085056,0.181577,0.759768,1.020413,0.076065 -181,1.977508,1.694187,2.089619,1.866047,1.262455,3.389643,2.007548,2.596960,2.129892,1.585220,0.810729,-0.823628,0.766278,0.904775,-0.929364,1.235822,0.226339,0.628378,-0.310239,0.567413 -427,-0.944998,0.626063,-0.954749,-0.838979,-0.594251,-0.889033,-0.661136,-0.899672,0.746185,-0.429195,-0.351797,0.733261,-0.309918,-0.443050,-0.166000,-0.223454,-0.065774,-0.218356,-0.254543,-0.417931 -471,-0.592820,2.059536,-0.622756,-0.583010,-0.629123,-0.840327,-0.817697,-0.648697,0.154732,-0.824707,0.725191,2.587520,0.610343,0.101712,0.158680,-0.554879,-0.571750,0.005493,-0.008754,0.033322 -515,-0.791630,-0.158159,-0.791224,-0.749959,0.607733,-0.366730,-0.574758,-0.592724,0.421251,-0.097476,-0.585313,-0.375303,-0.680696,-0.487274,0.512028,-0.506814,-0.361535,-0.117786,0.459820,-0.975096 -445,-0.607021,1.303239,-0.591451,-0.607469,0.472519,-0.230469,-0.431757,-0.159385,0.030600,0.808373,-0.515655,-0.023321,-0.495555,-0.460872,0.426358,-0.292198,-0.283616,0.787343,0.048153,-0.045732 -506,-0.541698,0.174612,-0.514838,-0.573909,0.942210,0.205794,-0.088504,-0.703122,1.140487,0.870748,-0.808723,-0.763754,-0.930835,-0.610703,-0.256337,-0.195509,0.257507,-0.308545,-0.888995,0.006845 -93,-0.192360,-0.230298,-0.221152,-0.283810,0.415586,-0.430028,-0.615938,-0.544747,-0.633872,-0.807695,-0.397634,0.283304,-0.379718,-0.332602,-0.385675,-0.590648,-0.436801,-0.394192,-0.206112,-0.748145 -453,0.114375,-1.235591,0.077889,-0.030401,0.963560,-0.225920,-0.249208,0.413500,-0.590060,-0.225060,-0.357933,-0.899104,-0.357936,-0.322041,-0.302672,-0.724783,-0.522015,-0.070746,0.184972,-0.087718 -357,-0.073075,-0.716655,-0.142066,-0.174028,-0.635527,-0.936601,-0.926297,-0.723241,-1.415174,-0.562450,-0.544529,0.265160,-0.558919,-0.431169,-0.467679,-0.980254,-0.883294,-0.933377,-0.617779,-0.646017 -544,-0.073075,0.328198,-0.090579,-0.199341,-0.041296,-0.048155,-0.651846,-0.650760,-0.699589,0.578721,-0.480646,-0.308173,-0.391104,-0.378805,-0.247670,-0.210041,-0.190445,-0.443666,-0.683161,-0.074101 -26,0.128576,0.521345,0.224115,-0.028694,0.643316,1.562720,0.674211,1.003666,1.607807,0.913276,-0.543807,-0.423928,-0.374272,-0.424349,-0.863028,0.283464,-0.168561,0.279627,-0.726749,-0.031737 -356,-0.305966,-0.162813,-0.283349,-0.406391,0.842579,0.493857,0.090781,0.183417,0.815553,0.313629,-0.341330,0.531869,-0.136658,-0.412908,0.255351,0.773615,0.637486,1.129606,0.975614,0.726279 -227,0.247862,-0.879550,0.225762,0.084216,-0.900263,0.099666,-0.298172,-0.286807,0.253307,-0.528427,-0.625736,-1.306787,-0.292097,-0.450091,-0.974032,0.368416,0.150741,-0.039926,-0.803029,-0.223510 -377,-0.189520,2.075826,-0.250397,-0.263902,-1.508016,-1.081769,-0.955299,-0.973701,-1.426127,-0.732562,-0.852755,-0.121295,-0.725744,-0.559439,-0.699688,-0.751610,-0.808558,-1.073364,-0.741279,-0.798452 -234,-1.295188,-0.786467,-1.308161,-1.067361,-0.834079,-1.202869,-0.907465,-0.831834,-0.951504,0.174704,-0.685649,-0.701704,-0.817325,-0.609383,1.533069,-0.842710,-0.664258,-0.352504,0.398070,-0.096418 -341,-1.284111,-0.570050,-1.249259,-1.064801,-0.821981,-0.228573,-0.057493,-0.670622,0.819204,1.198214,-0.796813,-0.497046,-0.711388,-0.621924,-0.362341,0.515964,0.609634,-0.533043,-0.368357,0.089304 -129,1.608290,1.356762,1.582979,1.527599,0.365770,1.033974,2.080367,1.701909,1.425260,-0.236401,0.325291,-0.032392,-0.050027,0.505884,-0.669353,0.352767,0.519116,-0.221601,-0.576612,-0.215945 -289,-0.783110,-0.093001,-0.815526,-0.736307,-0.656877,-1.028326,-0.813679,-0.701316,0.735232,-0.460382,-0.503745,1.373724,-0.451497,-0.502675,-0.167666,-0.644302,-0.601260,-0.303679,1.648811,-0.327908 -539,-1.827998,1.431228,-1.797089,-1.377937,-0.688901,0.294866,0.046713,-0.909990,0.822855,2.085634,-0.669769,0.475624,-0.703468,-0.629405,2.809788,2.184826,2.010187,0.299092,0.061472,1.420748 -145,-0.632582,-1.079677,-0.570856,-0.631644,1.340737,0.478696,-0.649083,-0.487226,0.669515,2.210384,-0.032382,-1.021573,0.076700,-0.336782,1.043049,1.225762,-0.132751,-0.112920,1.298894,2.128835 -138,0.233661,-0.400174,0.201048,0.066014,1.447485,0.495752,0.817337,0.962653,0.530779,-0.090388,3.215185,0.426637,2.748876,1.354490,0.986380,0.520995,1.123237,2.630045,3.770106,-0.343416 -410,-0.785950,-0.400174,-0.802345,-0.725500,-0.553687,-0.970524,-0.765468,-0.720146,-0.768957,-0.519922,-0.770826,0.613514,-0.746040,-0.586941,-0.550015,-0.868028,-0.671885,-0.960952,-0.863568,-0.807909 -56,1.443561,-0.167467,1.381147,1.413834,0.636199,0.423736,0.546150,1.058091,0.384741,-0.451877,1.163349,-0.043278,0.975676,1.367691,-0.194334,-0.135149,-0.081026,0.581337,-0.244857,-0.448948 -216,-0.635423,-0.218663,-0.603397,-0.633351,-0.194302,0.209585,-0.281725,-0.451373,0.567289,0.048537,-0.392942,0.628029,-0.385658,-0.415328,0.610365,0.918928,0.914016,0.981995,0.894492,0.370723 -281,-0.678025,-1.226283,-0.730262,-0.647287,-1.297366,-1.154922,-0.833014,-0.549132,1.056515,-0.236401,0.565664,0.092797,0.421242,-0.055160,0.331020,-0.618034,-0.555503,1.080943,1.049472,-0.677034 -16,0.156977,0.195555,0.114137,0.084216,0.164372,-0.612909,-0.186433,0.094686,-0.823721,-0.507163,0.243723,0.041996,0.162836,0.111393,-0.441011,-0.774525,-0.395023,-0.114542,-0.780024,-0.646773 -444,1.108425,-0.567723,1.051626,0.953090,-0.490350,0.357406,0.253619,0.351336,-0.334494,-0.708463,-0.384641,-1.136239,-0.467833,-0.100484,-0.974699,-0.552085,-0.071079,-0.331417,-0.919264,-0.514007 -547,-1.098366,-0.630553,-1.075848,-0.950184,-0.540166,-0.448790,-0.567727,-0.632962,-0.520693,0.615579,-1.049457,-0.351717,-0.929350,-0.726300,1.076384,0.299672,-0.191108,-0.134007,0.269727,0.792473 -270,0.046211,-0.574704,-0.068748,-0.063392,-2.282296,-1.470464,-1.023849,-1.100607,-1.108494,-1.281175,-0.992432,-0.901826,-1.001031,-0.650526,-1.183041,-1.216611,-0.897485,-1.328195,-0.627465,-0.913062 -44,-0.271884,0.586503,-0.269756,-0.350931,0.055489,0.006804,-0.077958,0.092880,-0.239570,-0.145674,-0.762886,-1.096868,-0.758416,-0.568680,-1.230376,-0.650450,-0.576061,-0.801825,-1.139627,-0.784835 -34,0.568798,-0.328035,0.619129,0.433188,0.543684,0.977120,0.585070,0.737730,0.680468,0.333476,-0.256875,-0.963695,-0.338135,-0.116765,-0.952032,0.178951,-0.174198,-0.442692,-0.425264,0.008358 -371,0.301824,-1.414775,0.234000,0.161860,-1.190618,-0.663320,-0.688883,-0.576473,-0.330843,-1.043018,-0.818829,-1.459374,-0.756436,-0.497614,-0.676354,-0.594002,-0.543235,-0.428418,-0.493068,-0.767057 -310,-0.689385,-0.041805,-0.726555,-0.671746,-0.585000,-0.981895,-0.916128,-0.965705,0.454109,-0.215137,-0.884517,0.386721,-0.869798,-0.639305,-0.325673,-0.914808,-0.712005,-0.674490,0.356903,-0.807152 -538,-1.817206,1.442863,-1.811917,-1.354046,-1.094545,-1.052963,-1.114873,-1.261820,0.213147,1.425031,-0.099152,0.444780,-0.185171,-0.466372,1.873083,-0.882280,-1.057501,-1.913447,1.002252,1.163916 -72,0.872693,1.217138,0.915698,0.780737,0.764297,1.490705,1.009428,0.787254,0.421251,0.293783,0.669609,-0.319059,0.415301,0.640974,-0.407009,1.714795,0.352336,-0.085345,-0.638362,0.947178 -73,-0.092956,-0.814392,-0.063393,-0.201331,0.308838,0.448373,-0.136966,0.045677,-0.546249,0.405774,-0.456464,-1.081990,-0.450012,-0.373745,-0.774691,-0.269842,-0.474269,-0.420145,-1.010073,-0.251501 -279,-0.078755,-0.956343,-0.122707,-0.191946,-0.085419,-0.520426,-0.552536,-0.304605,1.089373,-0.604978,-0.567628,-0.539501,-0.512881,-0.460432,-0.935364,-0.562704,-0.574403,-0.221601,-0.637151,-0.832873 -196,-0.101476,0.698202,-0.055155,-0.187680,1.682331,0.423736,0.623991,0.421496,0.081713,0.846648,0.772111,1.624104,1.009833,0.206000,2.253098,0.447220,0.489274,1.428072,0.768570,0.193323 -74,-0.516137,-0.644516,-0.526371,-0.523284,-0.330228,-0.683219,-0.691519,-0.675780,-0.334494,-0.518504,-0.558244,-0.348088,-0.557434,-0.454491,-0.729023,-0.407331,-0.452053,-0.621447,-0.809083,-0.563936 -23,1.997389,0.872732,1.863073,2.130548,-0.148044,-0.040575,0.262407,0.964717,-0.155598,-1.420100,1.034140,-0.163025,0.711330,1.180456,-0.771024,-0.720312,-0.488858,-0.229711,-1.175951,-0.683842 -334,-0.518977,-0.062749,-0.580330,-0.541771,-0.941539,-1.181075,-1.017496,-1.041668,-0.995315,-0.474559,-0.798256,0.571784,-0.825245,-0.596182,0.280018,-0.952701,-0.825269,-0.855516,-0.139519,-0.594953 -400,1.074343,0.402664,1.335838,0.964466,1.895827,2.904485,2.888907,1.828299,1.100326,1.184038,-0.007839,-0.802218,0.127194,0.025806,0.039342,0.654012,0.986630,-0.208624,-0.560872,0.493276 -431,-0.490575,-0.374576,-0.432457,-0.532101,0.643316,0.516599,-0.142993,-0.539846,-0.002259,1.165609,-0.824604,0.441152,-0.327740,-0.547998,0.986380,0.417599,0.554262,-0.020461,0.160756,0.835972 -254,1.511725,0.009390,1.422337,1.462184,0.508101,0.274020,0.616458,0.954141,-0.130042,-0.897005,0.688377,-1.057134,0.460844,0.674637,-0.797359,-0.417950,-0.146014,0.141749,-0.834509,-0.470887 -48,-0.589980,-1.084331,-0.573740,-0.584717,0.479635,-0.254348,-0.287249,-0.552743,-0.498787,-0.335633,-0.510963,-0.884408,-0.503970,-0.450311,-0.517681,-0.626418,-0.287595,-0.995341,-0.758230,-0.519303 -130,-0.550218,-1.396158,-0.530902,-0.566230,0.728714,-0.175320,-0.756428,-0.518437,0.249657,0.271101,-0.738704,-0.726742,-0.442092,-0.552179,-0.089330,-0.051314,-0.719962,-0.600522,0.735879,0.131668 -555,-1.089845,1.936202,-1.083262,-0.948477,-0.431283,-0.526112,-0.361700,-0.555580,-0.798164,-0.216555,-0.668686,1.854525,-0.707428,-0.569340,1.669741,0.105176,0.535363,0.878181,-0.255754,0.432378 -520,-1.372439,-1.254207,-1.318458,-1.129362,2.899260,0.344140,-0.696541,-0.637347,1.407006,2.007666,-0.185413,-0.157582,-0.236654,-0.455591,2.806454,-0.004367,-0.329041,0.649465,2.352277,0.040131 -122,2.874993,0.211845,3.057588,3.145893,3.440117,3.455973,4.243589,3.927930,3.079138,0.846648,3.983947,3.452962,3.435978,4.238914,5.429894,4.056567,3.179967,1.042013,3.018209,2.299805 -496,-0.419572,-0.260550,-0.382205,-0.481476,0.799880,0.550711,-0.108341,0.046967,-0.622919,0.814044,-0.623571,-1.058948,-0.579215,-0.482653,-0.445678,-0.025605,-0.183482,-0.239444,-0.357460,-0.087718 -225,0.060412,-1.354271,0.022283,-0.038933,0.192127,-0.532555,-0.396226,-0.074523,0.961590,-1.179108,0.421657,-0.734362,0.444013,0.174978,0.016008,-0.625859,-0.274995,0.172569,0.256408,-0.882046 -326,-0.004911,-1.491568,-0.079869,-0.109183,-0.232731,-0.971851,-0.893278,-0.556870,-1.601372,-0.821872,-0.534423,-0.246485,-0.647529,-0.361204,-0.116331,-0.791292,-0.867942,-0.838484,-0.772759,-0.498877 -106,-0.706426,-0.223317,-0.691956,-0.689379,1.269571,-0.050051,-0.227236,-0.362899,-0.038768,0.340564,-0.357933,0.798577,-0.351996,-0.433809,0.499694,-0.132913,-0.081026,0.354244,-0.592352,0.017058 -381,-0.876835,-1.014519,-0.877311,-0.802575,-1.173538,-0.635841,-0.669674,-0.726852,0.698723,-0.047860,-0.869719,-0.337202,-0.784653,-0.630505,-0.581683,-0.360384,-0.486205,-0.784468,-0.210955,-0.701242 -415,-0.635423,0.437570,-0.641704,-0.628800,0.097477,-0.438556,-0.794093,-0.699769,0.757138,0.014515,-0.470901,-0.025135,-0.463378,-0.457791,0.951379,0.282346,-0.517041,-0.408791,0.246722,-0.576040 -71,-1.488033,-1.082004,-1.366651,-1.168611,0.104593,0.924055,-0.034392,-0.521016,0.329977,3.827870,0.436815,-0.661607,0.149470,-0.327761,3.389811,3.811771,0.822834,0.951175,0.589374,6.859624 -558,0.131416,0.788958,0.182100,0.006288,-0.827674,0.543131,0.177034,-0.298156,-1.305645,-0.188203,-0.648835,-0.197498,-0.317839,-0.457571,-0.933031,1.168754,1.123568,0.691640,-0.503965,0.231148 -21,-1.313080,-1.593959,-1.302806,-1.083572,0.429819,-0.747086,-0.743748,-0.726337,0.012345,0.886341,-0.461517,-0.435539,-0.473774,-0.542058,0.855042,-0.623623,-0.399334,0.391552,-0.032970,-0.312777 -316,-0.553058,-1.212320,-0.606280,-0.550303,-1.353587,-1.368694,-0.973881,-1.131534,-0.506089,-0.894169,-0.699725,-1.119910,-0.706933,-0.539418,-0.566016,-1.101646,-0.685147,-1.094126,-0.093509,-1.076013 -463,-0.717787,-0.216335,-0.745091,-0.688811,-0.802766,-0.867807,-0.692147,-0.803461,-1.079287,-0.596472,-0.806919,-0.818729,-0.773762,-0.603883,-0.110664,-0.473280,-0.367835,-0.697362,0.084477,-0.388428 -454,-0.428092,-0.497911,-0.467469,-0.460714,-0.749392,-0.948351,-0.742492,-0.675780,-0.046070,-0.643253,-0.851673,-0.996897,-0.866333,-0.594422,-1.051036,-0.946721,-0.641048,-0.796147,-0.540288,-0.815852 -413,0.245021,0.656315,0.229057,0.110382,-0.797785,-0.034889,-0.253727,-0.262045,0.483317,-0.519922,-0.312457,0.216173,-0.275266,-0.260216,-0.864028,0.145417,0.040660,0.026580,-0.179475,0.083252 -330,0.540396,-0.879550,0.569700,0.393370,-0.103210,0.620832,0.396746,0.554335,-0.108136,-0.430613,-0.245686,-0.851387,-0.117352,-0.155488,-0.400676,0.389654,0.174283,0.453192,-0.711009,-0.254905 -468,0.986298,0.940217,1.113411,0.926071,-0.246964,1.820460,1.566869,1.322737,-0.422117,1.544110,1.890244,0.450223,1.452886,1.420495,-0.091663,2.502279,1.127879,1.835218,-0.461587,2.838813 -322,-0.359929,-1.389177,-0.376851,-0.426869,1.212639,-0.303242,-0.637784,-0.384824,-0.980711,0.278189,-0.663994,-0.317244,-0.619808,-0.522916,-0.377008,-0.297228,-0.426854,-0.273508,-1.031867,-0.260201 -307,-1.456224,-1.137854,-1.466743,-1.162069,-1.872383,-1.386888,-1.068658,-1.172263,-0.086230,0.784274,-0.832183,0.159928,-0.852472,-0.672110,0.116012,-1.151053,-0.935450,-1.350255,0.783100,-0.621052 -373,1.849702,-0.451369,1.764216,1.934305,-0.135234,0.061763,0.802271,1.044420,-0.878485,-1.136580,0.752621,-1.014859,0.620244,0.807087,-0.276671,-0.364855,-0.168561,0.084976,-0.942269,-0.788239 -304,-0.757549,-0.262877,-0.757036,-0.716114,-0.557245,-0.519289,-0.695035,-0.874394,-1.462636,-0.052113,-0.279252,-0.286400,-0.193586,-0.382985,-0.129665,0.058229,-0.321083,-0.647077,-0.194004,-0.145212 -529,-0.584300,-1.361252,-0.582390,-0.596377,0.970677,-0.270077,-0.640169,-0.540104,-0.564504,0.465313,-0.555357,-1.293361,-0.570305,-0.479573,0.095344,-0.779555,-0.461337,-0.618041,-0.111671,-0.590414 -40,-0.195201,0.532980,-0.238451,-0.261342,-1.048999,-0.834452,-0.724413,-0.737944,-0.100834,-0.982061,-0.601554,-0.708235,-0.640599,-0.435790,-1.253710,-0.808059,-0.596618,-0.797283,-0.816347,-0.948996 -115,-0.624062,0.521345,-0.635937,-0.615148,0.093918,-0.489914,-0.697043,-0.743876,-0.451325,-0.121575,-0.336999,-0.533695,-0.428726,-0.342062,0.254017,-0.022811,-0.449069,-0.662649,-0.939848,0.023110 -2,1.579888,0.456187,1.566503,1.558884,0.942210,1.052926,1.363478,2.037231,0.939685,-0.398008,1.228676,-0.780083,0.850928,1.181336,-0.297005,0.814974,0.213076,1.424827,0.237036,0.293559 -39,-0.183840,0.356123,-0.147009,-0.272150,0.372887,0.400995,0.219721,0.141115,-0.334494,0.197386,-0.693589,-1.134788,-0.653965,-0.480013,-0.558016,-0.172595,-0.046543,0.133639,-0.819980,-0.229940 diff --git a/fml_manager/Examples/Hetero_LR/data/breast_b.csv b/fml_manager/Examples/Hetero_LR/data/breast_b.csv deleted file mode 100644 index b78d88dbb..000000000 --- a/fml_manager/Examples/Hetero_LR/data/breast_b.csv +++ /dev/null @@ -1,570 +0,0 @@ -id,y,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9 -133,1,0.254879,-1.046633,0.209656,0.074214,-0.441366,-0.377645,-0.485934,0.347072,-0.287570,-0.733474 -273,1,-1.142928,-0.781198,-1.166747,-0.923578,0.628230,-1.021418,-1.111867,-0.959523,-0.096672,-0.121683 -175,1,-1.451067,-1.406518,-1.456564,-1.092337,-0.708765,-1.168557,-1.305831,-1.745063,-0.499499,-0.302893 -551,1,-0.879933,0.420589,-0.877527,-0.780484,-1.037534,-0.483880,-0.555498,-0.768581,0.433960,-0.200928 -199,0,0.426758,0.723479,0.316885,0.287273,1.000835,0.962702,1.077099,1.053586,2.996525,0.961696 -274,0,0.963102,1.467675,0.829202,0.772457,-0.038076,-0.468613,-0.307946,-0.015321,-0.641864,-0.247477 -420,1,-0.662496,0.212149,-0.620475,-0.632995,-0.327392,-0.385278,-0.077665,-0.730362,0.217178,-0.061280 -76,1,-0.453343,-2.147457,-0.473631,-0.483572,0.558093,-0.740244,-0.896170,-0.617229,-0.308601,-0.666975 -315,1,-0.606584,-0.971725,-0.678558,-0.591332,-0.963013,-1.302401,-1.212855,-1.321154,-1.591501,-1.230554 -399,1,-0.583805,-0.193332,-0.633283,-0.560041,-0.349310,-0.519504,-0.610669,-0.929526,-0.196974,-0.151608 -238,1,-0.107515,2.420311,-0.141817,-0.204943,-1.063835,-0.074206,0.164131,-0.493589,-1.635181,-0.331709 -246,1,-0.482335,0.348938,-0.565371,-0.489725,-0.976164,-0.658182,-0.203360,-0.988301,-0.216387,-0.663096 -253,0,0.741523,-0.095626,0.704101,0.600181,0.404667,-0.087565,0.314773,1.082516,0.383809,-0.156041 -550,1,-0.954483,-0.147736,-0.988330,-0.823201,-1.414523,-1.150045,-1.305831,-1.745063,-0.716282,-0.998915 -208,1,-0.356014,0.567149,-0.231770,-0.424155,0.110966,1.182806,0.211146,-0.030548,1.985412,1.310816 -185,1,-0.910995,-0.732345,-0.949311,-0.779780,0.864944,-0.969255,-1.272632,-1.586402,0.052164,-0.386571 -156,0,0.869914,-0.092369,0.763673,0.740814,0.413434,0.607736,0.413122,0.561767,-0.708193,-0.363850 -0,0,1.886690,-1.359293,2.303601,2.001237,1.307686,2.616665,2.109526,2.296076,2.750622,1.937015 -70,0,1.779007,0.147012,1.746605,1.732277,-0.572873,-0.131459,-0.016736,0.978975,-0.565828,-1.000578 -293,1,-0.664567,0.011851,-0.682430,-0.637741,0.198638,-0.499147,-0.674477,-0.353352,0.323951,-0.768940 -287,1,-0.548601,-1.650784,-0.591583,-0.533673,-1.587236,-0.887829,-0.736844,-0.928004,-0.957331,-0.819922 -222,1,-1.055953,-0.462024,-1.052072,-0.887716,0.360831,-0.701440,-0.990537,-0.896180,0.249533,0.223003 -262,0,0.853348,0.254488,0.912602,0.728509,-0.831505,0.206332,-0.203360,0.581561,0.268947,-0.504606 -309,1,-0.318739,-1.347894,-0.396188,-0.365968,-1.348769,-1.245530,-1.218324,-1.207259,-1.284123,-1.005565 -534,1,-0.962766,0.135613,-0.918334,-0.831639,0.457270,-0.020770,-0.287316,-0.243568,-0.989687,-0.064605 -54,0,0.379129,0.979143,0.310928,0.262662,0.286310,-0.308942,-0.004742,0.584607,-0.365223,-0.289039 -172,0,0.522016,-1.406518,0.528365,0.389232,0.908780,0.661808,1.491126,1.036837,0.509996,0.945071 -484,1,0.153409,-1.868995,0.156042,-0.046203,0.952616,0.277579,0.615098,0.465839,-0.556121,-0.118358 -102,1,-0.606584,1.166414,-0.675579,-0.585004,-0.879725,-1.053734,-0.756514,-0.613574,-0.334485,-0.840426 -458,1,-0.399502,1.010084,-0.482567,-0.443140,-0.463284,-0.922180,-1.091765,-0.843496,-0.962184,-1.165717 -495,1,-0.053674,0.456415,-0.100117,-0.170488,-0.472051,-0.734519,-0.490252,-0.196518,-0.860264,-0.995036 -158,1,-0.648001,-1.183422,-0.690472,-0.611372,-0.213419,-0.833757,-0.891517,-0.675394,-0.625686,-0.275185 -160,1,-0.610726,0.086759,-0.546606,-0.591860,0.150419,-0.413905,-0.367435,-0.540791,0.432343,-0.225865 -292,1,-0.523751,-0.935900,-0.549585,-0.518906,0.698367,-0.301944,-0.230706,-0.137134,0.775312,0.659126 -493,1,-0.637646,-1.517252,-0.715492,-0.609263,-1.664826,-1.205453,-1.225520,-1.336990,-1.004247,-0.757302 -394,1,-0.561026,0.019993,-0.563882,-0.564436,0.474804,-0.489605,-0.536788,-0.790964,0.239827,-0.727932 -528,1,-0.341518,-1.676839,-0.379508,-0.399544,0.308228,-0.749786,-0.557897,-0.199563,-1.198380,-0.632617 -335,0,0.977597,1.216895,1.070467,0.846289,0.549325,-0.311486,0.574799,1.036837,-0.449348,-0.440878 -311,1,0.039513,-0.639524,-0.106074,-0.069935,-1.370687,-1.166649,-1.078140,-0.859941,-0.599801,-1.495995 -25,0,1.238521,-0.696519,1.344497,1.020322,0.970150,0.894635,0.542655,2.137720,1.885110,1.216609 -522,1,-0.898570,0.122585,-0.919823,-0.781714,-0.945479,-1.126190,-1.184309,-1.313846,-0.556121,-0.433120 -512,0,0.029159,0.648570,0.179870,-0.063607,1.097274,0.835474,1.143785,1.377912,1.106957,1.493688 -230,0,0.687682,-0.128194,0.781544,0.542170,1.662757,0.885093,1.101567,2.127061,0.336894,0.369301 -548,1,-1.105653,-0.014204,-1.136664,-0.907756,-0.546572,-1.010222,-0.857262,-1.159448,-0.564210,-0.262993 -213,0,0.372916,0.389649,0.391350,0.246841,-0.353694,-0.476882,0.038915,-0.071660,-2.099484,-0.873676 -390,1,-1.012466,-1.632871,-1.013648,-0.854492,0.084665,-0.567850,-0.892524,-0.725490,0.058635,-0.372717 -275,1,-0.801242,-1.088973,-0.828082,-0.717550,0.154802,-1.085159,-0.962664,-0.382891,-1.101313,-1.308690 -219,0,2.408538,3.213360,2.172543,2.806362,0.369598,0.988783,0.610780,0.729259,-0.303748,-0.458057 -236,0,3.052564,1.438363,2.941018,3.627307,0.689600,1.007232,1.486328,2.203194,0.327187,0.156504 -336,1,-0.527893,-1.427688,-0.592179,-0.535431,-0.796437,-0.361105,-0.610190,-0.854307,-0.758344,0.939530 -20,1,-0.366368,-0.844707,-0.332744,-0.439624,-0.051226,0.148443,-0.399099,-0.636110,0.458227,-0.117250 -193,0,-0.128223,2.224898,-0.165645,-0.196329,2.022211,1.376193,0.817074,0.476498,0.508378,2.025680 -200,1,-0.378793,0.436874,-0.450100,-0.425737,0.461654,-0.318484,-0.645212,-0.100590,-0.376548,-0.122237 -83,0,0.840923,1.146872,1.013874,0.733782,0.299461,0.174525,-0.139073,1.058154,-0.954095,0.447992 -376,1,-1.122220,-0.465281,-0.915951,-0.929379,-0.792053,0.684709,1.587076,0.485634,-0.491410,1.997972 -295,1,-0.331164,-1.424431,-0.389933,-0.385832,-0.673696,-0.935539,-1.126787,-0.861616,-0.125792,-0.886975 -77,0,1.267513,-1.102000,1.275989,1.282251,0.676449,1.966530,0.510512,1.455568,1.375509,1.488146 -441,0,0.851277,1.593064,0.760694,0.709172,0.492339,1.004687,1.110202,0.902842,-0.648335,-0.249694 -109,1,-0.674921,0.565520,-0.693153,-0.637214,1.645223,-0.220518,0.190997,-0.484605,-0.116085,0.242399 -319,1,-0.697700,-0.890303,-0.759575,-0.641081,-2.116335,-1.317732,-1.198510,-1.313846,-1.617385,-1.364660 -349,1,-0.716338,-1.295784,-0.719960,-0.675712,-0.134515,-0.418358,-1.156052,-1.019363,0.361160,-0.445866 -89,1,0.014664,-1.211106,0.063706,-0.135330,-0.204652,0.347555,-0.056556,0.382093,0.404840,0.043456 -419,1,-0.809525,0.528066,-0.834040,-0.742864,-0.182734,-0.912638,-1.133983,-1.089406,0.121729,-0.704657 -562,0,0.259021,2.786709,0.638572,0.060502,0.409050,3.418837,4.307272,1.842324,1.922319,3.156163 -370,0,0.644194,0.871666,0.656444,0.499980,0.400283,1.350111,2.094174,1.677876,3.111387,0.675751 -546,1,-1.039387,-0.636267,-1.076496,-0.871368,-0.169583,-1.055006,-1.095507,-1.382518,-0.355517,-0.551710 -49,1,-0.231765,1.000313,-0.246067,-0.319559,-0.708765,-0.529046,-0.211036,0.206987,-0.048138,-0.818814 -412,1,-1.305488,0.376621,-1.210830,-1.018857,-1.041918,-0.417085,-0.409653,-1.354653,-0.848940,0.449654 -417,0,1.429037,0.321254,1.484490,1.524843,0.847409,0.928350,0.714407,1.504293,0.165409,1.155652 -440,1,-0.809525,0.194236,-0.509970,-0.710519,0.295077,0.979241,0.986906,0.622673,-0.583624,0.630310 -482,1,-0.298031,-1.198078,-0.366998,-0.387414,0.303844,-0.027768,-0.419248,0.287688,0.527792,0.516153 -291,1,-0.003974,0.083503,0.054770,-0.124431,-0.046843,0.310022,-0.440357,0.522178,0.099080,0.042902 -198,0,1.468383,1.039396,1.761498,1.419368,-0.007390,1.945538,0.548412,0.855640,0.472787,0.457966 -542,1,0.049868,1.076850,0.004134,-0.095249,-1.155891,-0.742153,-0.532950,-0.077750,-0.289188,-0.797202 -320,1,-1.033174,-0.825166,-1.064284,-0.861699,0.343297,-0.116191,-0.395261,-0.261383,-0.473615,0.724517 -94,0,0.406050,-0.235671,0.483686,0.253872,0.996451,1.056214,1.190321,1.475363,-0.107996,-0.088988 -116,1,-1.419590,-1.401633,-1.308230,-1.073352,-0.634244,-0.422174,-0.565093,-1.159448,-2.020213,-0.372717 -429,1,-0.507184,-0.768171,-0.547798,-0.516445,-1.120822,-1.006469,-1.139405,-1.195078,-1.190291,-1.313123 -518,1,-0.252473,-0.212873,-0.236834,-0.361925,0.580010,0.266129,-0.708059,-0.076228,-0.515677,0.276203 -323,0,1.870123,1.006827,1.901492,1.858847,1.176179,1.240059,1.257966,2.343279,4.298838,1.022654 -248,1,-0.832304,1.549097,-0.872165,-0.746907,0.768505,-0.728158,-0.766109,-0.810759,0.822228,-0.137199 -280,0,1.542933,1.664716,1.564912,1.482653,2.009061,0.825932,1.454664,1.105356,0.577943,0.734491 -391,1,-1.263036,-0.468538,-1.288274,-0.990730,0.597545,-0.784138,-1.305831,-1.745063,-0.737313,0.260686 -462,1,-0.179994,1.026368,-0.204367,-0.256626,-1.344385,-0.688717,-0.599635,-0.898007,-0.899091,-1.069848 -406,1,0.298367,-0.992895,0.257314,0.118337,-0.515887,-0.522048,-0.197603,-0.025980,-0.198592,-0.766169 -416,1,-1.122220,0.905864,-1.147684,-0.916194,0.886862,-0.858566,-1.011215,-1.171020,-0.046520,-0.050197 -411,1,-0.799171,0.124213,-0.814083,-0.719308,0.198638,-0.674722,-0.793935,-0.613574,0.157320,-0.284606 -474,1,-0.896500,-1.030349,-0.788765,-0.786636,0.036445,0.862192,0.308536,-0.532112,-0.517295,1.332982 -96,1,-0.712196,-0.774684,-0.748256,-0.677470,-0.805204,-1.022181,-1.066914,-0.849434,-1.089989,-0.564455 -232,1,-0.809525,2.622237,-0.858464,-0.720187,-1.421536,-1.179499,-1.194624,-1.287960,0.016573,-0.602692 -354,1,-0.859225,-1.605188,-0.823317,-0.750775,-1.916882,-0.818489,-0.729648,-1.147876,-0.525383,-0.762844 -240,1,-0.293889,-1.079202,-0.391720,-0.346631,-0.200268,-0.796225,-0.570370,-0.340866,-0.599801,-1.044356 -378,1,-0.358085,-0.983124,-0.277044,-0.393040,-0.213419,0.357097,-0.073347,-0.140179,0.786637,0.689050 -177,0,0.314933,0.451529,0.483686,0.176876,0.400283,1.351383,1.506478,1.353550,0.247916,0.623106 -33,0,1.631978,0.850497,1.612569,1.639107,0.812341,2.574680,1.616341,0.972885,1.247704,1.571270 -251,1,-0.683205,-0.523905,-0.719066,-0.653386,-0.616709,-0.950170,-0.916991,-0.748634,-0.260067,-1.057102 -224,1,-0.233835,-0.338263,-0.250833,-0.301980,-0.209036,-0.783502,-0.448993,-0.271433,-0.638628,-0.427578 -186,0,1.043864,0.111186,0.951324,0.930669,-0.393146,-0.062119,0.391533,0.647036,0.493818,-0.807177 -85,0,1.379337,0.326140,1.338539,1.269946,0.325762,-0.288585,0.208748,0.755144,1.284913,0.102197 -422,1,-0.751542,-0.978239,-0.754511,-0.711749,0.400283,-0.237058,-0.201441,-0.062524,-0.184032,-0.536193 -104,1,-0.979333,-0.385488,-0.984160,-0.839901,-0.458900,-0.672177,-0.922652,-1.257355,-0.120938,-0.466924 -461,0,4.094189,0.927033,4.287337,5.930172,0.146035,1.089930,1.972317,2.251919,-0.420228,-0.536193 -142,1,-0.722550,0.176323,-0.732768,-0.663758,0.391516,-0.477519,-0.936038,-0.770256,-0.512441,-0.165462 -380,1,-0.710125,-0.838193,-0.665154,-0.711046,1.255083,-0.072298,-0.227827,0.261803,0.715454,0.454641 -123,1,-0.117869,-1.579133,-0.132881,-0.237464,-0.046843,-0.480063,-0.077665,0.114105,-0.019018,-0.215336 -428,1,-0.950341,-0.877276,-0.980288,-0.807731,-1.287398,-1.221866,-1.086104,-1.129299,-0.837615,-0.726824 -396,1,-0.304244,0.247975,-0.295809,-0.361046,0.457270,0.017398,0.343558,0.467362,-0.379783,-0.392666 -66,1,-1.213336,0.957974,-1.198320,-0.966647,0.983301,-0.558944,-0.854288,-0.752745,-0.036814,0.452425 -170,1,-0.573451,-1.634499,-0.604391,-0.582718,0.268776,-0.812128,-0.709978,-0.315133,-0.119321,-0.899721 -481,1,0.029159,0.120957,-0.085224,-0.088042,-1.138356,-0.717343,-0.503205,-0.504095,-0.881295,-0.438661 -313,1,-0.813667,-2.085577,-0.775361,-0.725637,-1.015616,-0.583118,-0.906485,-1.027129,0.791490,-0.532314 -53,0,0.896835,-0.251956,0.829202,0.774214,-0.191501,-0.156268,-0.047440,0.272461,0.194529,-0.225865 -296,1,-1.014537,-1.768031,-1.037775,-0.858535,-1.720497,-1.139994,-1.167470,-1.258725,-1.225882,-0.970653 -114,1,-1.375274,-0.986381,-1.274274,-1.048038,1.754812,-0.113647,-0.127559,-0.146270,0.040840,0.983863 -243,1,-0.260756,0.107929,-0.275853,-0.306902,-1.695949,-0.700167,-0.653847,-0.815327,-0.384637,-1.149092 -140,1,-1.169849,-1.885279,-1.213213,-0.945200,-0.393146,-1.159206,-1.305831,-1.745063,0.330422,-0.134983 -152,1,-1.087016,-1.007551,-1.078879,-0.879102,-0.138898,0.145898,2.635815,0.647036,0.335276,2.324925 -268,1,-0.490618,-0.331749,-0.535883,-0.497635,-0.296707,-0.467340,-0.350164,-0.864965,1.137695,-0.738461 -91,0,0.033301,0.026507,0.007112,-0.087339,-0.292324,-0.347110,0.059545,0.502383,-0.557739,-0.868134 -255,0,0.025018,-0.587414,0.024984,-0.095952,0.825491,0.457607,0.233695,0.347072,0.270565,-0.242489 -559,1,-0.784675,1.869899,-0.744086,-0.714386,-0.112597,-0.016317,0.435670,-0.275239,-1.276034,0.186983 -14,0,-0.256615,1.031253,0.045834,-0.321493,1.434810,3.296698,2.025090,1.616970,1.124753,3.278077 -100,0,0.149267,1.562124,0.039877,0.045560,-0.257255,-0.381461,0.214505,0.057766,-0.404050,-0.552818 -353,0,0.464033,1.228294,0.415178,0.297820,1.474263,-0.118736,0.627092,0.578516,-0.399197,0.578219 -543,1,-0.393289,1.871527,-0.440271,-0.441206,-1.103288,-0.738972,-0.796334,-0.533330,-0.692015,-1.081485 -165,1,-0.059886,0.023250,-0.147774,-0.173125,-1.221645,-0.981659,-0.934023,-0.868924,-0.412139,-1.279874 -43,0,0.230029,0.378250,0.173913,0.046790,0.904396,0.751503,0.451982,0.526746,1.356095,1.039278 -201,0,0.859560,0.026507,0.960259,0.630066,0.251241,0.558117,0.377141,1.207374,0.044075,-0.292364 -108,0,2.512079,0.379878,2.964846,2.600686,1.653990,2.833589,3.303630,2.685877,1.867314,0.772728 -328,0,0.623486,0.765818,0.671337,0.422632,1.167411,0.257223,0.419839,0.665308,0.327187,-0.107829 -439,1,-0.281464,-1.036863,-0.319638,-0.336962,-1.269864,-0.970527,-1.005506,-0.494046,-1.237207,-0.933525 -162,0,2.166251,0.116071,2.014678,2.375673,0.501106,0.829112,1.961283,1.676354,1.200788,0.459075 -325,1,-0.529963,-0.745372,-0.552861,-0.538243,0.264392,-0.846480,-0.816483,-0.892069,-0.344192,-0.834885 -183,1,-0.807454,-1.299041,-0.838210,-0.726691,-0.888492,-0.593296,-0.515199,-0.786396,-1.762986,-0.536193 -197,0,0.722886,-0.159135,0.650487,0.610729,-1.935293,-0.368739,-0.089658,-0.347109,-0.860264,-1.017757 -498,0,1.342063,-0.455510,1.165782,1.264672,0.387133,0.347555,0.389134,0.787120,-0.632157,0.582098 -477,1,-0.233835,-0.631382,-0.180538,-0.284225,-1.688935,-0.341385,-0.641853,-0.797664,-0.358752,-0.386016 -421,1,0.039513,-1.194821,0.203699,-0.125485,-0.051226,0.694887,0.238492,-0.057956,-0.119321,0.450762 -374,1,-0.295960,-0.890303,-0.241301,-0.369132,-0.958629,-0.284132,-0.660084,-0.681180,0.683099,-0.384354 -470,1,-1.062166,-0.009318,-1.083645,-0.870840,-0.393146,-0.636553,-0.693187,-0.746198,0.442049,0.071718 -382,1,-0.766038,0.493869,-0.592774,-0.689424,-1.945375,0.427072,0.091208,-0.082318,-1.148229,0.528899 -510,1,-0.790887,-1.315326,-0.774766,-0.715089,-1.098904,0.159257,-0.015297,-0.137134,-0.480086,0.822603 -132,0,0.662832,0.977515,0.668358,0.517559,0.312612,0.325926,0.129588,0.252667,0.937090,-0.429795 -545,1,-0.190348,0.555750,-0.288363,-0.265064,-0.472051,-0.652457,-0.802570,-0.652707,-0.418610,-0.798864 -121,0,1.238521,-0.126566,1.135996,1.175019,0.786039,-0.160085,0.263919,0.803870,-0.010929,0.034035 -45,0,1.356558,-0.709547,1.290882,1.206661,1.557551,1.620470,2.217950,1.875822,1.453162,0.438017 -333,1,-0.726692,-0.589042,-0.750044,-0.681865,-0.691230,-0.994446,-1.279358,-1.491235,-0.138734,-0.541181 -134,0,1.294434,0.930290,1.141953,1.247093,0.619463,-0.170263,0.596387,0.354685,0.336894,-0.434782 -182,0,0.795365,1.163157,0.656444,0.682803,0.395900,0.638907,0.086411,0.601356,0.867526,0.131013 -519,1,-0.376722,-0.641152,-0.406017,-0.450875,0.663299,-0.358560,-0.623143,-0.520083,0.275418,0.090005 -207,0,0.731169,-0.102139,0.677294,0.579086,-0.932328,-0.672177,-0.378949,-0.076228,0.605445,-1.067077 -99,0,0.012593,0.843983,0.066684,-0.095249,0.470421,0.307478,0.226498,0.637900,-0.295659,0.531115 -366,0,1.640261,1.324372,1.570869,1.389484,-0.200268,0.555572,0.470692,1.531701,0.598974,-0.422591 -180,0,3.489508,1.168042,3.381848,4.105459,0.650148,0.948707,1.256047,2.347847,-0.072405,-0.173220 -318,1,-1.285815,-0.370832,-1.150961,-1.025712,-0.450133,0.766771,0.905348,-0.001617,0.378956,1.194443 -501,0,-0.053674,1.182698,-0.037566,-0.162753,2.061664,0.905449,0.316212,0.570903,1.213731,1.903765 -229,0,-0.221411,0.728364,-0.058416,-0.306902,1.987143,1.781414,1.707974,1.265235,0.818992,2.236260 -7,0,0.163763,0.401048,0.099449,0.028859,1.447961,0.724786,-0.021054,0.624196,0.477640,1.726435 -247,1,-0.389147,-1.299041,-0.067352,-0.424506,-0.305475,2.103300,2.401216,0.631809,-0.423463,1.876057 -494,1,-0.366368,0.453158,-0.356573,-0.408333,-0.901643,-0.570395,-0.936517,-1.106307,-0.345810,-0.535085 -174,1,-0.979333,-1.054776,-1.014542,-0.830233,-1.085753,-1.185478,-1.305831,-1.745063,-0.308601,-1.236095 -340,1,0.083001,-0.678606,0.123277,-0.032492,-0.130131,0.526946,0.495639,0.407978,0.246298,0.204716 -125,1,-0.161357,-0.341520,-0.207346,-0.271919,-0.730683,-0.758692,-0.916512,-0.967897,-0.868353,-0.671962 -209,1,0.230029,-1.588903,0.191785,0.091617,-0.445749,-0.226880,0.115196,-0.169110,-0.939535,-0.510148 -347,1,0.207250,-1.261587,0.206678,0.000381,-0.454517,-0.339476,-0.273883,0.159785,0.336894,-0.115033 -332,1,-0.888216,0.016737,-0.904036,-0.781363,0.439736,-1.002397,-1.241784,-1.437181,0.632947,-1.037706 -361,1,-0.428493,0.573662,-0.426569,-0.455973,-0.805204,-0.557036,-0.724371,-0.890242,-0.426699,-0.962341 -500,1,0.101638,-0.854478,0.072641,-0.041633,-0.827122,-0.233241,-0.415410,-0.194995,-1.170878,0.085572 -300,0,2.000585,0.091645,1.901492,2.061007,0.750970,1.000870,1.630733,1.269803,0.108787,0.850310 -283,0,0.472316,-0.095626,0.584958,0.264420,0.181104,1.376193,1.105405,0.892184,-0.211534,1.238775 -107,1,-0.616938,0.295199,-0.646389,-0.591508,-0.612326,-0.368739,-0.376550,-0.459633,0.133053,-0.670300 -131,0,0.619345,0.052562,0.525386,0.484159,0.974533,-0.094562,0.512911,0.560244,-0.103143,-0.208132 -303,1,-1.078732,-0.185190,-1.087219,-0.888068,0.391516,-0.953351,-0.901735,-0.751071,-1.112638,-0.306218 -567,0,1.961239,2.237926,2.303601,1.653171,1.430427,3.904848,3.197605,2.289985,1.919083,2.219635 -507,1,-0.948270,-0.803996,-0.928759,-0.825310,1.483030,-0.325481,-0.703262,-0.296404,-0.195356,1.820641 -3,0,-0.281464,0.133984,-0.249939,-0.550021,3.394275,3.893397,1.989588,2.175786,6.046041,4.935010 -516,0,1.157759,0.085131,1.040681,1.076575,0.737820,-0.004231,0.497558,0.554153,0.280271,-0.294580 -143,1,-0.370510,-0.628125,-0.300575,-0.416244,-0.051226,0.003403,-0.303148,-0.204131,1.048717,-0.153270 -398,1,-0.743259,-0.867505,-0.788467,-0.674833,-0.892875,-0.422174,-0.308426,-0.899073,-0.502735,0.423054 -530,1,-0.573451,0.374993,-0.558223,-0.577093,0.110966,-0.438078,-0.650489,-0.207176,-0.683926,-0.353321 -119,0,0.892693,0.350566,0.653465,0.668740,-1.103288,-0.852841,-0.226868,0.059289,3.205219,-1.265465 -259,0,0.459891,3.885905,0.567086,0.271451,2.451803,1.922000,1.430197,1.321574,0.988859,2.020138 -450,1,-0.720479,0.407562,-0.707450,-0.656375,-1.656935,0.544758,0.238012,-0.412736,-0.963802,0.863056 -192,1,-1.304866,-0.789340,-1.340697,-1.013934,-2.682695,-1.443878,-1.305831,-1.745063,-1.604443,-1.017203 -514,0,0.271446,0.388020,0.194763,0.151913,-0.340543,-0.280951,0.069140,-0.039684,-1.001011,-0.798310 -241,1,-0.635576,-0.864248,-0.697323,-0.592739,-1.256713,-1.122819,-1.006321,-1.128081,0.000395,-0.893071 -402,1,-0.442989,-0.173791,-0.326191,-0.454742,-1.713045,-0.142909,-0.536308,-0.738889,0.495436,-0.635942 -148,1,-0.086807,-0.948927,0.039877,-0.199845,-0.033692,0.122361,0.182841,0.689670,-0.339339,-0.394329 -395,1,-0.279394,-0.054915,-0.322915,-0.344697,-1.129589,-0.834393,-0.899960,-0.540487,-0.611126,-0.989495 -299,1,-1.105653,-0.237300,-1.106878,-0.910393,-0.792053,-1.069510,-1.106350,-1.269232,-1.089989,-0.896396 -149,1,-0.192419,-0.523905,-0.299980,-0.271919,-1.545592,-0.457162,-0.555498,-0.828574,-0.891002,-0.765061 -60,1,-1.087016,-1.339752,-1.114026,-0.900022,-0.213419,-0.989865,-1.201820,-1.352369,1.061659,-0.207578 -61,1,-1.388321,0.221920,-1.346356,-1.066496,1.382207,-0.537316,-0.874006,-1.322068,0.115258,-0.378258 -126,0,0.128559,1.622376,0.176892,-0.056048,0.645764,0.217146,0.515309,0.278552,0.920912,-0.274077 -372,0,1.329638,-0.624868,1.335561,1.150408,-0.577257,0.189156,0.624693,1.248486,-0.276245,0.150409 -82,0,2.843411,1.293432,3.110797,2.955784,1.092890,2.247704,1.801046,2.620403,-0.882913,1.172276 -425,1,-1.068378,0.531323,-1.112239,-0.886486,-0.866574,-1.166203,-1.246581,-1.352369,-0.892620,-0.184857 -337,0,1.712740,1.415565,1.603633,1.744582,0.764121,1.453165,0.917342,1.373344,1.259029,0.817615 -386,1,-0.650071,-1.040120,-0.584136,-0.616470,-1.304933,-0.071025,0.169888,-0.353352,-0.361988,0.237966 -184,0,0.317004,0.383135,0.194763,0.162637,-0.099446,0.481144,0.435670,0.121718,0.443667,0.763308 -58,1,-0.422281,-0.558102,-0.506991,-0.450875,-1.326851,-1.223647,-1.296979,-1.575895,-0.747019,-1.166825 -88,1,-0.505114,0.785359,-0.470652,-0.537716,-0.086295,-0.050669,-0.138113,0.089742,0.115258,0.480133 -117,0,0.526157,0.275658,0.590915,0.376926,2.429885,1.232425,0.950924,1.341368,1.106957,1.249858 -459,1,-1.159494,1.830816,-1.168535,-0.932895,-0.936711,-0.912002,-0.960889,-1.004137,-0.937918,-0.655891 -221,1,-0.266969,-1.391862,-0.183517,-0.341005,0.229323,0.098824,-0.069509,-0.360965,0.265711,-0.120575 -223,0,0.681469,0.751162,0.555172,0.364621,1.000835,1.232425,0.601665,0.506951,1.767012,1.244317 -128,1,-0.032965,-1.196450,-0.040545,-0.207404,0.273159,0.216510,-0.365516,0.421682,-0.502735,-0.341130 -294,1,-0.573451,-1.334867,-0.557627,-0.574632,-0.112597,-0.681083,-1.054776,-0.779240,-0.860264,-0.816043 -541,1,-0.010186,0.985657,0.185828,-0.126013,0.071514,1.055578,0.632369,0.089742,0.463080,1.017112 -111,1,-0.608655,-0.033745,-0.543926,-0.620865,-0.160816,-0.186167,-0.242699,-0.062524,-1.091607,0.050660 -266,1,-0.908925,-0.445740,-0.863230,-0.801227,-0.485202,-0.017590,-0.386625,-0.538203,0.063489,-0.447528 -568,1,-1.410893,0.764190,-1.432735,-1.075813,-1.859019,-1.207552,-1.305831,-1.745063,-0.048138,-0.751207 -362,1,-0.521680,0.050934,-0.579073,-0.528926,-0.112597,-0.447620,-0.703741,-0.479428,-0.253596,-0.640929 -271,1,-0.817808,-1.546564,-0.863528,-0.743743,0.150419,-0.658818,-0.694146,-0.412736,-0.271392,-0.206469 -187,1,-0.674921,-0.698148,-0.680345,-0.631237,-0.003007,-0.955896,-0.576127,-0.071660,-0.531855,-0.719066 -155,1,-0.554813,-0.074456,-0.615412,-0.556174,-0.467667,-0.480063,-0.373672,-0.494807,0.343365,-0.145512 -5,0,-0.165498,-0.313836,-0.115009,-0.244320,2.048513,1.721616,1.263243,0.905888,1.754069,2.241802 -511,1,-0.136507,-1.318583,-0.165645,-0.211623,-0.809587,-0.974344,-0.777623,-0.533787,-0.916886,-1.248287 -252,0,1.865981,-0.014204,1.564912,1.850057,1.693442,2.170731,2.766787,2.072245,-0.245507,2.535505 -480,1,-0.606584,0.357080,-0.548989,-0.585707,-0.507120,-0.167719,-0.528632,-0.878669,-0.800406,-0.368838 -513,1,0.101638,-1.373949,0.036898,-0.032668,-0.441366,-0.391004,-0.110288,-0.346348,-0.444494,-0.746219 -86,0,-0.012257,0.581805,0.033920,-0.126013,-0.077528,-0.360469,0.300860,0.120195,0.192911,-0.858159 -105,0,0.008451,-0.533675,-0.025652,-0.093843,2.359748,0.990056,1.753070,1.278939,0.398369,3.133996 -242,1,-0.763967,0.371736,-0.598731,-0.716671,0.102199,1.466524,2.261608,0.109537,0.658832,2.535505 -312,1,-0.430564,-1.510738,-0.453377,-0.460192,-0.568490,-0.212884,-0.457149,-0.464354,-0.544797,-0.078459 -473,1,-0.583805,2.014830,-0.660686,-0.565491,-1.672278,-1.285861,-1.305831,-1.745063,-0.795553,-0.915238 -364,1,-0.318739,-0.647666,-0.402145,-0.381613,-0.485202,-0.551311,-0.651448,-0.681180,-0.258450,-0.450299 -565,0,1.536720,2.047399,1.421940,1.494959,-0.691230,-0.394820,0.236573,0.733827,-0.531855,-0.973978 -447,1,0.033301,-0.478309,-0.040545,-0.089800,-0.428215,-0.420902,-0.317541,-0.480037,1.131224,-0.614884 -488,1,-0.610726,-0.665579,-0.616305,-0.581488,0.886862,-0.677903,-0.591000,-0.250572,-0.156530,-0.205361 -151,1,-1.486271,0.658341,-1.464904,-1.108862,1.342755,1.124281,1.275717,-0.545359,0.681481,3.582864 -263,0,0.339783,0.975886,0.257314,0.189884,-1.050685,-0.467976,-0.221590,-0.440448,-0.352281,-0.867580 -563,0,1.660970,0.607860,2.139779,1.649655,0.365215,1.045400,1.860055,2.125538,0.045693,0.819278 -194,0,-0.039178,0.342424,0.337735,-0.168554,-0.033692,1.339296,0.895753,0.884571,0.160555,0.169804 -433,0,1.323425,0.855382,1.133017,1.269946,0.290694,0.585471,0.570961,0.855640,0.171880,-0.044655 -244,0,1.114272,0.790245,1.121103,0.942974,0.610696,0.270582,0.353153,0.636377,0.031133,-0.432566 -489,0,0.602778,0.143755,0.596872,0.357589,-1.379454,0.240047,-0.117484,-0.414715,2.873574,-0.427578 -465,1,-0.171711,-0.028860,0.230506,-0.258559,-0.537805,1.974164,1.839426,0.321186,-0.090201,2.269510 -483,1,-0.271110,-0.349662,-0.341978,-0.341181,-0.546572,-0.761237,-0.470102,-0.362945,-0.619215,-0.794985 -397,1,-0.523751,-0.751886,-0.492694,-0.509062,-1.623181,-0.464796,-0.393821,-0.481864,-1.476638,-0.743448 -52,1,-0.656284,-0.707918,-0.702684,-0.621217,-0.787669,-1.050935,-0.864315,-0.786396,-0.187267,-0.546722 -147,1,-0.003974,-0.033745,-0.004802,-0.124606,-1.432057,-0.013773,-0.106450,-0.465267,-0.078876,0.456304 -27,0,1.043864,0.257745,0.972174,0.918363,0.062747,-0.270773,0.347396,0.523700,-0.905562,-0.539518 -436,1,-0.376722,-0.211245,-0.361040,-0.445953,-0.480818,-0.566578,-0.963911,-0.772997,0.666921,-0.365513 -434,1,0.008451,-0.836565,-0.147774,-0.181211,-0.463284,-0.631464,-0.720533,-0.531351,-0.607890,-0.868688 -190,0,-0.109586,1.873156,-0.025652,-0.207756,0.917547,4.315794,2.766307,0.953090,3.664668,3.361201 -407,1,-0.387077,0.217034,-0.465589,-0.412728,-1.681045,-0.385914,-0.424046,-0.892221,-0.667748,-0.134983 -202,0,1.832848,1.140359,2.077228,1.943226,0.930698,1.033313,2.480375,2.416366,0.480876,0.203608 -455,1,-0.252473,2.594554,-0.314872,-0.307605,-0.664929,-0.713526,-0.969860,-0.563022,-1.140140,-0.398762 -452,1,-0.658355,1.987146,-0.660984,-0.627369,-0.507120,-0.436806,-0.436999,-0.661538,-0.734077,-0.111154 -385,0,-0.099232,0.982400,-0.150752,-0.215139,-0.051226,-0.611744,-0.022493,0.324232,-0.685544,-0.863701 -188,1,-0.766038,0.130727,-0.824806,-0.687490,0.141652,-0.981341,-0.883505,-1.089406,0.484112,-1.007782 -159,1,-0.809525,-1.217619,-0.869485,-0.721769,-0.669312,-1.089867,-1.216885,-1.143155,-0.263303,-0.393221 -32,0,0.954818,1.044281,0.858987,0.814646,1.360289,0.646540,1.375025,1.067290,1.017979,0.048444 -264,0,1.099776,0.594832,0.990045,0.976374,1.027137,0.015490,0.559926,1.275894,0.509996,-0.456949 -245,1,-0.991758,0.616002,-1.000245,-0.839901,0.838642,-0.964802,-0.739243,-0.719399,-0.028725,-0.358309 -535,0,1.663040,-0.032117,1.576826,1.632076,-0.244104,0.376817,0.820912,1.525610,0.285125,-0.457503 -430,0,0.016734,0.308227,0.540279,-0.084174,0.417818,2.892750,3.021056,2.023520,-0.056227,1.748601 -443,1,-1.103582,-0.385488,-1.129217,-0.904065,-1.509208,-1.201318,-1.114601,-1.406728,-0.326396,-0.919117 -154,1,-0.310456,-0.843079,-0.285682,-0.357354,0.676449,-0.182350,0.137744,-0.264733,1.534051,0.132121 -9,0,-0.244190,2.443109,-0.286278,-0.297409,2.320295,5.112877,3.995433,1.620015,2.370444,6.846856 -63,1,-1.296169,-1.049890,-1.241212,-1.002860,-1.490797,-0.550038,-0.635617,-0.970486,0.616770,0.052877 -329,0,0.302508,-0.076084,0.191785,0.166328,0.448503,-0.271409,0.298461,-0.150838,-0.266539,-0.244706 -536,0,-0.202773,1.399280,-0.088202,-0.267700,0.246858,0.121089,0.725441,0.328800,-0.328014,-0.024151 -65,0,0.215534,1.255978,0.218592,0.078257,1.421660,0.555572,0.144940,0.712510,0.679863,0.286177 -438,1,-0.132365,0.379878,-0.189474,-0.231136,-0.901643,-0.891646,-1.077804,-0.848216,-0.627304,-0.822139 -269,1,-0.948270,-0.076084,-0.915951,-0.826541,0.049596,0.004675,-0.090138,-0.435575,-0.478468,0.169804 -36,0,-0.078524,0.762561,0.266249,-0.142361,0.536175,1.078479,1.182165,0.458226,1.116664,0.967238 -98,1,-0.664567,-1.386977,-0.723832,-0.647058,0.470421,-0.439986,-0.383747,-0.458568,-0.208298,0.200283 -392,0,1.021085,0.607860,1.037702,0.841015,1.566318,0.871734,1.358234,1.484498,0.463080,0.994946 -69,1,-0.581734,-0.963583,-0.643112,-0.572523,-0.121364,-1.168303,-0.807368,-0.849434,-0.837615,-1.099772 -517,0,1.545003,-0.072828,1.585762,1.345536,0.409050,0.487505,0.701933,0.710987,-0.569064,0.410863 -92,1,0.018805,-0.541818,-0.082245,-0.087866,-1.392605,-0.829940,-0.658165,-0.220880,-1.413545,-1.212820 -552,1,-0.496830,1.681000,-0.570733,-0.502558,-0.393146,-0.940628,-0.890701,-0.755639,-0.798788,-1.058764 -505,1,-1.173990,-1.243674,-1.125643,-0.971217,2.990984,0.712699,0.091688,-0.108204,-0.085347,2.906791 -566,0,0.561361,1.374854,0.579001,0.427906,-0.809587,0.350735,0.326767,0.414069,-1.104549,-0.318409 -163,1,-0.556884,0.488984,-0.592774,-0.575863,0.562476,-0.130186,-0.496009,-0.497395,-1.023660,0.380938 -358,1,-1.302174,-1.299041,-1.250743,-1.017099,-1.353152,-0.823579,-0.852896,-1.019972,-0.755108,-0.533977 -442,1,-0.206915,-1.331610,-0.278832,-0.305847,-1.103288,-0.936175,-1.137102,-1.240758,-1.685332,-0.878109 -15,0,0.246596,1.865014,0.501557,0.110075,1.553167,2.566410,2.064909,0.861731,2.131012,2.779335 -503,0,3.007006,-0.294295,3.104840,3.342525,-0.546572,0.688526,0.514350,1.702239,0.011720,-0.619317 -176,1,-1.037316,-0.209616,-1.018414,-0.862051,-0.099446,0.259131,0.366586,-0.236107,-0.463908,1.787392 -327,1,-0.662496,-0.558102,-0.730385,-0.627897,-1.361920,-1.147374,-1.268736,-1.319327,-1.180585,-0.752315 -1,0,1.805927,-0.369203,1.535126,1.890489,-0.375612,-0.430444,-0.146749,1.087084,-0.243890,0.281190 -404,1,-0.639717,-1.437458,-0.689578,-0.610845,-1.208494,-1.188468,-1.069745,-1.015252,-0.975127,-1.341385 -24,0,2.110339,0.957974,2.077228,2.345788,2.109883,0.658627,0.946607,1.444909,1.152255,0.648043 -139,1,-0.900641,-1.613330,-0.915355,-0.785054,0.189871,-0.458434,-0.889933,-0.433901,-1.292212,-0.892517 -101,1,-1.726901,-0.999409,-1.693361,-1.222423,1.141110,-0.852841,-1.305831,-1.745063,0.050546,0.547186 -350,1,-0.619009,-0.966840,-0.704471,-0.594321,-1.437317,-1.205517,-1.159698,-1.096105,-0.274627,-0.869797 -90,1,-0.032965,0.559006,-0.129902,-0.135154,-0.914793,-0.494058,-0.864986,-0.687423,-0.612744,-0.636496 -191,1,-0.521680,-0.354547,-0.542734,-0.529278,-1.687182,-1.046355,-1.057031,-1.018297,-1.167642,-0.844305 -383,1,-0.432635,-0.414799,-0.358360,-0.492362,0.452886,0.668169,0.232255,-0.252247,-0.132263,1.543562 -11,0,0.859560,0.261002,0.870902,0.735540,0.316995,1.950627,0.596387,1.010951,1.441838,1.155652 -314,1,-1.515262,-0.527162,-1.507497,-1.125913,0.102199,-1.123391,-1.305831,-1.745063,0.390280,-0.154378 -352,0,3.491579,-0.341520,3.635028,4.137101,0.904396,2.159281,1.789052,2.451387,1.276824,0.232978 -235,1,-0.194490,0.749534,-0.267811,-0.290202,-0.160816,-0.655002,-1.006897,-0.532569,-1.091607,-0.430903 -305,1,-0.792958,0.967744,-0.770596,-0.710343,-1.618359,-0.751695,-0.958538,-1.011902,0.555294,-0.914129 -375,1,0.145126,-1.064546,0.173913,-0.033546,-0.388763,0.004675,-0.291634,0.159785,0.408076,0.313331 -215,0,-0.107515,0.204007,-0.085224,-0.229378,0.597545,1.162450,0.918301,0.773416,1.179757,1.216609 -466,1,-0.304244,-0.035373,-0.189474,-0.336611,0.119734,0.640179,0.854974,0.053198,-0.546415,-0.122237 -486,1,0.039513,-0.038630,-0.037566,-0.087163,-0.796437,-0.300672,-0.136674,-0.553125,-0.721135,-0.996699 -250,0,1.928106,0.215406,1.728734,1.985416,-0.493969,0.400354,2.048118,1.460136,0.364396,-0.302339 -339,0,2.982156,0.822813,2.833789,3.560506,0.838642,0.086101,1.006096,1.435773,-0.497881,-0.363850 -103,1,-1.140857,0.187723,-1.043732,-0.913030,1.031520,-0.153087,-0.037365,-0.260622,-0.450966,0.052877 -206,1,-1.211265,-0.400144,-1.196831,-0.965064,0.400283,-0.824215,-1.007761,-0.894201,0.142760,-0.562239 -345,1,-1.116007,-1.009179,-1.083347,-0.920238,0.159186,-0.576756,-0.962232,-1.124731,-0.755108,0.051769 -346,1,-0.544459,0.225177,-0.617199,-0.558987,-0.152049,-0.757420,-1.089654,-0.969572,-0.033578,-0.172666 -317,0,1.153617,-0.110282,1.001959,1.062512,0.483572,0.140173,0.542655,0.959181,-0.143587,-0.108937 -499,0,1.571924,0.827699,1.666184,1.545938,0.615079,0.670714,1.178807,1.472317,-0.680690,0.334943 -478,1,-0.801242,-0.615097,-0.751235,-0.725988,0.124117,-0.338840,-0.060394,-0.613574,0.065106,0.435246 -47,0,-0.124082,0.370108,-0.132881,-0.213029,2.026595,1.032677,1.095810,1.434251,1.616558,1.881599 -456,1,-0.652142,2.138591,-0.632092,-0.620162,0.360831,-0.325481,0.096485,-0.704325,-0.027107,-0.650904 -211,1,-0.614867,-0.111910,-0.656516,-0.587641,-0.191501,-0.421538,-0.600115,-0.692448,-0.591712,-0.222540 -490,1,-0.434706,1.027997,-0.432526,-0.452984,-0.296707,-0.469885,-0.715735,-0.780458,0.322334,-0.106167 -113,1,-1.058024,-0.476680,-1.031818,-0.889826,-0.103829,-0.314031,-0.684551,-0.810759,-0.837615,0.349906 -359,1,-0.879933,-0.107025,-0.937396,-0.775210,0.040829,-0.950170,-0.756994,-0.975815,-0.722753,-0.143295 -344,1,-0.664567,-1.224133,-0.688089,-0.640202,0.597545,-0.908185,-0.784340,-0.547643,-0.219623,-0.326167 -532,1,-0.086807,-0.891932,-0.168624,-0.188419,-0.261639,-0.622558,-0.727249,-0.419740,-0.153294,-0.339467 -261,0,0.741523,0.943318,0.623679,0.593149,-0.366845,-0.672177,-0.724851,-0.491153,-0.725988,-1.041586 -4,0,1.298575,-1.466770,1.338539,1.220724,0.220556,-0.313395,0.613179,0.729259,-0.868353,-0.397100 -226,1,-0.983474,-0.957069,-1.006500,-0.852910,0.075898,-0.884012,-1.179224,-1.065348,-0.462290,-0.069592 -464,1,-0.283535,-0.291038,-0.362232,-0.339247,-0.182734,-0.367466,-0.405815,-0.153883,-1.077047,-0.814381 -277,0,0.764302,-0.224272,0.647508,0.624792,-0.353694,-0.879559,-0.245578,0.225259,-0.539944,-1.472721 -257,0,0.302508,-0.491336,0.373478,0.084761,1.934540,1.247056,0.818993,1.648946,0.577943,1.948098 -260,0,1.669253,2.195586,1.639376,1.693603,0.869327,0.255951,0.511471,0.838891,0.404840,-0.219215 -301,1,-0.581734,-0.424570,-0.569839,-0.578851,-1.199727,-0.244691,-0.392382,-0.584035,-0.349045,-0.349442 -537,1,-0.681134,1.060565,-0.629709,-0.690654,1.947690,0.450609,-0.636576,0.246576,-0.158147,0.873031 -403,1,-0.498901,-0.432712,-0.523373,-0.526817,-0.664929,-0.371919,-0.437479,-0.467856,0.641036,-0.310651 -233,0,1.698245,1.905725,1.651291,1.742824,-0.441366,0.138901,0.683223,0.634854,-0.750255,-0.036897 -487,0,1.592632,0.767446,1.389175,1.510780,0.834259,0.752140,1.541979,1.391616,0.590885,0.340485 -141,0,0.756019,-0.066314,0.647508,0.619518,-0.042459,-0.195073,0.038435,0.106491,-0.175943,-0.131104 -68,1,-1.234044,-0.492965,-1.243893,-0.977194,0.693984,1.159269,4.700669,0.919592,2.147190,1.859432 -363,1,0.385341,-0.037002,0.296035,0.225746,0.062747,-0.549402,-0.508002,-0.355940,-0.819820,-1.067077 -379,0,-0.627292,1.163157,-0.461717,-0.654793,3.771263,4.348873,2.725049,2.098130,2.027474,3.122913 -51,1,-0.331164,-0.405029,-0.333042,-0.393567,-1.028767,-0.611108,-0.802091,-0.437707,-0.897473,-0.204807 -195,1,-0.494759,-0.598813,-0.490013,-0.492186,-0.993698,-0.659455,-0.459548,-0.491153,0.199382,-0.801081 -239,0,1.292363,3.125425,1.010895,0.927153,0.181104,0.758501,0.249047,1.400752,-0.077258,0.056202 -161,0,1.192963,-1.281128,1.171739,1.080091,-0.875341,-0.335023,-0.219671,0.960704,-0.740548,-1.187883 -169,1,-0.032965,-0.435969,-0.079266,-0.152733,-0.472051,-0.576120,-0.986891,-0.453086,-0.803642,-1.089797 -212,0,2.452025,-1.173652,2.419765,2.845036,-0.796437,-0.653093,0.229857,0.683579,-2.026684,-1.590202 -120,1,-0.714267,-1.580761,-0.700599,-0.650574,0.983301,-0.097107,-0.297391,-0.381064,0.186440,0.071164 -367,1,-0.409856,-0.266612,-0.399464,-0.449996,0.194255,-0.237058,-0.148188,-0.400707,0.513232,-0.512365 -171,0,0.354279,0.682768,0.278164,0.198674,0.338913,-0.634009,-0.037365,0.021222,-0.027107,-0.567226 -348,1,-0.778463,-0.795854,-0.821827,-0.711573,0.908780,-0.905004,-0.834570,-0.748025,-0.080494,0.204162 -409,1,-0.449201,0.521552,-0.543926,-0.475310,-0.366845,-0.475610,-0.645212,-0.293664,0.896646,-0.830451 -112,1,-0.200702,-0.317093,-0.007780,-0.301628,-1.879621,1.049853,1.948330,0.546540,-0.813348,1.344065 -524,1,-1.041457,-0.437598,-0.981182,-0.886134,0.417818,-0.190620,-0.901208,-0.751071,-0.645099,0.451317 -62,0,0.290083,0.624144,0.352628,0.138729,1.386591,2.356484,2.015015,0.972885,-0.091818,1.621144 -393,0,2.062710,0.498754,1.928299,2.110228,0.781656,2.019330,2.077863,1.942819,1.500078,0.928447 -75,0,0.724957,-0.181933,0.641551,0.601939,0.772888,-0.316575,0.051389,0.569380,-0.405668,-1.112518 -124,1,-0.416068,-0.476680,-0.454866,-0.436812,-1.309316,-0.007411,0.281190,-0.378019,-1.379572,-0.424808 -231,1,-0.867508,1.314602,-0.817360,-0.752884,-1.768278,-0.706529,-0.783380,-1.427893,-0.083729,-0.724607 -285,1,-0.573451,-0.422942,-0.646389,-0.556350,-1.252330,-1.196102,-1.279065,-1.611495,-0.640246,-1.088135 -448,1,0.006380,0.441759,0.024984,-0.088042,-1.028767,0.067653,0.507153,-0.284223,-0.695250,-0.516244 -460,0,1.385550,1.435106,1.335561,1.349052,1.211247,-0.062755,-0.039764,0.622673,0.176733,0.368747 -218,0,1.959169,0.482470,1.877663,1.983658,0.128501,0.440431,0.429434,1.026178,0.273800,-0.077350 -467,1,-1.060095,-0.172162,-1.076794,-0.879630,0.281926,-0.819126,-0.998357,-1.364398,0.252769,-0.287930 -525,1,-1.407372,-1.176908,-1.309422,-1.063508,1.390974,-0.195709,-0.464345,-0.448975,0.133053,1.161193 -80,1,-0.654213,1.055680,-0.677068,-0.624908,1.022753,-0.551311,-0.463865,-0.812129,-0.224476,0.252928 -472,1,0.188613,-1.214362,0.141149,0.045735,-1.133973,0.157985,0.205869,0.001428,-0.344192,-0.067376 -549,1,-0.670780,0.940061,-0.695833,-0.659188,-0.524654,-0.578665,-1.008672,-1.248067,0.256005,-0.425916 -527,1,-0.550672,-1.043377,-0.596944,-0.554943,-0.138898,-0.298127,-0.446594,-0.115817,0.338511,-0.444757 -168,0,1.422825,1.083363,1.430876,1.370147,0.229323,0.818934,1.040158,0.875435,-1.198380,0.501745 -351,0,0.225888,-0.245442,0.361564,0.061029,0.992068,1.592480,1.991028,1.505816,2.174692,1.166735 -57,0,0.331500,0.817928,0.251356,0.184435,0.194255,1.111558,0.415041,1.047495,1.289766,1.410564 -560,1,-0.200702,1.220152,-0.210324,-0.305671,-0.362461,-0.177261,-0.669679,-0.149315,-1.052780,-0.040776 -424,1,-1.047670,-0.408286,-1.056540,-0.878399,0.325762,-0.757420,-1.205802,-1.046314,0.477640,-0.213673 -355,1,-0.600372,-0.528790,-0.543330,-0.585707,-0.998082,-0.343929,-0.160182,-0.334319,-1.261474,-0.668637 -42,0,1.619553,1.220152,2.089143,1.354326,-0.336160,3.117943,2.168536,2.050928,2.862249,1.100236 -217,1,-0.991758,-0.196589,-0.949013,-0.838319,-1.626250,-0.728794,-0.382307,-1.201321,-0.052992,-0.324505 -286,1,-0.627292,0.262631,-0.448611,-0.587992,-0.910410,0.171980,-0.171216,0.013609,-0.704957,0.879127 -135,0,-0.368439,1.252721,-0.453377,-0.399017,0.417818,-0.648640,-0.261410,-0.324269,-0.116085,-0.181532 -521,0,2.826844,0.204007,2.932082,3.096417,0.080281,1.046672,0.928856,2.023520,0.414547,0.707338 -278,1,-0.159286,0.068847,-0.248748,-0.248715,-1.199727,-1.132615,-0.797293,-0.955564,-0.915269,-1.181233 -504,1,-1.240257,-1.513995,-1.138153,-1.020263,2.535091,0.571476,-0.298830,-0.184337,0.222031,2.286134 -97,1,-1.107724,0.099787,-1.145302,-0.904416,-0.033692,-1.014866,-1.207529,-1.382518,-1.563999,0.328848 -84,1,-0.538247,-0.126566,-0.580264,-0.549670,0.233707,-0.343293,-0.218232,-0.582969,0.773694,-0.260777 -553,1,-1.330337,-0.102139,-1.322527,-1.027998,-0.967396,-1.089612,-0.922365,-1.354653,-0.753491,-0.555035 -469,1,-0.602442,-0.045144,-0.569541,-0.619635,2.000293,0.213329,0.222660,0.411023,-0.389490,0.485120 -87,0,1.716882,0.770703,1.359390,1.305104,-0.327392,0.421983,1.455144,1.233260,1.707154,0.495095 -22,0,0.372916,-1.074317,0.531343,0.176348,0.290694,2.170095,1.719008,1.898662,2.857396,0.859731 -19,1,-0.240048,-1.045005,-0.225217,-0.297761,0.509873,-0.489605,-0.159223,0.216123,0.123347,-0.629292 -265,0,3.359046,3.498337,3.179304,4.485168,0.338913,0.064472,0.345477,0.781030,-0.052992,-0.097854 -38,0,-0.264898,-0.077713,-0.349126,-0.319559,-1.687620,-1.291078,-1.190786,-1.303644,-2.160960,-1.601839 -479,0,0.232100,-0.427827,0.441986,0.103922,0.233707,1.220974,1.523269,0.957658,0.675010,0.410863 -17,0,0.971385,0.944946,0.879838,0.763667,2.039746,1.075298,0.989305,1.411411,1.302709,1.676560 -189,1,-0.604513,-0.991267,-0.613922,-0.586937,-0.998082,-0.567850,-0.623143,-1.011902,-0.677455,-1.157405 -178,1,-0.469910,0.544350,-0.568350,-0.477771,-2.240829,-1.399158,-1.267561,-1.604080,-0.979980,-1.413980 -164,0,2.431317,0.414075,2.291686,2.676276,-0.419448,0.661808,0.588232,1.827097,1.113428,0.439125 -451,0,1.070784,0.860267,0.969195,0.950006,0.895629,-0.443803,0.602144,0.487156,-0.983215,-1.276549 -290,1,-0.103373,-0.577643,-0.165645,-0.199142,-1.426358,-0.044944,-0.240780,-0.190427,-1.017189,0.224112 -67,1,-0.815737,-0.299180,-0.871570,-0.727570,-0.147665,-1.035540,-0.613068,-0.685139,-0.810113,-0.971762 -110,1,-1.080803,-0.685120,-1.059816,-0.902834,0.628230,-0.494694,-0.682153,-0.932876,-0.594948,0.040685 -556,1,-1.163636,-0.455510,-1.173002,-0.937465,-0.257255,-0.854113,-1.257616,-1.405205,-1.033367,-0.915792 -150,1,-0.436776,-0.255213,-0.489715,-0.463884,-0.116980,-0.914547,-0.916656,-0.786396,0.477640,-1.085918 -321,0,1.406258,-0.431084,1.278968,1.364873,-1.182192,-0.639734,-0.055596,0.424727,0.249533,-1.364106 -526,1,-0.190348,-0.084227,-0.159688,-0.282643,1.316453,0.369820,-0.032568,0.427773,0.998566,0.149854 -18,0,2.288430,0.847240,2.369129,2.667486,0.825491,0.386359,1.271399,1.891049,-0.214770,-0.432012 -446,0,1.089422,2.094623,1.135996,0.978132,0.338913,0.775677,1.764105,1.254577,0.115258,0.377059 -432,0,1.192963,-0.098883,1.153867,1.051965,1.496181,0.254042,1.240695,1.563677,0.212324,-0.177099 -523,1,-0.240048,-0.007690,-0.233259,-0.314109,0.444119,0.014854,-0.377510,0.210032,-0.083729,0.352676 -418,1,-0.542388,-1.426059,-0.570137,-0.551428,-0.042459,-0.595204,-0.855583,-0.492827,-0.203445,0.592627 -272,0,2.468592,0.407562,2.640181,2.642876,-0.226570,1.388279,1.480091,1.058154,-0.109614,0.256807 -153,1,-0.886145,-1.527023,-0.923695,-0.773100,0.075898,-1.046800,-0.964439,-0.906686,-0.067552,-0.899167 -509,0,0.174117,1.734739,0.310928,0.050658,1.789881,1.542225,1.529986,1.548450,0.181587,1.260942 -288,1,-0.913066,-0.545075,-0.863528,-0.778726,-1.296166,-0.445075,-0.564134,-0.326858,0.087755,-0.767831 -258,0,0.741523,0.971001,1.085360,0.607213,0.790423,1.672634,1.659519,2.003725,0.608681,0.994946 -228,1,-0.428493,0.917263,-0.494183,-0.451051,-0.423831,0.579746,0.570481,0.051676,-0.120938,0.659680 -476,1,0.037443,0.257745,0.144127,-0.091558,-0.748217,0.563842,-0.100693,0.293779,-0.593330,-0.297351 -423,1,-0.233835,-0.028860,-0.174581,-0.301980,-0.774519,0.397174,0.450063,0.397319,-0.253596,0.246278 -64,0,0.169975,1.269005,0.135192,0.013566,2.311528,0.965882,0.624693,0.867821,0.780165,1.061445 -137,1,-0.817808,-0.595556,-0.814083,-0.735833,-0.586024,-0.569123,-0.634657,-0.454456,-0.363605,-0.903046 -343,0,1.342063,1.462789,1.499383,1.159197,-0.463284,0.582290,0.965317,1.688535,1.851136,-0.264102 -302,0,1.534649,0.611116,1.535126,1.433432,0.102199,0.539669,1.060308,1.183012,0.636183,0.595398 -561,1,-0.900641,2.055541,-0.955268,-0.775210,-1.740223,-1.267986,-1.305831,-1.745063,-2.159342,-1.379622 -8,0,-0.161357,0.822813,-0.031609,-0.248363,1.662757,1.818310,1.280035,1.391616,2.389857,1.288650 -502,1,-0.558955,-0.696519,-0.613327,-0.577620,1.123576,-0.503600,-0.399578,-0.464354,0.411311,-0.474682 -28,0,0.828498,1.796619,1.252161,0.682803,1.390974,2.269333,1.733401,1.336800,1.822016,0.820940 -457,1,-0.397431,1.392767,-0.475716,-0.435405,-0.152049,-0.941264,-0.638975,-0.830706,-0.738931,-0.890300 -136,1,-0.608655,-0.032117,-0.628517,-0.586937,-0.230954,-0.963529,-0.804010,-0.684074,-1.923146,-0.582743 -365,0,1.665111,0.112814,1.606612,1.581096,0.014527,-0.106013,-0.009540,0.942432,-0.471997,-0.919671 -384,1,-0.420210,-1.352780,-0.317851,-0.451578,-0.691230,0.090554,0.069140,-0.348327,-0.266539,-0.595488 -81,1,-0.153073,-0.405029,-0.315766,-0.467048,0.930698,1.430264,1.024806,0.855640,1.013126,0.978321 -306,1,-0.385006,-0.851221,-0.454568,-0.428374,-0.857807,-0.761237,-1.252098,-1.364398,-0.404050,-0.005310 -282,0,1.557428,0.484098,1.344497,1.313894,0.851793,0.767407,0.764781,1.683967,1.115046,-0.336696 -475,1,-0.451272,-1.030349,-0.418229,-0.483045,0.010144,0.042844,0.361789,-0.255445,0.170262,-0.328384 -118,0,0.811931,0.785359,0.686230,0.688077,2.329063,1.515507,2.223228,1.352027,0.603827,2.286134 -55,1,-0.710125,-0.522276,-0.758086,-0.658133,-0.327392,-1.062767,-0.870407,-0.783351,0.655596,-0.752869 -557,1,-1.196769,1.394395,-1.214107,-0.966822,-1.098904,-1.162132,-1.305831,-1.745063,-0.688779,-0.789998 -284,1,-0.490618,-0.974982,-0.450994,-0.500975,-1.451345,-0.143545,0.298461,-0.196518,-1.458843,-0.702441 -256,0,1.818352,1.724968,2.124886,1.837752,-0.187118,1.772508,0.733597,1.210420,-0.133881,0.917364 -426,1,-0.857154,-0.668836,-0.770000,-0.773804,0.014527,0.288394,0.104162,-0.327467,0.192911,0.693484 -387,1,-0.157215,-0.929386,-0.226408,-0.237816,-2.083458,-0.833121,-0.782421,-1.054232,-0.580388,-0.981736 -497,1,-0.457485,-0.217758,-0.430144,-0.480408,-0.209036,-0.023315,-0.332893,-0.141702,0.217178,-0.406520 -298,1,-0.010186,-0.067942,-0.043523,-0.107027,-1.662195,-0.238966,-0.555018,-0.598500,-0.428317,-0.398208 -157,1,0.403979,0.389649,0.388371,0.266178,-1.956334,-0.529682,-0.402937,-0.460547,-0.604655,-1.342493 -249,1,-0.749471,-0.730716,-0.785787,-0.683447,0.286310,-0.611108,-0.440357,-0.282091,-0.383019,-0.324505 -342,1,-0.900641,-0.940785,-0.819147,-0.774507,0.413434,-0.211612,-0.202880,-0.108204,0.647507,0.379830 -508,1,0.217604,-1.289271,0.075620,0.083706,0.132884,-0.751695,-0.371753,0.321186,-0.971891,-0.645362 -267,1,-0.304244,0.710451,-0.285980,-0.385129,-1.396988,-0.516959,-0.608750,-0.802689,-0.735695,-0.759519 -338,1,-1.058024,0.189351,-1.050880,-0.872950,0.343297,-0.725613,-0.799692,-0.755486,-0.010929,-0.404858 -360,1,-0.527893,-0.764914,-0.608859,-0.518379,-1.728826,-1.342223,-1.288651,-1.496108,-1.080282,-1.592419 -12,0,0.971385,0.694167,1.323647,0.793551,-1.256713,0.865372,0.439988,0.945477,0.445285,1.017112 -210,0,1.443533,0.352195,1.520233,1.363115,-0.638627,0.240047,0.546493,1.178444,0.013337,-1.401788 -533,0,1.441462,0.239833,1.332582,1.343778,-0.993698,-0.005503,0.177564,0.710987,0.516467,-1.113072 -167,0,0.782940,0.101415,0.698144,0.666982,-0.682463,-0.269501,-0.193765,0.499338,-0.146823,-0.646471 -79,1,-0.420210,-0.139593,-0.458142,-0.454391,-0.152049,-0.255506,-0.475379,-0.538203,-0.196974,-0.264102 -369,0,2.358838,0.019993,2.613373,2.366883,-0.130131,0.853922,0.975872,1.958046,-0.258450,0.099426 -437,1,-0.126153,-0.667207,-0.180538,-0.229554,-0.564106,-0.821034,-0.769947,-0.610225,-0.284334,-0.643146 -485,1,-0.515468,-0.756771,-0.281214,-0.527344,-0.651778,0.965882,1.043037,0.298347,0.534263,1.078069 -408,0,0.996235,-0.043516,0.918559,0.823436,0.693984,0.758501,0.277832,1.260668,0.257622,0.060081 -308,1,-0.269040,-1.422803,-0.350913,-0.319735,-1.847183,-1.246230,-1.239673,-1.408555,-1.025278,-1.220579 -414,0,0.205179,1.829188,0.084556,0.089332,-0.770135,-0.989865,-0.563654,-0.743914,0.537498,-1.235541 -41,0,-0.710125,1.573523,-0.596944,-0.644421,2.565776,0.098824,0.624213,0.423205,0.102315,0.671317 -276,1,-0.842658,-1.088973,-0.890335,-0.742864,-0.283557,-1.150045,-1.282059,-1.575895,-0.230947,-1.113072 -146,0,-0.523751,0.114443,-0.456653,-0.507831,0.268776,0.985603,0.854974,1.094698,4.648279,1.055903 -405,1,-0.801242,-0.015832,-0.729789,-0.717550,0.172337,-0.571667,-0.628420,-0.544141,-1.051162,-0.367175 -35,0,0.774656,0.544350,0.781544,0.612486,1.049055,0.822115,1.289150,1.015519,3.174481,0.132121 -13,0,0.118205,0.322883,0.141149,-0.007178,-0.844656,-0.393548,-0.191846,-0.041207,-0.148441,-1.167934 -166,1,-0.966908,-2.223994,-1.000840,-0.820212,0.492339,-0.817853,-0.803530,-1.044183,-1.311625,-0.385462 -173,1,-1.018678,-1.442344,-1.049987,-0.850976,-0.472051,-1.093302,-1.116904,-1.089406,-1.615768,-0.599367 -531,1,-0.604513,0.510153,-0.603497,-0.580082,0.992068,0.268037,0.017326,-0.508663,0.493818,0.307790 -388,1,-0.875791,-1.098743,-0.820040,-0.756928,-0.971780,0.169436,0.143501,-0.485519,-1.203234,1.127944 -205,0,0.310792,-0.885418,0.310928,0.191466,0.733436,0.501500,0.290305,0.161307,0.831934,0.745575 -6,0,1.368983,0.322883,1.368325,1.275220,0.518640,0.021215,0.509552,1.196716,0.262476,-0.014730 -144,1,-0.894429,-0.807253,-0.877825,-0.772397,-1.085753,-0.839482,-0.837832,-1.225379,-0.971891,-0.900829 -540,1,-0.830233,-0.976611,-0.848337,-0.743216,0.093432,-0.270137,-0.443716,-0.691687,-0.924975,-0.144403 -30,0,1.424896,1.356941,1.585762,1.387726,0.733436,1.090566,1.636491,1.068812,0.878850,0.768849 -435,0,0.159621,0.834212,0.197742,-0.019835,1.268234,0.652266,0.646282,1.036837,0.450138,1.194443 -31,0,0.114063,0.397791,0.361564,0.014269,1.373440,2.056226,2.031327,0.608969,3.009467,3.117372 -203,0,0.606920,2.633636,0.632615,0.478885,3.955374,1.696171,0.923099,1.320051,2.477217,1.366232 -59,1,-1.400331,-1.673582,-1.410693,-1.064738,1.794265,-0.829304,-0.745480,-1.072199,0.516467,0.349906 -95,0,1.646474,0.962859,1.454704,1.528359,-0.586024,0.633818,0.660194,0.650081,1.275206,-0.014730 -37,1,-0.614867,-0.466909,-0.679153,-0.588344,-1.549975,-1.323648,-1.073966,-0.981753,-1.478256,-1.233324 -331,1,-0.382935,-0.606955,-0.239812,-0.432944,-0.156432,0.451882,0.344038,-0.244025,1.124753,0.427488 -237,0,1.646474,0.080246,1.621505,1.528359,-0.419448,-0.147362,0.209227,0.455181,-1.072193,-0.702441 -10,0,0.604849,1.335771,0.492622,0.473611,-0.625477,-0.630828,-0.605872,-0.226210,0.076431,0.031819 -389,0,0.942393,0.775589,1.034724,0.760151,-0.318625,-0.081840,0.531141,1.033791,-0.525383,-0.439216 -324,1,-0.521680,-0.699776,-0.481077,-0.522949,-0.296707,-0.391004,-0.745960,-0.899073,-0.387872,-0.240273 -179,1,-0.546530,-1.551449,-0.612433,-0.544747,-0.708765,-1.271103,-1.173515,-1.137522,-1.808284,-0.596042 -220,1,-0.192419,-1.518880,-0.224919,-0.306550,-0.055610,-0.043671,-0.461946,-0.518408,-0.842468,0.179225 -50,1,-0.681134,0.006966,-0.723236,-0.640026,-1.046301,-1.069447,-1.040863,-1.179395,-0.756726,-1.014986 -401,1,-0.511326,-0.901702,-0.584434,-0.511699,0.220556,-0.615561,-0.579486,-0.698082,-0.713046,-0.627629 -368,0,2.998723,0.124213,2.747410,3.977131,0.172337,-0.581845,0.066741,1.026178,-0.632157,-1.053223 -29,0,0.774656,-1.002666,0.823244,0.608971,-0.301091,0.171344,-0.111727,0.471930,-0.234183,-0.263547 -78,0,1.470454,0.984029,1.877663,1.305104,1.382207,2.303684,2.379147,2.073768,4.107940,0.869706 -204,1,-0.269040,-0.168905,-0.333935,-0.356299,0.448503,-0.104741,-0.024412,-0.199563,0.183204,0.196958 -214,0,0.122346,1.493730,0.230506,-0.121794,1.031520,0.964610,0.490362,0.953090,2.949609,1.033737 -127,0,1.253017,0.008594,1.219396,1.155681,-1.326851,-0.177261,0.232735,0.109537,-0.096672,-1.027177 -46,1,-1.512777,-0.605327,-1.489328,-1.122222,-0.116980,-0.754239,-0.975761,-1.354653,0.330422,-0.546168 -554,1,-0.492689,1.638661,-0.548691,-0.500800,-0.423831,-0.586935,-0.135715,-0.756400,-0.855411,-0.638713 -449,0,1.948814,1.041024,1.815113,2.006511,0.194255,0.355188,0.804600,1.726601,-1.023660,-0.537302 -564,0,1.901185,0.117700,1.752563,2.015301,0.378365,-0.273318,0.664512,1.629151,-1.360158,-0.709091 -297,0,-0.602442,-0.372460,-0.660090,-0.574808,-0.818354,-1.110223,-1.012222,-0.654838,-1.492816,-0.819922 -491,1,0.735311,-1.181794,0.590915,0.579086,-1.479400,-0.982868,-0.803050,-0.475012,-1.808284,-1.398463 -492,0,1.089422,0.062333,1.076424,0.958795,-0.064377,-0.137184,-0.085341,0.522178,0.566618,-0.426470 -181,0,2.155897,1.270634,2.062335,2.124291,0.733436,3.207003,1.946890,2.675218,1.936879,2.463465 -427,1,-0.726692,1.036139,-0.702088,-0.687490,-0.090679,-0.538588,-0.381348,-0.605352,0.103933,-0.405966 -471,1,-0.552743,1.246207,-0.596349,-0.550197,-1.239179,-0.998771,-1.040815,-0.900443,-0.803642,-0.972870 -515,1,-0.786746,-0.431084,-0.837316,-0.706651,0.698367,-0.616197,-0.526713,-0.444407,0.257622,-0.893071 -445,1,-0.681134,0.762561,-0.678558,-0.644597,-0.055610,-0.458434,-0.533909,0.085174,-0.488175,-0.079567 -506,1,-0.643859,-0.245442,-0.659197,-0.642136,0.343297,-0.144817,0.390094,-0.513536,-0.310219,0.246278 -93,1,-0.242119,0.042792,-0.288065,-0.318504,0.067131,-0.503600,-0.643293,-0.540487,-0.360370,-0.992820 -453,1,-0.097161,-1.424431,-0.123945,-0.229730,0.102199,-0.677266,-0.647131,-0.117340,-0.476850,-0.323951 -357,1,-0.240048,-0.015832,-0.313383,-0.327294,-0.748217,-0.976252,-1.052282,-0.899073,-0.871589,-0.710199 -544,1,-0.252473,-0.150993,-0.241004,-0.337490,-0.261639,-0.321664,-0.645212,-0.702802,-1.054398,0.053985 -26,0,0.279729,1.226666,0.450921,0.028684,0.882478,2.608395,1.351518,2.367641,2.205430,2.413591 -356,1,-0.430564,-0.134708,-0.388443,-0.508710,0.084665,0.073378,-0.071428,0.170443,0.343365,-0.042993 -227,1,0.029159,-1.036863,0.206678,-0.127243,-0.822738,0.689798,0.326287,0.354685,0.086138,-0.018055 -377,1,-0.327023,1.620748,-0.302362,-0.351553,-0.945479,-0.690625,-0.925195,-0.864813,-0.334485,-0.739015 -234,1,-1.192628,-1.061289,-1.236744,-0.957505,0.790423,-1.012194,-0.962280,-0.645398,-0.232565,-0.120021 -341,1,-1.142928,-0.424570,-1.072624,-0.926390,-0.397530,0.555572,0.776774,-0.508663,0.131435,0.792678 -129,0,1.317213,1.286918,1.234289,1.245335,-0.213419,0.838655,1.415804,0.892184,0.653979,0.039023 -289,1,-0.809525,0.075360,-0.833146,-0.740579,-0.901643,-0.999916,-0.944625,-0.800557,0.592503,-0.776144 -539,1,-1.572003,1.011712,-1.571835,-1.154919,1.193713,0.331651,0.321969,-0.983733,-0.179178,1.255400 -145,1,-0.645930,-1.492825,-0.625539,-0.652156,0.439736,-0.016317,-0.853904,-0.825072,-0.281099,1.089153 -138,0,0.472316,-0.691634,0.421136,0.159648,0.382749,-0.240875,0.303738,0.793211,0.830317,-0.691358 -410,1,-0.666638,1.733110,-0.660984,-0.631588,0.566860,-0.585662,-0.436999,-0.420653,0.116875,-0.359971 -56,0,2.044072,0.401048,1.871706,2.222734,1.316453,0.616006,0.555129,1.438819,1.029304,-0.055738 -216,1,-0.625221,0.231690,-0.627326,-0.614185,0.356447,0.320201,0.040354,-0.012276,0.802814,-0.016393 -281,1,-0.612797,-1.207849,-0.672005,-0.609790,-1.261097,-1.076762,-0.982717,-0.482778,0.323951,-0.945716 -16,0,0.579999,0.847240,0.480707,0.452516,0.615079,-0.427264,0.092168,0.704897,0.207471,-0.098963 -444,0,0.851277,-0.595556,0.775587,0.723235,-0.266022,0.078468,0.752307,0.592220,-0.095054,-0.093975 -547,1,-1.126361,-0.592299,-1.077688,-0.919710,0.601928,-0.188711,-0.450432,-0.476230,-0.339339,0.600939 -270,1,-0.281464,-0.818652,-0.381891,-0.344521,-2.047074,-1.297121,-1.120358,-1.237560,-0.716282,-1.260478 -44,0,-0.008116,0.686025,-0.052459,-0.245902,0.786039,0.866009,0.482686,0.701851,1.281677,0.677967 -34,0,0.816073,0.257745,0.757716,0.668740,0.536175,2.074674,1.224383,1.093175,2.215137,2.180844 -371,1,-0.014328,-1.619844,-0.082245,-0.108082,-0.866574,-0.512506,-0.652408,-0.499832,-0.669366,-0.902492 -310,1,-0.757754,0.142126,-0.784595,-0.698741,-0.441366,-0.925997,-0.926107,-0.870904,0.948415,-0.796094 -538,1,-1.489377,0.853754,-1.492009,-1.112026,-0.296707,-1.086940,-1.305831,-1.745063,0.254387,0.855298 -72,0,1.460100,1.326001,1.320668,1.407063,1.145493,3.086136,1.844223,1.146468,0.666921,2.768252 -73,0,0.062293,-0.784455,0.090513,-0.119860,0.382749,0.635726,0.027401,0.360776,-0.504352,1.055903 -279,1,-0.266969,-0.641152,-0.264832,-0.370187,-0.607942,-0.520776,-0.607311,-0.223469,0.087755,-0.821585 -196,0,0.025018,1.356941,0.129234,-0.129529,1.811799,0.368547,0.521546,0.802347,0.289978,0.520032 -74,1,-0.447130,-0.401772,-0.522778,-0.473728,-0.647394,-0.445075,-0.488812,-0.426439,-0.457437,-0.435337 -23,0,2.671532,1.614234,2.404872,3.048953,0.338913,0.036482,0.207788,1.313961,-0.127409,-0.481332 -334,1,-0.604513,0.453158,-0.677068,-0.591156,-0.445749,-1.041647,-1.132208,-1.138587,-0.560975,-0.658108 -400,0,0.938252,0.342424,1.261096,0.744330,2.407967,2.146558,3.028252,1.245441,0.556912,1.986889 -431,1,-0.701842,-0.450625,-0.525756,-0.641257,0.553709,0.054930,-0.152986,-0.622863,-0.557739,0.534440 -254,0,1.952956,-0.180304,1.663205,1.918616,0.759738,0.393357,0.765260,1.298734,0.773694,0.307790 -48,1,-0.519609,-0.810510,-0.517714,-0.523828,0.746587,-0.245964,0.157414,-0.748025,-0.248743,-0.051859 -130,1,-0.606584,-1.281128,-0.473035,-0.589574,0.452886,0.026940,-0.830156,-0.498461,0.919295,0.469050 -555,1,-1.124290,1.503500,-1.122664,-0.919359,0.264392,-0.529682,-0.346326,-0.355331,-1.091607,-0.061834 -520,1,-1.180203,-1.276243,-1.174194,-0.973854,2.307145,-0.283496,-0.826270,-0.639307,1.262264,0.325523 -122,0,2.019222,-0.274754,2.193393,2.096165,1.632072,1.082296,1.478172,1.677876,0.519703,-0.213673 -496,1,-0.391218,-0.574386,-0.356573,-0.433999,0.917547,0.826568,0.412642,0.397319,0.532645,1.072528 -225,1,0.103709,-1.429316,0.093491,-0.012979,-0.116980,-0.647368,-0.522875,-0.089932,0.260858,-1.287078 -326,1,-0.153073,-1.250188,-0.263939,-0.229730,-0.187118,-0.912002,-1.051226,-0.848216,-1.295447,-0.726824 -106,1,-0.648001,0.583433,-0.647878,-0.630885,1.597003,0.074651,0.072498,0.109537,-0.153294,0.389251 -381,1,-0.865437,-0.789340,-0.820040,-0.762026,-1.002465,-0.356652,-0.560775,-0.716658,0.487347,-0.613775 -415,1,-0.666638,0.249603,-0.660388,-0.628776,0.448503,-0.226243,-0.747399,-0.486889,0.281889,-0.578309 -71,1,-1.353531,-1.629614,-1.331463,-1.048038,-0.511503,-0.067845,-0.617866,-1.016318,-1.046309,1.355149 -558,1,-0.163427,0.259374,-0.040545,-0.258559,-1.304933,0.399718,0.451022,-0.062524,-1.039838,-0.216444 -21,1,-1.250611,-1.631243,-1.254913,-0.994422,0.001377,-0.887193,-0.880434,-0.796903,-0.729224,-0.344455 -316,1,-0.708054,-1.499339,-0.764341,-0.646003,-1.414523,-1.278291,-1.108365,-1.463066,-0.983215,-1.306473 -463,1,-0.724621,-0.269869,-0.732172,-0.677646,0.080281,-0.467340,-0.413491,-0.483082,0.500289,-0.294580 -454,1,-0.399502,-0.574386,-0.465887,-0.434351,-0.432599,-0.652457,-0.400538,-0.245091,0.597356,-0.589947 -413,1,0.101638,0.956345,0.087534,-0.023702,-1.081370,0.510406,0.188118,0.246576,0.424254,0.474591 -330,0,0.515803,-0.602070,0.507515,0.332978,0.487955,1.231153,1.071822,1.271326,0.191293,0.404213 -468,0,1.097705,0.519924,1.082381,0.978132,-0.511503,1.426448,1.172090,1.294166,-0.970273,2.130970 -322,1,-0.461626,-0.748629,-0.430739,-0.494120,0.978917,-0.198253,-0.446594,0.013609,-0.839233,0.087789 -307,1,-1.360572,-0.913101,-1.380908,-1.046104,-1.479838,-1.284653,-1.235211,-1.533565,0.145995,-0.327276 -373,0,1.884619,-0.408286,1.773413,1.872910,1.044671,0.325926,0.689459,1.470795,-0.342574,-0.742340 -304,1,-0.743259,-0.662322,-0.731874,-0.686963,-0.787669,-0.479427,-0.717654,-0.906230,-1.120727,-0.419266 -529,1,-0.583805,-1.613330,-0.605880,-0.581312,0.864944,-0.579301,-0.527672,-0.619360,-0.193738,-0.189844 -40,0,-0.070240,0.744648,-0.141817,-0.162929,-1.006849,-0.317847,-0.305547,-0.051865,0.150849,-0.691912 -115,1,-0.538247,0.076989,-0.587413,-0.523125,0.772888,-0.091382,-0.584763,-0.641591,-0.748637,0.081139 -2,0,1.511870,-0.023974,1.347475,1.456285,0.527407,1.082932,0.854974,1.955000,1.152255,0.201391 -39,0,-0.153073,0.055819,0.001155,-0.246430,1.255083,1.070209,1.107324,1.693103,-0.151676,1.283108 diff --git a/fml_manager/Examples/Hetero_LR/hetero_lr.ipynb b/fml_manager/Examples/Hetero_LR/hetero_lr.ipynb deleted file mode 100644 index a1dec44aa..000000000 --- a/fml_manager/Examples/Hetero_LR/hetero_lr.ipynb +++ /dev/null @@ -1,297 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "import json\n", - "import time\n", - "import os\n", - "import requests\n", - "\n", - "import fml_manager\n", - "from fml_manager import *\n", - "\n", - "manager = fml_manager.FMLManager()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## For more details about the FMLManager, please refer to this [document](https://kubefate.readthedocs.io/README.html)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = manager.load_data(url='./data/breast_b.csv', namespace='fate_flow_test_breast', table_name='breast_b', work_mode=1, head=1, partition=10)\n", - "output = json.loads(response.content)\n", - "print(output)\n", - "guest_job_id = output['jobId']\n", - "guest_query_condition = QueryCondition(job_id=guest_job_id)\n", - "\n", - "manager.query_job_status(guest_query_condition, max_tries=20)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "# Pipeline\n", - "data_io = ComponentBuilder(name='dataio_0',\n", - " module='DataIO')\\\n", - " .add_input_data('args.train_data')\\\n", - " .add_output_data('train')\\\n", - " .add_output_model('dataio').build()\n", - " \n", - "hetero_feature_binning = ComponentBuilder(name='hetero_feature_binning_0',\n", - " module='HeteroFeatureBinning')\\\n", - " .add_input_data('dataio_0.train')\\\n", - " .add_output_data('train')\\\n", - " .add_output_model('hetero_feature_binning').build()\n", - "\n", - "hetero_feature_selection = ComponentBuilder(name='hetero_feature_selection_0',\n", - " module='HeteroFeatureSelection')\\\n", - " .add_input_data('hetero_feature_binning_0.train')\\\n", - " .add_input_isometric_model('hetero_feature_binning_0.hetero_feature_binning')\\\n", - " .add_output_data('train')\\\n", - " .add_output_model('selected').build()\n", - "\n", - "hetero_lr = ComponentBuilder(name='hetero_lr_0',\n", - " module='HeteroLR')\\\n", - " .add_input_train_data('hetero_feature_selection_0.train')\\\n", - " .add_output_data('train')\\\n", - " .add_output_model('hetero_lr').build()\n", - "\n", - "evaluation = ComponentBuilder(name='evaluation_0',\n", - " module='Evaluation',\n", - " need_deploy=False)\\\n", - " .add_input_data('hetero_lr_0.train')\\\n", - " .add_output_data('evaluate').build()\n", - "pipeline = Pipeline(\n", - " data_io, \n", - " hetero_feature_selection, \n", - " hetero_feature_binning, \n", - " hetero_lr, \n", - " evaluation)\n", - "\n", - "# Configuration\n", - "\n", - "initiator = Initiator(role='guest', party_id=10000)\n", - "\n", - "job_parameters = JobParameters(work_mode=1)\n", - "\n", - "role = RoleBuilder()\\\n", - " .add_guest(party_id=10000)\\\n", - " .add_host(party_id=9999)\\\n", - " .add_arbiter(party_id=9999).build()\n", - "\n", - "guest_data_io_config = {\n", - " 'with_label': [True],\n", - " 'label_name': ['y'],\n", - " 'label_type': ['int'],\n", - " 'output_format': ['dense']\n", - "}\n", - "\n", - "host_data_io_config = {\n", - " 'with_label': [False],\n", - " 'output_format': ['dense']\n", - "}\n", - "\n", - "role_parameters = RoleParametersBuilder()\\\n", - " .add_guest_train_data(namespace='fate_flow_test_breast', name='breast_b')\\\n", - " .add_guest_module_config(module='dataio_0', config=guest_data_io_config)\\\n", - " .add_host_train_data(namespace='fate_flow_test_breast', name='breast_a')\\\n", - " .add_host_module_config(module='dataio_0', config=host_data_io_config).build()\n", - "\n", - "hetero_lr_params = {\n", - " 'penalty': 'L2',\n", - " 'optimizer': 'rmsprop',\n", - " 'eps': 1e-5,\n", - " 'alpha': 0.01,\n", - " 'max_iter': 3,\n", - " 'converge_func': 'diff',\n", - " 'batch_size': 320,\n", - " 'learning_rate': 0.15,\n", - " 'init_param': {\n", - " 'init_method': 'random_uniform'\n", - " }\n", - "}\n", - "\n", - "algorithm_parameters = AlgorithmParametersBuilder()\\\n", - " .add_module_config(module='hetero_lr_0', config=hetero_lr_params).build()\n", - "\n", - "config = Config(initiator, job_parameters, role, role_parameters, algorithm_parameters)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = manager.submit_job(pipeline.to_dict(),config.to_dict())\n", - "manager.prettify(response, verbose=True)\n", - "stdout = json.loads(response.content)\n", - "job_id = stdout['jobId']\n", - "query_condition = QueryCondition(job_id)\n", - "model_id, model_version = '', ''\n", - "manager.query_job_status(query_condition, max_tries=20)\n", - "\n", - "manager.prettify(response, verbose=True)\n", - "output = json.loads(response.content)\n", - "model_id, model_version = output['data']['model_info']['model_id'], output['data']['model_info']['model_version']" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Click [here](/fateboard-10000/) to view jobs in FATE Board" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "# Pipline\n", - "data_io = ComponentBuilder(name='dataio_0',\n", - " module='DataIO')\\\n", - " .add_input_data('args.train_data')\\\n", - " .add_output_data('train')\\\n", - " .add_output_model('dataio').build()\n", - "\n", - "hetero_feature_binning = ComponentBuilder(name='hetero_feature_binning_0',\n", - " module='HeteroFeatureBinning')\\\n", - " .add_input_data('dataio_0.train')\\\n", - " .add_output_data('train')\\\n", - " .add_output_model('hetero_feature_binning').build() \n", - "\n", - "hetero_feature_selection = ComponentBuilder(name='hetero_feature_selection_0',\n", - " module='HeteroFeatureSelection')\\\n", - " .add_input_data('hetero_feature_binning_0.train')\\\n", - " .add_input_isometric_model('hetero_feature_binning_0.hetero_feature_binning')\\\n", - " .add_output_data('eval')\\\n", - " .add_output_model('selected').build()\n", - "\n", - "evaluation = ComponentBuilder(name='evaluation_0',\n", - " module='Evaluation',\n", - " need_deploy=False)\\\n", - " .add_input_data('hetero_feature_selection_0.eval')\\\n", - " .add_output_data('evaluate').build()\n", - "\n", - "pipeline = Pipeline(\n", - " data_io, \n", - " hetero_feature_selection, \n", - " hetero_feature_binning, \n", - " evaluation)\n", - "\n", - "# Configuration\n", - "initiator = Initiator(role='guest', party_id=10000)\n", - "\n", - "job_parameters = JobParameters(work_mode=1,\n", - " job_type='predict',\n", - " model_id='arbiter-9999#guest-10000#host-9999#model',\n", - " model_version='2020060802475836992436')\n", - "\n", - "role = RoleBuilder()\\\n", - " .add_guest(party_id=10000)\\\n", - " .add_host(party_id=9999)\\\n", - " .add_arbiter(party_id=9999).build()\n", - "\n", - "role_parameters = RoleParametersBuilder()\\\n", - " .add_guest_eval_data(namespace='fate_flow_test_breast', name='breast_b')\\\n", - " .add_host_eval_data(namespace='fate_flow_test_breast', name='breast_a').build()\n", - "\n", - "config = Config(initiator, job_parameters, role, role_parameters)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = manager.submit_job(pipeline.to_dict(), config.to_dict())\n", - "manager.prettify(response, verbose=True)\n", - "stdout = json.loads(response.content)\n", - "job_id = stdout['jobId']\n", - "query_condition = QueryCondition(job_id)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(query_condition)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model_id, model_version = '', ''\n", - "manager.query_job_status(query_condition, max_tries=20)\n", - "\n", - "manager.prettify(response, verbose=True)\n", - "output = json.loads(response.content)\n", - "model_id, model_version = output['data']['model_info']['model_id'], output['data']['model_info']['model_version']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "# data is a pandas.DataFrame\n", - "data = manager.track_component_output_data(job_id='2020060806580950942753', role='guest', party_id='10000', component_name='hetero_lr_0')\n", - "\n", - "print(data.head(4))" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.9" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/fml_manager/Examples/Homo_LR/data/breast_homo_guest.csv b/fml_manager/Examples/Homo_LR/data/breast_homo_guest.csv deleted file mode 100644 index 4163c1279..000000000 --- a/fml_manager/Examples/Homo_LR/data/breast_homo_guest.csv +++ /dev/null @@ -1,228 +0,0 @@ -id,y,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11,x12,x13,x14,x15,x16,x17,x18,x19,x20,x21,x22,x23,x24,x25,x26,x27,x28,x29 -133,1,0.254879,-1.046633,0.20965599999999998,0.074214,-0.441366,-0.377645,-0.48593400000000003,0.347072,-0.28757,-0.733474,0.44951199999999997,-1.247226,0.413178,0.30378099999999997,-0.123848,-0.184227,-0.21907600000000002,0.26853699999999997,0.015996,-0.789267,-0.33736,-0.728193,-0.442587,-0.272757,-0.6080180000000001,-0.577235,-0.5011260000000001,0.143371,-0.46643100000000004,-0.5541020000000001 -273,1,-1.1429280000000002,-0.781198,-1.166747,-0.923578,0.6282300000000001,-1.021418,-1.111867,-0.9595229999999999,-0.09667200000000001,-0.121683,-1.245485,-0.842317,-1.255026,-1.038066,-0.42630100000000004,-1.088781,-0.976392,-0.898898,0.9834959999999999,0.045702,-0.49363900000000005,0.34862,-0.552483,-0.526877,2.253098,-0.82762,-0.7807390000000001,-0.37699699999999997,-0.310239,0.17630099999999999 -175,1,-1.451067,-1.406518,-1.456564,-1.0923370000000001,-0.708765,-1.168557,-1.305831,-1.745063,-0.49949899999999997,-0.30289299999999997,-1.549664,-1.126219,-1.5466520000000001,-1.216392,-0.35442399999999996,-1.167051,-1.114873,-1.2618200000000002,-0.32719299999999996,0.6297550000000001,-0.6668810000000001,-0.779358,-0.708418,-0.637545,0.7103689999999999,-0.976454,-1.057501,-1.9134470000000001,0.795207,-0.149751 -551,1,-0.879933,0.42058900000000005,-0.8775270000000001,-0.7804840000000001,-1.037534,-0.48388000000000003,-0.5554979999999999,-0.7685810000000001,0.43396,-0.200928,-0.851273,0.733108,-0.843535,-0.786363,-0.049836,-0.42453199999999996,-0.509221,-0.679649,0.797298,0.385927,-0.451772,0.45385200000000003,-0.431696,-0.49475399999999997,-1.182041,0.28122800000000003,0.084759,-0.25242,1.038575,0.351054 -199,0,0.42675799999999997,0.723479,0.316885,0.287273,1.0008350000000001,0.9627020000000001,1.077099,1.0535860000000001,2.996525,0.9616959999999999,0.091654,0.21649899999999997,0.10383900000000001,-0.034667,0.16793,0.308132,0.366614,0.280661,0.505223,0.264013,-0.707304,-1.026834,-0.702973,-0.460212,-0.999033,-0.531406,-0.39436,-0.72883,-0.644416,-0.6880029999999999 -274,0,0.963102,1.467675,0.829202,0.772457,-0.038076,-0.468613,-0.307946,-0.015321000000000001,-0.641864,-0.247477,1.080023,1.20783,0.956888,0.978402,-0.555822,-0.645696,-0.399365,-0.038153,-0.998966,-1.091216,0.057848000000000004,0.392164,-0.050026999999999995,0.12041400000000001,-0.5323479999999999,-0.770613,-0.519694,-0.531097,-0.769127,-0.394858 -420,1,-0.662496,0.21214899999999998,-0.620475,-0.632995,-0.327392,-0.385278,-0.077665,-0.730362,0.217178,-0.061279999999999994,-0.7263069999999999,-0.05809500000000001,-0.7319100000000001,-0.697343,-0.7757229999999999,-0.513983,-0.426233,-0.893482,0.800949,-0.018090000000000002,-0.42867299999999997,0.40486500000000003,-0.32675,-0.44085,0.07901,-0.279903,0.41699200000000003,-0.48616499999999996,-0.22548400000000002,-0.17244600000000002 -76,1,-0.45334300000000005,-2.147457,-0.473631,-0.483572,0.5580930000000001,-0.740244,-0.89617,-0.617229,-0.308601,-0.666975,-0.169639,-1.9430189999999998,-0.167192,-0.27215,2.3299369999999997,0.006804000000000001,-0.251467,0.429234,2.1591,0.5120939999999999,0.017786,-0.368046,-0.105966,-0.169129,2.11976,0.162743,-0.672216,-0.577002,0.626908,0.896114 -315,1,-0.606584,-0.971725,-0.678558,-0.591332,-0.9630129999999999,-1.302401,-1.212855,-1.3211540000000002,-1.591501,-1.2305540000000001,-0.46501400000000004,-0.567723,-0.526371,-0.49285200000000007,-0.8006310000000001,-1.250816,-1.058714,-1.0961450000000001,-2.1782209999999997,-0.8601469999999999,-0.8430110000000001,-0.9103530000000001,-0.90049,-0.608283,-0.7043550000000001,-1.255622,-0.9706290000000001,-1.363557,-0.800607,-0.9270579999999999 -399,1,-0.583805,-0.193332,-0.633283,-0.560041,-0.34931,-0.519504,-0.6106689999999999,-0.9295260000000001,-0.196974,-0.151608,-0.660984,-0.47231300000000004,-0.688248,-0.634204,-0.390718,-0.7963600000000001,-0.75668,-0.8393139999999999,0.129175,-0.369656,-0.221505,-0.139439,-0.317344,-0.336122,-0.526014,-0.326291,-0.368166,-1.0378399999999999,-0.698901,-0.273818 -238,1,-0.10751500000000001,2.420311,-0.141817,-0.204943,-1.063835,-0.074206,0.164131,-0.49358900000000006,-1.635181,-0.33170900000000003,0.026330000000000003,1.9920509999999998,0.02393,-0.088136,-1.005588,-0.008357,0.26994,-0.124821,-1.714551,-0.213719,-0.251822,2.0087450000000002,-0.376748,-0.22831300000000002,-0.24466999999999997,0.166096,0.219045,-0.273508,-1.052451,-0.07788300000000001 -246,1,-0.482335,0.348938,-0.565371,-0.489725,-0.9761639999999999,-0.6581819999999999,-0.20335999999999999,-0.988301,-0.216387,-0.663096,-0.263364,-0.432753,-0.322891,-0.322206,-1.7229349999999999,-1.1200510000000001,-0.570489,-0.9767959999999999,-1.1851639999999999,-0.9140159999999999,-0.87405,0.696974,-0.986625,-0.589142,-0.260004,-0.5470550000000001,-0.036596,-1.040273,-0.111671,-0.5843619999999999 -253,0,0.7415229999999999,-0.095626,0.704101,0.6001810000000001,0.404667,-0.087565,0.314773,1.082516,0.383809,-0.156041,0.9010940000000001,-0.5142,0.8662700000000001,0.777324,0.31595500000000004,-0.004567,0.47458599999999995,0.892752,0.005043,-0.945203,-0.346022,-0.653261,-0.333185,-0.147567,-0.761357,-0.583942,-0.284279,0.133639,-0.799396,-0.541998 -550,1,-0.954483,-0.14773599999999998,-0.9883299999999999,-0.8232010000000001,-1.414523,-1.150045,-1.305831,-1.745063,-0.716282,-0.998915,-0.9279569999999999,0.509709,-0.966282,-0.8372729999999999,-1.569218,-1.1763370000000002,-1.114873,-1.2618200000000002,-0.5499,-0.47030600000000006,-0.320758,0.158114,-0.371797,-0.432709,0.8460409999999999,-0.806941,-1.057501,-1.9134470000000001,1.149967,-0.592683 -208,1,-0.356014,0.567149,-0.23176999999999998,-0.424155,0.11096600000000001,1.182806,0.211146,-0.030548000000000002,1.985412,1.310816,-0.288925,0.756379,-0.203852,-0.356904,0.273255,0.8330879999999999,-0.021963,0.054189,0.140128,1.4604709999999999,-0.765413,-0.534421,-0.680696,-0.555479,-0.596684,0.27563899999999997,-0.21995399999999998,-0.566458,0.556683,0.152472 -185,1,-0.9109950000000001,-0.732345,-0.949311,-0.77978,0.864944,-0.969255,-1.272632,-1.586402,0.052164,-0.386571,-1.1494879999999998,-0.9726319999999999,-1.161936,-0.959569,-0.26261999999999996,-1.087644,-1.0948229999999999,-1.199811,-0.39655999999999997,-0.328545,0.069759,0.092797,-0.09210499999999999,-0.30598000000000003,2.449773,-0.753287,-1.004549,-1.523495,0.585742,-0.122895 -156,0,0.869914,-0.092369,0.7636729999999999,0.7408140000000001,0.413434,0.6077359999999999,0.41312200000000004,0.561767,-0.7081930000000001,-0.36385,1.00902,0.337507,1.047507,0.87829,1.077425,1.178005,1.2140739999999999,1.456866,0.581892,-0.161268,1.4658010000000001,0.332291,1.323683,1.178696,0.665367,1.344806,0.668654,1.072832,-0.334455,0.443725 -0,0,1.8866900000000002,-1.359293,2.303601,2.0012369999999997,1.3076860000000001,2.6166650000000002,2.109526,2.296076,2.750622,1.9370150000000002,1.097064,-2.0733349999999997,1.2699340000000001,0.984375,1.568466,3.283515,2.652874,2.532475,2.217515,2.255747,2.489734,-0.565265,2.833031,2.487578,-0.214002,1.316862,0.7240260000000001,0.66082,1.148757,0.907083 -70,0,1.779007,0.14701199999999998,1.746605,1.732277,-0.572873,-0.131459,-0.016736,0.978975,-0.565828,-1.000578,1.3668770000000001,0.470149,1.302886,1.351264,-0.44622700000000004,-0.027308999999999996,0.241064,0.78906,-0.838325,-1.160679,1.384594,-0.760851,1.296951,1.2257799999999999,-0.8656950000000001,-0.5006659999999999,-0.305168,0.308825,-0.809083,-0.793157 -293,1,-0.664567,0.011851,-0.68243,-0.637741,0.198638,-0.499147,-0.674477,-0.353352,0.32395100000000004,-0.76894,-0.646783,-0.425771,-0.676715,-0.631929,-0.899551,-0.9081739999999999,-0.777395,-0.673717,0.231402,-0.800607,-0.715245,0.038367,-0.8079189999999999,-0.582101,0.18468099999999998,-0.5856180000000001,-0.589324,-0.5220130000000001,-0.317504,-0.760627 -287,1,-0.548601,-1.650784,-0.591583,-0.533673,-1.587236,-0.8878290000000001,-0.7368439999999999,-0.928004,-0.9573309999999999,-0.8199219999999999,-0.351408,-1.435719,-0.415157,-0.39529899999999996,-1.907966,-1.270715,-0.8311299999999999,-0.959772,-1.732806,-0.990566,-0.9094200000000001,-1.356863,-0.866828,-0.608503,-0.7700239999999999,-0.672247,-0.5097470000000001,-0.955599,-0.5269699999999999,-0.6482859999999999 -222,1,-1.0559530000000001,-0.462024,-1.052072,-0.887716,0.360831,-0.70144,-0.9905370000000001,-0.89618,0.249533,0.223003,-1.1210870000000002,-0.409482,-1.105917,-0.972083,0.693131,-0.366161,-0.8929010000000001,-0.7678649999999999,0.359185,0.890594,-0.571959,0.000266,-0.606442,-0.556359,0.286019,-0.641508,-0.774539,-0.676761,0.705609,-0.012824 -262,0,0.853348,0.254488,0.912602,0.728509,-0.8315049999999999,0.20633200000000002,-0.20335999999999999,0.581561,0.268947,-0.504606,0.8982540000000001,0.6609689999999999,0.9239360000000001,0.8330690000000001,-0.45334399999999997,0.43510699999999997,0.102583,0.674535,1.082072,-1.156426,1.550617,0.755033,1.6236709999999999,1.113351,-0.10799700000000001,1.9187919999999998,0.480322,1.572438,0.8387959999999999,0.772804 -309,1,-0.318739,-1.3478940000000001,-0.396188,-0.36596799999999996,-1.348769,-1.24553,-1.218324,-1.207259,-1.284123,-1.005565,-0.30596599999999996,-1.26817,-0.38138099999999997,-0.353491,-0.913784,-1.269578,-1.057635,-1.0340850000000001,-1.309296,-1.079875,-0.02769,-0.704607,-0.148043,-0.161208,-0.9643649999999999,-1.160666,-0.963633,-1.131435,-0.765494,-0.575662 -534,1,-0.962766,0.135613,-0.918334,-0.8316389999999999,0.45726999999999995,-0.02077,-0.28731599999999996,-0.24356799999999998,-0.9896870000000001,-0.064605,-0.8995559999999999,-0.388538,-0.8723690000000001,-0.822768,0.036274,-0.129268,-0.45410500000000004,-0.542683,-0.70324,0.181792,-0.918443,0.664316,-0.842076,-0.6654869999999999,0.82004,0.463987,0.40174,0.269895,-0.8865729999999999,-0.098687 -54,0,0.379129,0.979143,0.31092800000000004,0.262662,0.28631,-0.308942,-0.0047420000000000006,0.584607,-0.365223,-0.289039,0.27626300000000004,0.635371,0.21793600000000002,0.164705,-0.41278000000000004,-0.635462,-0.45536000000000004,-0.401848,-0.7141930000000001,-0.8445530000000001,-0.34169099999999997,-0.694809,-0.380708,-0.22941399999999998,-0.7886920000000001,-0.848299,-0.525994,-0.41755,-1.1614209999999998,-0.819256 -172,0,0.522016,-1.406518,0.528365,0.389232,0.9087799999999999,0.661808,1.491126,1.036837,0.509996,0.9450709999999999,0.378508,-1.721948,0.43377299999999996,0.23324699999999998,2.087974,0.9695389999999999,1.436297,1.56778,0.563638,1.1188280000000002,0.056764999999999996,-1.013408,-0.030226,0.094672,-0.549349,-0.12676500000000002,0.369246,0.22771999999999998,-0.317504,0.141124 -484,1,0.153409,-1.868995,0.15604200000000001,-0.046202999999999994,0.952616,0.277579,0.6150979999999999,0.46583900000000006,-0.556121,-0.11835799999999999,0.455192,-1.8638990000000002,0.44613,0.262542,0.565034,0.48438100000000006,0.380424,0.34024499999999996,-0.10083400000000001,-0.029431,-0.87405,-1.505458,-0.852967,-0.582321,-0.335673,-0.40677199999999997,0.048618,-0.18267,-1.064558,-0.582092 -102,1,-0.606584,1.1664139999999998,-0.675579,-0.585004,-0.879725,-1.053734,-0.756514,-0.613574,-0.334485,-0.840426,-0.5530579999999999,0.28631100000000004,-0.607516,-0.557982,-1.155035,-1.2121549999999999,-0.815688,-0.805266,-0.265127,-0.854476,-0.767939,0.642544,-0.833166,-0.5645,-0.653686,-1.0831469999999999,-0.703052,-0.810908,-0.735225,-0.8559459999999999 -458,1,-0.399502,1.010084,-0.482567,-0.44314,-0.46328400000000003,-0.9221799999999999,-1.0917649999999999,-0.8434959999999999,-0.962184,-1.1657170000000001,-0.32016700000000003,1.359089,-0.3855,-0.38306999999999997,-0.9016860000000001,-1.0160069999999999,-0.96346,-0.80733,-0.527994,-1.17769,-0.5163770000000001,0.027481000000000002,-0.598521,-0.42126800000000003,-0.329006,-0.922409,-0.869136,-0.8856870000000001,-1.016127,-0.8627549999999999 -495,1,-0.053674,0.45641499999999996,-0.100117,-0.170488,-0.472051,-0.7345189999999999,-0.490252,-0.196518,-0.8602639999999999,-0.995036,0.21094000000000002,0.21417199999999997,0.170979,0.073978,-0.034891000000000005,-0.395915,-0.25812199999999996,0.01524,-1.1851639999999999,-0.753827,-0.6239319999999999,0.760476,-0.628718,-0.406967,-0.5420149999999999,-0.657716,-0.34363,0.005493,-0.115303,-0.7500359999999999 -158,1,-0.648001,-1.183422,-0.690472,-0.611372,-0.213419,-0.833757,-0.891517,-0.6753939999999999,-0.625686,-0.275185,-0.58714,-1.524147,-0.623168,-0.586707,-0.23130799999999999,-0.9841690000000001,-0.867289,-0.755484,-0.809117,-0.5284270000000001,-0.8047529999999999,-0.886041,-0.8391059999999999,-0.595962,-0.504347,-0.876859,-0.7809050000000001,-0.895906,-0.713431,-0.47845200000000004 -160,1,-0.610726,0.086759,-0.546606,-0.59186,0.150419,-0.413905,-0.367435,-0.540791,0.43234300000000003,-0.225865,-0.675185,0.207191,-0.6536489999999999,-0.668618,0.892395,0.184948,-0.255736,-0.297641,0.662213,0.24558400000000002,0.34875100000000003,0.863894,0.5247029999999999,-0.043939,0.797373,-0.079818,0.324152,-0.044792,1.6258059999999999,0.47890299999999997 -292,1,-0.523751,-0.9359,-0.549585,-0.5189060000000001,0.698367,-0.301944,-0.230706,-0.13713399999999998,0.775312,0.659126,-0.334368,-0.7608689999999999,-0.36367,-0.40155599999999997,0.294605,-0.4721,-0.342114,-0.39256199999999997,-0.297985,0.26968400000000003,-0.7065819999999999,-0.822358,-0.809404,-0.49871400000000005,0.5613630000000001,-0.304494,-0.28328400000000004,-0.077234,0.69108,0.352188 -493,1,-0.637646,-1.517252,-0.715492,-0.609263,-1.664826,-1.205453,-1.2255200000000002,-1.33699,-1.004247,-0.757302,-0.47353500000000004,-1.503204,-0.541199,-0.5050819999999999,-1.611206,-1.211208,-1.024816,-0.9654469999999999,-0.7251449999999999,-0.378161,-0.279974,0.488324,-0.375263,-0.34624299999999997,1.116386,-0.863389,-0.844765,-0.6328020000000001,0.986511,0.38812199999999997 -394,1,-0.561026,0.019993,-0.563882,-0.5644359999999999,0.474804,-0.489605,-0.536788,-0.790964,0.23982699999999998,-0.727932,-0.5757800000000001,-0.365268,-0.572504,-0.593533,0.46540200000000004,-0.128131,-0.514369,-0.403912,0.45776000000000006,-0.168356,-0.436974,0.789506,-0.493575,-0.398607,0.36835500000000004,-0.5124029999999999,-0.446417,-0.694766,-0.047499,-0.755332 -528,1,-0.341518,-1.676839,-0.379508,-0.399544,0.308228,-0.749786,-0.557897,-0.19956300000000002,-1.19838,-0.632617,-0.053194000000000005,-1.424083,-0.06833600000000001,-0.17260599999999998,2.023925,-0.128699,0.153179,0.44445200000000007,0.600147,0.251255,0.5086390000000001,2.5730060000000003,0.606383,0.096872,0.999714,0.39077199999999995,0.521768,2.714394,-0.410734,0.655546 -335,0,0.9775969999999999,1.216895,1.070467,0.846289,0.549325,-0.311486,0.574799,1.036837,-0.44934799999999997,-0.440878,0.832931,0.39801,0.816841,0.750021,1.105891,0.02386,0.778417,1.300555,-0.308938,-0.29594000000000004,1.483125,1.6549479999999999,1.589019,1.030404,-0.19533399999999998,-0.419626,0.43522799999999995,0.915489,-0.27270500000000003,-0.023415000000000002 -311,1,0.039513,-0.639524,-0.106074,-0.069935,-1.370687,-1.166649,-1.0781399999999999,-0.859941,-0.599801,-1.495995,0.137096,-0.8376620000000001,0.029285000000000002,0.028472000000000004,-1.4361389999999998,-1.311272,-0.9332020000000001,-0.777667,-0.655777,-1.452705,-0.321841,-0.554016,-0.45149700000000004,-0.251635,-0.67002,-1.087451,-0.88094,-0.887634,-0.678318,-1.096968 -25,0,1.238521,-0.696519,1.3444969999999998,1.020322,0.97015,0.8946350000000001,0.542655,2.1377200000000003,1.8851099999999998,1.216609,0.8556520000000001,-0.672441,0.9898399999999999,0.733241,1.582699,2.335941,1.68363,2.351917,4.484751,1.606484,2.3128830000000002,-0.43699099999999996,2.183056,1.5635059999999998,0.329354,0.699282,0.179919,1.974718,0.307261,1.380276 -522,1,-0.8985700000000001,0.12258499999999999,-0.919823,-0.781714,-0.9454790000000001,-1.12619,-1.184309,-1.3138459999999998,-0.556121,-0.43312,-0.8143520000000001,0.125743,-0.8513620000000001,-0.758776,-0.8006310000000001,-1.1410870000000002,-1.0512569999999999,-1.1157219999999999,-0.637523,0.089648,-0.977273,-0.242856,-0.9330629999999999,-0.6821649999999999,-0.947031,-1.0942129999999999,-0.9299790000000001,-1.254065,-0.686793,-0.567341 -512,0,0.029158999999999997,0.64857,0.17987,-0.06360700000000001,1.097274,0.8354739999999999,1.143785,1.377912,1.106957,1.493688,-0.20656100000000002,0.28631100000000004,-0.137124,-0.27926,1.013376,0.806556,0.6993199999999999,0.846065,1.1112790000000001,1.481735,-0.05259400000000001,-0.519362,0.11234300000000001,-0.14668699999999998,-0.5423479999999999,-0.158063,0.08707999999999999,0.250429,-0.422842,0.07946900000000001 -230,0,0.687682,-0.128194,0.7815439999999999,0.5421699999999999,1.662757,0.885093,1.101567,2.127061,0.336894,0.36930100000000005,0.830091,-0.048787,0.8827459999999999,0.6829,1.2624549999999999,1.001757,1.2831270000000001,1.5497239999999999,1.166043,0.064131,-0.394386,-0.975851,-0.352986,-0.18387,-0.503013,-0.301699,-0.044554,0.331534,-1.062137,-0.551832 -548,1,-1.105653,-0.014204,-1.136664,-0.9077559999999999,-0.5465720000000001,-1.010222,-0.8572620000000001,-1.159448,-0.56421,-0.262993,-1.262242,0.011717,-1.2735610000000002,-1.0500120000000002,-0.814864,-1.024157,-0.8214629999999999,-1.01381,-0.845627,-0.063453,-0.395108,0.26516,-0.401994,-0.486174,0.133012,-0.796322,-0.282621,-0.353802,0.18012899999999998,0.135829 -213,0,0.37291599999999997,0.38964899999999997,0.39135,0.24684099999999998,-0.353694,-0.476882,0.038915,-0.07166,-2.099484,-0.873676,0.935176,1.4591530000000001,0.928055,0.8336370000000001,0.301721,0.194424,0.996873,0.439809,-1.838683,-0.586549,0.449087,0.816721,0.44599300000000003,0.400275,8.029999,3.357389,3.7104790000000003,4.456526,0.146227,3.315409 -390,1,-1.012466,-1.632871,-1.0136479999999999,-0.854492,0.084665,-0.56785,-0.892524,-0.72549,0.058635,-0.372717,-1.098366,-1.645155,-1.079967,-0.947908,0.256176,-0.548095,-0.873441,-0.7541939999999999,-0.042419,0.410026,-0.7726310000000001,-1.2140739999999999,-0.751486,-0.626104,-0.45301099999999994,-0.6610689999999999,-0.775799,-0.789497,-0.140729,-0.5385939999999999 -275,1,-0.801242,-1.088973,-0.828082,-0.71755,0.154802,-1.085159,-0.962664,-0.382891,-1.101313,-1.30869,-0.635423,-0.449042,-0.64953,-0.623681,1.8602450000000001,-0.6110140000000001,-0.370489,0.647967,0.742534,-0.5737909999999999,0.851874,1.9525,0.571731,0.187079,2.37977,-0.591766,-0.637401,1.184757,0.755252,-0.299917 -219,0,2.408538,3.2133599999999998,2.172543,2.806362,0.369598,0.988783,0.61078,0.729259,-0.303748,-0.458057,1.534446,3.067156,1.4841229999999999,1.615766,-0.865392,0.164101,0.322671,0.45012700000000005,-1.40057,-1.370484,1.2055770000000001,0.188958,0.918747,1.5305030000000002,-0.50068,0.05375800000000001,-0.174198,-0.164827,-0.8744649999999999,-0.582092 -236,0,3.052564,1.4383629999999998,2.941018,3.627307,0.6896,1.007232,1.486328,2.203194,0.327187,0.156504,2.579618,1.7872689999999998,2.5344729999999998,2.88708,-0.0904,1.210223,1.3333469999999998,1.928896,0.355534,0.041449,2.356193,-0.45966999999999997,2.1687,2.540382,-0.204335,0.176156,0.433571,0.8700700000000001,-0.562083,-0.28062600000000004 -336,1,-0.5278930000000001,-1.4276879999999998,-0.592179,-0.535431,-0.796437,-0.361105,-0.61019,-0.8543069999999999,-0.7583439999999999,0.93953,-0.32300700000000004,-1.177414,-0.32495100000000005,-0.39985,-0.123848,-0.088901,-0.645568,-0.720662,-0.582759,1.3584040000000002,-0.8076399999999999,-1.044433,-0.962418,-0.57044,-0.603351,-0.135708,-0.025323,-0.726884,-0.502754,0.720983 -20,1,-0.36636799999999997,-0.844707,-0.332744,-0.43962399999999996,-0.051226,0.14844300000000002,-0.399099,-0.6361100000000001,0.45822700000000005,-0.11725,-0.297446,-0.833008,-0.261106,-0.38363800000000003,0.792763,0.429422,-0.541362,-0.45962700000000006,0.567289,0.753087,-0.793925,-0.8512059999999999,-0.73416,-0.56472,-0.981366,-0.363178,-0.49449399999999993,-0.860707,-0.455534,-0.5181680000000001 -193,0,-0.128223,2.224898,-0.16564500000000001,-0.196329,2.022211,1.376193,0.817074,0.476498,0.508378,2.02568,-0.507616,1.7616720000000001,-0.44563800000000003,-0.504797,0.500985,0.586719,0.247342,-0.08509800000000001,0.479666,0.9317049999999999,0.000462,1.074358,-0.11091600000000001,-0.129746,0.685701,0.7249909999999999,0.190198,0.22934200000000002,-0.213376,0.710014 -200,1,-0.378793,0.436874,-0.4501,-0.425737,0.461654,-0.318484,-0.645212,-0.10059,-0.376548,-0.12223699999999998,-0.538858,0.062913,-0.553145,-0.551441,-0.035602999999999996,-0.44481000000000004,-0.589196,-0.202461,0.6111,-0.378161,-0.186856,0.19802899999999998,-0.276256,-0.28815799999999997,0.15768,-0.429687,-0.592971,-0.06425700000000001,-0.667421,-0.172825 -83,0,0.840923,1.146872,1.013874,0.7337819999999999,0.29946100000000003,0.174525,-0.139073,1.058154,-0.954095,0.447992,1.41232,1.629029,1.529432,1.356952,1.789079,1.416794,1.317025,2.527316,-0.648476,1.338557,0.410829,3.07195,1.4528860000000001,0.58883,0.168014,1.957356,-0.34993,1.076077,1.212928,2.4946040000000003 -376,1,-1.12222,-0.46528100000000006,-0.9159510000000001,-0.9293790000000001,-0.792053,0.684709,1.587076,0.485634,-0.49141,1.997972,-1.0103209999999998,0.21649899999999997,-0.8987299999999999,-0.9004120000000001,-0.400681,1.1685299999999998,1.7476610000000001,0.27060100000000004,1.374147,3.07654,-1.059924,0.025667000000000002,-0.24903000000000003,-0.728456,0.48602700000000004,2.847676,4.032102,2.821453,-0.529391,3.179239 -295,1,-0.331164,-1.424431,-0.389933,-0.385832,-0.673696,-0.9355389999999999,-1.126787,-0.8616159999999999,-0.12579200000000001,-0.886975,-0.101476,-1.4008129999999999,-0.16101400000000002,-0.20531300000000002,-0.311725,-0.7984439999999999,-0.9814139999999999,-0.767349,-0.8018149999999999,-0.521339,-0.671573,-0.9475469999999999,-0.6866369999999999,-0.497174,-0.8976959999999999,-0.9682940000000001,-0.9158540000000001,-0.805718,0.1208,-0.7538189999999999 -77,0,1.267513,-1.102,1.275989,1.282251,0.676449,1.96653,0.510512,1.455568,1.375509,1.488146,1.114105,-0.730617,1.162839,0.9985950000000001,0.721598,2.089571,0.999384,1.523931,1.242713,0.557457,2.0768400000000002,-1.2089940000000001,1.7053509999999998,2.078346,0.299686,1.839429,0.48695299999999997,1.444293,0.649913,1.2327569999999999 -441,0,0.8512770000000001,1.593064,0.760694,0.709172,0.492339,1.004687,1.1102020000000001,0.902842,-0.648335,-0.249694,0.892574,1.426574,0.841556,0.779031,-0.9287290000000001,0.124303,0.396746,0.217723,-1.258183,-1.2372299999999998,0.378346,0.838493,0.206398,0.396975,0.356021,0.9837600000000001,0.5811189999999999,0.9122450000000001,-0.557239,-0.021146 -109,1,-0.674921,0.56552,-0.693153,-0.6372140000000001,1.6452229999999999,-0.220518,0.190997,-0.484605,-0.116085,0.24239899999999998,-0.7916300000000001,0.45851400000000003,-0.8027569999999999,-0.734885,-0.624141,-0.731356,-0.470426,-0.771992,-1.1851639999999999,0.35332199999999997,-0.616352,-0.418666,-0.628223,-0.526437,0.690702,-0.553761,-0.247475,-0.8696280000000001,-0.58872,-0.49849899999999997 -319,1,-0.6977,-0.8903030000000001,-0.759575,-0.641081,-2.116335,-1.317732,-1.19851,-1.3138459999999998,-1.617385,-1.36466,-0.482055,-0.532817,-0.550673,-0.5050819999999999,-1.47955,-1.322832,-0.946385,-0.82358,-1.2399280000000001,-1.018918,-0.09879099999999999,1.783766,-0.187646,-0.20191099999999998,0.10534500000000001,-0.8209129999999999,-0.72729,-0.09669900000000001,1.6693939999999998,-0.315425 -349,1,-0.716338,-1.295784,-0.71996,-0.675712,-0.134515,-0.418358,-1.156052,-1.019363,0.36116,-0.445866,-0.618382,-1.0075379999999998,-0.607104,-0.648993,1.383436,0.308132,-0.9678540000000001,-0.8008810000000001,2.363553,0.42703800000000003,-0.159426,-0.30272899999999997,-0.203487,-0.301139,-0.41367600000000004,-0.073111,-0.798345,-0.207002,0.823055,-0.25755300000000003 -89,1,0.014663999999999998,-1.211106,0.063706,-0.13533,-0.204652,0.347555,-0.056555999999999995,0.38209299999999996,0.40484000000000003,0.043456,0.145616,-0.9423799999999999,0.156563,-0.008501,1.1984059999999999,0.560187,0.136356,0.560267,1.1112790000000001,0.093901,0.38376,-0.8702559999999999,0.46926,0.053308,-0.5110140000000001,1.041885,0.41334499999999996,0.7192149999999999,0.45134399999999997,0.395687 -419,1,-0.809525,0.528066,-0.83404,-0.7428640000000001,-0.18273399999999998,-0.912638,-1.133983,-1.0894059999999999,0.12172899999999999,-0.704657,-0.8427530000000001,0.49341999999999997,-0.865778,-0.78096,0.38711999999999996,-0.8444969999999999,-1.002443,-0.984276,-0.717843,-0.192455,-0.42831199999999997,0.836678,-0.444567,-0.46967200000000003,-0.044328,-0.896252,-0.8263299999999999,-0.9143979999999999,0.774624,-0.65623 -562,0,0.259021,2.786709,0.638572,0.060502,0.40905,3.4188370000000003,4.307272,1.842324,1.9223189999999999,3.1561630000000003,0.310345,2.6366490000000002,0.470844,0.17636500000000002,0.600616,1.9777580000000001,2.086645,1.170295,1.15509,1.2364899999999999,-0.523235,-0.021506,-0.249525,-0.389146,-0.805359,1.283328,1.382525,0.6948840000000001,0.100217,0.887792 -370,0,0.6441939999999999,0.8716659999999999,0.6564439999999999,0.49998000000000004,0.400283,1.350111,2.094174,1.677876,3.1113869999999997,0.675751,0.6312810000000001,0.930909,0.701509,0.527612,0.075415,0.85962,1.1588319999999999,1.0010860000000001,1.3266850000000001,-0.08755299999999999,0.09394,-0.353531,0.052444000000000005,0.113593,-0.468679,0.765232,0.955794,0.772745,1.383649,0.109729 -546,1,-1.039387,-0.636267,-1.076496,-0.871368,-0.16958299999999998,-1.0550059999999999,-1.095507,-1.3825180000000001,-0.355517,-0.55171,-1.081325,-0.684076,-1.098091,-0.938523,-0.143774,-1.030979,-0.9878170000000001,-1.120082,0.267911,-0.111652,-0.702973,-0.45332,-0.747526,-0.602123,0.015008,-1.01893,-0.721952,-1.0221049999999998,-0.598406,-0.449704 -49,1,-0.231765,1.000313,-0.246067,-0.31955900000000004,-0.708765,-0.529046,-0.211036,0.20698699999999998,-0.048138,-0.818814,-0.181,0.700529,-0.20838299999999998,-0.26703000000000005,-0.629123,-0.5185310000000001,-0.518386,-0.38895100000000005,-0.009559999999999999,-0.796355,-0.618518,0.24701700000000001,-0.559909,-0.44305,-0.862028,-0.651568,-0.362861,0.007115000000000001,-0.500333,-0.6955680000000001 -412,1,-1.305488,0.376621,-1.2108299999999999,-1.018857,-1.0419180000000001,-0.41708500000000004,-0.409653,-1.354653,-0.8489399999999999,0.449654,-1.3434700000000002,0.556251,-1.327108,-1.098077,-1.186348,-0.830283,-0.645945,-1.129548,-1.9628150000000002,0.6297550000000001,-1.034299,-0.063236,-0.837621,-0.737829,-0.50868,0.070525,0.18091400000000002,-1.081636,-0.12498900000000002,0.298098 -417,0,1.4290370000000001,0.321254,1.4844899999999999,1.524843,0.847409,0.92835,0.714407,1.504293,0.165409,1.1556520000000001,0.389869,0.416627,0.45024899999999995,0.42152700000000004,1.113008,0.9998610000000001,0.795994,0.925768,0.9981,0.8282200000000001,3.482267,-0.006991,3.246381,2.9958169999999997,0.385689,0.749583,0.43224399999999996,1.556217,-0.043867,0.764104 -440,1,-0.809525,0.194236,-0.50997,-0.710519,0.295077,0.979241,0.986906,0.6226729999999999,-0.583624,0.63031,-0.896716,-0.486275,-0.83365,-0.805988,-0.513123,0.131884,0.072451,-0.329883,-1.1778620000000002,0.510676,-0.5333399999999999,0.288747,-0.029731,-0.488154,0.5080279999999999,1.168195,1.074496,0.954419,-0.6516810000000001,0.446751 -482,1,-0.298031,-1.198078,-0.366998,-0.387414,0.303844,-0.027768,-0.41924799999999995,0.287688,0.527792,0.516153,-0.18667999999999998,-1.216974,-0.191495,-0.308839,0.764297,0.21148000000000003,-0.388442,0.096492,-0.11908900000000001,0.509259,-0.889208,-1.167627,-0.8732629999999999,-0.6049829999999999,-0.8636950000000001,-0.612445,-0.615186,-0.487462,-0.43131800000000003,-0.381241 -291,1,-0.003974,0.083503,0.054770000000000006,-0.124431,-0.046843,0.31002199999999996,-0.440357,0.5221779999999999,0.09908,0.042901999999999996,0.23650100000000002,-0.044132,0.20846199999999998,0.09218,-0.458325,-0.11581199999999998,-0.369108,-0.018808000000000002,0.246006,-0.606396,-0.42398100000000005,-0.48779300000000003,-0.34407600000000005,-0.340302,-0.5696829999999999,-0.241898,-0.5482090000000001,0.011981,-0.644416,-0.37065 -198,0,1.468383,1.039396,1.761498,1.419368,-0.007390000000000001,1.9455380000000002,0.548412,0.85564,0.472787,0.457966,1.435041,0.744743,1.4635280000000002,1.402458,-0.792091,0.728855,0.28375100000000003,0.48494899999999996,-0.1629,-1.064282,0.11018199999999999,-0.261,0.47866499999999995,0.305448,-0.50568,0.642834,-0.160272,0.067133,-0.7739699999999999,-0.150508 -542,1,0.049868,1.07685,0.0041340000000000005,-0.095249,-1.155891,-0.742153,-0.53295,-0.07775,-0.289188,-0.7972020000000001,0.174018,1.426574,0.112489,0.038995,-0.9685819999999999,-0.610256,-0.599491,-0.48103599999999996,0.103619,-0.850224,-0.368399,0.305076,-0.341105,-0.284418,-0.755357,-0.768936,-0.411933,0.144993,-0.223063,-0.44213900000000006 -320,1,-1.033174,-0.825166,-1.064284,-0.861699,0.34329699999999996,-0.116191,-0.39526100000000003,-0.261383,-0.47361499999999995,0.724517,-1.101206,-0.723636,-1.048251,-0.940514,0.693131,0.128093,-0.270425,-0.23908800000000002,-0.250523,1.416525,-0.135245,0.461109,-0.628223,-0.388486,1.14972,0.959727,0.27010700000000004,0.59107,0.804894,1.43777 -94,0,0.40605,-0.235671,0.48368599999999995,0.253872,0.996451,1.056214,1.190321,1.475363,-0.10799600000000001,-0.088988,0.26490199999999997,0.125743,0.343154,0.144227,0.536567,0.965749,1.019472,1.0119200000000002,0.158383,0.0060090000000000005,0.25851999999999997,-0.45803699999999997,0.415796,0.149676,0.73637,0.652335,0.5562520000000001,1.089053,-0.673475,-0.10398299999999999 -116,1,-1.41959,-1.401633,-1.30823,-1.073352,-0.6342439999999999,-0.422174,-0.5650930000000001,-1.159448,-2.020213,-0.372717,-1.470424,-0.8213729999999999,-1.368711,-1.165198,-0.123848,0.378253,0.048094,-0.6664939999999999,-1.849636,1.252083,-0.331946,-0.431729,0.204913,-0.514776,3.769827,2.354171,2.013834,1.830352,0.399281,0.683158 -429,1,-0.507184,-0.768171,-0.547798,-0.5164449999999999,-1.120822,-1.006469,-1.139405,-1.195078,-1.190291,-1.313123,-0.39969099999999996,-0.37690300000000004,-0.45264,-0.43682299999999996,-1.238299,-1.12043,-0.938852,-0.7885,-1.287391,-1.043018,-0.39619,-0.604636,-0.374767,-0.376165,0.098678,-0.767818,-0.879016,-1.001343,-0.138308,-0.9890909999999999 -518,1,-0.252473,-0.212873,-0.23683400000000002,-0.361925,0.58001,0.266129,-0.708059,-0.076228,-0.515677,0.27620300000000003,-0.354249,-0.248914,-0.30971,-0.46014499999999997,1.810429,1.170425,-0.509095,0.10603499999999999,-0.374655,1.3796680000000001,0.13508499999999998,-0.086823,0.15342999999999998,-0.13128599999999999,-0.58935,-0.122294,-0.591645,0.104441,-0.288445,-0.187576 -323,0,1.870123,1.0068270000000001,1.9014919999999997,1.8588470000000001,1.176179,1.240059,1.257966,2.343279,4.298838,1.0226540000000002,1.7644970000000002,0.516691,1.809525,1.732374,1.4688350000000001,1.575986,2.105477,2.617595,2.765157,0.553204,0.595621,-0.351717,0.567276,0.631953,-0.5186810000000001,-0.065287,0.00020800000000000001,0.307203,0.8642219999999999,-0.170177 -248,1,-0.832304,1.549097,-0.872165,-0.746907,0.768505,-0.728158,-0.766109,-0.810759,0.8222280000000001,-0.137199,-0.9876,1.380033,-0.986877,-0.875668,0.014925,-0.606466,-0.81619,-0.845247,0.31172300000000003,0.069801,-0.561131,0.501025,-0.677726,-0.5213760000000001,0.049342000000000004,-0.845505,-0.6990729999999999,-0.9004479999999999,0.125643,-0.444787 -280,0,1.5429329999999999,1.664716,1.564912,1.482653,2.009061,0.825932,1.454664,1.105356,0.577943,0.734491,1.429361,1.701168,1.40998,1.374017,0.401353,0.776234,1.296937,1.230911,0.329977,-0.084717,0.833467,-0.39163200000000004,0.72024,0.644934,0.11701199999999999,-0.055227,0.264801,0.183924,-0.749754,-0.131973 -391,1,-1.263036,-0.468538,-1.288274,-0.9907299999999999,0.597545,-0.784138,-1.305831,-1.745063,-0.737313,0.26068600000000003,-1.531771,-0.57005,-1.511641,-1.196199,0.536567,-0.5697,-1.114873,-1.2618200000000002,0.6330060000000001,1.1599389999999998,0.40325,1.56423,0.148975,-0.252735,2.926459,-0.325173,-1.057501,-1.9134470000000001,-0.22911700000000002,1.112474 -462,1,-0.17999400000000002,1.026368,-0.20436700000000002,-0.256626,-1.344385,-0.688717,-0.599635,-0.898007,-0.899091,-1.069848,0.077453,1.791923,0.011573,-0.024997,-1.8794990000000003,-0.9875799999999999,-0.678462,-0.813778,-0.381957,-1.200372,-0.626819,-0.55456,-0.563869,-0.43601,-0.561682,-0.485576,-0.403976,-0.880172,-0.775181,-0.719776 -406,1,0.298367,-0.9928950000000001,0.257314,0.11833699999999998,-0.515887,-0.522048,-0.197603,-0.02598,-0.19859200000000002,-0.766169,0.571638,-1.030809,0.507915,0.41271,-0.100363,-0.36635100000000004,-0.424349,-0.093868,-0.27973000000000003,-0.5737909999999999,-0.600833,-1.051691,-0.562879,-0.40718699999999997,-1.027701,-0.727578,-0.450396,-0.4946,-0.671053,-0.822282 -416,1,-1.12222,0.9058639999999999,-1.147684,-0.9161940000000001,0.886862,-0.8585659999999999,-1.011215,-1.1710200000000002,-0.04652,-0.050197000000000006,-1.341198,0.560905,-1.333287,-1.091251,0.57215,-0.810194,-0.857873,-0.9375889999999999,0.779043,0.45538999999999996,0.090331,3.013891,-0.052998,-0.333702,2.5664439999999997,-0.488371,-0.604244,-0.506441,1.198399,0.300746 -411,1,-0.799171,0.12421300000000002,-0.814083,-0.719308,0.198638,-0.674722,-0.7939350000000001,-0.613574,0.15732000000000002,-0.28460599999999997,-0.876835,-0.572377,-0.867014,-0.801153,0.806996,-0.498443,-0.732448,-0.622129,-0.3564,0.085395,-0.752419,0.308704,-0.754456,-0.5895819999999999,-0.627685,-0.901115,-0.707363,-0.699633,-0.407102,-0.604031 -474,1,-0.8965,-1.030349,-0.7887649999999999,-0.786636,0.036445,0.862192,0.30853600000000003,-0.532112,-0.517295,1.332982,-0.922277,-0.853952,-0.888021,-0.841823,0.308838,0.048497000000000005,-0.47268599999999994,-0.8565959999999999,0.180289,0.789944,-0.927466,-1.231674,-0.774752,-0.676334,-0.8556940000000001,0.304702,-0.143362,-0.828103,-1.019759,0.33176300000000003 -96,1,-0.712196,-0.774684,-0.748256,-0.67747,-0.805204,-1.022181,-1.066914,-0.849434,-1.0899889999999999,-0.564455,-0.5530579999999999,-0.337343,-0.584037,-0.579597,0.579267,-0.6400100000000001,-0.802254,-0.503219,0.322675,0.503588,-0.141019,0.533683,-0.22576300000000002,-0.349763,-0.5360149999999999,-0.7650239999999999,-0.682495,0.550517,0.200712,-0.146347 -232,1,-0.809525,2.6222369999999997,-0.8584639999999999,-0.720187,-1.4215360000000001,-1.179499,-1.194624,-1.28796,0.016572999999999997,-0.602692,-0.8257120000000001,3.3789830000000003,-0.8723690000000001,-0.762473,-1.320851,-1.30009,-1.0525120000000001,-1.095861,0.121873,-0.640418,-0.654249,0.780434,-0.681686,-0.547338,-0.8940290000000001,-1.0431860000000002,-0.950636,-1.358853,-0.16736700000000002,-0.47693900000000006 -354,1,-0.859225,-1.6051879999999998,-0.823317,-0.750775,-1.916882,-0.818489,-0.729648,-1.1478760000000001,-0.525383,-0.762844,-0.848433,-1.214647,-0.853833,-0.76873,-1.6809479999999999,-0.828198,-0.549271,-0.88239,-0.44402299999999995,-0.278929,0.061457000000000005,-0.739624,0.229665,-0.25295500000000004,-0.500013,0.469017,0.43622299999999997,0.472657,1.268624,0.391905 -240,1,-0.293889,-1.079202,-0.39171999999999996,-0.34663099999999997,-0.200268,-0.796225,-0.5703699999999999,-0.340866,-0.599801,-1.0443559999999998,-0.138398,-0.858606,-0.189023,-0.22635999999999998,-0.151602,-0.720933,-0.524161,-0.299446,-0.345447,-0.878576,-0.292245,-1.008146,-0.43070600000000003,-0.289258,-0.190334,-0.72646,-0.457359,-0.124275,-0.189161,-0.756845 -378,1,-0.358085,-0.983124,-0.277044,-0.39304,-0.213419,0.357097,-0.07334700000000001,-0.140179,0.786637,0.68905,-0.132717,-0.963324,-0.152364,-0.211286,-0.973563,-0.5469579999999999,-0.581412,-0.62445,-0.071627,-0.542604,-0.95634,-1.224961,-0.873758,-0.6377649999999999,-0.609684,0.24378200000000003,-0.247475,-0.5580229999999999,-0.286024,0.4059 -177,0,0.314933,0.451529,0.48368599999999995,0.176876,0.400283,1.351383,1.506478,1.35355,0.247916,0.623106,0.662522,0.190901,0.713866,0.5062810000000001,0.138753,0.971434,1.136233,1.025075,-0.064325,0.061296,-0.366234,0.121827,-0.190121,-0.192451,-0.137999,0.8641559999999999,0.723363,1.027414,-0.45069,0.298477 -33,0,1.6319780000000002,0.850497,1.6125690000000001,1.6391069999999999,0.812341,2.57468,1.616341,0.972885,1.2477040000000001,1.5712700000000002,1.460602,1.670916,1.480004,1.442275,-0.167259,1.280343,0.9654860000000001,0.696717,0.151081,-0.026595999999999998,0.5436479999999999,-1.1079350000000001,0.327681,0.612372,-0.675354,0.430453,0.102,-0.34926,-0.618989,0.038239999999999996 -251,1,-0.6832050000000001,-0.5239050000000001,-0.719066,-0.653386,-0.6167090000000001,-0.9501700000000001,-0.916991,-0.748634,-0.260067,-1.057102,-0.746188,-0.195392,-0.7698050000000001,-0.7038840000000001,-0.20711100000000002,-0.842033,-0.783673,-0.728142,0.081713,-0.49015200000000003,-0.045014,-0.67848,-0.090125,-0.293659,-0.220335,-0.828738,-0.644696,-0.425174,0.287889,-0.8828020000000001 -224,1,-0.233835,-0.33826300000000004,-0.250833,-0.30198,-0.209036,-0.783502,-0.44899300000000003,-0.271433,-0.638628,-0.42757799999999996,-0.243483,-0.528162,-0.305591,-0.308554,-0.8476,-1.030979,-0.668669,-0.628319,-1.152306,-0.8587290000000001,-0.405935,-0.591754,-0.40694400000000003,-0.344483,-0.33634000000000003,-0.806941,-0.30848400000000004,-0.44431499999999996,-0.692847,-0.49131199999999997 -186,0,1.043864,0.111186,0.951324,0.9306690000000001,-0.393146,-0.06211900000000001,0.391533,0.647036,0.49381800000000003,-0.807177,1.187949,-0.16514,1.096935,1.098139,-0.745834,-0.372605,-0.089257,0.23784299999999997,-0.6959380000000001,-1.2117129999999998,-0.532258,-1.3447069999999999,-0.519316,-0.251195,-1.391716,-0.91084,-0.589324,-0.8235610000000001,-1.1929020000000001,-1.024268 -85,0,1.379337,0.32614,1.338539,1.269946,0.325762,-0.288585,0.208748,0.7551439999999999,1.284913,0.102197,1.230551,-0.179102,1.19991,1.1948379999999998,0.169354,0.018175,0.5612159999999999,1.006761,1.169694,-0.36540300000000003,1.063013,0.46836700000000003,0.948449,0.8858530000000001,-0.190001,-0.502343,-0.127115,0.38993,0.38233,-0.015094 -422,1,-0.751542,-0.978239,-0.7545109999999999,-0.711749,0.400283,-0.237058,-0.201441,-0.062524,-0.184032,-0.536193,-0.714947,-0.7608689999999999,-0.68001,-0.701609,0.8852780000000001,0.236117,-0.223847,-0.101864,0.27156199999999997,0.057042999999999996,-0.575929,-0.876244,-0.593571,-0.537878,-0.385675,-0.30337600000000003,-0.185139,0.201767,-0.2497,-0.685733 -104,1,-0.979333,-0.385488,-0.98416,-0.839901,-0.4589,-0.672177,-0.922652,-1.257355,-0.12093800000000002,-0.466924,-1.033042,8.2e-05,-1.011592,-0.9066690000000001,0.25119400000000003,-0.351758,-0.738851,-0.952034,1.480024,0.285277,-0.181081,0.5754130000000001,-0.279227,-0.378585,0.18468099999999998,-0.183773,-0.102578,-0.516173,0.7939970000000001,-0.130082 -461,0,4.094189,0.927033,4.287337,5.930172,0.146035,1.0899299999999998,1.9723169999999999,2.251919,-0.420228,-0.536193,3.775318,1.624375,3.910226,5.250529,0.8568120000000001,1.790138,3.4488589999999997,3.094784,0.9104770000000001,-0.931027,7.730307000000001,0.161743,7.813534,11.041841999999999,0.203015,1.579542,1.613298,2.30076,-0.432529,0.288642 -142,1,-0.72255,0.176323,-0.732768,-0.663758,0.391516,-0.47751899999999997,-0.9360379999999999,-0.770256,-0.512441,-0.165462,-0.766069,-0.46067700000000006,-0.754153,-0.7306189999999999,0.913744,-0.179679,-0.8598809999999999,-0.781794,-0.6083149999999999,0.40010300000000004,-0.436253,1.253978,-0.459913,-0.417088,-0.125665,-0.45427799999999996,-0.673542,-0.369211,0.275781,-0.101713 -380,1,-0.710125,-0.8381930000000001,-0.665154,-0.711046,1.255083,-0.072298,-0.227827,0.261803,0.715454,0.45464099999999996,-0.811511,-1.472952,-0.774748,-0.763895,1.945643,0.128093,-0.12303,0.169746,0.753487,0.8991,-0.537671,-0.41921,-0.523277,-0.5345770000000001,-0.13533199999999998,-0.430804,-0.36087199999999997,-0.025327000000000002,0.065104,-0.027954000000000003 -123,1,-0.117869,-1.579133,-0.132881,-0.23746399999999998,-0.046843,-0.480063,-0.077665,0.114105,-0.019018,-0.215336,0.10585399999999999,-1.954655,0.095189,-0.040355,0.977793,0.105352,-0.004762,0.228557,0.16203399999999998,0.173287,-0.405213,-0.652898,-0.464368,-0.355263,-1.07437,-0.710811,-0.10191499999999999,0.033069,-0.17463099999999998,-0.373298 -428,1,-0.950341,-0.8772760000000001,-0.9802879999999999,-0.8077310000000001,-1.287398,-1.221866,-1.0861040000000002,-1.129299,-0.837615,-0.726824,-0.851273,-0.6212449999999999,-0.885549,-0.778684,-1.056827,-1.250816,-0.942995,-0.908442,-1.097542,-0.18678499999999998,-0.9516479999999999,-0.453139,-0.9395969999999999,-0.67398,-0.386009,-1.073925,-0.745891,-0.909532,-0.054764,-0.536324 -396,1,-0.304244,0.247975,-0.295809,-0.361046,0.45726999999999995,0.017398,0.343558,0.467362,-0.379783,-0.39266599999999996,-0.17532,-0.093001,-0.159366,-0.275278,0.678898,0.196319,-0.037656,0.126155,-0.020513,-0.2846,-0.6914239999999999,0.20891500000000002,-0.669806,-0.46307200000000004,-0.533015,-0.330203,0.038007,0.30395900000000003,-0.895048,-0.503416 -66,1,-1.213336,0.957974,-1.19832,-0.9666469999999999,0.983301,-0.558944,-0.8542879999999999,-0.752745,-0.036814,0.452425,-1.324157,0.400337,-1.31228,-1.096371,0.57215,-0.504318,-0.8421790000000001,-0.873878,-0.345447,0.8778360000000001,-0.613826,1.440855,-0.597036,-0.57506,1.159721,-0.4431,-0.48918900000000004,-0.399545,0.272148,0.167223 -170,1,-0.573451,-1.634499,-0.604391,-0.5827180000000001,0.268776,-0.812128,-0.709978,-0.315133,-0.11932100000000001,-0.8997209999999999,-0.513297,-1.605595,-0.540376,-0.542624,0.45828500000000005,-0.654413,-0.614306,-0.307442,0.538081,-0.46038199999999996,-0.610577,-1.000163,-0.592086,-0.503995,0.334687,-0.764465,-0.499468,0.099574,-0.15768,-0.585118 -481,1,0.029158999999999997,0.120957,-0.085224,-0.088042,-1.138356,-0.7173430000000001,-0.503205,-0.504095,-0.881295,-0.43866099999999997,-0.064554,-0.011554,-0.133416,-0.147862,-1.170692,-0.96806,-0.738851,-0.727884,-0.8492770000000001,-0.972138,-0.26553699999999997,-0.526982,-0.40100399999999997,-0.262416,-1.112371,-0.8192370000000001,-0.550198,-0.895582,-1.2013770000000001,-0.5968439999999999 -313,1,-0.813667,-2.0855770000000002,-0.775361,-0.725637,-1.015616,-0.583118,-0.906485,-1.0271290000000002,0.7914899999999999,-0.5323140000000001,-0.7348279999999999,-1.994215,-0.751269,-0.699049,-0.739429,-0.846202,-0.9432459999999999,-1.032073,0.078062,-0.25483,-0.988823,-1.5542639999999999,-0.8707879999999999,-0.679833,-0.972366,-0.675042,-0.7249359999999999,-1.156415,-0.026916000000000002,-0.6970810000000001 -53,0,0.896835,-0.251956,0.829202,0.7742140000000001,-0.191501,-0.15626800000000002,-0.047439999999999996,0.272461,0.194529,-0.225865,1.162387,-0.137215,1.166958,1.075386,1.312271,0.836879,1.109868,1.472343,1.023657,0.042867,1.5466469999999999,0.682459,0.9954770000000001,1.286504,-1.047369,0.23092800000000002,-0.12346700000000001,-0.41755,0.750408,0.5034890000000001 -296,1,-1.014537,-1.7680310000000001,-1.037775,-0.858535,-1.7204970000000002,-1.139994,-1.16747,-1.258725,-1.225882,-0.9706530000000001,-0.9137569999999999,-1.614903,-0.940332,-0.8281719999999999,-0.79565,-1.082717,-0.959693,-0.9087,-1.3239,-0.352645,-0.829657,-0.344459,-0.791583,-0.643486,-1.187708,-0.74155,-0.7020569999999999,-0.389812,1.073688,-0.138782 -114,1,-1.3752739999999999,-0.986381,-1.274274,-1.048038,1.7548119999999998,-0.113647,-0.12755899999999998,-0.14627,0.040839999999999994,0.9838629999999999,-1.534044,-0.805083,-1.488162,-1.205869,1.3265040000000001,-0.423205,-0.596101,-0.7655430000000001,-0.593711,1.9183569999999999,-0.861418,-1.143859,-0.748516,-0.690217,0.40669,-0.187126,0.023087,-0.226467,-0.41921,0.004197 -243,1,-0.260756,0.107929,-0.275853,-0.306902,-1.6959490000000002,-0.700167,-0.6538470000000001,-0.815327,-0.384637,-1.149092,-0.107156,1.042608,-0.141243,-0.184551,-1.133686,-0.687389,-0.525166,-0.657209,-0.140994,-1.206042,0.106573,-0.290029,-0.018345,-0.008956,-0.8966959999999999,0.066613,0.060223,-0.279996,0.658389,-0.267766 -140,1,-1.169849,-1.8852790000000001,-1.2132129999999999,-0.9452,-0.393146,-1.159206,-1.305831,-1.745063,0.330422,-0.134983,-1.246621,-1.7033310000000002,-1.265735,-1.0420479999999999,-0.274719,-1.200026,-1.114873,-1.2618200000000002,0.33362800000000004,0.201639,-0.74484,-1.307876,-0.81584,-0.617744,-0.333673,-1.10785,-1.057501,-1.9134470000000001,0.269727,-0.21745799999999998 -152,1,-1.087016,-1.007551,-1.0788790000000001,-0.879102,-0.138898,0.145898,2.635815,0.647036,0.335276,2.324925,-1.248609,-0.91911,-1.1611120000000001,-1.008772,0.771413,1.052926,4.042708999999999,0.764814,2.688487,4.275833,1.513443,2.625622,0.5974729999999999,0.20930100000000001,1.309727,3.93361,12.07268,6.649601,1.806213,9.851593 -268,1,-0.49061800000000005,-0.33174899999999996,-0.535883,-0.49763500000000005,-0.296707,-0.46734,-0.350164,-0.864965,1.1376950000000001,-0.7384609999999999,-0.357089,-0.716655,-0.394974,-0.40582199999999996,-0.150179,-0.798824,-0.625229,-0.845247,0.724279,-0.724057,-0.615991,0.0038950000000000005,-0.65347,-0.486174,-0.50768,-0.20668699999999998,-0.19906500000000002,-0.88666,0.652335,-0.6195390000000001 -91,0,0.033301,0.026507,0.007112,-0.087339,-0.292324,-0.34711,0.05954500000000001,0.502383,-0.557739,-0.868134,0.35294699999999996,0.807574,0.339035,0.208504,-0.31030100000000005,-0.014043,0.293795,0.6683439999999999,-0.345447,-0.259083,-0.333029,-0.681383,-0.391599,-0.239754,0.947045,-0.058021,0.44484399999999996,0.9463090000000001,0.5046189999999999,-0.625213 -255,0,0.025018000000000002,-0.587414,0.024984,-0.095952,0.825491,0.457607,0.23369499999999999,0.347072,0.270565,-0.24248899999999998,-0.047513,-0.521181,-0.022203,-0.149284,0.94221,0.446478,0.114133,0.091333,0.351883,-0.212302,0.071563,-0.738535,-0.150024,-0.10114400000000001,-0.230002,0.07331900000000001,-0.023333,0.263406,0.009408,-0.41603999999999997 -559,1,-0.784675,1.8698990000000002,-0.7440859999999999,-0.714386,-0.112597,-0.016316999999999998,0.43567,-0.275239,-1.2760340000000001,0.18698299999999998,-0.743348,1.079841,-0.718729,-0.7149760000000001,-0.26689,-0.04247,0.28124,-0.202977,-1.546608,0.411444,-0.600472,3.061064,-0.460408,-0.514116,0.386356,0.24266500000000002,0.8450489999999999,0.141749,-0.6855829999999999,0.356727 -14,0,-0.256615,1.031253,0.045834,-0.321493,1.4348100000000001,3.296698,2.02509,1.61697,1.1247530000000001,3.2780769999999997,-0.11283599999999999,0.772668,0.06717999999999999,-0.217827,1.191289,2.368158,1.556825,0.8081470000000001,0.939685,1.9878200000000001,-0.696838,-0.086823,-0.398529,-0.46483199999999997,-0.204001,1.8936419999999998,0.766467,0.727326,-0.112881,1.625761 -100,0,0.149267,1.562124,0.039876999999999996,0.04556,-0.257255,-0.381461,0.21450500000000003,0.057766,-0.40405,-0.552818,-0.146918,1.3241829999999999,-0.161426,-0.20531300000000002,-0.10534500000000001,-0.364456,-0.032007,-0.10392799999999999,-0.739749,-0.579461,0.185253,0.132713,-0.0025039999999999997,0.06166900000000001,-0.389675,-0.592325,-0.179834,-0.304165,-0.713431,-0.544646 -353,0,0.464033,1.228294,0.415178,0.29782,1.4742629999999999,-0.118736,0.627092,0.578516,-0.39919699999999997,0.5782189999999999,0.270583,1.50104,0.248417,0.175512,0.42981899999999995,-0.126046,0.435666,0.42846000000000006,-0.601013,0.261178,0.895907,0.5246109999999999,0.647471,0.506764,1.159721,-0.065287,0.571172,0.918733,0.07963300000000001,0.405522 -543,1,-0.393289,1.871527,-0.440271,-0.44120600000000004,-1.1032879999999998,-0.7389720000000001,-0.796334,-0.53333,-0.6920149999999999,-1.081485,-0.260524,2.0409200000000003,-0.29199899999999995,-0.331307,-0.686767,-0.6741229999999999,-0.7398560000000001,-0.417067,-0.670381,-0.707046,-0.613826,0.689717,-0.656935,-0.49475399999999997,-0.689354,-0.657157,-0.560808,-0.43458199999999997,-0.399837,-0.927436 -165,1,-0.059886,0.02325,-0.147774,-0.173125,-1.221645,-0.9816590000000001,-0.9340229999999999,-0.868924,-0.41213900000000003,-1.279874,0.239341,0.109454,0.14544100000000001,0.100428,-0.8646799999999999,-0.963133,-0.8704280000000001,-0.761674,-1.082938,-1.4371120000000002,-0.7982560000000001,-0.275514,-0.7821779999999999,-0.5213760000000001,-1.1357059999999999,-0.9777950000000001,-0.783358,-0.870115,-0.15768,-0.860485 -43,0,0.23002899999999998,0.37825,0.173913,0.04679,0.9043959999999999,0.7515029999999999,0.451982,0.526746,1.356095,1.0392780000000001,-0.240643,0.230461,-0.191495,-0.311967,0.550801,0.744016,0.12141500000000001,0.326574,0.592845,0.711976,-0.1255,-0.711138,-0.217348,-0.198171,-0.656353,-0.22401300000000002,-0.33302,-0.362723,-0.405891,-0.180768 -201,0,0.85956,0.026507,0.960259,0.630066,0.251241,0.558117,0.377141,1.207374,0.044075,-0.292364,0.969258,0.007063,0.9527690000000001,0.8438760000000001,-0.475405,0.292971,0.18582200000000001,0.6696340000000001,-1.115796,-1.1181510000000001,-0.029133999999999997,-0.705151,0.109867,0.008645,-0.317006,0.011840999999999999,-0.157951,0.268273,-0.557239,-0.43003500000000006 -108,0,2.512079,0.379878,2.964846,2.600686,1.65399,2.833589,3.3036300000000005,2.685877,1.867314,0.772728,2.3126450000000003,0.08851,2.50564,2.429179,2.579016,3.268353,4.238567,3.4404230000000005,2.717694,1.0763,2.9228389999999997,0.595371,3.5562709999999997,2.852806,-0.175333,3.420544,2.39083,2.109352,1.280732,0.469825 -328,0,0.623486,0.765818,0.671337,0.42263199999999995,1.167411,0.257223,0.419839,0.665308,0.327187,-0.107829,0.60856,0.330525,0.6150100000000001,0.45167399999999996,1.461718,0.522284,0.7407520000000001,0.927574,0.497921,-0.003914,0.116678,0.027481000000000002,0.199963,0.089611,-0.11466400000000002,-0.259782,0.019438999999999998,0.344511,-0.6274649999999999,-0.380484 -439,1,-0.281464,-1.0368629999999999,-0.31963800000000003,-0.336962,-1.2698639999999999,-0.970527,-1.005506,-0.494046,-1.237207,-0.933525,-0.030472000000000003,-0.8446440000000001,-0.097993,-0.137624,-1.188483,-0.919734,-0.852851,-0.577763,-0.812768,-0.9834780000000001,-0.689258,-1.019577,-0.623768,-0.46395200000000003,-0.734689,-0.9076540000000001,-0.752025,-0.168071,-1.069402,-0.639587 -162,0,2.166251,0.11607100000000001,2.014678,2.375673,0.501106,0.8291120000000001,1.961283,1.6763540000000001,1.200788,0.459075,1.551487,-0.265204,1.595336,1.590169,1.113008,1.1799,2.0339139999999998,2.055287,0.786345,-0.28034699999999996,1.195471,-0.306358,0.9534,1.2482209999999998,-0.9947,-0.151357,0.278396,0.20014400000000002,-0.44584700000000005,-0.18039000000000002 -325,1,-0.529963,-0.745372,-0.5528609999999999,-0.538243,0.264392,-0.8464799999999999,-0.816483,-0.892069,-0.344192,-0.834885,-0.413892,-0.463005,-0.44151899999999994,-0.469246,0.45828500000000005,-0.524975,-0.713993,-0.71834,-0.381957,-0.41927200000000003,-0.704417,-0.483257,-0.643569,-0.500035,-0.077329,-0.892228,-0.616844,-0.863302,0.0033539999999999998,-0.760627 -183,1,-0.807454,-1.299041,-0.83821,-0.726691,-0.8884920000000001,-0.593296,-0.515199,-0.786396,-1.7629860000000002,-0.536193,-0.771749,-1.0168469999999998,-0.7595069999999999,-0.7192430000000001,-0.410645,-0.43192299999999995,-0.33885,-0.652824,-2.353466,-0.08897000000000001,-0.255431,-0.197498,-0.477239,-0.386506,0.105011,0.659601,0.9037370000000001,0.863582,0.13411900000000002,0.374127 -197,0,0.7228859999999999,-0.159135,0.650487,0.610729,-1.935293,-0.368739,-0.089658,-0.347109,-0.8602639999999999,-1.017757,1.122625,0.593484,1.047507,1.0497889999999999,-1.611917,-0.339629,0.26994,0.228557,-0.151947,-1.332209,0.833828,0.159928,0.715785,0.792566,-0.50368,1.535949,0.9683930000000001,0.42886,0.103849,0.46944600000000003 -498,0,1.342063,-0.45551,1.1657819999999999,1.264672,0.387133,0.347555,0.38913400000000004,0.7871199999999999,-0.632157,0.582098,1.2390709999999998,-0.41180900000000004,1.208148,1.17493,0.344421,0.518494,0.757073,1.106842,0.074411,0.59148,1.397226,-0.311801,0.982606,1.219619,0.31101999999999996,0.372328,0.413013,0.638111,-0.529391,0.552283 -477,1,-0.233835,-0.631382,-0.180538,-0.284225,-1.6889349999999999,-0.341385,-0.641853,-0.797664,-0.358752,-0.38601599999999997,-0.064554,-0.6212449999999999,-0.12353099999999999,-0.157817,-1.998346,-0.9693870000000001,-0.83565,-0.916438,0.005043,-1.054359,-0.901119,-1.162366,-0.729704,-0.578801,-1.244377,-0.689014,-0.729114,-1.12965,-0.8938379999999999,-0.642991 -421,1,0.039513,-1.194821,0.203699,-0.12548499999999999,-0.051226,0.694887,0.238492,-0.057955999999999994,-0.11932100000000001,0.450762,0.15981700000000001,-1.2355909999999999,0.257479,0.0034439999999999996,0.47963500000000003,1.502076,0.705598,0.363201,1.001751,1.5965610000000001,0.508999,0.533683,0.954885,0.2005,0.9783799999999999,1.506886,0.692527,0.649465,0.724982,0.6237729999999999 -374,1,-0.29596,-0.8903030000000001,-0.24130100000000002,-0.369132,-0.9586290000000001,-0.284132,-0.660084,-0.68118,0.683099,-0.384354,-0.124197,-0.7492340000000001,-0.170076,-0.215552,-0.949367,-0.7694489999999999,-0.793968,-0.7379439999999999,0.22044899999999998,-0.8658170000000001,-0.846981,-1.2886440000000001,-0.7396050000000001,-0.579461,-0.937031,-0.536994,-0.669895,-0.885849,-0.134675,-0.611974 -470,1,-1.062166,-0.009318,-1.083645,-0.87084,-0.393146,-0.6365529999999999,-0.693187,-0.7461979999999999,0.44204899999999997,0.07171799999999999,-1.266786,-0.186084,-1.2554370000000001,-1.040342,-0.491062,-0.791432,-0.7447520000000001,-0.8712989999999999,1.556694,0.18888,-0.099513,0.24157399999999998,-0.147053,-0.387386,0.153347,-0.312319,-0.157619,-0.310329,-0.11409200000000001,0.04467 -382,1,-0.766038,0.49386899999999995,-0.592774,-0.6894239999999999,-1.945375,0.427072,0.091208,-0.08231799999999999,-1.1482290000000002,0.528899,-0.5899800000000001,0.798266,-0.544495,-0.588983,-1.9221990000000002,0.056077999999999996,-0.117631,-0.493675,-2.222032,0.537611,-1.031411,0.393979,-0.538623,-0.6773899999999999,-0.666353,1.124602,0.369909,0.790588,0.503408,1.3477459999999999 -510,1,-0.790887,-1.315326,-0.774766,-0.7150890000000001,-1.098904,0.15925699999999998,-0.015297,-0.13713399999999998,-0.48008599999999996,0.8226030000000001,-0.678025,-1.0703690000000001,-0.644999,-0.650984,-1.093833,-0.146514,-0.270425,-0.581116,-1.141353,0.677954,-0.767939,-1.043526,-0.7529710000000001,-0.600582,-0.01966,0.7646729999999999,0.274417,0.565116,0.24914299999999998,1.145003 -132,0,0.662832,0.9775149999999999,0.668358,0.517559,0.312612,0.32592600000000005,0.129588,0.25266700000000003,0.93709,-0.429795,0.577318,0.523672,0.586176,0.440582,0.31595500000000004,0.45595399999999997,0.19461099999999998,0.185997,1.2719209999999999,-0.551109,0.101159,0.087354,-0.010920000000000001,0.07355,-0.721356,-0.332998,-0.321746,-0.41608999999999996,-0.628676,-0.537837 -545,1,-0.190348,0.55575,-0.28836300000000004,-0.26506399999999997,-0.472051,-0.652457,-0.80257,-0.6527069999999999,-0.41861000000000004,-0.798864,-0.14407799999999998,0.9169459999999999,-0.196849,-0.232332,-0.277565,-0.69876,-0.741488,-0.6316729999999999,-0.5389470000000001,-0.6786939999999999,-0.21356399999999998,0.216173,-0.396054,-0.200151,-0.391009,-0.25084,-0.387397,-0.443179,0.039678,-0.458404 -121,0,1.238521,-0.126566,1.135996,1.175019,0.786039,-0.16008499999999998,0.263919,0.80387,-0.010929000000000001,0.034035,1.287353,-0.504892,1.212267,1.2005270000000001,0.643316,0.10724700000000001,0.714386,0.9732290000000001,0.563638,-0.094641,1.1102940000000001,0.660687,1.0043879999999998,1.10301,0.353688,-0.24972199999999997,0.050607,0.6835289999999999,-0.011175,0.29318099999999997 -45,0,1.356558,-0.709547,1.2908819999999999,1.206661,1.5575510000000001,1.62047,2.21795,1.875822,1.453162,0.438017,1.284513,-0.39319299999999996,1.307005,1.197683,0.9635600000000001,1.217803,1.363478,1.3407930000000001,0.34823200000000004,-0.32712800000000003,0.807481,-1.0043360000000001,0.706379,0.686958,-0.24900300000000003,0.808267,0.78404,0.8360059999999999,0.45255500000000004,-0.09830900000000001 -333,1,-0.726692,-0.589042,-0.7500439999999999,-0.6818649999999999,-0.69123,-0.9944459999999999,-1.279358,-1.491235,-0.138734,-0.541181,-0.817192,-1.049425,-0.848066,-0.753372,-0.94652,-1.132559,-1.1026479999999999,-1.185959,-0.140994,-0.281764,-0.6885359999999999,-0.400523,-0.6618850000000001,-0.5559189999999999,-0.474679,-1.0258040000000002,-1.025216,-1.436389,-0.428896,-0.6675770000000001 -134,0,1.294434,0.93029,1.141953,1.247093,0.619463,-0.170263,0.596387,0.354685,0.336894,-0.43478199999999995,1.227711,0.609773,1.162839,1.1948379999999998,-0.146621,-0.137417,0.332715,0.5042939999999999,-0.436721,-0.783596,0.688377,-0.026949,0.44549799999999995,0.616332,-0.346674,-0.629212,-0.11086800000000001,-0.42955299999999996,-0.68074,-0.601383 -182,0,0.795365,1.163157,0.6564439999999999,0.6828029999999999,0.3959,0.638907,0.086411,0.601356,0.867526,0.13101300000000002,0.446671,0.237443,0.380225,0.317717,-0.027775,-0.309875,-0.287124,0.07662999999999999,-0.7068909999999999,-1.03593,-0.127304,-0.121295,-0.22774299999999997,0.014145,-0.805026,-0.159181,-0.409612,-0.329146,-0.614146,-0.5162770000000001 -519,1,-0.376722,-0.641152,-0.406017,-0.450875,0.663299,-0.35856,-0.623143,-0.520083,0.275418,0.090005,-0.39116999999999996,-0.602629,-0.389619,-0.458154,1.14859,0.139464,-0.62774,-0.48928999999999995,1.125883,0.48657700000000004,-0.07858,-0.388004,-0.183686,-0.257796,0.156013,-0.5515260000000001,-0.401986,-0.421281,-0.301764,-0.062753 -207,0,0.731169,-0.102139,0.677294,0.579086,-0.932328,-0.672177,-0.378949,-0.076228,0.605445,-1.067077,0.8187300000000001,0.225807,0.730342,0.70935,-0.614889,-0.5932,-0.242302,0.128476,0.782694,-1.498069,0.651924,-0.655801,0.613809,0.6187520000000001,-0.667687,-0.583942,-0.41226499999999994,-0.091833,0.29031,-0.459161 -99,0,0.012593,0.843983,0.06668400000000001,-0.095249,0.470421,0.30747800000000003,0.226498,0.6379,-0.295659,0.531115,0.083133,0.11178099999999999,0.103427,-0.035236,0.08253200000000001,0.184948,0.063788,0.24429099999999998,0.246006,0.156275,-0.41748500000000005,1.15056,-0.242594,-0.296739,0.321353,0.19404100000000002,0.043644,0.396418,-0.717063,0.248547 -366,0,1.640261,1.3243719999999999,1.570869,1.389484,-0.200268,0.5555720000000001,0.470692,1.531701,0.598974,-0.422591,1.724735,1.75469,1.718907,1.6470509999999998,0.191415,1.185586,0.945398,2.0011189999999996,0.231402,-0.368238,2.0605990000000003,1.224948,2.109791,1.391892,0.466026,1.188316,0.90009,2.20019,2.004782,0.185758 -180,0,3.4895080000000003,1.168042,3.381848,4.105459,0.650148,0.948707,1.256047,2.3478470000000002,-0.072405,-0.17322,3.718515,0.6004649999999999,3.712513,4.536658999999999,0.927977,1.6498970000000002,2.489659,3.58229,-0.042419,-0.722639,1.555309,0.479253,1.462291,1.944135,-0.803359,-0.006044,-0.026649000000000003,0.099574,-0.580244,-0.39637100000000003 -318,1,-1.285815,-0.370832,-1.1509610000000001,-1.0257120000000002,-0.450133,0.766771,0.9053479999999999,-0.0016170000000000002,0.378956,1.194443,-1.444295,-0.090674,-1.313928,-1.167189,0.236249,1.759816,1.3647340000000001,0.004149,1.892581,3.4918980000000004,0.21701399999999998,1.259421,0.446983,-0.355043,0.9347120000000001,2.259159,2.3477259999999998,2.185591,1.7396200000000002,1.544058 -501,0,-0.053674,1.182698,-0.037566,-0.162753,2.061664,0.905449,0.316212,0.5709029999999999,1.213731,1.9037650000000002,-0.087275,1.210157,0.014868000000000001,-0.167771,1.411902,1.208328,0.5888369999999999,0.481596,1.6917790000000001,1.356986,0.25238499999999997,0.564527,0.053434,-0.028318,0.879709,0.731139,0.095037,0.707861,0.45982,1.2104409999999999 -229,0,-0.22141100000000002,0.728364,-0.058415999999999996,-0.306902,1.987143,1.7814139999999998,1.707974,1.265235,0.818992,2.2362599999999997,-0.36844899999999997,0.7075100000000001,-0.276346,-0.43141899999999994,0.8852780000000001,1.4319549999999999,1.013195,0.5079060000000001,1.136836,1.3810850000000001,-0.357572,-0.268257,-0.301503,-0.334582,-0.019327,0.7322569999999999,0.49524300000000004,0.518075,-0.453112,0.68921 -7,0,0.16376300000000002,0.401048,0.099449,0.028858999999999996,1.447961,0.7247859999999999,-0.021054,0.624196,0.47763999999999995,1.726435,-0.11851700000000001,0.35845,-0.072867,-0.21896500000000002,1.604049,1.140102,0.061026,0.28195,1.403355,1.660353,0.643623,0.290561,0.490051,0.23372199999999999,0.5880310000000001,0.268933,-0.23255399999999998,0.435349,-0.688004,0.6116689999999999 -247,1,-0.38914699999999997,-1.299041,-0.067352,-0.42450600000000005,-0.305475,2.1033,2.401216,0.6318090000000001,-0.42346300000000003,1.8760569999999999,-0.351408,-1.205339,-0.289115,-0.40582199999999996,-0.623429,0.573453,0.6101800000000001,-0.235219,-0.787211,0.18320999999999998,-0.731486,-1.409116,-0.234179,-0.5277569999999999,-0.513347,1.701382,1.6474490000000002,0.308825,-0.954377,1.422261 -494,1,-0.36636799999999997,0.453158,-0.35657300000000003,-0.408333,-0.9016430000000001,-0.570395,-0.936517,-1.106307,-0.34581,-0.535085,-0.274725,0.29096500000000003,-0.325775,-0.33045399999999997,-1.637537,-0.977725,-0.888883,-0.937847,-0.360051,-0.555362,-0.29405,0.464738,-0.26734600000000003,-0.3139,0.253684,-0.27710799999999997,-0.612865,-1.010913,0.038467,-0.413771 -174,1,-0.979333,-1.054776,-1.014542,-0.830233,-1.085753,-1.185478,-1.305831,-1.745063,-0.308601,-1.2360950000000002,-0.9847600000000001,-0.963324,-1.0082959999999999,-0.868274,-0.6006560000000001,-1.162123,-1.114873,-1.2618200000000002,0.42490200000000006,-0.43203,-0.268063,1.284821,-0.35199600000000003,-0.403887,0.557363,-0.8555649999999999,-1.057501,-1.9134470000000001,1.465982,-1.0564200000000001 -340,1,0.083001,-0.678606,0.123277,-0.032492,-0.130131,0.526946,0.49563900000000005,0.407978,0.246298,0.204716,0.083133,-0.639862,0.089834,-0.038932999999999995,0.08182,0.18115699999999998,-0.109596,-0.17254,0.366487,0.18746300000000002,-0.202376,-0.809657,-0.09358999999999999,-0.18035,-0.8213600000000001,0.282346,0.215729,0.10281900000000001,-0.21943000000000001,-0.15958599999999998 -125,1,-0.161357,-0.34152,-0.207346,-0.271919,-0.730683,-0.758692,-0.9165120000000001,-0.9678969999999999,-0.868353,-0.671962,-0.07875499999999999,-0.48394799999999993,-0.145362,-0.188249,-0.605638,-0.814553,-0.936592,-0.96751,-0.721494,-0.552527,-0.6737390000000001,-0.6545310000000001,-0.678716,-0.49343400000000004,-0.814026,-0.91151,-0.754877,-1.13257,-0.977382,-0.789374 -209,1,0.23002899999999998,-1.5889030000000002,0.19178499999999998,0.09161699999999999,-0.44574899999999995,-0.22688000000000003,0.115196,-0.16910999999999998,-0.939535,-0.5101479999999999,0.324545,-1.484587,0.255419,0.200825,-1.034766,-0.7967390000000001,-0.375134,-0.447504,-1.6524849999999998,-1.068535,-0.692146,-1.549547,-0.6638649999999999,-0.44745100000000004,-0.916697,-0.733167,-0.44708000000000003,-0.717476,-1.332263,-0.8090430000000001 -347,1,0.20725,-1.261587,0.20667800000000003,0.000381,-0.454517,-0.339476,-0.273883,0.15978499999999998,0.336894,-0.115033,0.179698,-1.058734,0.11949100000000001,0.03928,-0.541589,-0.502991,-0.5363399999999999,-0.351808,-1.061032,-0.521339,-0.22511399999999998,-1.4855,-0.162894,-0.248115,-0.769691,-0.5822649999999999,-0.442438,-0.18267,0.131697,-0.382754 -332,1,-0.888216,0.016737000000000002,-0.9040360000000001,-0.781363,0.43973599999999996,-1.002397,-1.241784,-1.437181,0.632947,-1.037706,-0.8257120000000001,0.132725,-0.825,-0.761051,0.643316,-0.692695,-1.052023,-1.066224,0.468713,-0.35689699999999996,-0.38825,1.35921,-0.44902200000000003,-0.45581099999999997,1.9497529999999998,-0.806941,-0.948182,-1.107752,2.65013,-0.69292 -361,1,-0.428493,0.573662,-0.426569,-0.45597299999999996,-0.805204,-0.557036,-0.724371,-0.890242,-0.426699,-0.962341,-0.234963,0.5306529999999999,-0.27716999999999997,-0.30940700000000004,-0.750104,-0.769638,-0.6950350000000001,-0.6365729999999999,0.012345,-0.827542,-0.5163770000000001,0.584485,-0.414865,-0.425889,-0.514347,-0.28102,-0.46233199999999997,-0.875468,-0.27270500000000003,-0.6955680000000001 -500,1,0.101638,-0.8544780000000001,0.072641,-0.041632999999999996,-0.8271219999999999,-0.23324099999999998,-0.41541000000000006,-0.194995,-1.170878,0.08557200000000001,0.25922199999999995,-0.59332,0.278486,0.098153,0.175759,0.6075659999999999,-0.145504,0.322447,-0.524343,0.8353079999999999,-0.119725,-0.679569,-0.278237,-0.12094500000000001,-0.972699,-0.407331,-0.395687,-0.284862,-1.209853,-0.21102800000000002 -300,0,2.000585,0.091645,1.9014919999999997,2.061007,0.75097,1.0008700000000001,1.630733,1.269803,0.108787,0.85031,1.534446,-0.090674,1.545908,1.598701,1.3265040000000001,1.134417,1.6434540000000002,1.477501,-0.071627,0.385927,2.5474810000000003,-0.101337,2.163749,2.038743,-0.32834,0.366181,0.8118920000000001,0.8976459999999999,-0.206112,0.375262 -283,0,0.47231599999999996,-0.095626,0.584958,0.26442,0.181104,1.376193,1.105405,0.892184,-0.21153400000000003,1.238775,0.600039,-0.120926,0.693271,0.42721499999999996,0.7287140000000001,1.437641,1.3308360000000001,1.0730520000000001,0.235053,0.573051,-0.425425,-0.543493,-0.199032,-0.269457,-0.826026,0.5215529999999999,0.22633899999999998,0.047667,-0.806661,0.108216 -107,1,-0.616938,0.295199,-0.646389,-0.591508,-0.612326,-0.368739,-0.37655,-0.459633,0.133053,-0.6703,-0.5019359999999999,-0.174448,-0.5337850000000001,-0.535229,-0.8248270000000001,-0.685873,-0.783045,-0.766317,-0.765306,-0.303028,-1.029607,-0.585041,-0.9988030000000001,-0.684475,-1.194708,-0.8387979999999999,-0.670558,-1.011886,-0.9725389999999999,-0.9225190000000001 -131,0,0.619345,0.052562,0.525386,0.484159,0.974533,-0.09456200000000001,0.512911,0.560244,-0.10314300000000001,-0.208132,0.378508,0.044295999999999995,0.40081999999999995,0.267377,0.913744,0.34035,0.7256859999999999,0.82414,0.435855,-0.685782,0.24949699999999997,-0.781898,0.11283800000000001,0.17541800000000002,-0.267004,-0.594561,-0.124794,-0.140496,-0.795764,-0.504551 -303,1,-1.078732,-0.18519000000000002,-1.0872190000000002,-0.8880680000000001,0.391516,-0.9533510000000001,-0.9017350000000001,-0.751071,-1.112638,-0.306218,-1.033042,-0.158159,-1.034246,-0.9117879999999999,0.742947,-0.7118359999999999,-0.826485,-0.802687,-1.203419,0.45397200000000004,-0.926383,0.628029,-0.90643,-0.6657069999999999,0.6113649999999999,-0.900724,-0.45802200000000004,-0.421281,-0.318715,-0.180768 -567,0,1.961239,2.237926,2.303601,1.653171,1.4304270000000001,3.9048480000000003,3.197605,2.2899849999999997,1.919083,2.219635,1.838341,2.336457,1.9825240000000002,1.735218,1.525767,3.272144,3.2969440000000003,2.658866,2.137194,1.043695,1.157935,0.686088,1.4385299999999999,1.009503,-0.173,2.017716,1.3022850000000001,0.785721,0.32663400000000004,0.904057 -507,1,-0.9482700000000001,-0.803996,-0.928759,-0.82531,1.4830299999999998,-0.325481,-0.7032619999999999,-0.296404,-0.195356,1.820641,-0.8711540000000001,-0.504892,-0.8534209999999999,-0.820208,1.639632,0.052287,-0.6047640000000001,-0.160932,0.519826,2.404595,-0.8202729999999999,-0.339016,-0.7663369999999999,-0.6168640000000001,1.859749,-0.111675,-0.460674,0.17094700000000002,0.200712,1.7089759999999998 -3,0,-0.281464,0.133984,-0.249939,-0.550021,3.394275,3.893397,1.9895880000000001,2.175786,6.046041000000001,4.93501,-0.7689090000000001,0.253732,-0.592687,-0.764464,3.283553,3.4029089999999997,1.9158970000000002,1.451707,2.8673830000000002,4.910919,0.326373,-0.110409,0.286593,-0.288378,0.689702,2.74428,0.8195180000000001,1.115007,4.73268,2.047511 -516,0,1.157759,0.085131,1.040681,1.076575,0.73782,-0.004231,0.497558,0.554153,0.280271,-0.29458,1.187949,0.300273,1.187553,1.129424,0.742947,0.387729,0.855002,1.1759700000000002,0.17663800000000002,-0.480229,0.504307,-0.534058,0.17422100000000001,0.59455,-0.288338,-0.374915,-0.091637,-0.216734,-0.599617,-0.40469299999999997 -143,1,-0.37051,-0.628125,-0.300575,-0.41624399999999995,-0.051226,0.003403,-0.30314800000000003,-0.204131,1.048717,-0.15327000000000002,-0.348568,-0.78414,-0.338956,-0.40582199999999996,-0.682497,-0.17532,-0.500433,-0.465301,-0.12274000000000002,-0.063453,-0.688897,-0.808569,-0.58268,-0.5213760000000001,-0.57235,-0.550408,-0.55683,-0.683249,0.060261,-0.7216670000000001 -398,1,-0.743259,-0.867505,-0.788467,-0.674833,-0.892875,-0.422174,-0.30842600000000003,-0.899073,-0.502735,0.423054,-0.8711540000000001,-1.03779,-0.8921399999999999,-0.786932,-1.3486049999999998,-1.073809,-0.774382,-1.074916,-1.009919,-0.093223,-0.7928430000000001,-0.959341,-0.793563,-0.601903,-0.927364,-0.6029439999999999,-0.414254,-1.237358,-1.045186,-0.097552 -530,1,-0.573451,0.37499299999999997,-0.5582229999999999,-0.5770930000000001,0.11096600000000001,-0.43807799999999997,-0.650489,-0.20717600000000003,-0.683926,-0.353321,-0.675185,-0.402501,-0.662299,-0.659801,0.77853,-0.136659,-0.451719,-0.116567,-0.77991,0.563128,0.11992699999999999,1.252163,0.14006400000000002,-0.212912,-0.151332,-0.409566,-0.481563,0.221232,-0.26544,-0.5586409999999999 -119,0,0.8926930000000001,0.350566,0.653465,0.66874,-1.1032879999999998,-0.852841,-0.226868,0.059289,3.205219,-1.265465,1.085703,0.167631,0.9156979999999999,0.9303370000000001,-0.878202,-0.703498,-0.199239,0.181612,1.158741,-1.7787540000000002,0.52488,-0.005177,0.243031,0.30148800000000003,-1.0057,-0.95326,-0.297873,-0.31357399999999996,3.5836449999999997,-0.715993 -259,0,0.459891,3.885905,0.567086,0.271451,2.451803,1.922,1.430197,1.321574,0.988859,2.020138,0.398389,3.320807,0.48320100000000005,0.256,0.707364,1.128731,1.083503,0.904617,1.020006,0.5248520000000001,-0.589283,0.11094100000000001,-0.47674399999999995,-0.38100500000000004,-0.565349,0.004575,-0.099594,-0.25566500000000003,-1.2828629999999999,-0.164881 -450,1,-0.720479,0.407562,-0.70745,-0.656375,-1.6569349999999998,0.544758,0.238012,-0.412736,-0.963802,0.8630559999999999,-0.641103,0.523672,-0.62358,-0.633919,-2.151351,0.039021,-0.012922999999999999,-0.646375,-1.6889939999999999,0.470983,-0.538393,0.6117,-0.45100200000000007,-0.44217,-0.062328999999999996,1.964621,1.151752,0.605669,0.36295700000000003,1.940466 -192,1,-1.304866,-0.7893399999999999,-1.340697,-1.0139340000000001,-2.6826950000000003,-1.443878,-1.305831,-1.745063,-1.604443,-1.017203,-1.251733,-0.248914,-1.286742,-1.0431860000000002,-1.9115240000000002,-1.533193,-1.114873,-1.2618200000000002,-0.579108,0.23707899999999998,-0.185052,6.655278999999999,-0.314869,-0.410268,-1.7760650000000002,-1.04749,-1.057501,-1.9134470000000001,2.112542,-0.7969390000000001 -514,0,0.271446,0.38802,0.19476300000000002,0.15191300000000002,-0.340543,-0.280951,0.06914,-0.039684,-1.001011,-0.7983100000000001,0.26206199999999996,-0.051114,0.21793600000000002,0.133704,-0.299627,-0.348157,-0.175008,-0.14365,-0.9149940000000001,-0.517087,-0.06919600000000001,-0.034207,-0.11685699999999999,-0.040639,-0.6963550000000001,-0.512962,-0.073732,-0.384946,-1.092406,-0.785592 -241,1,-0.635576,-0.8642479999999999,-0.6973229999999999,-0.592739,-1.256713,-1.122819,-1.006321,-1.128081,0.000395,-0.893071,-0.48489499999999996,-0.988922,-0.550261,-0.507357,-1.216949,-1.334392,-0.982669,-0.976022,-0.969759,-0.745321,-1.046209,-0.984016,-1.044049,-0.689337,-1.2587110000000001,-1.148426,-0.8422120000000001,-1.303215,-0.404681,-0.921006 -402,1,-0.442989,-0.173791,-0.326191,-0.454742,-1.7130450000000002,-0.14290899999999998,-0.536308,-0.738889,0.495436,-0.635942,-0.331527,-0.232625,-0.320832,-0.368849,-1.62615,-0.48043900000000006,-0.605518,-0.776119,0.227751,-0.539768,-0.61166,0.149042,-0.232199,-0.44283,-1.1373719999999998,0.651217,0.08707999999999999,-0.185915,0.699555,-0.034006 -148,1,-0.08680700000000001,-0.9489270000000001,0.039876999999999996,-0.199845,-0.033692,0.12236099999999998,0.182841,0.68967,-0.339339,-0.394329,0.088814,-0.956343,0.08242000000000001,-0.042062,0.237673,-0.04247,-0.049332,0.165104,-0.319891,-0.281764,-0.593975,-0.866265,-0.369322,-0.421048,-0.445011,-0.140179,-0.024991,0.508343,-0.726749,-0.47920799999999997 -395,1,-0.27939400000000003,-0.054915,-0.322915,-0.344697,-1.129589,-0.834393,-0.89996,-0.5404869999999999,-0.6111260000000001,-0.9894950000000001,-0.019112,-0.490929,-0.091402,-0.13022899999999998,-1.132262,-0.961427,-0.778274,-0.423257,-0.622919,-0.7311449999999999,-0.9195260000000001,0.8493790000000001,-0.806434,-0.6087229999999999,-0.556682,-0.712488,-0.682163,-0.428418,-0.405891,-0.8892329999999999 -299,1,-1.105653,-0.2373,-1.106878,-0.910393,-0.792053,-1.06951,-1.10635,-1.269232,-1.0899889999999999,-0.8963959999999999,-1.027362,0.884367,-1.034658,-0.9120729999999999,0.36577,-0.689284,-0.8016260000000001,-0.7781819999999999,-0.42576800000000004,0.391598,-0.42723,-0.133996,-0.28566199999999997,-0.43513,1.043049,-0.617475,-0.44044799999999995,0.114173,1.7069290000000001,-0.689894 -149,1,-0.192419,-0.5239050000000001,-0.29998,-0.271919,-1.545592,-0.457162,-0.5554979999999999,-0.828574,-0.8910020000000001,-0.765061,-0.10999600000000001,-0.32105300000000003,-0.15854200000000002,-0.198772,-1.2041389999999998,-0.76907,-0.7531640000000001,-0.919018,-1.236277,-0.991984,-0.560049,-0.833607,-0.640104,-0.415108,-1.40105,-0.5342,-0.46730600000000005,-0.96809,-0.8780969999999999,-0.6880029999999999 -60,1,-1.087016,-1.339752,-1.114026,-0.900022,-0.213419,-0.989865,-1.20182,-1.3523690000000002,1.061659,-0.207578,-1.1239270000000001,-1.026155,-1.1293950000000001,-0.9754959999999999,1.212639,-0.449737,-0.978777,-0.929077,3.4004209999999997,0.96431,0.399279,0.406679,0.220754,-0.12578599999999998,0.15768,-0.809735,-0.8033520000000001,-0.584464,2.577483,0.816303 -61,1,-1.388321,0.22191999999999998,-1.3463559999999999,-1.0664959999999999,1.382207,-0.537316,-0.874006,-1.322068,0.115258,-0.378258,-1.570397,0.393356,-1.536767,-1.23175,1.9883419999999998,-0.27879499999999996,-0.738224,-1.022993,0.059807000000000006,0.6765359999999999,-0.169532,1.5424579999999999,-0.184676,-0.48287299999999994,1.62974,0.34326599999999996,-0.06279,-0.41154799999999997,1.577375,-0.282517 -126,0,0.12855899999999998,1.622376,0.176892,-0.056048,0.645764,0.217146,0.515309,0.278552,0.9209120000000001,-0.274077,-0.146918,1.2566979999999999,-0.173371,-0.234039,-0.269025,-0.487451,-0.45134300000000005,-0.466075,-0.184806,-0.212302,-0.628624,-0.384375,-0.551493,-0.451191,-0.984366,-0.767818,-0.462001,-0.797445,-0.8853620000000001,-0.881289 -372,0,1.3296379999999999,-0.624868,1.335561,1.150408,-0.577257,0.189156,0.624693,1.248486,-0.27624499999999996,0.150409,2.057032,-0.974959,2.031952,2.079354,0.266139,0.893733,1.310748,1.975325,0.589194,-0.137169,-0.230167,0.167186,-0.227248,-0.028098,-0.8716950000000001,0.07108400000000001,0.082106,0.26502800000000004,-0.45916599999999996,0.216396 -82,0,2.843411,1.293432,3.1107970000000003,2.955784,1.09289,2.247704,1.801046,2.620403,-0.882913,1.172276,3.150487,1.3078940000000001,3.275896,3.478653,0.707364,3.073153,3.077232,3.4971699999999997,0.063458,0.711976,1.7761930000000001,0.466552,2.235529,1.75272,0.37502199999999997,1.75783,0.842397,1.379409,-1.197745,0.793608 -425,1,-1.068378,0.531323,-1.112239,-0.8864860000000001,-0.866574,-1.166203,-1.246581,-1.3523690000000002,-0.8926200000000001,-0.184857,-1.163689,0.463168,-1.185414,-0.9885790000000001,-1.081023,-1.236034,-1.083862,-1.1287479999999999,-0.663079,0.22573800000000002,-0.7942859999999999,0.22524499999999997,-0.8326709999999999,-0.632265,-0.439011,-1.105559,-0.988733,-1.341334,-0.737646,-0.523463 -337,0,1.7127400000000002,1.415565,1.603633,1.744582,0.764121,1.453165,0.917342,1.3733440000000001,1.2590290000000002,0.817615,1.3185950000000002,0.49807399999999996,1.274053,1.243188,-0.37008,0.679581,0.215954,0.30903400000000003,0.516175,-0.278929,0.855484,0.568156,0.744002,1.054166,0.169014,0.754054,0.240265,1.0322799999999999,0.137751,0.2459 -386,1,-0.650071,-1.0401200000000002,-0.584136,-0.61647,-1.304933,-0.071025,0.169888,-0.353352,-0.36198800000000003,0.23796599999999998,-0.544538,-1.209993,-0.543259,-0.548596,-1.087428,-0.494842,-0.256238,-0.6082,-0.604664,-0.178279,-0.5001359999999999,-0.7002520000000001,-0.380708,-0.44833100000000004,-0.8786950000000001,0.267256,0.38284,-0.150228,-0.161313,0.31285 -184,0,0.317004,0.383135,0.19476300000000002,0.16263699999999998,-0.099446,0.48114399999999996,0.43567,0.121718,0.44366700000000003,0.763308,0.327386,0.7261270000000001,0.286312,0.158448,-0.41206800000000005,0.016280000000000003,-0.440043,-0.420162,-0.308938,0.05279,-0.721019,-1.308602,-0.753466,-0.457791,-1.250377,-0.644302,-0.46929499999999996,-0.93873,-1.068191,-0.46143 -58,1,-0.422281,-0.5581020000000001,-0.506991,-0.450875,-1.326851,-1.223647,-1.296979,-1.575895,-0.747019,-1.166825,-0.30596599999999996,0.004736,-0.3855,-0.363161,-1.1215870000000001,-1.259344,-1.106185,-1.154336,0.026949,-1.103975,-0.004229999999999999,-0.005177,-0.134183,-0.16230799999999998,0.150013,-0.9437030000000001,-1.034556,-1.2375200000000002,0.164388,-0.304456 -88,1,-0.5051140000000001,0.785359,-0.47065200000000007,-0.537716,-0.08629500000000001,-0.050669,-0.138113,0.089742,0.115258,0.480133,-0.5019359999999999,0.584176,-0.502069,-0.536936,-0.614889,-0.187449,-0.359691,-0.295835,0.432204,0.176122,-0.38752800000000004,0.517354,-0.328235,-0.42654899999999996,0.023675,-0.030635000000000003,-0.16126600000000002,0.183924,-0.116514,0.252708 -117,0,0.526157,0.275658,0.590915,0.37692600000000004,2.429885,1.232425,0.950924,1.3413680000000001,1.106957,1.249858,0.21094000000000002,-0.60961,0.274778,0.078528,1.411902,1.147683,1.006917,1.039777,1.260968,0.69213,0.077338,-0.48616000000000004,0.060859,0.018546,-0.01866,0.008487,-0.059142999999999994,0.14823699999999998,-0.547553,0.033701 -459,1,-1.159494,1.830816,-1.168535,-0.932895,-0.9367110000000001,-0.9120020000000001,-0.960889,-1.004137,-0.937918,-0.655891,-1.241793,2.073499,-1.247611,-1.0352219999999999,-1.175673,-1.1007209999999998,-0.921401,-0.9927879999999999,-0.6959380000000001,-0.464635,-0.819551,0.853008,-0.8034640000000001,-0.639305,-0.150999,-0.714164,-0.576724,-0.92332,-0.581455,-0.577553 -221,1,-0.266969,-1.3918620000000002,-0.183517,-0.341005,0.229323,0.098824,-0.069509,-0.36096500000000004,0.26571100000000003,-0.120575,-0.161119,-1.254207,-0.139183,-0.266177,0.621966,0.2816,-0.128052,-0.113729,0.549034,0.032944,-0.535145,-1.304066,-0.42328000000000005,-0.424789,-0.39667600000000003,-0.130678,-0.213986,-0.557375,-0.256965,-0.33169 -223,0,0.681469,0.751162,0.555172,0.36462100000000003,1.0008350000000001,1.232425,0.601665,0.506951,1.7670119999999998,1.2443170000000001,0.460872,0.22348,0.437892,0.302644,0.436936,0.304342,0.325182,0.404988,0.45045799999999997,0.032944,-0.20887199999999997,-0.536961,-0.307938,-0.17925,-0.758357,-0.09714400000000001,-0.26703699999999997,-0.512443,-0.34171999999999997,-0.251123 -128,1,-0.032964999999999994,-1.19645,-0.040545,-0.207404,0.27315900000000004,0.21650999999999998,-0.365516,0.42168199999999995,-0.502735,-0.34113000000000004,0.27626300000000004,-0.674768,0.31349699999999997,0.055775,1.3265040000000001,1.447116,0.31388299999999997,0.9394389999999999,0.691421,0.265431,0.09285800000000001,-0.270071,-0.034681000000000003,-0.010937,0.655034,0.9150159999999999,0.00485,1.7492470000000002,0.361746,0.44977700000000004 -294,1,-0.573451,-1.334867,-0.5576270000000001,-0.5746319999999999,-0.112597,-0.681083,-1.054776,-0.7792399999999999,-0.8602639999999999,-0.8160430000000001,-0.39969099999999996,-1.282132,-0.419688,-0.46298900000000004,0.022040999999999998,-0.386818,-0.9531649999999999,-0.7655430000000001,-0.633872,-0.25483,-0.810167,-0.950269,-0.755446,-0.593102,-0.325673,-0.764465,-0.839858,-0.6193390000000001,-0.8236120000000001,-0.9089020000000001 -541,1,-0.010186,0.985657,0.185828,-0.12601300000000001,0.071514,1.0555780000000001,0.632369,0.089742,0.46308,1.017112,0.097334,1.32651,0.15821,0.004297,-0.568632,0.353616,0.151924,-0.258434,0.22044899999999998,0.086813,-0.54489,-0.250114,-0.12428199999999999,-0.379025,0.032342,1.176579,0.21208200000000002,-0.028571,0.016673,0.876066 -111,1,-0.6086550000000001,-0.033745,-0.543926,-0.620865,-0.16081600000000001,-0.186167,-0.24269899999999997,-0.062524,-1.091607,0.050660000000000004,-0.425252,0.342161,-0.404448,-0.49626499999999996,0.21134099999999997,0.313818,0.222232,0.291236,-0.27973000000000003,1.1202459999999999,-0.22655799999999998,1.0634709999999998,-0.07675900000000001,-0.43688999999999995,1.95642,0.835094,0.633839,1.809264,0.10869200000000001,0.792851 -266,1,-0.908925,-0.44573999999999997,-0.8632299999999999,-0.801227,-0.485202,-0.01759,-0.386625,-0.538203,0.063489,-0.447528,-1.001801,-0.079038,-0.934566,-0.877375,0.036986,0.196319,-0.312987,-0.580343,0.402996,0.299453,0.163598,-0.036021,0.27916799999999997,-0.291238,0.143013,0.577443,0.054586,0.300714,1.7553599999999998,-0.18039000000000002 -568,1,-1.410893,0.7641899999999999,-1.432735,-1.075813,-1.859019,-1.207552,-1.305831,-1.745063,-0.048138,-0.751207,-1.808401,1.221792,-1.814389,-1.347789,-3.112085,-1.150752,-1.114873,-1.2618200000000002,-0.8200700000000001,-0.561032,-0.070279,0.383092,-0.157449,-0.46615200000000007,0.049342000000000004,-1.163516,-1.057501,-1.9134470000000001,0.75283,-0.382754 -362,1,-0.52168,0.050934,-0.579073,-0.528926,-0.112597,-0.44761999999999996,-0.703741,-0.47942799999999997,-0.253596,-0.640929,-0.38833,-0.10463599999999999,-0.41598100000000005,-0.45019,0.028446,-0.470394,-0.777395,-0.802429,-0.192108,-0.137169,-0.663633,0.12364100000000001,-0.6589149999999999,-0.507735,-0.477679,-0.50402,-0.55053,-0.291837,-0.175842,-0.628617 -271,1,-0.817808,-1.546564,-0.8635280000000001,-0.743743,0.150419,-0.658818,-0.694146,-0.412736,-0.271392,-0.206469,-0.8058310000000001,-1.454335,-0.813055,-0.7590600000000001,0.140888,-0.535587,-0.7049529999999999,-0.551195,-0.15559800000000001,-0.013837,-0.775157,-1.247458,-0.842571,-0.597722,-0.189667,-0.7968810000000001,-0.632428,-0.478054,-0.438583,-0.36989299999999997 -187,1,-0.674921,-0.698148,-0.680345,-0.6312369999999999,-0.003007,-0.955896,-0.5761270000000001,-0.07166,-0.531855,-0.719066,-0.686545,-0.48860200000000004,-0.7121390000000001,-0.667195,0.098188,-0.813606,-0.6366539999999999,-0.42635200000000006,-1.079287,-0.261918,-0.577734,-0.81891,-0.5564439999999999,-0.49453400000000003,-0.045328,-0.937499,-0.40165500000000004,0.008737,-0.190372,-0.803369 -155,1,-0.554813,-0.074456,-0.615412,-0.556174,-0.467667,-0.480063,-0.373672,-0.494807,0.34336500000000003,-0.145512,-0.5331779999999999,-0.31407199999999996,-0.5642659999999999,-0.553431,-0.698865,-0.711647,-0.627112,-0.660562,0.578241,-0.073377,-0.668325,-0.42556099999999997,-0.6841619999999999,-0.5242359999999999,-0.50768,-0.550967,-0.39635,-0.6280979999999999,-0.309029,-0.49585100000000004 -5,0,-0.165498,-0.313836,-0.11500899999999999,-0.24431999999999998,2.0485130000000003,1.721616,1.2632430000000001,0.9058879999999999,1.7540689999999999,2.241802,-0.476375,-0.835335,-0.387148,-0.50565,2.237421,1.244335,0.866302,0.824656,1.0054020000000001,1.890005,-0.25506999999999996,-0.592662,-0.321304,-0.289258,0.156347,0.44554399999999994,0.160025,-0.069124,0.13411900000000002,0.486846 -511,1,-0.136507,-1.318583,-0.16564500000000001,-0.211623,-0.809587,-0.9743440000000001,-0.777623,-0.533787,-0.9168860000000001,-1.2482870000000001,0.193899,-1.068042,0.11084100000000001,0.073409,-0.8283860000000001,-1.02681,-0.685995,-0.606394,-0.5572020000000001,-1.320868,-0.674821,-1.077091,-0.588621,-0.431609,-0.110997,-0.7549640000000001,-0.566114,-0.20051300000000002,-0.57419,-0.762518 -252,0,1.8659810000000001,-0.014204,1.564912,1.850057,1.693442,2.170731,2.766787,2.072245,-0.245507,2.535505,1.591249,0.12341600000000001,1.595336,1.567416,0.700248,1.526713,1.919664,1.250514,-0.287032,0.59148,1.302665,-0.792602,0.618264,1.1544940000000001,0.48036,1.4023729999999999,1.197177,1.2837049999999999,-0.711009,1.3102989999999999 -480,1,-0.606584,0.35708,-0.5489890000000001,-0.585707,-0.50712,-0.167719,-0.528632,-0.878669,-0.8004060000000001,-0.368838,-0.558739,-0.293129,-0.563442,-0.567652,-0.390718,-0.49199899999999996,-0.7487699999999999,-0.8679450000000001,-1.269136,0.0060090000000000005,-0.6704899999999999,-0.048721,-0.588126,-0.529737,-0.710022,-0.492842,-0.5942970000000001,-1.076283,-0.726749,-0.732636 -513,1,0.101638,-1.373949,0.036898,-0.032668,-0.441366,-0.391004,-0.11028800000000001,-0.346348,-0.44449399999999994,-0.746219,0.128576,-1.310057,0.095601,0.011123000000000001,0.139464,-0.287323,-0.082603,-0.140039,-0.265127,-0.9069280000000001,0.040885000000000005,-1.076184,-0.151014,-0.071001,-0.696021,-0.411243,-0.051185,-0.510658,-0.623833,-0.5726359999999999 -86,0,-0.012256999999999999,0.581805,0.03392,-0.12601300000000001,-0.077528,-0.360469,0.30086,0.12019500000000001,0.192911,-0.858159,0.100174,0.505055,0.093953,-0.019024,-0.136658,-0.092312,0.396746,0.011887,0.96159,-0.9125979999999999,0.054961,1.820053,0.21530900000000003,-0.032277999999999994,0.776038,0.24322399999999997,0.723363,0.9430649999999999,0.44044700000000003,-0.20648899999999998 -105,0,0.008451,-0.533675,-0.025651999999999998,-0.093843,2.3597479999999997,0.9900559999999999,1.7530700000000001,1.278939,0.39836900000000003,3.133996,-0.288925,-0.867914,-0.196026,-0.35462899999999997,3.091407,1.36752,1.485262,1.214661,0.41394899999999996,2.001996,-0.051872,-0.5317,-0.22576300000000002,-0.124905,0.040342,0.203542,0.7571829999999999,0.338023,-0.614146,1.249401 -242,1,-0.7639670000000001,0.371736,-0.598731,-0.7166710000000001,0.102199,1.466524,2.2616080000000003,0.109537,0.658832,2.535505,-0.802991,-0.255896,-0.743031,-0.755078,-0.031333,0.533655,0.8286370000000001,-0.525659,0.884921,1.969391,-0.5860350000000001,0.771362,-0.24606,-0.526877,-0.125998,1.881346,1.886843,0.21798800000000002,-0.071715,1.8459029999999998 -312,1,-0.43056400000000006,-1.510738,-0.45337700000000003,-0.460192,-0.5684899999999999,-0.21288400000000002,-0.45714899999999997,-0.464354,-0.544797,-0.078459,-0.38833,-1.377542,-0.398681,-0.42886,-0.599233,-0.47115200000000007,-0.6061449999999999,-0.604589,-0.768957,-0.198126,-0.28394400000000003,-1.011412,-0.257445,-0.333482,-0.182334,0.12306099999999999,-0.017365000000000002,-0.179426,-0.391362,0.225852 -473,1,-0.583805,2.01483,-0.660686,-0.565491,-1.672278,-1.2858610000000001,-1.305831,-1.745063,-0.795553,-0.9152379999999999,-0.527497,2.485389,-0.599278,-0.538926,-1.378495,-1.333445,-1.114873,-1.2618200000000002,-0.403862,-0.453294,0.145552,4.409122,0.008881,-0.11456500000000001,0.099344,-0.9632639999999999,-1.057501,-1.9134470000000001,1.3158450000000002,-0.249231 -364,1,-0.318739,-0.647666,-0.402145,-0.38161300000000004,-0.485202,-0.551311,-0.6514479999999999,-0.68118,-0.25845,-0.450299,-0.20656100000000002,-0.544452,-0.267285,-0.29149,-1.209121,-0.89794,-0.8410489999999999,-0.8818739999999999,-0.59006,-0.8204540000000001,-0.890652,-1.096686,-0.905935,-0.596622,-0.8823620000000001,-0.725342,-0.576392,-1.02389,-0.924107,-0.650934 diff --git a/fml_manager/Examples/Homo_LR/data/breast_homo_host.csv b/fml_manager/Examples/Homo_LR/data/breast_homo_host.csv deleted file mode 100644 index 503643565..000000000 --- a/fml_manager/Examples/Homo_LR/data/breast_homo_host.csv +++ /dev/null @@ -1,229 +0,0 @@ -id,y,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11,x12,x13,x14,x15,x16,x17,x18,x19,x20,x21,x22,x23,x24,x25,x26,x27,x28,x29 -565,0,1.53672,2.047399,1.42194,1.494959,-0.69123,-0.39482,0.236573,0.733827,-0.531855,-0.973978,1.704854,2.085134,1.615931,1.7238419999999999,0.102458,-0.017833,0.6930430000000001,1.2636690000000002,-0.217664,-1.058611,1.300499,2.260938,1.156857,1.291565,-0.42401000000000005,-0.069758,0.252202,0.808431,-0.189161,-0.49055600000000005 -447,1,0.033301,-0.47830900000000004,-0.040545,-0.0898,-0.42821499999999996,-0.42090200000000005,-0.31754099999999996,-0.480037,1.131224,-0.614884,0.19105899999999998,-0.37923,0.16109300000000001,0.056629,-0.325246,-0.292629,-0.604011,-0.678876,0.297119,-0.558197,-0.6668810000000001,-1.0790870000000001,-0.6851520000000001,-0.45295100000000005,-0.7483569999999999,-0.7694949999999999,-0.4746,-0.794687,0.24187899999999998,-0.689894 -488,1,-0.610726,-0.665579,-0.616305,-0.581488,0.886862,-0.6779029999999999,-0.591,-0.25057199999999996,-0.15653,-0.20536100000000002,-0.695066,-0.725963,-0.678775,-0.6666270000000001,1.16994,-0.22194,-0.577646,-0.453952,0.151081,0.171869,-0.122251,-0.11403800000000001,-0.154479,-0.280898,0.652367,-0.701869,-0.44376400000000005,-0.020461,0.118379,-0.220106 -151,1,-1.486271,0.658341,-1.464904,-1.108862,1.342755,1.1242809999999999,1.275717,-0.545359,0.681481,3.582864,-1.6780389999999998,0.328198,-1.594021,-1.282659,-0.164412,0.495752,0.543639,-0.7026060000000001,1.4982790000000001,2.808612,-0.763969,1.351952,-0.8034640000000001,-0.6628470000000001,1.796413,1.603016,1.5131629999999998,-0.25566500000000003,0.30847199999999997,3.0203729999999998 -263,0,0.339783,0.975886,0.257314,0.189884,-1.050685,-0.467976,-0.22159,-0.44044799999999995,-0.352281,-0.8675799999999999,0.42111000000000004,0.021025,0.330797,0.294964,-1.278151,-0.913101,-0.586434,-0.527465,-0.966108,-1.1861959999999998,-0.632955,-0.395624,-0.65941,-0.399487,-1.40505,-0.9150870000000001,-0.622812,-1.074175,-1.2607059999999999,-0.9266790000000001 -563,0,1.66097,0.6078600000000001,2.139779,1.6496549999999999,0.365215,1.0454,1.860055,2.125538,0.045693,0.8192780000000001,1.9292259999999999,1.349781,2.1019759999999996,1.968434,0.9635600000000001,2.260135,2.870075,2.540213,1.23176,0.849484,2.010431,-0.34627399999999997,2.916691,1.726318,-0.214002,0.984878,1.543668,2.342934,0.0033539999999999998,0.9146479999999999 -194,0,-0.039178,0.34242399999999995,0.337735,-0.168554,-0.033692,1.3392959999999998,0.895753,0.8845709999999999,0.160555,0.16980399999999998,0.2081,0.912292,0.347273,0.046959,0.57215,1.774977,1.015706,1.02817,-0.272428,0.55604,-0.453216,-0.462029,0.35886799999999996,-0.333042,0.346688,1.438701,0.783046,1.1409610000000001,0.594217,0.19559200000000002 -433,0,1.323425,0.855382,1.133017,1.269946,0.290694,0.585471,0.5709609999999999,0.85564,0.17188,-0.044655,1.3327959999999999,0.623736,1.307005,1.294382,0.38711999999999996,0.654944,0.88639,0.993606,0.479666,-0.20946700000000001,1.493953,1.295707,0.805385,1.398493,0.344354,0.8608030000000001,0.706785,1.0614780000000001,0.39807,0.272377 -244,0,1.114272,0.790245,1.121103,0.942974,0.610696,0.270582,0.353153,0.6363770000000001,0.031132999999999997,-0.432566,1.497524,0.979777,1.529432,1.422367,0.451169,0.975224,1.457641,1.030234,0.607449,-0.39659,0.429958,1.061657,0.5796520000000001,0.44163900000000006,1.189722,0.39356599999999997,0.24059699999999998,0.615401,0.159545,0.058287 -489,0,0.6027779999999999,0.143755,0.5968720000000001,0.357589,-1.3794540000000002,0.24004699999999998,-0.11748399999999999,-0.41471499999999994,2.873574,-0.42757799999999996,0.727846,0.21184499999999998,0.6232479999999999,0.576531,-1.522249,-0.629587,-0.656742,-0.666752,0.125524,-1.353473,-0.569793,-1.177425,-0.540108,-0.38254499999999997,-1.4580520000000001,-0.615799,-0.585676,-1.054223,-0.112881,-0.7924 -465,1,-0.171711,-0.028860000000000004,0.230506,-0.25855900000000004,-0.537805,1.974164,1.8394259999999998,0.321186,-0.090201,2.26951,-0.25200300000000003,0.195555,-0.21003000000000002,-0.31850900000000004,-0.9621770000000001,0.34035,0.153179,-0.531076,-0.768957,0.215815,-0.44816300000000003,-0.731822,0.248971,-0.363624,-0.704022,2.296605,1.489953,0.305581,-0.639573,1.6408909999999999 -483,1,-0.27111,-0.349662,-0.341978,-0.341181,-0.5465720000000001,-0.7612369999999999,-0.470102,-0.36294499999999996,-0.619215,-0.794985,-0.12135699999999999,-0.383884,-0.173371,-0.23830500000000002,0.223439,-0.469447,-0.5438729999999999,-0.44673,-0.29068299999999997,-0.271841,-0.584952,-0.491058,-0.644559,-0.43336899999999995,-1.2653780000000001,-0.96645,-0.49449399999999993,-0.41576599999999997,-0.931372,-0.858972 -397,1,-0.523751,-0.7518859999999999,-0.49269399999999997,-0.509062,-1.623181,-0.464796,-0.393821,-0.481864,-1.476638,-0.743448,-0.37696999999999997,-0.425771,-0.367377,-0.416914,-1.132974,-0.291682,-0.18706,-0.20865100000000003,-0.867532,-0.750991,-0.14895999999999998,0.087354,-0.09804500000000001,-0.214892,-0.5400149999999999,0.5193180000000001,0.44948599999999994,0.331534,-0.22427399999999997,0.102921 -52,1,-0.656284,-0.707918,-0.702684,-0.621217,-0.787669,-1.050935,-0.8643149999999999,-0.786396,-0.18726700000000002,-0.546722,-0.621222,-0.24425999999999998,-0.669713,-0.617993,-0.9785450000000001,-1.0770309999999998,-0.867289,-0.913859,0.20584499999999997,-0.240654,-0.6419779999999999,-1.059492,-0.666341,-0.503115,0.056343,-0.955607,-0.622812,-0.6157699999999999,-0.070504,-0.43873500000000004 -147,1,-0.003974,-0.033745,-0.004802000000000001,-0.124606,-1.432057,-0.013772999999999999,-0.10645,-0.465267,-0.078876,0.456304,0.233661,-0.120926,0.241826,0.098437,-1.066078,0.23422199999999999,0.021352,-0.343038,-0.24687199999999998,0.302288,0.060736,1.255792,0.200458,-0.019957,-0.41701000000000005,1.301771,0.700816,0.5634939999999999,1.576164,2.108031 -27,0,1.043864,0.257745,0.972174,0.9183629999999999,0.062747,-0.270773,0.347396,0.5237,-0.9055620000000001,-0.539518,1.273153,0.22348,1.241101,1.248876,-0.139504,0.042812,0.755818,0.732313,-0.418466,-0.8232889999999999,1.615944,1.146931,1.369225,1.170555,1.2363899999999999,0.097352,0.627208,1.186379,0.289099,0.159658 -436,1,-0.376722,-0.211245,-0.36104,-0.445953,-0.480818,-0.566578,-0.9639110000000001,-0.772997,0.666921,-0.36551300000000003,-0.357089,0.058258000000000004,-0.383029,-0.41435500000000003,-0.35584699999999997,-0.483471,-0.88926,-0.722725,0.180289,0.095318,-0.139576,-0.812016,-0.133193,-0.30443899999999996,-0.377008,-0.6627460000000001,-0.823213,-0.8587600000000001,0.204344,-0.535946 -434,1,0.008451,-0.836565,-0.147774,-0.18121099999999998,-0.46328400000000003,-0.631464,-0.720533,-0.531351,-0.6078899999999999,-0.8686879999999999,0.2081,-0.546779,0.120315,0.0535,-0.506718,-0.636788,-0.6947840000000001,-0.519727,-0.871183,-0.817619,-0.36948200000000003,-0.995265,-0.620798,-0.36120399999999997,-0.42834399999999995,-0.49340100000000003,-0.5730770000000001,-0.5701890000000001,-0.6093029999999999,-0.615379 -190,0,-0.10958599999999999,1.873156,-0.025651999999999998,-0.207756,0.9175469999999999,4.315794,2.766307,0.95309,3.664668,3.3612010000000003,0.026330000000000003,0.891349,0.098896,-0.12795399999999998,0.792763,2.595576,1.372267,0.445226,2.089732,1.789355,-0.430117,1.620475,-0.373282,-0.18959,0.309686,6.143482,2.808608,0.788966,3.7035129999999996,2.997678 -407,1,-0.387077,0.217034,-0.46558900000000003,-0.412728,-1.6810450000000001,-0.385914,-0.424046,-0.8922209999999999,-0.667748,-0.134983,-0.362769,0.484112,-0.384677,-0.399281,-1.483819,-0.401411,-0.34575500000000003,-0.780246,-0.845627,-0.234983,0.33972800000000003,1.0544,-0.155469,0.019865999999999998,-0.34334000000000003,1.07989,0.658375,0.261784,0.744354,1.488834 -202,0,1.832848,1.1403590000000001,2.077228,1.943226,0.9306979999999999,1.033313,2.480375,2.416366,0.480876,0.20360799999999998,2.6023389999999997,1.717457,2.7569,2.929741,1.2624549999999999,1.972072,3.308243,2.9168060000000002,1.417958,-0.07195900000000001,0.53679,0.622586,0.8915209999999999,0.942177,0.762038,1.438142,1.9127049999999999,2.085021,0.146227,0.5295880000000001 -455,1,-0.252473,2.594554,-0.314872,-0.307605,-0.664929,-0.713526,-0.96986,-0.563022,-1.14014,-0.398762,-0.21224099999999999,2.65992,-0.231861,-0.27783800000000003,-0.278277,-0.570079,-0.760948,-0.419904,-1.59407,-0.373909,-0.232332,1.283007,-0.286652,-0.250975,-0.400009,-0.727578,-0.794367,-0.43279799999999996,-0.5935630000000001,-0.306347 -452,1,-0.658355,1.9871459999999999,-0.660984,-0.627369,-0.50712,-0.436806,-0.43699899999999997,-0.661538,-0.7340770000000001,-0.111154,-0.6041810000000001,2.08048,-0.626051,-0.604056,-0.853294,-0.755046,-0.605769,-0.760127,-0.717843,-0.24915900000000002,-0.77227,0.885666,-0.668321,-0.582541,0.097678,0.023019,-0.082353,-0.426634,-0.374411,0.19180999999999998 -385,0,-0.099232,0.9824,-0.150752,-0.215139,-0.051226,-0.611744,-0.022493,0.324232,-0.6855439999999999,-0.863701,0.134256,0.930909,0.08242000000000001,0.027902999999999997,-0.678938,-0.719796,-0.061511,0.09778099999999999,-0.674032,-1.2244709999999999,0.037998000000000004,0.744147,0.023732,-0.161208,0.423691,-0.450366,0.066191,0.642977,-0.380465,-0.349468 -188,1,-0.766038,0.130727,-0.8248059999999999,-0.6874899999999999,0.141652,-0.981341,-0.8835049999999999,-1.0894059999999999,0.484112,-1.007782,-0.658144,-0.442061,-0.687837,-0.642736,0.308838,-0.923335,-0.8194540000000001,-0.861239,-0.341796,-0.7084630000000001,-0.791399,1.2866360000000001,-0.9183110000000001,-0.56912,0.263351,-0.933475,-0.5415770000000001,-0.901259,1.313423,-0.6834640000000001 -159,1,-0.809525,-1.217619,-0.8694850000000001,-0.721769,-0.669312,-1.089867,-1.216885,-1.143155,-0.263303,-0.393221,-0.9165969999999999,-1.472952,-0.958868,-0.8193549999999999,-1.509439,-1.2728,-1.076078,-1.0918889999999999,-1.349457,-0.760915,-0.445276,-0.826349,-0.523772,-0.47957299999999997,-0.299672,-1.081135,-0.996658,-1.333386,-0.5051760000000001,-0.427388 -32,0,0.9548180000000001,1.044281,0.8589870000000001,0.814646,1.3602889999999999,0.64654,1.375025,1.06729,1.0179790000000002,0.048444,0.82157,1.0914760000000001,0.858032,0.69513,1.660981,0.857725,1.919664,1.8411959999999998,1.593204,0.14493399999999998,0.7064229999999999,0.328662,0.56084,0.603791,0.409024,0.298554,0.614276,-0.109676,0.057839,0.022353 -264,0,1.099776,0.5948319999999999,0.9900450000000001,0.976374,1.0271370000000002,0.01549,0.559926,1.275894,0.509996,-0.456949,0.8698530000000001,0.6470060000000001,0.808603,0.777609,0.064029,-0.27273000000000003,0.022733,0.421754,0.202194,-0.991984,0.0546,-0.86826,-0.023296,0.11183299999999999,-0.849361,-0.749934,-0.378445,-0.311627,-1.102093,-0.8396819999999999 -245,1,-0.991758,0.616002,-1.000245,-0.839901,0.838642,-0.964802,-0.739243,-0.719399,-0.028725,-0.358309,-1.0358829999999999,0.132725,-1.0400129999999999,-0.902118,0.75718,-0.845823,-0.508342,-0.469944,-0.272428,0.22715500000000002,-0.12008599999999998,2.5312759999999996,-0.172795,-0.376605,2.999795,-0.649332,-0.439122,-0.075612,1.7214580000000002,-0.088853 -535,0,1.66304,-0.032117,1.576826,1.632076,-0.244104,0.376817,0.8209120000000001,1.5256100000000001,0.285125,-0.457503,1.82414,0.365431,1.8877869999999999,1.8575139999999999,0.586383,1.3182459999999998,1.502839,2.148145,1.15144,-0.040772,1.059043,-0.411409,0.910827,1.043825,-0.821026,0.038109,0.270438,0.391552,-0.12862200000000001,-0.41831 -430,0,0.016734,0.308227,0.540279,-0.084174,0.417818,2.89275,3.021056,2.02352,-0.056227,1.7486009999999998,0.21946,0.7540520000000001,0.41729700000000003,0.085638,0.22130500000000003,2.239288,2.316401,1.243034,0.837458,0.876418,-0.549221,-0.620421,0.296989,-0.355263,-0.025327000000000002,2.0484560000000003,1.570525,1.710316,-0.6722640000000001,0.752378 -443,1,-1.103582,-0.385488,-1.129217,-0.904065,-1.5092079999999999,-1.201318,-1.114601,-1.406728,-0.326396,-0.919117,-1.0103209999999998,-0.225644,-1.035894,-0.8930170000000001,-1.063232,-1.131801,-0.8646520000000001,-0.975248,2.045921,-0.725474,-0.806197,2.404272,-0.786633,-0.5988220000000001,1.2263899999999999,-0.680072,-0.396681,-0.11129800000000001,-0.40831300000000004,0.263678 -154,1,-0.310456,-0.8430790000000001,-0.285682,-0.357354,0.676449,-0.18235,0.137744,-0.264733,1.534051,0.13212100000000002,-0.277565,-0.91911,-0.274287,-0.329885,-0.179357,-0.366919,0.051861000000000004,-0.363415,0.037902,-0.103146,-0.484255,-0.76956,-0.5183260000000001,-0.38606599999999996,0.514361,-0.296669,-0.047206,-0.366616,0.865433,-0.11949100000000001 -9,0,-0.24419000000000002,2.443109,-0.28627800000000003,-0.29740900000000003,2.320295,5.112877,3.9954330000000002,1.620015,2.370444,6.846856,-0.47353500000000004,1.1054389999999998,-0.329482,-0.509063,1.582699,2.563358,1.7388720000000002,0.94176,0.797298,2.783096,-0.38825,0.693345,-0.40942,-0.360764,0.036008,2.609587,1.5098479999999999,0.409395,-0.32113600000000003,2.3773459999999997 -63,1,-1.2961690000000001,-1.04989,-1.241212,-1.00286,-1.490797,-0.550038,-0.635617,-0.9704860000000001,0.6167699999999999,0.052877,-1.407089,-1.263516,-1.3497629999999998,-1.1205450000000001,-1.362838,-0.318972,-0.363081,-0.699511,1.932741,0.968562,0.016703,1.9016979999999999,-0.127747,-0.370004,0.565697,0.7769689999999999,0.37223,0.617023,2.58838,0.766752 -329,0,0.302508,-0.076084,0.19178499999999998,0.166328,0.448503,-0.271409,0.29846100000000003,-0.150838,-0.26653899999999997,-0.244706,0.605719,0.602792,0.639724,0.488932,1.433252,0.454059,1.143766,0.796798,0.20949600000000002,0.357575,0.597064,0.435708,0.046999,0.382454,1.173055,0.67525,0.8785379999999999,0.010359,2.382547,0.9048139999999999 -536,0,-0.202773,1.3992799999999999,-0.088202,-0.2677,0.246858,0.12108900000000002,0.725441,0.3288,-0.328014,-0.024151,0.040531,0.758706,0.07418200000000001,-0.07135599999999999,0.529451,0.209585,0.721919,0.321673,0.4176,-0.42210699999999995,-0.7307640000000001,1.15056,-0.480704,-0.47957299999999997,-0.309339,0.019665000000000002,0.48264300000000004,0.15634800000000001,-0.730382,-0.014715 -65,0,0.215534,1.255978,0.218592,0.07825700000000001,1.4216600000000001,0.5555720000000001,0.14494,0.7125100000000001,0.679863,0.286177,0.185378,1.082168,0.223703,0.038142,1.483068,0.825508,0.475842,1.0671190000000002,0.516175,0.530523,-0.171337,0.11638399999999999,-0.20596199999999998,-0.11214400000000001,-0.11266400000000001,-0.132913,-0.289916,0.007115000000000001,-0.18673900000000002,-0.21594499999999997 -438,1,-0.132365,0.379878,-0.189474,-0.231136,-0.9016430000000001,-0.8916459999999999,-1.077804,-0.848216,-0.627304,-0.822139,-0.07875499999999999,0.07222100000000001,-0.13547599999999999,-0.17715699999999998,-0.677515,-0.777787,-0.946385,-0.6703640000000001,-0.9369,-0.8601469999999999,-0.22836199999999998,0.836678,-0.264871,-0.23557399999999998,-0.40167600000000003,-0.811971,-0.864792,-0.771654,-0.04871,-0.555615 -269,1,-0.9482700000000001,-0.076084,-0.9159510000000001,-0.826541,0.049596,0.004675,-0.090138,-0.435575,-0.478468,0.16980399999999998,-0.9705590000000001,0.25605900000000004,-0.9255040000000001,-0.881641,0.8425790000000001,0.46543,-0.054229,-0.522306,-0.524343,0.825384,-0.308126,0.493767,-0.314869,-0.43116899999999997,0.620366,1.250353,1.375893,0.917111,0.8157909999999999,1.443065 -36,0,-0.078524,0.7625609999999999,0.26624899999999996,-0.142361,0.536175,1.0784790000000002,1.182165,0.458226,1.1166639999999999,0.9672379999999999,0.034851,0.565559,0.06841599999999999,-0.062255,0.13305899999999998,0.10345599999999999,0.5411279999999999,0.18212799999999998,0.267911,-0.21939,-0.430117,-0.35897399999999996,-0.10349100000000001,-0.339422,-0.387675,0.24993,0.53901,-0.030193,-0.031758999999999996,0.085899 -98,1,-0.664567,-1.3869770000000001,-0.7238319999999999,-0.647058,0.470421,-0.439986,-0.383747,-0.45856800000000003,-0.208298,0.200283,-0.717787,-1.500877,-0.726143,-0.689095,-0.46473000000000003,-0.5513170000000001,-0.588066,-0.397721,-0.699589,0.42845500000000003,-0.626819,-1.229678,-0.688617,-0.5409579999999999,-0.296005,-0.680631,-0.49615200000000004,-0.796796,-0.488225,-0.470508 -392,0,1.021085,0.6078600000000001,1.0377020000000001,0.841015,1.566318,0.871734,1.358234,1.4844979999999999,0.46308,0.9949459999999999,0.38702800000000004,0.158322,0.429654,0.255431,1.397669,0.9828049999999999,1.259272,1.088786,0.42855299999999996,0.658107,0.872808,0.20710100000000004,0.895481,0.58465,0.07601000000000001,0.212484,0.591066,0.745169,-0.24485700000000002,0.165332 -69,1,-0.581734,-0.963583,-0.643112,-0.572523,-0.12136400000000001,-1.168303,-0.807368,-0.849434,-0.837615,-1.099772,-0.38265,-0.651497,-0.436576,-0.43341,0.138753,-0.9854959999999999,-0.6562399999999999,-0.52308,-0.809117,-0.8884989999999999,-0.60769,-0.623505,-0.690597,-0.48419300000000004,0.307019,-1.110309,-0.5319619999999999,-0.508387,0.24187899999999998,-0.71448 -517,0,1.5450030000000001,-0.072828,1.5857620000000001,1.345536,0.40905,0.487505,0.701933,0.710987,-0.569064,0.41086300000000003,1.636691,0.225807,1.587098,1.590169,0.522334,0.505228,0.656634,1.170811,-0.035117,-0.130081,0.370767,-0.622598,0.39005500000000004,0.42601700000000003,-0.650686,-0.136825,-0.045548000000000005,-0.002618,-1.207431,-0.152777 -92,1,0.018805000000000002,-0.541818,-0.082245,-0.087866,-1.3926049999999999,-0.82994,-0.658165,-0.22088000000000002,-1.413545,-1.21282,-0.243483,-1.05408,-0.297765,-0.29348,-1.623304,-1.019419,-0.705455,-0.578795,-1.5539100000000001,-1.3633959999999998,0.001905,-0.115852,-0.081709,-0.087723,-0.853361,-0.843828,-0.607228,-0.158339,-1.1929020000000001,-0.8926370000000001 -552,1,-0.49683,1.681,-0.570733,-0.502558,-0.393146,-0.940628,-0.8907010000000001,-0.7556390000000001,-0.798788,-1.058764,-0.38549,2.359728,-0.4374,-0.41805200000000003,-0.9678700000000001,-1.17501,-0.86415,-0.8751680000000001,-0.995315,-0.9111809999999999,-0.592892,0.272418,-0.687627,-0.474733,0.5980310000000001,-0.734843,-0.617175,-0.404087,-0.190372,-0.782566 -505,1,-1.1739899999999999,-1.243674,-1.125643,-0.971217,2.990984,0.712699,0.091688,-0.108204,-0.085347,2.906791,-1.26423,-1.431064,-1.1471069999999999,-1.0875540000000001,2.073741,2.19949,0.376658,0.553561,0.8958729999999999,4.671344,-0.471984,0.31414699999999995,-0.5341670000000001,-0.49871400000000005,4.909873,1.307919,0.663017,0.438593,0.699555,2.906898 -566,0,0.561361,1.374854,0.579001,0.42790600000000006,-0.809587,0.35073499999999996,0.32676700000000003,0.41406899999999996,-1.104549,-0.318409,0.702284,2.0455740000000002,0.672676,0.5779529999999999,-0.840484,-0.03868,0.046588,0.105777,-0.809117,-0.8955870000000001,0.184892,-0.257371,0.27669299999999997,0.180698,-0.379342,0.661277,0.510827,0.612157,-0.891416,0.036726999999999996 -163,1,-0.556884,0.48898400000000003,-0.592774,-0.575863,0.562476,-0.130186,-0.49600900000000003,-0.497395,-1.02366,0.380938,-0.507616,0.681912,-0.49918500000000005,-0.5414859999999999,0.344421,-0.053841,-0.440671,-0.533913,-0.9515040000000001,0.6822060000000001,-0.397995,0.796763,-0.45100200000000007,-0.41334799999999994,1.433065,0.350531,-0.021344,-0.072368,-0.212166,0.587461 -358,1,-1.302174,-1.299041,-1.250743,-1.017099,-1.3531520000000001,-0.8235790000000001,-0.8528959999999999,-1.0199719999999999,-0.755108,-0.533977,-1.490873,-0.8842040000000001,-1.451091,-1.177143,-0.955772,-0.5185310000000001,-0.522153,-0.6476649999999999,0.432204,0.483742,0.47976499999999994,-0.030577999999999998,0.698459,-0.223473,1.2963930000000001,0.196276,0.008166,0.5294300000000001,0.947766,0.143394 -442,1,-0.20691500000000002,-1.33161,-0.278832,-0.305847,-1.1032879999999998,-0.936175,-1.137102,-1.240758,-1.685332,-0.878109,-0.098636,-0.814392,-0.14824500000000002,-0.196212,-0.582865,-0.704256,-0.982418,-1.0055049999999999,-1.484542,-0.612066,-0.17639000000000002,-1.330918,-0.312394,-0.24195500000000003,-0.203001,-0.777878,-0.800832,-0.995827,-1.001598,-0.465591 -15,0,0.246596,1.865014,0.5015569999999999,0.110075,1.553167,2.56641,2.064909,0.8617309999999999,2.1310119999999997,2.7793349999999997,0.11721500000000001,1.9199119999999998,0.196105,0.011123000000000001,1.248222,1.045345,0.942887,0.637649,1.7940060000000002,1.130169,-0.126943,-0.333573,0.006406,-0.171329,-0.478012,0.9457549999999999,0.514474,-0.145362,-0.23880300000000002,0.6320939999999999 -503,0,3.007006,-0.294295,3.1048400000000003,3.342525,-0.5465720000000001,0.688526,0.51435,1.7022389999999998,0.01172,-0.619317,2.5455360000000002,0.125743,2.476807,2.921209,-0.20924600000000002,0.43889799999999995,0.98934,1.325317,-1.1194469999999999,-1.128074,3.197139,-0.855742,3.350833,3.077223,-0.42934399999999995,0.451691,0.260823,1.583793,0.177707,-0.339633 -176,1,-1.037316,-0.209616,-1.018414,-0.862051,-0.099446,0.259131,0.366586,-0.236107,-0.463908,1.7873919999999999,-1.199475,-0.286147,-1.1273360000000001,-1.002515,0.044814,0.474905,0.526062,-0.303315,-0.5206930000000001,2.6030599999999997,0.093579,1.894441,0.131649,-0.282878,1.9397520000000001,3.498789,2.911395,2.07691,2.210615,5.342837 -327,1,-0.662496,-0.5581020000000001,-0.7303850000000001,-0.627897,-1.36192,-1.147374,-1.268736,-1.3193270000000001,-1.180585,-0.752315,-0.595661,-0.316399,-0.654061,-0.594102,-1.389881,-1.239824,-1.0954629999999999,-1.11758,-1.5685129999999998,-0.297358,-0.6196010000000001,-0.5572819999999999,-0.693072,-0.514116,-0.770691,-1.03905,-1.018243,-1.272557,-0.71222,-0.771596 -1,0,1.805927,-0.369203,1.535126,1.8904889999999999,-0.375612,-0.43044399999999994,-0.146749,1.087084,-0.24389,0.28119,1.829821,-0.353632,1.6859549999999999,1.908708,-0.8269620000000001,-0.487072,-0.023846,0.548144,0.001392,-0.8686520000000001,0.499255,-0.876244,0.263327,0.742402,-0.605351,-0.692926,-0.44078,0.260162,-0.80545,-0.099444 -404,1,-0.639717,-1.437458,-0.689578,-0.610845,-1.208494,-1.188468,-1.0697450000000002,-1.015252,-0.9751270000000001,-1.341385,-0.507616,-1.009865,-0.563442,-0.528403,-0.678938,-1.1111440000000001,-0.850089,-0.732011,-0.8784850000000001,-0.810531,-0.078941,-0.560729,-0.130717,-0.224133,0.220349,-0.9494030000000001,-0.624138,-0.24268800000000001,1.108801,-0.8937719999999999 -24,0,2.1103389999999997,0.957974,2.077228,2.345788,2.109883,0.6586270000000001,0.946607,1.444909,1.152255,0.648043,0.716485,0.486439,0.742699,0.710203,1.120124,0.7838149999999999,0.79976,1.103489,0.669515,0.071219,1.44956,-0.571797,1.2816049999999999,1.369891,-0.33100599999999997,-0.37212,-0.148667,-0.080478,-0.7097979999999999,-0.375945 -139,1,-0.900641,-1.61333,-0.9153549999999999,-0.785054,0.189871,-0.458434,-0.889933,-0.43390100000000004,-1.2922120000000001,-0.892517,-0.808671,-1.3728879999999999,-0.781338,-0.768161,1.426135,0.175472,-0.53295,-0.02474,-0.148296,-0.294523,-0.240994,0.228873,-0.5024850000000001,-0.30818,1.4097309999999998,0.531055,-0.332357,1.2739719999999999,-0.57419,-0.133486 -101,1,-1.7269009999999998,-0.999409,-1.6933610000000001,-1.222423,1.14111,-0.852841,-1.305831,-1.745063,0.050546,0.547186,-2.029648,-1.36358,-1.984504,-1.454443,1.4688350000000001,-0.543168,-1.114873,-1.2618200000000002,0.432204,2.180614,-0.6535270000000001,0.5282399999999999,-0.650005,-0.671142,1.049716,-0.8181189999999999,-1.057501,-1.9134470000000001,0.732247,0.11540299999999999 -350,1,-0.619009,-0.9668399999999999,-0.704471,-0.594321,-1.437317,-1.205517,-1.159698,-1.0961049999999999,-0.274627,-0.8697969999999999,-0.700746,-0.5165270000000001,-0.752505,-0.665205,-1.476703,-1.289477,-1.010591,-0.962094,-0.5133909999999999,-0.777926,-0.186856,-0.987826,-0.317344,-0.31478,-0.152666,-1.0332379999999999,-0.86025,-0.892175,0.19586900000000002,-0.426253 -90,1,-0.032964999999999994,0.559006,-0.129902,-0.135154,-0.9147930000000001,-0.494058,-0.864986,-0.687423,-0.612744,-0.636496,0.13993599999999998,1.1007850000000001,0.107134,0.022215000000000002,-0.471135,-0.346452,-0.725418,-0.499092,-0.46227700000000005,-0.586549,-0.11936400000000001,-0.192055,-0.290612,-0.144707,-0.724356,-0.40789000000000003,-0.68581,-0.517471,0.037256,-0.341146 -191,1,-0.52168,-0.354547,-0.542734,-0.529278,-1.687182,-1.046355,-1.057031,-1.018297,-1.167642,-0.844305,-0.38549,0.49341999999999997,-0.409803,-0.41947399999999996,-0.631257,-0.726429,-0.724162,-0.52308,-0.42941899999999994,0.010262,1.176342,0.963683,1.1147799999999999,0.29290700000000003,-0.8233600000000001,-0.42353900000000005,-0.33600399999999997,0.9365760000000001,1.596747,0.786799 -383,1,-0.43263500000000005,-0.414799,-0.35836,-0.49236199999999997,0.45288599999999996,0.668169,0.23225500000000002,-0.252247,-0.13226300000000002,1.543562,-0.49341599999999997,-0.421117,-0.466645,-0.546037,0.557917,0.480591,-0.375134,-0.518953,-0.11908900000000001,0.436961,-0.521069,-0.623868,-0.370807,-0.465052,-0.10866400000000001,0.646746,0.519779,-0.20862399999999998,-0.261808,1.168455 -11,0,0.85956,0.261002,0.8709020000000001,0.73554,0.31699499999999997,1.950627,0.596387,1.010951,1.441838,1.1556520000000001,0.469393,-0.325708,0.47908199999999995,0.358672,0.052641999999999994,0.47111499999999995,0.134849,0.44213100000000005,0.110921,-0.28034699999999996,0.363187,-0.420843,0.345502,0.304128,-0.423343,0.8457129999999999,-0.132088,0.16608,-0.055974,0.132046 -314,1,-1.515262,-0.527162,-1.5074969999999999,-1.125913,0.102199,-1.123391,-1.305831,-1.745063,0.39028,-0.154378,-1.570681,-0.160486,-1.560245,-1.2334559999999999,0.785646,-0.869323,-1.114873,-1.2618200000000002,1.282873,1.529934,-0.24676900000000002,2.8306419999999997,-0.318829,-0.495634,4.569858999999999,-0.639831,-1.057501,-1.9134470000000001,4.954254,1.144247 -352,0,3.491579,-0.34152,3.6350279999999997,4.137101,0.9043959999999999,2.159281,1.789052,2.451387,1.276824,0.23297800000000002,3.2953339999999995,-0.425771,3.38711,3.8540739999999998,1.319387,2.500819,3.113641,3.672569,0.5271279999999999,-0.22505999999999998,2.128091,-0.6639649999999999,2.156324,2.480977,-0.224002,0.9474319999999999,0.356978,0.532674,0.339952,-0.155047 -235,1,-0.19449,0.749534,-0.267811,-0.290202,-0.16081600000000001,-0.6550020000000001,-1.006897,-0.532569,-1.091607,-0.430903,-0.027632,0.456187,-0.089755,-0.14644000000000001,-0.402816,-0.6612359999999999,-0.931319,-0.772766,-1.075636,-0.630495,-0.527927,0.5191680000000001,-0.593571,-0.401907,0.11601199999999999,-0.6510090000000001,-0.815388,-0.284862,-0.958009,-0.329042 -305,1,-0.7929579999999999,0.967744,-0.7705960000000001,-0.7103430000000001,-1.618359,-0.751695,-0.9585379999999999,-1.011902,0.555294,-0.9141290000000001,-0.717787,1.210157,-0.7306739999999999,-0.676012,-1.538617,-0.8994559999999999,-0.867038,-0.9231450000000001,0.45045799999999997,-0.569538,-0.555717,1.032628,-0.448032,-0.486834,-0.306339,-0.117823,-0.528315,-0.778304,1.380017,-0.599114 -375,1,0.145126,-1.064546,0.173913,-0.033546,-0.388763,0.004675,-0.291634,0.15978499999999998,0.408076,0.31333099999999997,0.580158,-0.7492340000000001,0.590295,0.38000300000000004,0.173624,0.747807,-0.279842,0.13028199999999998,0.6512600000000001,0.414279,-0.8325440000000001,-1.320576,-0.750991,-0.559439,-0.843694,-0.411243,-0.41060699999999994,0.02658,-0.145573,-0.037411 -215,0,-0.10751500000000001,0.204007,-0.085224,-0.22937800000000003,0.597545,1.16245,0.918301,0.773416,1.179757,1.216609,-0.075915,-0.5491060000000001,-0.041562,-0.216121,0.44405200000000006,0.897523,0.128195,0.183159,1.07477,0.901935,-0.537311,-0.041464,-0.46189300000000005,-0.388266,-0.36034099999999997,0.497521,0.238607,0.414261,-0.139519,0.28939899999999996 -466,1,-0.304244,-0.035373,-0.189474,-0.336611,0.119734,0.6401789999999999,0.854974,0.053198,-0.546415,-0.12223699999999998,-0.280405,0.337507,-0.246689,-0.335573,-0.68392,0.0864,0.247342,-0.356451,-0.911343,-0.368238,-0.324728,-0.777362,-0.274276,-0.284638,0.084677,0.352767,0.472696,0.12066199999999999,-0.597195,-0.21367600000000003 -486,1,0.039513,-0.03863,-0.037566,-0.087163,-0.796437,-0.300672,-0.136674,-0.553125,-0.7211350000000001,-0.996699,0.145616,-0.567723,0.092306,0.031601,-0.708116,-0.708046,-0.46301899999999996,-0.54191,-1.469938,-1.310945,-0.6668810000000001,-0.38256100000000004,-0.690597,-0.447891,-1.168707,-0.64542,-0.460674,-0.916831,-0.9689059999999999,-0.974339 -250,0,1.928106,0.215406,1.728734,1.985416,-0.49396899999999994,0.400354,2.048118,1.460136,0.364396,-0.30233899999999997,1.9349060000000002,0.9937389999999999,1.933096,2.016784,0.308838,1.066192,2.290035,2.1171919999999997,1.436213,-0.541186,2.161296,-0.718577,1.7355479999999999,2.1465509999999997,-0.586017,0.760202,2.098385,1.110141,0.419864,0.456586 -339,0,2.9821560000000003,0.8228129999999999,2.833789,3.5605059999999997,0.838642,0.086101,1.0060959999999999,1.435773,-0.497881,-0.36385,2.664822,1.1589610000000001,2.600377,3.106075,0.7500640000000001,0.454059,1.782814,2.375132,-0.053372,-1.0968870000000002,2.1793419999999997,-0.5304300000000001,1.780101,2.7229959999999997,-0.24966999999999998,-0.32237899999999997,0.130184,0.1969,-0.69648,-0.25604 -103,1,-1.140857,0.187723,-1.0437319999999999,-0.9130299999999999,1.03152,-0.153087,-0.037364999999999995,-0.26062199999999996,-0.450966,0.052877,-1.207427,0.02568,-1.15411,-1.014176,0.294605,-0.139691,-0.34224,-0.48052,0.486968,0.059878,-0.81161,0.009337999999999999,-0.66238,-0.628525,0.6723680000000001,-0.196627,-0.053174,-0.109676,-0.539078,-0.08507 -206,1,-1.211265,-0.400144,-1.196831,-0.9650639999999999,0.400283,-0.8242149999999999,-1.0077610000000001,-0.894201,0.14276,-0.562239,-1.207427,-0.46998599999999996,-1.196536,-1.022424,0.892395,-0.606845,-0.894408,-0.758321,0.44680699999999995,0.007427,-0.691063,0.227059,-0.667826,-0.616204,0.89271,-0.725901,-0.7930729999999999,-0.692495,0.188604,-0.50039 -345,1,-1.116007,-1.009179,-1.0833469999999998,-0.9202379999999999,0.159186,-0.576756,-0.962232,-1.124731,-0.755108,0.051769,-1.098366,-1.065715,-1.061432,-0.947908,0.175047,-0.24165,-0.665279,-0.736396,-0.652127,1.028102,-0.24243800000000001,2.344398,-0.233684,-0.462192,3.4398129999999996,1.186639,-0.191771,0.18879,1.962404,1.120795 -346,1,-0.544459,0.225177,-0.6171989999999999,-0.558987,-0.152049,-0.75742,-1.0896540000000001,-0.969572,-0.033578,-0.17266600000000001,-0.58714,-0.090674,-0.630582,-0.596093,-0.8895879999999999,-0.879367,-1.020585,-1.04288,-0.9369,-0.328545,-0.585313,-0.117666,-0.647034,-0.491014,0.046342,-0.811412,-0.864129,-1.0221049999999998,-0.08745499999999999,-0.393723 -317,0,1.1536170000000001,-0.11028199999999999,1.001959,1.0625120000000001,0.483572,0.140173,0.542655,0.959181,-0.143587,-0.10893699999999999,1.162387,-0.09765499999999999,1.101054,1.058322,0.078262,0.139464,0.30383899999999997,0.788802,-0.016862000000000002,-0.8729049999999999,-0.0038689999999999996,-1.209357,-0.157944,0.188399,-0.740023,-0.496754,-0.25908000000000003,-0.059391,-0.9434799999999999,-0.508333 -499,0,1.5719239999999999,0.827699,1.666184,1.5459379999999998,0.615079,0.670714,1.178807,1.472317,-0.68069,0.334943,1.8355009999999998,0.45386000000000004,1.8877869999999999,1.8916439999999999,0.863928,1.138207,1.632155,1.629686,0.132826,-0.08188200000000001,0.6685270000000001,-0.0015480000000000001,0.663312,0.764624,-0.124998,0.135916,0.289338,0.485634,-1.134784,-0.025685000000000003 -478,1,-0.801242,-0.615097,-0.751235,-0.725988,0.124117,-0.33884000000000003,-0.060394,-0.613574,0.065106,0.435246,-0.749028,-1.09364,-0.74056,-0.7109949999999999,0.586383,-0.418088,-0.448455,-0.7539359999999999,-0.11908900000000001,0.41711400000000004,-0.728238,-0.092266,-0.6430739999999999,-0.57198,-0.694688,-0.242457,0.32050500000000004,-0.6096060000000001,-0.255754,-0.068427 -47,0,-0.124082,0.370108,-0.132881,-0.213029,2.026595,1.032677,1.09581,1.434251,1.616558,1.8815990000000002,-0.271884,-0.14652300000000001,-0.246689,-0.342115,1.383436,0.355511,0.424367,0.631459,1.15509,0.704888,-0.426147,-0.5863109999999999,-0.47971400000000003,-0.353943,-0.169667,-0.11838199999999999,-0.094289,0.0574,-0.376832,-0.057458 -456,1,-0.652142,2.138591,-0.632092,-0.620162,0.360831,-0.325481,0.096485,-0.704325,-0.027107,-0.650904,-0.709266,2.3271490000000004,-0.704313,-0.6819850000000001,-0.198572,-0.352516,-0.215937,-0.741555,-0.04607,-0.161268,-0.330864,2.193808,-0.354471,-0.378585,0.940045,-0.07255199999999999,0.359962,-0.41868500000000003,0.509462,-0.630509 -211,1,-0.614867,-0.11191,-0.656516,-0.587641,-0.191501,-0.421538,-0.600115,-0.692448,-0.591712,-0.22254000000000002,-0.649623,-0.081366,-0.677951,-0.645296,-0.5444359999999999,-0.669764,-0.7797810000000001,-0.902509,-1.017221,-0.315787,-0.660385,-0.63802,-0.703963,-0.510815,-0.5080140000000001,-0.45874899999999996,-0.37944,-0.8190189999999999,-0.530602,-0.330177 -490,1,-0.43470600000000004,1.0279969999999998,-0.432526,-0.452984,-0.296707,-0.46988500000000005,-0.715735,-0.780458,0.322334,-0.10616700000000001,-0.5331779999999999,0.733108,-0.567973,-0.535798,-1.027649,-0.9919389999999999,-0.899681,-0.936557,-0.97706,-0.430613,-0.654249,-0.141253,-0.638124,-0.49057399999999995,-0.648353,-0.750492,-0.7454930000000001,-1.175231,-0.540288,-0.528003 -113,1,-1.058024,-0.47668,-1.0318180000000001,-0.889826,-0.10382899999999999,-0.314031,-0.684551,-0.810759,-0.837615,0.349906,-1.027362,0.20951799999999998,-0.9609280000000001,-0.9120729999999999,1.127241,0.491962,-0.301813,-0.47046000000000004,0.402996,2.12958,-0.258319,1.1668889999999998,-0.40843,-0.449431,1.6130719999999998,0.6702199999999999,0.464738,0.59107,0.281835,1.133656 -359,1,-0.879933,-0.107025,-0.9373959999999999,-0.7752100000000001,0.040829000000000004,-0.9501700000000001,-0.756994,-0.975815,-0.722753,-0.143295,-1.332393,-0.225644,-1.324225,-1.0702049999999999,0.323071,-0.848666,-0.774633,-0.899156,-1.115796,0.962892,0.370767,0.054696,0.198478,-0.216873,-0.068329,-0.9219620000000001,-0.278974,-0.848541,-0.13588599999999998,-0.40923200000000004 -344,1,-0.664567,-1.224133,-0.6880890000000001,-0.640202,0.597545,-0.908185,-0.7843399999999999,-0.547643,-0.219623,-0.326167,-0.686545,-0.8935120000000001,-0.6977220000000001,-0.667195,1.3265040000000001,-0.5975590000000001,-0.611921,-0.423515,0.7206279999999999,0.32071700000000003,-0.218617,-0.866083,-0.25299,-0.347783,0.831707,-0.8108529999999999,-0.5100779999999999,-0.095077,-0.12256800000000001,-0.262848 -532,1,-0.08680700000000001,-0.891932,-0.168624,-0.188419,-0.261639,-0.622558,-0.7272489999999999,-0.41973999999999995,-0.153294,-0.339467,-0.12703699999999998,-0.6887300000000001,-0.173371,-0.225791,-0.255504,-0.602486,-0.89491,-0.7768930000000001,-0.659428,-0.176862,-0.723546,-1.336724,-0.73911,-0.507955,-1.071036,-1.019992,-0.822815,-1.089909,-1.211063,-0.793535 -261,0,0.7415229999999999,0.943318,0.623679,0.5931489999999999,-0.366845,-0.672177,-0.724851,-0.49115299999999995,-0.725988,-1.0415860000000001,0.9152950000000001,0.8773860000000001,0.7838890000000001,0.7912600000000001,-0.693171,-0.7853680000000001,-0.751909,-0.530044,-0.904042,-1.37899,-0.016141,0.1817,-0.143093,0.089611,-0.43834399999999996,-0.805823,-0.644364,-0.669137,-0.778813,-0.8385469999999999 -4,0,1.298575,-1.4667700000000001,1.338539,1.220724,0.220556,-0.313395,0.613179,0.729259,-0.868353,-0.3971,1.7502970000000002,-1.151816,1.776573,1.826229,0.28037199999999995,0.5393399999999999,1.371011,1.428493,-0.009559999999999999,-0.56245,1.270543,-0.790244,1.273189,1.1903569999999999,1.483067,-0.04852,0.828471,1.144205,-0.361092,0.499328 -226,1,-0.983474,-0.9570690000000001,-1.0065,-0.8529100000000001,0.07589800000000001,-0.8840120000000001,-1.179224,-1.065348,-0.46229,-0.069592,-1.047243,-0.891185,-1.044132,-0.925156,0.636199,-0.513983,-1.03147,-0.9481649999999999,-0.08623,0.24133200000000002,-0.7719090000000001,-0.569982,-0.8207899999999999,-0.626544,-0.176,-0.9734360000000001,-0.964097,-1.106941,-0.668632,-0.36838000000000004 -464,1,-0.283535,-0.291038,-0.362232,-0.33924699999999997,-0.18273399999999998,-0.36746599999999996,-0.40581500000000004,-0.153883,-1.0770469999999999,-0.814381,-0.271884,-0.248914,-0.316713,-0.334435,-1.54431,-0.841464,-0.504827,-0.521532,-1.3056450000000002,-1.03593,-0.732208,-0.964965,-0.806929,-0.515876,-0.35734099999999996,-0.589531,-0.538925,-0.540667,-1.163843,-0.802991 -277,0,0.764302,-0.22427199999999997,0.647508,0.624792,-0.353694,-0.879559,-0.24557800000000002,0.22525900000000001,-0.539944,-1.472721,1.329956,0.16064900000000001,1.1916719999999998,1.2716290000000001,-0.50743,-0.862311,-0.107964,0.245323,-0.955155,-1.819865,-0.277447,-0.7055140000000001,-0.24903000000000003,-0.07914199999999999,0.176681,-0.8013520000000001,-0.18779200000000001,0.45968000000000003,-0.15041600000000002,-0.801478 -257,0,0.302508,-0.491336,0.37347800000000003,0.084761,1.9345400000000001,1.247056,0.8189930000000001,1.648946,0.577943,1.9480979999999999,0.338746,-0.46998599999999996,0.462606,0.166127,2.643065,2.351102,1.958584,1.941793,2.140845,1.865906,0.91684,-0.2864,0.591532,0.42073699999999997,1.036382,1.1402510000000001,0.5947140000000001,1.5367520000000001,-0.206112,1.8402290000000001 -260,0,1.6692529999999999,2.195586,1.639376,1.693603,0.8693270000000001,0.25595100000000004,0.511471,0.8388909999999999,0.40484000000000003,-0.21921500000000002,1.7559770000000001,1.808213,1.6859549999999999,1.800632,0.259022,0.084505,0.792227,1.145533,0.008694,-1.003325,-0.026968000000000002,-0.333573,-0.13814300000000002,0.264085,-0.66602,-0.542025,-0.355567,-0.585761,-0.9350040000000001,-0.719776 -301,1,-0.581734,-0.42457,-0.569839,-0.578851,-1.199727,-0.244691,-0.392382,-0.5840350000000001,-0.349045,-0.34944200000000003,-0.47353500000000004,0.139706,-0.47529499999999997,-0.522146,-0.8433299999999999,-0.055736,-0.257368,-0.46246400000000004,-0.11178699999999998,-0.043607,-0.147877,-0.320873,-0.142103,-0.264396,-0.17033299999999998,0.45895699999999995,0.504858,0.362354,0.8303200000000001,0.32382 -537,1,-0.681134,1.060565,-0.629709,-0.690654,1.9476900000000001,0.45060900000000004,-0.636576,0.24657600000000002,-0.158147,0.873031,-0.692225,1.198521,-0.6425270000000001,-0.706729,1.9385270000000001,0.9638540000000001,-0.548016,-0.09309400000000001,1.166043,1.595143,-0.395108,1.380982,-0.350511,-0.42654899999999996,1.946419,0.529378,-0.439122,0.951175,-0.598406,0.767508 -403,1,-0.49890100000000004,-0.432712,-0.523373,-0.526817,-0.664929,-0.371919,-0.437479,-0.46785600000000005,0.6410359999999999,-0.310651,-0.337208,-0.725963,-0.36202199999999995,-0.418905,0.17291199999999998,-0.302863,-0.7010609999999999,-0.645343,-0.27973000000000003,-0.113069,-0.9361280000000001,-0.565809,-0.9249940000000001,-0.637545,-1.384716,-0.705781,-0.522678,-0.728019,-0.223063,-0.689516 -233,0,1.698245,1.905725,1.651291,1.7428240000000002,-0.441366,0.138901,0.6832229999999999,0.634854,-0.750255,-0.036897000000000006,1.8127799999999998,1.982743,1.74774,1.8888,-0.339479,0.057973000000000004,0.8361700000000001,0.8893989999999999,-1.327551,-0.974973,0.428875,-0.050536000000000005,0.44599300000000003,0.652855,-0.673687,-0.271519,0.088738,-0.14374,-0.915632,-0.343416 -487,0,1.592632,0.767446,1.389175,1.51078,0.834259,0.7521399999999999,1.541979,1.391616,0.590885,0.340485,1.508885,-0.10929000000000001,1.488242,1.456496,0.892395,0.7667579999999999,1.717529,1.817982,0.041553,-0.233566,0.580101,0.346806,0.378669,0.602911,-0.58435,0.159389,0.353662,-0.005862,-0.40831300000000004,-0.220862 -141,0,0.756019,-0.066314,0.647508,0.619518,-0.042459,-0.195073,0.038435000000000004,0.106491,-0.17594300000000002,-0.131104,0.563117,-0.288475,0.540867,0.44968400000000003,0.060471000000000004,0.177367,0.07119500000000001,0.271117,0.180289,-0.045025,1.0817809999999999,0.20891500000000002,0.8251870000000001,0.742402,-0.09033,-0.34082199999999996,-0.040575,-0.020461,-0.217009,-0.136512 -68,1,-1.2340440000000001,-0.492965,-1.2438930000000001,-0.977194,0.693984,1.159269,4.700669,0.919592,2.14719,1.859432,-1.4479870000000001,-0.45602299999999996,-1.366651,-1.150124,0.7287140000000001,0.7004279999999999,2.814833,-0.13333299999999998,1.093024,2.503828,-0.280696,-0.041464,-0.485654,-0.49871400000000005,0.836041,3.3858919999999997,9.015603,3.475158,2.594434,2.1802770000000002 -363,1,0.385341,-0.037002,0.296035,0.225746,0.062747,-0.5494020000000001,-0.5080020000000001,-0.35594000000000003,-0.8198200000000001,-1.067077,0.673883,-0.232625,0.6026520000000001,0.521071,0.035563,-0.372605,-0.3789,-0.014681,-1.155957,-0.973555,-0.23919,0.40305,-0.25843499999999997,-0.148667,0.07201,-0.415155,-0.450064,-0.23782199999999998,-0.43616099999999997,-0.678547 -379,0,-0.627292,1.163157,-0.461717,-0.6547930000000001,3.771263,4.348873,2.7250490000000003,2.09813,2.027474,3.122913,-0.865474,-0.10696300000000002,-0.768981,-0.834144,1.796196,2.104733,1.005662,0.380483,1.403355,2.367738,-0.699364,-0.344459,-0.567829,-0.579681,0.12134500000000001,1.118454,0.46374399999999993,0.25854,-0.382886,0.242117 -51,1,-0.331164,-0.405029,-0.333042,-0.393567,-1.028767,-0.611108,-0.802091,-0.43770699999999996,-0.897473,-0.20480700000000002,-0.138398,-0.686403,-0.196026,-0.236314,-1.388458,-0.8291459999999999,-0.881727,-0.817389,-1.674391,-0.463218,-0.7867069999999999,-0.532425,-0.701488,-0.5673600000000001,-0.8546940000000001,-0.766142,-0.6997359999999999,-0.622907,-0.882941,-0.470508 -195,1,-0.494759,-0.598813,-0.49001300000000003,-0.49218599999999996,-0.993698,-0.659455,-0.45954799999999996,-0.49115299999999995,0.199382,-0.801081,-0.34572800000000004,-0.6887300000000001,-0.38879600000000003,-0.393877,-1.206274,-0.9604799999999999,-0.6286189999999999,-0.6486970000000001,0.063458,-0.8686520000000001,-0.761442,-0.5592779999999999,-0.679706,-0.5409579999999999,-0.581017,-0.536994,-0.287926,-0.547642,-0.243646,-0.6214310000000001 -239,0,1.292363,3.125425,1.010895,0.9271530000000001,0.181104,0.758501,0.249047,1.400752,-0.07725800000000001,0.056201999999999995,0.946536,4.651889,0.8827459999999999,0.755709,0.12523099999999998,0.48248599999999997,0.664167,1.010888,-0.009559999999999999,-0.44478900000000005,0.474351,-0.6545310000000001,0.06729500000000001,0.190599,-0.7270220000000001,0.13256199999999999,-0.19475499999999998,0.315313,-1.002808,-0.391832 -161,0,1.192963,-1.281128,1.1717389999999999,1.080091,-0.875341,-0.335023,-0.219671,0.9607040000000001,-0.740548,-1.187883,1.437881,-0.779486,1.414099,1.4280549999999999,-0.6703979999999999,0.268334,0.38293499999999997,1.2316850000000001,-0.257825,-1.5646959999999999,2.146859,-1.058222,2.0320709999999997,1.7373189999999998,0.7883720000000001,0.283464,0.38284,2.618691,1.334007,-0.163746 -169,1,-0.032964999999999994,-0.43596899999999994,-0.079266,-0.15273299999999998,-0.472051,-0.57612,-0.986891,-0.453086,-0.803642,-1.089797,0.239341,-0.544452,0.175098,0.088198,0.155832,-0.483092,-0.788192,-0.286549,-0.11543800000000001,-0.892752,-0.48317200000000005,0.000266,-0.48169399999999996,-0.353283,-0.653686,-0.658275,-0.81625,-0.441719,-0.852671,-0.790131 -212,0,2.452025,-1.1736520000000001,2.419765,2.845036,-0.796437,-0.653093,0.229857,0.683579,-2.026684,-1.5902020000000001,3.971288,-0.19073800000000002,3.9761300000000004,5.244841,1.269571,0.8956280000000001,2.903973,2.852321,-0.597362,-1.069952,8.906908999999999,0.470181,9.461986,10.676614,2.136427,0.125297,1.060902,0.368843,3.303954,0.257626 -120,1,-0.714267,-1.580761,-0.700599,-0.650574,0.983301,-0.097107,-0.29739099999999996,-0.381064,0.18644000000000002,0.071164,-0.771749,-1.9709439999999998,-0.767334,-0.715545,-0.187185,-0.7105100000000001,-0.673942,-0.5852430000000001,-0.527994,-0.23640100000000003,-0.954174,-1.3719219999999999,-0.872768,-0.656467,-0.333673,-0.5694100000000001,-0.555503,-0.8655729999999999,-0.859936,-0.601005 -367,1,-0.40985600000000005,-0.266612,-0.399464,-0.449996,0.19425499999999998,-0.237058,-0.14818800000000001,-0.40070700000000004,0.513232,-0.512365,-0.544538,-0.295456,-0.5626180000000001,-0.5588350000000001,-0.28824,-0.6176470000000001,-0.563459,-0.7389760000000001,-0.42576800000000004,-0.5156689999999999,-0.550304,-0.795142,-0.4911,-0.47891300000000003,-0.40267600000000003,-0.648215,-0.39436,-0.763868,-0.140729,-0.694055 -171,0,0.354279,0.682768,0.27816399999999997,0.198674,0.338913,-0.634009,-0.037364999999999995,0.021222,-0.027107,-0.567226,-0.198041,0.079202,-0.252456,-0.25451599999999996,-0.418473,-0.785747,-0.37940300000000005,-0.37502199999999997,-0.77991,-0.8629819999999999,0.232895,-0.12673800000000002,0.136599,0.067389,-0.346007,-0.8298559999999999,-0.34429299999999996,-0.381702,-0.646838,-0.728854 -348,1,-0.778463,-0.795854,-0.8218270000000001,-0.711573,0.9087799999999999,-0.9050040000000001,-0.83457,-0.748025,-0.08049400000000001,0.20416199999999998,-0.754709,-0.758542,-0.780514,-0.717252,-0.398546,-0.8619319999999999,-0.790076,-0.662883,-0.648476,0.13075799999999999,-0.846259,-0.826168,-0.8792040000000001,-0.6179640000000001,0.716703,-0.946218,-0.745825,-0.889093,-0.36230300000000004,-0.297269 -409,1,-0.449201,0.521552,-0.543926,-0.47531,-0.366845,-0.47561000000000003,-0.645212,-0.293664,0.8966459999999999,-0.830451,-0.527497,-0.318726,-0.558499,-0.536936,-0.6768029999999999,-0.740642,-0.711733,-0.577505,0.563638,-0.967885,-0.256153,1.023556,-0.38961799999999996,-0.32006100000000004,-0.384342,-0.132913,-0.374798,-0.169694,0.634173,-0.577932 -112,1,-0.200702,-0.31709299999999996,-0.00778,-0.301628,-1.879621,1.0498530000000001,1.94833,0.5465399999999999,-0.813348,1.344065,0.037691,0.083856,0.241414,-0.071072,-1.280286,2.254449,2.655385,0.749595,-0.39291,2.111151,-0.15292999999999998,0.49558199999999997,0.263822,-0.24393499999999999,-0.581017,2.737574,3.700532,1.804398,0.619644,3.474275 -524,1,-1.041457,-0.43759799999999993,-0.981182,-0.886134,0.417818,-0.19062,-0.901208,-0.751071,-0.645099,0.451317,-1.2156639999999999,-0.839989,-1.1932399999999999,-1.028681,-0.102498,-0.381891,-0.822342,-0.638637,-1.550259,0.8664950000000001,-0.56077,-0.0015480000000000001,-0.44060699999999997,-0.552179,0.563697,-0.282697,-0.7053729999999999,-0.8102600000000001,-0.27875900000000003,-0.113817 -62,0,0.290083,0.6241439999999999,0.352628,0.138729,1.386591,2.356484,2.015015,0.972885,-0.091818,1.6211440000000001,0.034851,0.665623,0.183336,-0.026135000000000002,0.607733,1.828041,1.565613,0.970134,0.501572,1.434954,1.077089,0.092797,1.241012,0.449779,0.788706,2.519604,1.230002,1.084187,-0.428896,0.8768229999999999 -393,0,2.06271,0.498754,1.9282990000000002,2.1102279999999998,0.781656,2.01933,2.0778630000000002,1.9428189999999999,1.500078,0.9284469999999999,2.125195,0.695875,2.159642,2.139081,1.4474850000000001,1.9777580000000001,2.413074,2.7672,1.279223,0.462478,0.790518,-0.536961,0.63955,0.8944340000000001,-0.608684,0.658483,0.506848,0.175813,-0.011175,0.08816900000000001 -75,0,0.724957,-0.18193299999999998,0.641551,0.601939,0.772888,-0.316575,0.051389,0.56938,-0.405668,-1.1125180000000001,0.5517569999999999,0.083856,0.49967700000000004,0.46305100000000005,-0.333074,-0.380943,0.111622,0.45038500000000004,-0.049721,-1.259911,1.2351729999999999,-0.364417,1.070722,0.856151,1.259725,-0.192715,0.102995,1.020925,-0.610514,-0.6985939999999999 -124,1,-0.41606800000000005,-0.47668,-0.454866,-0.436812,-1.309316,-0.007411,0.28119,-0.378019,-1.379572,-0.424808,-0.21508200000000002,-0.674768,-0.241747,-0.28836100000000003,-1.794101,-0.58922,-0.098925,-0.539588,-1.422476,-0.647506,-0.8708020000000001,-0.139439,-0.813365,-0.56494,-0.374008,0.40362600000000004,0.586093,-0.22971100000000003,-1.024602,0.106325 -231,1,-0.867508,1.314602,-0.8173600000000001,-0.752884,-1.7682779999999998,-0.706529,-0.78338,-1.427893,-0.083729,-0.724607,-0.797311,1.8128669999999998,-0.832414,-0.73716,-1.9592049999999999,-1.254796,-0.90985,-1.1812129999999998,0.20949600000000002,-0.923939,-1.0256370000000001,-0.588126,-0.89455,-0.69816,-1.129372,-0.5034609999999999,-0.516046,-1.406542,-0.626254,-0.6592560000000001 -285,1,-0.573451,-0.422942,-0.646389,-0.55635,-1.25233,-1.196102,-1.279065,-1.6114950000000001,-0.640246,-1.088135,-0.439453,-0.207027,-0.500009,-0.47180600000000006,-0.884606,-1.1784219999999999,-1.091521,-1.186398,-0.418466,-0.602143,-0.48100699999999996,0.24157399999999998,-0.566839,-0.393546,-0.219335,-0.976398,-0.9958290000000001,-1.4391459999999998,0.625698,-0.673251 -448,1,0.006379999999999999,0.441759,0.024984,-0.088042,-1.028767,0.06765299999999999,0.507153,-0.284223,-0.69525,-0.5162439999999999,0.114375,0.011717,0.093953,0.013683,-0.888165,-0.499201,-0.007901,-0.507346,-1.236277,-0.756662,-0.547416,0.248831,-0.431696,-0.380565,-0.964699,-0.279344,0.062875,-0.478216,-0.799396,-0.555237 -460,0,1.38555,1.435106,1.335561,1.3490520000000001,1.211247,-0.06275499999999999,-0.039764,0.6226729999999999,0.176733,0.368747,0.8386110000000001,1.829157,0.792127,0.785003,0.186433,0.126198,0.14941300000000002,0.396991,-0.067976,0.001756,1.890966,-0.117666,1.576643,1.647112,0.5663630000000001,-0.183773,-0.155298,0.45156899999999994,-0.011175,0.23530900000000002 -218,0,1.959169,0.48246999999999995,1.877663,1.9836580000000001,0.128501,0.440431,0.42943400000000004,1.026178,0.2738,-0.07735,1.61113,0.5283260000000001,1.5541459999999998,1.635674,-0.180069,0.497647,0.48211899999999996,0.979935,1.030958,-0.990566,1.985527,-0.055978999999999994,1.792477,1.8495279999999998,-0.078996,0.34717800000000004,0.08509,0.863582,-0.190372,0.094599 -467,1,-1.060095,-0.17216199999999998,-1.076794,-0.8796299999999999,0.281926,-0.819126,-0.9983569999999999,-1.364398,0.252769,-0.28793,-1.266502,-0.276839,-1.273149,-1.0483049999999998,-0.9429620000000001,-0.94873,-0.929185,-1.113014,-0.48053199999999996,0.18746300000000002,-0.22944499999999998,0.172629,-0.292592,-0.425889,1.3130600000000001,-0.72143,-0.7156520000000001,-1.275315,0.773413,-0.30861700000000003 -525,1,-1.4073719999999998,-1.176908,-1.3094219999999999,-1.063508,1.390974,-0.195709,-0.464345,-0.448975,0.133053,1.161193,-1.578066,-1.440373,-1.5421209999999999,-1.233172,0.5152180000000001,-0.531039,-0.792838,-0.8723299999999999,-0.487834,1.199632,-1.005064,-0.9753069999999999,-0.889599,-0.727884,0.285352,-0.439188,-0.460343,-0.726073,-0.559661,0.049208999999999996 -80,1,-0.654213,1.05568,-0.677068,-0.624908,1.022753,-0.551311,-0.46386499999999997,-0.812129,-0.224476,0.25292800000000004,-0.760389,0.391029,-0.747974,-0.720665,0.9849100000000001,-0.203179,-0.538474,-0.68584,0.110921,1.028102,-0.288997,1.736593,-0.390608,-0.345803,1.1097190000000001,-0.47048599999999996,-0.20006,-0.695902,-0.288445,0.0685 -472,1,0.188613,-1.2143620000000002,0.141149,0.045735000000000005,-1.133973,0.157985,0.205869,0.001428,-0.344192,-0.067376,0.22514,-1.014519,0.184572,0.091042,-1.094545,-0.357254,-0.41945299999999996,-0.43099499999999996,-0.45497600000000005,-0.8658170000000001,-0.579538,-1.421454,-0.514861,-0.374625,-1.2567110000000001,-0.434717,-0.291573,-0.551048,-1.09725,-0.535568 -549,1,-0.67078,0.940061,-0.695833,-0.659188,-0.524654,-0.578665,-1.008672,-1.248067,0.25600500000000004,-0.42591599999999996,-0.939318,1.1449989999999999,-0.95063,-0.834144,-1.027649,-0.7262390000000001,-0.920522,-1.051341,0.600147,0.068384,0.41299399999999997,1.2721209999999998,0.345502,-0.161428,0.407357,-0.378827,-0.6340859999999999,-0.9536530000000001,0.498565,-0.309373 -527,1,-0.550672,-1.043377,-0.5969439999999999,-0.5549430000000001,-0.138898,-0.298127,-0.44659399999999994,-0.115817,0.338511,-0.44475699999999996,-0.507616,-1.633519,-0.536668,-0.5301100000000001,-0.45049700000000004,-0.782146,-0.7434970000000001,-0.5790529999999999,-0.44767399999999996,-0.66877,-1.041517,-1.3084200000000001,-1.03692,-0.690459,-1.1200379999999999,-0.91151,-0.7679739999999999,-0.9823639999999999,-1.120254,-0.9187360000000001 -168,0,1.422825,1.0833629999999999,1.430876,1.370147,0.229323,0.818934,1.040158,0.8754350000000001,-1.19838,0.501745,0.949377,1.254371,0.993959,0.937732,0.607733,1.060506,1.595745,1.428493,-0.998966,0.12083499999999998,2.464469,0.350434,2.213253,1.8033240000000001,-0.289005,0.607064,0.482311,0.631622,-1.100882,0.501219 -351,0,0.225888,-0.245442,0.361564,0.06102899999999999,0.9920680000000001,1.59248,1.9910279999999998,1.505816,2.174692,1.166735,0.460872,-0.016208,0.6232479999999999,0.294964,1.9883419999999998,2.502714,2.543646,1.941793,2.056873,1.8758290000000002,0.415882,0.194401,0.30243400000000004,0.239443,0.762705,2.241833,2.2426180000000002,1.7897990000000001,4.224151,1.3371549999999999 -57,0,0.3315,0.817928,0.25135599999999997,0.184435,0.19425499999999998,1.111558,0.415041,1.047495,1.289766,1.410564,0.165497,0.535307,0.147501,0.005719,1.233989,0.6094609999999999,0.5084850000000001,0.833426,0.786345,0.677954,0.062901,-0.121295,-0.06487799999999999,-0.005436,-1.127372,0.171685,-0.204702,0.149859,-0.287234,0.11842899999999999 -560,1,-0.200702,1.2201520000000001,-0.21032399999999998,-0.305671,-0.36246100000000003,-0.177261,-0.669679,-0.149315,-1.0527799999999998,-0.040776,-0.021952,1.829157,-0.024262,-0.154973,0.20849499999999999,0.156521,-0.55467,-0.151647,-1.002617,-0.15417999999999998,-0.146794,0.49921099999999996,0.010860999999999999,-0.230954,0.071677,0.07276,-0.370819,0.724082,0.031201999999999997,0.570818 -424,1,-1.04767,-0.40828600000000004,-1.05654,-0.8783989999999999,0.325762,-0.75742,-1.205802,-1.046314,0.47763999999999995,-0.213673,-1.245485,-0.039478,-1.237314,-1.038635,0.792763,-0.39818899999999996,-1.002707,-0.754452,2.651977,1.062124,1.051464,0.9618680000000001,0.861819,0.07003,2.0097549999999997,-0.370444,-0.8578620000000001,-0.207002,1.266203,0.162684 -355,1,-0.600372,-0.52879,-0.54333,-0.585707,-0.998082,-0.343929,-0.160182,-0.33431900000000003,-1.261474,-0.668637,-0.445133,-0.051114,-0.413922,-0.480907,-0.623429,-0.010251999999999999,0.178289,-0.129206,-1.017221,-0.135751,-0.162314,0.47381,0.171251,-0.282658,0.937378,0.942961,1.021776,1.275595,0.7080310000000001,0.155119 -42,0,1.619553,1.2201520000000001,2.089143,1.354326,-0.33616,3.117943,2.168536,2.050928,2.8622490000000003,1.100236,1.403799,1.2846229999999998,1.49648,1.277317,-0.394988,2.1729580000000004,1.53046,1.307519,1.819562,0.089648,2.078645,0.814906,2.9523330000000003,1.420495,-0.16433299999999998,4.198526,2.166357,2.365644,3.9698860000000002,1.456682 -217,1,-0.991758,-0.196589,-0.9490129999999999,-0.838319,-1.62625,-0.7287939999999999,-0.382307,-1.2013209999999999,-0.052992,-0.32450500000000004,-1.115407,-0.421117,-1.1088,-0.949046,-1.1258569999999999,-0.857952,-0.389949,-0.9855659999999999,0.5563359999999999,0.049955,-0.174946,1.2793780000000001,-0.058938,-0.38606599999999996,-0.787025,0.319792,0.856986,-0.176182,0.612379,0.309446 -286,1,-0.627292,0.262631,-0.448611,-0.587992,-0.91041,0.17198,-0.171216,0.013609,-0.704957,0.879127,-0.621222,0.342161,-0.580742,-0.608323,-0.7337359999999999,-0.061421,-0.289509,-0.28397,-0.816419,0.689294,-0.472706,0.31414699999999995,0.164321,-0.405427,-0.107331,1.457703,0.397098,0.735436,-0.22064099999999998,1.596257 -135,0,-0.36843899999999996,1.252721,-0.45337700000000003,-0.399017,0.417818,-0.64864,-0.26141,-0.32426900000000003,-0.116085,-0.181532,-0.38549,0.740089,-0.42216000000000004,-0.42260299999999995,-0.413491,-0.8856209999999999,-0.523408,-0.56435,-0.8273719999999999,-0.304446,-0.608051,0.296004,-0.697527,-0.45031099999999996,0.15267999999999998,-0.752169,-0.284279,-0.466537,-0.493068,-0.44100500000000004 -521,0,2.826844,0.204007,2.932082,3.0964169999999998,0.08028099999999999,1.0466719999999998,0.9288559999999999,2.02352,0.414547,0.707338,2.982918,0.5376340000000001,3.028755,3.373421,0.47251899999999997,2.013766,1.785325,2.532475,0.654911,0.651019,2.116181,-0.574155,2.071179,2.190554,-0.684021,0.371211,0.12653699999999998,0.677041,-0.212166,0.36504899999999996 -278,1,-0.15928599999999998,0.068847,-0.248748,-0.24871500000000002,-1.199727,-1.132615,-0.797293,-0.955564,-0.915269,-1.181233,-0.152598,-0.337343,-0.23598000000000002,-0.23489200000000002,-1.201293,-1.209502,-0.86415,-0.9424899999999999,-0.871183,-1.07704,-0.531175,-0.092266,-0.585651,-0.398607,-1.1000379999999998,-1.1292,-0.704378,-1.0952620000000002,-0.859936,-1.009516 -504,1,-1.240257,-1.513995,-1.138153,-1.020263,2.535091,0.571476,-0.29883000000000004,-0.184337,0.222031,2.286134,-1.380108,-1.493895,-1.2554370000000001,-1.155243,4.770911,2.26582,0.106726,0.09287999999999999,2.0678259999999997,4.5678589999999994,0.008763,-0.22471300000000002,0.07323500000000001,-0.44656999999999997,0.9140440000000001,1.114541,0.097358,1.6357,0.590585,2.517299 -97,1,-1.107724,0.099787,-1.145302,-0.904416,-0.033692,-1.014866,-1.207529,-1.3825180000000001,-1.563999,0.32884800000000003,-1.232704,0.151341,-1.2299,-1.0249840000000001,0.42981899999999995,-0.9727979999999999,-1.0291350000000001,-1.057093,-1.685343,0.8650770000000001,-0.253266,1.4989139999999999,-0.36338200000000004,-0.44635,1.363062,-0.606297,-0.881503,-1.061847,-0.306607,0.708123 -84,1,-0.538247,-0.126566,-0.580264,-0.54967,0.23370700000000003,-0.34329299999999996,-0.218232,-0.582969,0.773694,-0.260777,-0.6041810000000001,-0.8469709999999999,-0.618637,-0.601781,0.061894000000000005,-0.6195430000000001,-0.593716,-0.781278,0.976194,-0.441954,-0.642699,0.069211,-0.705448,-0.531937,-0.35734099999999996,-0.411243,-0.39203899999999997,-0.7736,-0.099563,-0.449326 -553,1,-1.330337,-0.102139,-1.322527,-1.027998,-0.9673959999999999,-1.089612,-0.922365,-1.354653,-0.753491,-0.5550350000000001,-1.3616469999999998,0.616755,-1.357589,-1.1117290000000002,-0.281835,-0.915186,-0.613176,-0.931141,-0.436721,0.41995,-0.374896,1.201362,-0.368827,-0.49453400000000003,1.299726,-0.398947,0.26745399999999997,0.16608,2.06411,0.313228 -469,1,-0.602442,-0.045144,-0.569541,-0.619635,2.000293,0.213329,0.22266,0.41102299999999997,-0.38949,0.48512,-0.712106,-0.258223,-0.642115,-0.6999029999999999,1.5044170000000001,0.8330879999999999,0.165734,0.17335799999999998,0.530779,1.382503,0.017786,0.949168,0.079671,-0.274737,2.5164419999999996,0.367857,0.5874189999999999,1.072832,-0.299342,0.53791 -87,0,1.716882,0.770703,1.3593899999999999,1.305104,-0.327392,0.421983,1.455144,1.23326,1.707154,0.495095,1.389599,1.233427,1.236982,1.197683,-0.43199399999999993,0.308132,0.728197,0.8716010000000001,0.516175,-0.9225209999999999,0.52091,-1.003792,0.09353099999999999,0.380914,-1.0563690000000001,-0.39447600000000005,0.172625,0.033069,-0.109249,-0.173203 -22,0,0.37291599999999997,-1.074317,0.531343,0.176348,0.290694,2.170095,1.719008,1.8986619999999998,2.857396,0.8597309999999999,0.344426,-1.170433,0.43377299999999996,0.14081400000000002,0.77853,2.068725,1.492795,1.254641,2.589911,1.0663770000000001,0.12137,-0.9203319999999999,0.256397,0.10061200000000001,-0.083996,1.553833,1.079801,1.739514,1.9587720000000002,0.226609 -19,1,-0.240048,-1.045005,-0.225217,-0.297761,0.5098729999999999,-0.489605,-0.159223,0.216123,0.123347,-0.629292,-0.166799,-1.147162,-0.185728,-0.25195700000000004,0.101747,-0.43685,-0.27821,-0.028609,0.267911,-0.72831,-0.488225,-0.776999,-0.400014,-0.36912399999999995,0.47369300000000003,-0.607974,-0.266043,0.21961,-0.089876,-0.565449 -265,0,3.3590459999999998,3.4983370000000003,3.179304,4.485168,0.338913,0.064472,0.345477,0.78103,-0.052992,-0.097854,1.8752630000000001,2.753002,1.8012869999999999,2.1732099999999996,-0.11886600000000001,0.18873800000000002,0.601392,0.9683280000000001,-0.15559800000000001,-0.8587290000000001,2.767643,0.726003,2.417205,3.506257,-0.8300270000000001,-0.5979140000000001,-0.346946,-0.408142,-0.8320879999999999,-0.565828 -38,0,-0.264898,-0.077713,-0.34912600000000005,-0.31955900000000004,-1.6876200000000001,-1.2910780000000002,-1.1907860000000001,-1.303644,-2.1609599999999998,-1.601839,0.24502100000000002,1.375378,0.147089,0.12488699999999998,-0.177222,-1.005016,-0.813805,-0.5140520000000001,-0.9003909999999999,-1.0997219999999999,2.9192299999999998,1.7619939999999998,2.579575,1.444697,-0.052662,-0.81253,-0.45470600000000005,1.196112,-1.5328899999999999,-0.771975 -479,0,0.2321,-0.42782700000000007,0.441986,0.103922,0.23370700000000003,1.220974,1.523269,0.957658,0.67501,0.41086300000000003,0.6028789999999999,0.051276999999999996,0.7344609999999999,0.457647,0.44405200000000006,1.610099,1.692419,1.109679,1.239062,0.422785,-0.326533,-0.419392,0.100957,-0.158788,0.718703,1.6331959999999999,1.621255,1.679497,0.870276,0.96836 -17,0,0.971385,0.944946,0.8798379999999999,0.763667,2.039746,1.0752979999999999,0.9893049999999999,1.411411,1.3027090000000001,1.6765599999999998,0.5687979999999999,0.323544,0.664438,0.40929699999999997,1.4688350000000001,1.854573,1.047093,1.389802,1.286524,1.5256809999999998,0.592011,-0.261,0.48906099999999997,0.304568,-0.004993,-0.026164,-0.000455,0.190412,-0.44221499999999997,0.13129000000000002 -189,1,-0.604513,-0.991267,-0.6139220000000001,-0.586937,-0.998082,-0.56785,-0.623143,-1.011902,-0.677455,-1.157405,-0.518977,-0.788794,-0.541199,-0.5437609999999999,-1.1073540000000002,-0.602865,-0.63226,-0.8351870000000001,-0.527994,-1.14225,-0.602637,-0.691906,-0.58367,-0.48441300000000004,-0.34834,-0.187685,-0.355567,-0.8699530000000001,-0.035391000000000006,-0.782944 -178,1,-0.46991000000000005,0.54435,-0.56835,-0.477771,-2.240829,-1.399158,-1.2675610000000002,-1.60408,-0.97998,-1.41398,-0.317327,0.681912,-0.41021499999999994,-0.36543600000000004,-2.40897,-1.610136,-1.094848,-1.214049,-1.521051,-1.482475,-0.8375969999999999,-0.13581,-0.873758,-0.57198,-1.207709,-1.298098,-1.004615,-1.613035,-0.534235,-1.069167 -164,0,2.431317,0.414075,2.291686,2.6762759999999997,-0.41944799999999993,0.661808,0.588232,1.8270970000000002,1.113428,0.439125,2.596659,0.640025,2.476807,2.932585,-0.8518700000000001,0.192528,0.547405,1.240713,-0.038768000000000004,-1.030259,0.9348860000000001,-0.6545310000000001,0.8598389999999999,1.265383,-0.7103550000000001,-0.002132,-0.12180899999999999,0.7192149999999999,-0.11893499999999999,-0.020766999999999997 -451,0,1.070784,0.860267,0.969195,0.9500059999999999,0.8956290000000001,-0.44380299999999995,0.602144,0.48715600000000003,-0.983215,-1.276549,1.551487,1.328837,1.471766,1.5247540000000002,0.486752,-0.106715,0.962975,1.0758889999999999,-0.5425979999999999,-1.259911,0.22459400000000002,0.286932,0.024722,0.348571,1.619739,-0.345852,0.5695140000000001,0.518075,-0.500333,-0.751927 -290,1,-0.103373,-0.577643,-0.16564500000000001,-0.199142,-1.426358,-0.044944,-0.24078000000000002,-0.19042699999999999,-1.017189,0.22411199999999998,0.08029299999999999,0.10247300000000001,0.167272,-0.011061,-0.625564,1.198852,0.595114,0.44109899999999996,-0.3564,1.293194,1.7177240000000003,1.003598,0.739546,0.809067,0.24035,4.522686,2.244939,2.581382,2.4491400000000003,7.211398 -67,1,-0.8157369999999999,-0.29918,-0.8715700000000001,-0.7275699999999999,-0.14766500000000002,-1.03554,-0.6130680000000001,-0.685139,-0.810113,-0.9717620000000001,-0.8001510000000001,-0.05809500000000001,-0.830767,-0.741711,-1.065367,-1.086507,-0.649209,-0.6866140000000001,-1.079287,-0.8686520000000001,-0.47811899999999996,-0.497046,-0.512386,-0.488154,0.747037,-0.908883,-0.373471,-0.45923800000000004,0.15591300000000002,-0.6237 -110,1,-1.080803,-0.68512,-1.059816,-0.902834,0.6282300000000001,-0.49469399999999997,-0.682153,-0.932876,-0.5949479999999999,0.040685,-1.2355450000000001,-0.535144,-1.213835,-1.037213,0.522334,-0.384734,-0.5707399999999999,-0.803203,-0.831023,1.113158,-0.007839,0.375835,-0.058938,-0.384306,2.269766,0.21472,-0.154967,-0.25404299999999996,1.485355,0.318902 -556,1,-1.1636360000000001,-0.45551,-1.173002,-0.937465,-0.257255,-0.8541129999999999,-1.257616,-1.405205,-1.033367,-0.915792,-1.126767,0.06989400000000001,-1.121981,-0.976065,0.28037199999999995,-0.555297,-1.051784,-0.973959,-0.075277,0.07263700000000001,-0.581343,1.584188,-0.6029770000000001,-0.5178560000000001,1.95642,-0.182096,-0.9191030000000001,-0.764679,0.626908,-0.573771 -150,1,-0.436776,-0.255213,-0.48971499999999996,-0.463884,-0.11698,-0.9145469999999999,-0.9166559999999999,-0.786396,0.47763999999999995,-1.085918,-0.32016700000000003,0.346815,-0.348429,-0.385345,1.2197559999999998,-0.539188,-0.7211489999999999,-0.579569,2.659279,-0.27325900000000003,0.054238999999999996,0.190772,0.003436,-0.12226500000000001,-0.007993,-0.7857029999999999,-0.41126999999999997,-0.04317,1.085796,-0.855568 -321,0,1.406258,-0.431084,1.278968,1.364873,-1.182192,-0.639734,-0.055596000000000007,0.424727,0.249533,-1.364106,1.713375,0.08618300000000001,1.611812,1.7608150000000002,-1.1500540000000001,-0.35441100000000003,0.335226,0.731023,0.42490200000000006,-1.678104,0.676106,-0.962606,0.49599099999999996,0.759343,-0.8350270000000001,-0.654921,-0.180497,0.10930699999999999,0.168021,-0.8343860000000001 -526,1,-0.190348,-0.084227,-0.15968800000000002,-0.282643,1.316453,0.36982,-0.032568,0.42777299999999996,0.998566,0.149854,-0.18952,-0.12558,-0.186552,-0.295187,0.792763,0.17926199999999998,-0.587439,-0.44879300000000005,-0.323542,0.05279,-0.7412310000000001,-1.106847,-0.704458,-0.533917,-0.876028,-0.617475,-0.557161,-0.718124,-0.847828,-0.7594920000000001 -18,0,2.28843,0.84724,2.369129,2.6674860000000002,0.825491,0.386359,1.271399,1.8910490000000002,-0.21477,-0.432012,1.6139700000000001,0.665623,1.566503,1.720997,0.138753,-0.031099,0.742007,1.188093,-0.838325,-1.254241,1.274152,-0.362603,1.484567,1.585507,-0.182334,-0.36597199999999996,0.066854,0.553762,-0.845406,-0.68006 -446,0,1.089422,2.094623,1.135996,0.978132,0.338913,0.7756770000000001,1.764105,1.254577,0.115258,0.37705900000000003,1.028901,2.033938,1.043388,0.929199,0.256887,0.512808,1.016961,0.877275,-0.360051,-0.5156689999999999,-0.055841999999999996,-0.253742,0.003436,0.07948999999999999,-0.775691,-0.29778699999999997,0.16831400000000002,-0.11292,-0.98949,-0.468617 -432,0,1.192963,-0.098883,1.153867,1.051965,1.496181,0.254042,1.240695,1.563677,0.21232399999999998,-0.177099,1.719055,0.058258000000000004,1.723026,1.6925560000000002,1.205523,0.844459,1.563102,1.985643,-0.319891,-0.321457,0.100798,-0.39163200000000004,0.070265,0.267385,0.6820350000000001,0.09344,0.781388,1.184757,0.48040299999999997,0.07946900000000001 -523,1,-0.240048,-0.007690000000000001,-0.23325900000000002,-0.314109,0.444119,0.014854,-0.37751,0.21003200000000002,-0.083729,0.35267600000000005,-0.11851700000000001,-0.141869,-0.133416,-0.238589,0.199243,0.050392,-0.438788,-0.286033,-0.3564,0.79845,-0.310652,0.058325,-0.288137,-0.305539,-0.100664,-0.165888,-0.365845,0.279627,-0.223063,-0.018119999999999997 -418,1,-0.542388,-1.426059,-0.570137,-0.551428,-0.042459,-0.595204,-0.855583,-0.49282700000000007,-0.20344500000000001,0.592627,-0.405371,-1.6567900000000002,-0.45676000000000005,-0.454741,-0.605638,-0.879367,-0.818576,-0.6422479999999999,-0.834674,-0.006749,-0.649196,-1.036269,-0.662875,-0.505315,-0.30333899999999997,-0.7180770000000001,-0.756601,-0.5783,-0.413156,0.23417399999999997 -272,0,2.4685919999999997,0.407562,2.640181,2.642876,-0.22657,1.388279,1.480091,1.058154,-0.109614,0.256807,2.164957,0.395683,2.2790939999999997,2.377985,-0.167259,1.7389689999999998,1.640943,1.5445659999999999,-0.330843,-0.121575,2.749597,0.245202,2.9706490000000003,2.562383,-0.451345,1.34816,1.041007,0.618645,-0.15768,0.309824 -153,1,-0.8861450000000001,-1.527023,-0.923695,-0.7731,0.07589800000000001,-1.0468,-0.9644389999999999,-0.9066860000000001,-0.067552,-0.899167,-0.845593,-1.445027,-0.869073,-0.776409,0.083955,-1.008427,-0.8660329999999999,-0.801139,0.067109,-0.24774200000000002,-0.649918,-0.789881,-0.711388,-0.546898,0.659367,-0.921794,-0.6609430000000001,-0.578137,0.404124,-0.823039 -509,0,0.174117,1.734739,0.31092800000000004,0.050658,1.789881,1.542225,1.529986,1.54845,0.181587,1.260942,0.378508,1.084495,0.48732,0.21731999999999999,1.56135,1.566511,1.433786,0.9358280000000001,-0.016862000000000002,1.138675,-0.260123,1.3501379999999998,0.035118,-0.171989,0.832374,1.336982,0.9382209999999999,1.397252,0.061472000000000006,0.834081 -288,1,-0.9130659999999999,-0.545075,-0.8635280000000001,-0.778726,-1.296166,-0.445075,-0.564134,-0.326858,0.087755,-0.767831,-0.8143520000000001,0.155995,-0.751681,-0.741711,-1.1500540000000001,0.260754,0.049475,0.179548,2.860081,-0.066289,0.29389099999999996,1.248535,0.005416,-0.124465,2.899791,3.193632,1.627887,3.742804,1.651232,1.0296370000000001 -258,0,0.7415229999999999,0.971001,1.08536,0.607213,0.790423,1.672634,1.659519,2.003725,0.608681,0.9949459999999999,0.43531099999999995,0.9099649999999999,0.750937,0.33734200000000003,1.034726,3.924075,2.872586,2.290012,2.494987,2.586049,3.200748,2.244609,3.5909230000000005,2.159752,1.773079,1.9266159999999999,1.6719849999999998,2.991773,0.34237399999999996,0.849967 -228,1,-0.428493,0.9172629999999999,-0.494183,-0.45105100000000004,-0.423831,0.579746,0.570481,0.051676,-0.12093800000000002,0.65968,-0.42809200000000003,1.089149,-0.4374,-0.450759,-1.233317,-0.550559,-0.432134,-0.736654,-1.086589,-0.369656,-0.5784560000000001,-0.2737,-0.703468,-0.48023299999999997,-0.6240180000000001,-0.141855,-0.057485,-0.503683,-0.8345090000000001,-0.146725 -476,1,0.037443000000000004,0.257745,0.144127,-0.091558,-0.748217,0.563842,-0.10069299999999999,0.293779,-0.59333,-0.29735100000000003,0.02065,0.288638,0.018164,-0.103779,-0.501736,0.12240799999999999,-0.47921499999999995,-0.47303999999999996,-1.115796,-0.383832,-0.20706799999999997,-0.36078899999999997,-0.057948,-0.20521199999999998,-0.978033,0.41368699999999997,-0.12214100000000001,0.276383,-0.537867,-0.39750599999999997 -423,1,-0.233835,-0.028860000000000004,-0.174581,-0.30198,-0.774519,0.39717399999999997,0.450063,0.397319,-0.253596,0.24627800000000002,-0.132717,-0.037150999999999997,-0.103348,-0.22635999999999998,-0.41206800000000005,0.196319,0.097561,-0.020613,0.132826,-0.140004,-0.6524439999999999,-0.5839529999999999,-0.525752,-0.461532,-1.020368,0.145976,0.15903,0.153104,-0.57298,0.060935 -64,0,0.169975,1.269005,0.135192,0.013566,2.311528,0.965882,0.624693,0.867821,0.780165,1.061445,-0.411051,1.058897,-0.382205,-0.44336499999999995,1.127241,0.414261,0.30132800000000004,0.511001,0.34093,0.439796,0.07336799999999999,-0.070493,0.030168,-0.085302,0.24668400000000001,0.055994,-0.071742,0.179057,-0.5075970000000001,-0.073344 -137,1,-0.817808,-0.5955560000000001,-0.814083,-0.735833,-0.586024,-0.5691229999999999,-0.6346569999999999,-0.454456,-0.363605,-0.9030459999999999,-0.766069,-0.907474,-0.778867,-0.7255,0.0021149999999999997,-0.671849,-0.675072,-0.520243,-0.283381,-0.587967,-0.827491,-0.404696,-0.852967,-0.6087229999999999,-0.636019,-0.573882,-0.582029,-0.51812,-0.669842,-0.834764 -343,0,1.342063,1.462789,1.499383,1.159197,-0.46328400000000003,0.58229,0.965317,1.688535,1.8511360000000001,-0.264102,1.577048,0.556251,1.562384,1.533287,0.11455599999999999,0.560187,1.224118,1.583257,0.9871469999999999,-0.800607,0.7847430000000001,1.9361709999999999,1.1420059999999999,0.601151,-0.761691,0.45839799999999997,0.38317199999999996,1.016059,2.060478,-0.191737 -302,0,1.534649,0.611116,1.535126,1.433432,0.102199,0.539669,1.060308,1.1830120000000002,0.6361829999999999,0.595398,1.6934939999999998,1.063551,1.760097,1.6840240000000002,0.8283459999999999,1.505866,1.751427,2.0398099999999997,1.596855,1.68587,2.4067220000000002,0.9546110000000001,2.444432,1.9903389999999999,0.307686,1.220732,1.47868,1.226932,0.825477,0.806846 -561,1,-0.900641,2.055541,-0.955268,-0.7752100000000001,-1.7402229999999999,-1.267986,-1.305831,-1.745063,-2.1593419999999997,-1.379622,-0.831392,2.345765,-0.8773110000000001,-0.764748,-1.556408,-1.303122,-1.114873,-1.2618200000000002,-2.7441169999999997,-1.102557,-0.328698,4.860893,-0.40843,-0.385626,0.184348,-0.927774,-1.057501,-1.9134470000000001,-0.078979,-0.764788 -8,0,-0.161357,0.8228129999999999,-0.031609,-0.248363,1.662757,1.8183099999999999,1.280035,1.391616,2.389857,1.28865,-0.32016700000000003,0.58883,-0.18408,-0.384207,2.201839,1.6840099999999998,1.219096,1.150692,1.9656,1.572462,-0.35685,-0.389818,-0.22774299999999997,-0.352403,-0.43667700000000004,0.5332899999999999,0.12056800000000001,0.07524299999999999,0.107482,-0.017363 -502,1,-0.558955,-0.696519,-0.6133270000000001,-0.57762,1.1235760000000001,-0.5036,-0.39957800000000004,-0.464354,0.411311,-0.474682,-0.450813,-0.6910569999999999,-0.44151899999999994,-0.507926,1.383436,0.07882,-0.370614,-0.41603500000000004,0.479666,0.470983,-0.532258,-0.221084,-0.643569,-0.48067299999999996,0.887043,-0.548172,-0.203707,-0.030193,0.904178,-0.49736400000000003 -28,0,0.828498,1.796619,1.252161,0.6828029999999999,1.390974,2.269333,1.7334009999999997,1.3368,1.822016,0.82094,0.333066,1.3916680000000001,0.429654,0.22044899999999998,0.8425790000000001,1.23865,0.998129,0.9954120000000001,0.4176,0.36891599999999997,0.122092,-0.371675,0.31283,0.06959,-0.602684,0.284582,0.128194,-0.156717,-0.346563,-0.313156 -457,1,-0.397431,1.392767,-0.47571599999999997,-0.43540500000000004,-0.152049,-0.941264,-0.638975,-0.8307059999999999,-0.738931,-0.8903,-0.260524,1.387014,-0.324127,-0.332729,-0.601368,-0.990992,-0.766849,-0.7284,-0.70324,-0.9863139999999999,-0.710192,0.24157399999999998,-0.768317,-0.500695,-0.42434399999999994,-0.9722620000000001,-0.55683,-0.8670329999999999,-0.856303,-0.743984 -136,1,-0.6086550000000001,-0.032117,-0.628517,-0.586937,-0.230954,-0.9635290000000001,-0.80401,-0.684074,-1.923146,-0.582743,-0.686545,-0.60961,-0.710491,-0.65781,0.621966,-0.8223229999999999,-0.663898,-0.591176,-1.7255040000000001,-0.474559,0.157823,2.342584,0.194023,-0.13128599999999999,-0.154333,-0.652686,-0.174861,0.20663299999999998,-0.841774,-0.03325 -365,0,1.665111,0.11281400000000001,1.6066120000000002,1.5810959999999998,0.014527000000000002,-0.10601300000000001,-0.00954,0.9424319999999999,-0.471997,-0.9196709999999999,1.7928990000000002,0.579522,1.723026,1.8148529999999998,-0.345884,0.165996,0.115389,0.746242,-0.7068909999999999,-1.024589,0.6241329999999999,-0.5444,0.6692520000000001,0.7063189999999999,-0.277671,-0.358707,-0.270021,0.45643599999999995,-0.737646,-0.71448 -384,1,-0.42021000000000003,-1.3527799999999999,-0.317851,-0.451578,-0.69123,0.090554,0.06914,-0.348327,-0.26653899999999997,-0.595488,-0.240643,-1.296095,-0.25451599999999996,-0.321637,-0.905956,-0.352327,-0.47745699999999996,-0.52308,-0.710542,-0.972138,-0.800783,-1.244737,-0.630698,-0.551739,-0.923364,-0.265371,-0.11982000000000001,-0.539856,-0.718274,-0.447057 -81,1,-0.15307300000000001,-0.405029,-0.315766,-0.46704799999999996,0.9306979999999999,1.430264,1.0248059999999999,0.85564,1.013126,0.9783209999999999,-0.223602,-0.7981020000000001,-0.22568200000000002,-0.38363800000000003,0.814113,0.931636,0.35280300000000003,0.540406,0.47601499999999997,0.8820889999999999,-0.430117,-0.364417,-0.6589149999999999,-0.6023430000000001,-0.08233,0.57409,0.26214899999999997,0.329912,0.096584,0.305663 -306,1,-0.385006,-0.8512209999999999,-0.454568,-0.428374,-0.857807,-0.7612369999999999,-1.252098,-1.364398,-0.40405,-0.0053100000000000005,-0.263364,-0.8074100000000001,-0.325363,-0.334435,-0.8006310000000001,-0.982274,-1.09653,-1.177705,-0.655777,-0.546856,-0.775518,-1.1672639999999999,-0.8227700000000001,-0.546458,-1.136372,-0.984614,-1.0201,-1.5264149999999999,-0.859936,-0.45764799999999994 -282,0,1.557428,0.484098,1.3444969999999998,1.3138940000000001,0.851793,0.767407,0.764781,1.6839669999999998,1.115046,-0.336696,1.497524,-0.258223,1.451171,1.393926,0.522334,0.755387,0.926565,1.179323,0.297119,-0.549692,0.23722600000000002,-0.402337,0.018287,0.28212600000000004,-0.462345,-0.19495,-0.043227,0.518075,-0.522127,-0.692164 -475,1,-0.451272,-1.030349,-0.418229,-0.483045,0.010144,0.042844,0.36178899999999997,-0.255445,0.170262,-0.328384,-0.36844899999999997,-0.828354,-0.373967,-0.420896,-0.424166,-0.410318,-0.38229,-0.46788100000000005,-0.389259,-0.519922,-0.92133,-1.3232979999999999,-0.827225,-0.6313850000000001,-0.722689,-0.42018500000000003,0.042649,-0.557375,-0.548764,-0.5696100000000001 -118,0,0.8119310000000001,0.785359,0.68623,0.688077,2.329063,1.515507,2.2232279999999998,1.352027,0.603827,2.286134,0.469393,0.8424799999999999,0.565581,0.363223,1.362087,1.342883,1.563102,1.183192,1.03826,1.490241,0.529933,-0.262814,0.362333,0.402476,0.552696,0.798207,0.915342,0.341267,-0.677107,0.8280290000000001 -55,1,-0.710125,-0.5222760000000001,-0.7580859999999999,-0.658133,-0.327392,-1.062767,-0.8704069999999999,-0.783351,0.655596,-0.752869,-0.7405079999999999,-0.12558,-0.767334,-0.699334,-0.079725,-0.9402020000000001,-0.733704,-0.674233,0.395694,-0.5284270000000001,-0.289719,-0.467653,-0.33813499999999996,-0.371105,0.42902399999999996,-0.936493,-0.610212,-0.5070899999999999,1.409075,-0.53292 -557,1,-1.196769,1.394395,-1.214107,-0.966822,-1.098904,-1.162132,-1.305831,-1.745063,-0.688779,-0.789998,-1.336086,1.999032,-1.347292,-1.090967,-1.076753,-1.0353379999999999,-1.114873,-1.2618200000000002,-0.25417399999999996,-0.312952,0.477599,3.102794,0.372234,-0.247015,1.516402,-0.795763,-1.057501,-1.9134470000000001,1.149967,-0.17812 -284,1,-0.49061800000000005,-0.974982,-0.45099399999999995,-0.500975,-1.451345,-0.143545,0.29846100000000003,-0.196518,-1.4588430000000001,-0.702441,-0.351408,-0.835335,-0.32495100000000005,-0.393308,-1.293808,-0.161864,0.285006,-0.38740399999999997,-1.385966,-0.48873500000000003,-0.410988,0.31233299999999997,-0.25695,-0.375065,-0.207668,0.7898229999999999,1.570857,0.964152,-0.213376,-0.037411 -256,0,1.818352,1.724968,2.124886,1.837752,-0.187118,1.7725080000000002,0.733597,1.21042,-0.133881,0.917364,1.540126,2.206141,1.7147880000000002,1.57026,-0.267602,1.9322740000000003,1.124934,1.689012,0.297119,-0.06770599999999999,1.578769,-0.032392000000000004,2.1246419999999997,1.453497,-0.228336,1.239175,0.22335500000000003,0.550517,-0.143151,0.551149 -426,1,-0.8571540000000001,-0.668836,-0.77,-0.773804,0.014527000000000002,0.288394,0.10416199999999999,-0.327467,0.192911,0.693484,-1.0358829999999999,-1.002884,-1.0082959999999999,-0.9137790000000001,0.128078,-0.057630999999999995,-0.319515,-0.689709,0.41394899999999996,0.900517,-0.279974,-0.163025,-0.149529,-0.43050900000000003,0.107678,0.737287,0.687553,0.136883,0.129276,0.391527 -387,1,-0.15721500000000002,-0.929386,-0.226408,-0.23781599999999997,-2.0834580000000003,-0.8331209999999999,-0.782421,-1.054232,-0.580388,-0.981736,-0.070234,-0.72829,-0.14824500000000002,-0.16577999999999998,-1.8574380000000001,-1.06187,-0.858124,-1.04239,-0.747051,-1.14225,-0.545251,-1.079632,-0.5727800000000001,-0.378805,-1.104371,-0.633124,-0.398339,-0.775384,-0.493068,-0.690272 -497,1,-0.45748500000000003,-0.21775799999999998,-0.43014399999999997,-0.480408,-0.209036,-0.023315,-0.332893,-0.141702,0.217178,-0.40652,-0.47069399999999995,-0.46067700000000006,-0.474471,-0.497118,-0.503871,-0.5314180000000001,-0.661764,-0.65076,-1.042777,-0.331381,-0.9094200000000001,-0.790788,-0.7985140000000001,-0.625444,-1.081703,-0.657716,-0.611207,-0.762408,-0.6274649999999999,-0.852542 -298,1,-0.010186,-0.067942,-0.043523,-0.107027,-1.662195,-0.23896599999999998,-0.555018,-0.5985,-0.428317,-0.398208,0.037691,-0.26055,-0.030853,-0.061970000000000004,-2.177683,-0.9881489999999999,-0.804137,-0.90741,-0.644825,-0.9834780000000001,-0.632233,-0.9939950000000001,-0.596541,-0.43513,-1.290712,-0.654362,-0.6997359999999999,-1.0629819999999999,-1.151735,-0.69519 -157,1,0.403979,0.38964899999999997,0.388371,0.266178,-1.956334,-0.529682,-0.402937,-0.46054700000000004,-0.604655,-1.342493,0.7704479999999999,0.039642000000000004,0.676795,0.640807,-1.5592549999999998,-0.6085510000000001,-0.468292,-0.547068,0.11822300000000001,-1.434276,0.2661,1.529758,0.30342399999999997,0.138015,-1.199375,0.06325900000000001,-0.044222000000000004,-0.11292,-0.646838,-0.860864 -249,1,-0.749471,-0.730716,-0.785787,-0.683447,0.28631,-0.611108,-0.440357,-0.282091,-0.383019,-0.32450500000000004,-0.7405079999999999,-1.014519,-0.745503,-0.707013,0.351537,-0.49768500000000004,-0.571494,-0.506314,0.26060900000000004,-0.158433,-0.537671,-0.324502,-0.5841649999999999,-0.477813,-0.126331,-0.737638,-0.359545,-0.281618,-0.456744,-0.38237600000000005 -342,1,-0.900641,-0.940785,-0.8191470000000001,-0.774507,0.413434,-0.211612,-0.20288,-0.108204,0.6475069999999999,0.37983,-0.8711540000000001,-1.0075379999999998,-0.843535,-0.799162,0.49386800000000003,-0.2534,-0.43728100000000003,-0.400043,-0.13004200000000002,0.889177,-0.884517,-0.715493,-0.748021,-0.6498659999999999,0.12501199999999998,-0.374915,-0.14303,-0.27513000000000004,0.355692,-0.332068 -508,1,0.217604,-1.2892709999999998,0.07562,0.08370599999999999,0.132884,-0.751695,-0.371753,0.321186,-0.971891,-0.645362,0.6170800000000001,-0.835335,0.5243909999999999,0.469024,-0.148756,-0.705393,-0.42108500000000004,-0.08484,-0.36735300000000004,-0.8828280000000001,-0.716327,-1.35396,-0.851482,-0.432709,0.11767799999999999,-0.75161,-0.23852199999999998,0.407773,-0.859936,-0.4637 -267,1,-0.304244,0.710451,-0.28598,-0.385129,-1.396988,-0.5169590000000001,-0.60875,-0.802689,-0.735695,-0.7595189999999999,-0.152598,0.593484,-0.19808499999999998,-0.26703000000000005,-1.195599,-0.412213,-0.603634,-0.7093119999999999,-0.644825,-0.596472,-0.24243800000000001,1.268492,-0.136163,-0.298719,-0.535015,-0.07925900000000001,-0.029965,-0.30449,-0.029337000000000002,-0.297269 -338,1,-1.058024,0.189351,-1.05088,-0.87295,0.34329699999999996,-0.7256130000000001,-0.799692,-0.755486,-0.010929000000000001,-0.404858,-1.158009,-0.409482,-1.135162,-0.978625,0.308838,-0.5890310000000001,-0.799618,-0.803976,0.286166,0.07263700000000001,-0.517099,1.448112,-0.538623,-0.516756,0.254017,-0.6141220000000001,-0.497147,-0.608795,0.055417999999999995,-0.38464499999999996 -360,1,-0.5278930000000001,-0.764914,-0.608859,-0.518379,-1.7288259999999998,-1.342223,-1.288651,-1.496108,-1.080282,-1.592419,-0.450813,-0.28381999999999996,-0.516897,-0.46355799999999997,-1.56566,-1.4752020000000001,-1.099882,-1.121268,-1.035476,-1.5519379999999998,-0.195157,-0.47926499999999994,-0.265861,-0.264836,-0.419343,-1.161672,-1.03121,-1.326735,-0.013597,-1.0352379999999999 -12,0,0.971385,0.694167,1.323647,0.793551,-1.256713,0.8653719999999999,0.439988,0.945477,0.44528500000000004,1.017112,1.432201,1.2822959999999999,1.66536,1.3313549999999998,0.073992,2.680858,1.477729,1.621948,2.137194,2.155097,1.9862490000000002,4.265788,4.061202,1.669114,-1.300712,3.2131939999999997,1.890159,4.720928,2.941929,3.4213199999999997 -210,0,1.443533,0.352195,1.520233,1.3631149999999999,-0.6386270000000001,0.24004699999999998,0.546493,1.178444,0.013337,-1.4017879999999998,1.832661,0.663296,1.760097,1.8063209999999998,-0.388583,0.5772430000000001,0.944142,1.204343,-0.170202,-1.780172,1.64193,0.47743800000000003,2.0607830000000003,1.570106,0.36102199999999995,0.59421,0.762488,2.57165,1.358222,-0.540863 -533,0,1.441462,0.239833,1.332582,1.3437780000000001,-0.993698,-0.005503,0.177564,0.710987,0.516467,-1.1130719999999998,1.8014189999999999,0.32121700000000003,1.760097,1.8319169999999998,-0.341614,0.5109130000000001,0.797249,1.35627,1.293826,-1.220218,1.546286,0.94191,1.1395309999999998,1.321487,-0.701021,0.302466,0.299616,0.8424950000000001,0.9223399999999999,-0.406962 -167,0,0.78294,0.10141499999999999,0.698144,0.666982,-0.682463,-0.26950100000000005,-0.19376500000000002,0.499338,-0.146823,-0.646471,0.7534069999999999,-0.113944,0.713866,0.6581560000000001,-0.548706,-0.23729099999999997,-0.057492999999999995,0.43439300000000003,0.297119,-1.057194,0.699566,0.31596199999999997,0.6251939999999999,0.59411,-0.306005,-0.04349,-0.186797,0.688396,0.044521,-0.114196 -79,1,-0.42021000000000003,-0.139593,-0.458142,-0.454391,-0.152049,-0.255506,-0.475379,-0.538203,-0.196974,-0.264102,-0.359929,-0.30011,-0.36161,-0.42260299999999995,0.21205300000000002,-0.16830799999999999,-0.62661,-0.6646890000000001,-0.341796,-0.400843,-0.504106,-0.221084,-0.538623,-0.43975,-0.582683,-0.49563599999999997,-0.370819,-0.586735,-0.37077899999999997,-0.358168 -369,0,2.358838,0.019993,2.613373,2.366883,-0.130131,0.853922,0.975872,1.958046,-0.25845,0.099426,2.238801,0.607446,2.274975,2.352388,0.707364,1.725703,1.958584,2.609857,0.045204,-0.198126,2.175733,-0.937931,2.3241389999999997,1.977138,-1.021034,0.15268199999999998,0.128194,0.472657,-0.649259,0.00041500000000000006 -437,1,-0.126153,-0.667207,-0.180538,-0.229554,-0.564106,-0.821034,-0.769947,-0.610225,-0.28433400000000003,-0.643146,-0.024791999999999998,-0.770177,-0.090167,-0.12425599999999999,-0.838349,-0.860226,-0.67118,-0.502445,-0.3564,-0.541186,-0.057647000000000004,-0.309987,-0.109926,-0.167149,0.311686,-0.700192,-0.5243359999999999,-0.44609899999999997,-0.05960700000000001,-0.364976 -485,1,-0.515468,-0.756771,-0.28121399999999996,-0.5273439999999999,-0.651778,0.965882,1.043037,0.298347,0.534263,1.078069,-0.476375,-0.670114,-0.37561500000000003,-0.506788,-0.086842,0.8861530000000001,0.823615,-0.011843000000000001,0.9871469999999999,1.481735,-0.04718,-0.017878,1.058346,-0.223253,0.064343,2.751546,2.636192,2.500278,1.426026,2.205998 -408,0,0.996235,-0.043516,0.918559,0.8234360000000001,0.693984,0.758501,0.277832,1.2606680000000001,0.25762199999999996,0.060080999999999996,1.097064,0.31889,1.063983,0.957925,0.5152180000000001,0.493857,0.392979,1.014241,0.658562,-0.29877600000000004,0.175148,-0.623324,0.096502,0.20842,0.063343,0.125297,-0.22559099999999999,0.487256,-0.775181,-0.17358099999999999 -308,1,-0.26904,-1.422803,-0.35091300000000003,-0.319735,-1.8471830000000002,-1.24623,-1.239673,-1.408555,-1.0252780000000001,-1.220579,-0.17815999999999999,-1.531128,-0.258635,-0.252241,-1.608359,-1.29251,-1.0802459999999998,-1.147836,-1.630579,-1.339297,-0.6524439999999999,-0.962425,-0.671786,-0.43887,-1.234376,-1.214599,-0.990192,-1.387564,-0.6952689999999999,-0.8430860000000001 -414,0,0.205179,1.829188,0.08455599999999999,0.08933200000000001,-0.770135,-0.989865,-0.563654,-0.7439140000000001,0.5374979999999999,-1.235541,0.284783,2.448156,0.19528099999999998,0.18375999999999998,-0.936557,-1.1047,-0.526547,-0.555322,0.14743,-1.397419,0.22712,0.744147,0.087591,0.110953,-0.069996,-0.626418,-0.23222199999999998,-0.43944799999999995,1.3279530000000002,-0.7734880000000001 -41,0,-0.710125,1.573523,-0.5969439999999999,-0.644421,2.565776,0.098824,0.624213,0.423205,0.102315,0.671317,-0.9023959999999999,0.479458,-0.8266479999999999,-0.807125,1.8744779999999999,0.330874,0.195866,0.200441,0.304421,0.836725,-0.608412,0.383092,-0.516841,-0.514116,0.341021,-0.43806999999999996,-0.197076,-0.23133299999999998,-0.844196,-0.285543 -276,1,-0.842658,-1.088973,-0.890335,-0.7428640000000001,-0.283557,-1.150045,-1.282059,-1.575895,-0.23094699999999999,-1.1130719999999998,-0.794471,-1.193704,-0.8311780000000001,-0.7346010000000001,-0.18291500000000002,-1.243615,-1.0962040000000002,-1.175848,0.519826,-0.650341,-0.605164,0.11457,-0.6440640000000001,-0.511475,0.461692,-0.921067,-1.0081959999999999,-1.372803,0.3678,-0.8200129999999999 diff --git a/fml_manager/Examples/Homo_LR/data/breast_homo_test.csv b/fml_manager/Examples/Homo_LR/data/breast_homo_test.csv deleted file mode 100644 index aa997deb1..000000000 --- a/fml_manager/Examples/Homo_LR/data/breast_homo_test.csv +++ /dev/null @@ -1,115 +0,0 @@ -id,y,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11,x12,x13,x14,x15,x16,x17,x18,x19,x20,x21,x22,x23,x24,x25,x26,x27,x28,x29 -146,0,-0.523751,0.114443,-0.456653,-0.507831,0.268776,0.9856030000000001,0.854974,1.094698,4.6482790000000005,1.055903,-0.660984,-0.6305529999999999,-0.534609,-0.633919,0.9066280000000001,1.244335,0.9679969999999999,0.650804,3.163109,1.546945,-0.308487,0.37946399999999997,-0.289622,-0.343602,-0.5380149999999999,0.606506,0.483969,1.076077,4.3270669999999996,0.317767 -405,1,-0.801242,-0.015832,-0.729789,-0.71755,0.172337,-0.571667,-0.62842,-0.544141,-1.0511620000000002,-0.367175,-0.905236,-0.162813,-0.8888450000000001,-0.810254,0.287488,-0.5636359999999999,-0.494155,-0.50554,-1.188815,0.475236,-0.092295,0.9546110000000001,0.075215,-0.320281,0.826041,-0.23127899999999998,-0.39767600000000003,-0.039925999999999996,0.029991000000000004,-0.413771 -35,0,0.774656,0.54435,0.7815439999999999,0.612486,1.0490549999999998,0.8221149999999999,1.28915,1.015519,3.174481,0.13212100000000002,0.742046,0.535307,0.746818,0.610375,-0.018522999999999998,0.554502,0.577537,0.290462,0.30807199999999996,-0.8842459999999999,0.203299,-0.539139,0.070265,0.106773,-0.421677,-0.027282,0.167651,0.024958,0.889649,-0.427388 -13,0,0.11820499999999999,0.322883,0.141149,-0.007178,-0.844656,-0.393548,-0.19184600000000002,-0.041207,-0.14844100000000002,-1.167934,0.489274,1.084495,0.48320100000000005,0.363507,-0.8789129999999999,-0.07847799999999999,0.13284,0.12177,0.129175,-1.3350440000000001,-0.006756999999999999,-0.25192800000000004,0.018287,-0.082662,0.909377,0.323146,0.617261,1.317769,1.122119,-0.299917 -166,1,-0.966908,-2.223994,-1.00084,-0.820212,0.492339,-0.8178529999999999,-0.80353,-1.044183,-1.311625,-0.38546199999999997,-0.944998,-2.2292490000000003,-0.955573,-0.845521,-0.029908999999999998,-0.890359,-0.797107,-0.823838,-1.572164,0.170451,-0.83868,-1.4704409999999999,-0.861383,-0.634905,0.256017,-0.87535,-0.693105,-1.046599,-0.9689059999999999,-0.6335350000000001 -173,1,-1.018678,-1.442344,-1.049987,-0.8509760000000001,-0.472051,-1.093302,-1.1169040000000001,-1.0894059999999999,-1.615768,-0.599367,-0.865474,-1.065715,-0.896259,-0.802575,0.301721,-0.889033,-0.818199,-0.595561,-0.89674,0.551787,-0.714162,1.0671,-0.7371300000000001,-0.467692,2.639781,-0.238544,-0.575729,0.654332,1.244409,0.37450500000000003 -531,1,-0.604513,0.510153,-0.6034970000000001,-0.580082,0.9920680000000001,0.26803699999999997,0.017325999999999998,-0.508663,0.49381800000000003,0.30779,-0.697906,0.169958,-0.690308,-0.678856,0.37288699999999997,-0.185933,-0.5875640000000001,-0.705443,0.172987,0.256925,-0.716327,-0.621147,-0.729209,-0.549978,-0.596684,-0.45874899999999996,-0.44741099999999995,-1.0540610000000001,-0.732803,-0.42511800000000005 -388,1,-0.875791,-1.098743,-0.82004,-0.7569279999999999,-0.97178,0.169436,0.143501,-0.485519,-1.203234,1.127944,-0.811511,-0.881877,-0.765686,-0.747684,-0.904533,0.133779,0.14941300000000002,-0.550679,-0.00591,1.37825,-0.269507,-0.271886,-0.147053,-0.38210500000000003,1.1130520000000002,2.303312,2.083133,1.406984,-1.011284,3.228412 -205,0,0.310792,-0.885418,0.31092800000000004,0.191466,0.733436,0.5015,0.29030500000000004,0.161307,0.8319340000000001,0.745575,0.281943,-0.607283,0.280545,0.175512,-0.540877,-0.160348,-0.166973,-0.20968299999999998,-0.794513,-0.416437,-0.48389399999999994,-1.5508170000000001,-0.441597,-0.305759,-0.523014,-0.351441,-0.381429,-0.573596,-0.643205,-0.34568499999999996 -6,0,1.368983,0.322883,1.368325,1.27522,0.51864,0.021215,0.509552,1.196716,0.26247600000000004,-0.014730000000000002,1.170908,0.16064900000000001,1.138125,1.0952950000000001,-0.123136,0.088295,0.300072,0.646935,-0.064325,-0.762332,0.149883,-0.80494,0.15541,0.298627,-0.9090299999999999,-0.651568,-0.310141,-0.228089,-0.829666,-0.611218 -144,1,-0.8944290000000001,-0.807253,-0.877825,-0.772397,-1.085753,-0.839482,-0.837832,-1.225379,-0.971891,-0.9008290000000001,-0.959199,-1.005211,-0.9765799999999999,-0.8520620000000001,-1.311599,-1.003499,-0.83226,-1.058692,-1.5064469999999999,-0.838883,-0.551026,0.040181,-0.524762,-0.497174,-0.164666,-0.428569,-0.388392,-1.003127,-0.464009,-0.542754 -540,1,-0.830233,-0.976611,-0.8483370000000001,-0.743216,0.093432,-0.270137,-0.443716,-0.6916869999999999,-0.924975,-0.144403,-0.7348279999999999,-1.1285459999999998,-0.713374,-0.716683,0.24763600000000002,0.14515,-0.269044,-0.592724,0.023298,0.711976,-0.45754700000000004,0.999969,-0.612877,-0.428529,1.7030759999999998,0.874216,0.783709,0.509965,-0.259386,0.6494939999999999 -30,0,1.424896,1.356941,1.5857620000000001,1.387726,0.733436,1.090566,1.636491,1.068812,0.87885,0.768849,1.2788329999999999,1.354435,1.352314,1.2318120000000001,0.714481,1.5987280000000001,1.796625,1.946952,1.355892,-0.11732200000000001,1.53582,0.452038,1.340514,1.422695,-0.264337,0.46175200000000005,0.665338,-0.03506,-0.057185,0.28939899999999996 -435,0,0.159621,0.8342120000000001,0.197742,-0.019835,1.268234,0.652266,0.646282,1.036837,0.45013800000000004,1.194443,-0.041832999999999995,0.076875,-0.034972,-0.157532,0.6860149999999999,0.169787,0.298817,0.40524499999999997,-0.5206930000000001,0.37458600000000003,-0.665437,-0.478177,-0.625748,-0.47275200000000006,-0.575683,-0.42298,-0.33302,-0.361588,-1.006441,-0.35892399999999997 -31,0,0.11406300000000001,0.397791,0.361564,0.014269,1.37344,2.0562259999999997,2.031327,0.608969,3.009467,3.117372,-0.649623,-0.137215,-0.578271,-0.60946,1.034726,0.8956280000000001,0.41432299999999994,0.074825,1.786704,2.153679,0.279093,-0.339016,0.301444,0.014584999999999999,-0.49668,0.48410699999999995,0.336752,-0.219979,0.264884,0.708123 -203,0,0.60692,2.633636,0.6326149999999999,0.47888500000000006,3.955374,1.6961709999999999,0.923099,1.320051,2.477217,1.366232,-0.090115,1.037954,-0.016848,-0.162367,2.5576659999999998,1.373206,0.841192,1.1050360000000001,1.604157,1.617825,0.5761310000000001,1.293893,0.516288,0.272445,0.594364,0.313085,-0.025654000000000003,0.180679,-0.068082,0.268973 -59,1,-1.400331,-1.673582,-1.410693,-1.064738,1.7942650000000002,-0.829304,-0.74548,-1.072199,0.516467,0.349906,-1.564717,-1.7452189999999999,-1.549947,-1.224071,0.08253200000000001,-0.978294,-0.856115,-1.0606520000000002,-0.46957899999999997,1.286106,-0.899675,-1.156197,-0.900985,-0.704386,1.0230489999999999,-0.834327,-0.40066,-0.9820399999999999,0.04331,-0.380863 -95,0,1.6464740000000002,0.962859,1.454704,1.528359,-0.586024,0.633818,0.660194,0.650081,1.2752059999999998,-0.014730000000000002,1.7417759999999998,0.8704049999999999,1.66536,1.732374,-0.397123,0.5109130000000001,0.72443,0.977872,1.0346090000000001,-0.8941690000000001,1.271986,0.530054,0.8355819999999999,1.045806,-0.341673,0.522112,0.34570500000000004,0.144993,0.729825,0.233039 -37,1,-0.614867,-0.466909,-0.679153,-0.588344,-1.549975,-1.323648,-1.073966,-0.9817530000000001,-1.478256,-1.233324,-0.311646,-0.202373,-0.3855,-0.37283099999999997,-0.46473000000000003,-1.263703,-0.7932140000000001,-0.507861,-1.258183,-0.590802,-0.798617,2.041403,-0.839601,-0.57594,-0.8963629999999999,-1.150159,-0.612202,-0.025327000000000002,0.746776,-0.763275 -331,1,-0.38293499999999997,-0.606955,-0.239812,-0.43294399999999994,-0.156432,0.45188199999999995,0.344038,-0.244025,1.1247530000000001,0.42748800000000003,-0.325847,0.014044,-0.306827,-0.40070300000000003,-0.040583999999999995,0.154625,-0.22259099999999998,-0.500897,-0.184806,0.36891599999999997,-0.49363900000000005,-1.180146,-0.198537,-0.43314899999999995,-0.438011,0.395243,0.39908699999999997,-0.322008,0.8436389999999999,0.29393800000000003 -237,0,1.6464740000000002,0.080246,1.621505,1.528359,-0.41944799999999993,-0.147362,0.209227,0.45518100000000006,-1.072193,-0.702441,1.804259,0.505055,1.6694790000000002,1.8518259999999998,-0.9116489999999999,-0.395347,0.020347999999999998,0.29149400000000003,-1.258183,-1.5632780000000002,1.01862,-0.31905900000000004,1.12765,0.949658,0.306019,0.327058,0.35399400000000003,0.797076,-0.863568,0.052235000000000004 -10,0,0.604849,1.3357709999999998,0.492622,0.47361099999999995,-0.6254770000000001,-0.6308279999999999,-0.6058720000000001,-0.22621,0.076431,0.031819,0.537556,0.919273,0.442011,0.406453,-1.017686,-0.713542,-0.700684,-0.404686,-1.035476,-0.826124,-0.09265599999999999,-0.054164,-0.198042,0.003805,-1.004034,-0.9059209999999999,-0.692442,-0.682114,-0.719485,-0.284787 -389,0,0.942393,0.7755890000000001,1.034724,0.760151,-0.318625,-0.08184,0.531141,1.033791,-0.525383,-0.439216,1.540126,0.912292,1.5211940000000002,1.476405,0.33018800000000004,0.520389,1.21533,1.371746,0.647609,-0.561032,0.741794,2.9376889999999998,1.245963,0.6548350000000001,1.39973,0.865833,1.4189969999999998,3.668187,0.8642219999999999,0.924861 -324,1,-0.52168,-0.6997760000000001,-0.48107700000000003,-0.522949,-0.296707,-0.391004,-0.7459600000000001,-0.899073,-0.387872,-0.24027300000000001,-0.5473779999999999,-0.949362,-0.574975,-0.560257,-0.685343,-0.737042,-0.864527,-0.825385,-0.633872,-0.213719,-0.532979,-0.743071,-0.44902200000000003,-0.469232,-0.5460149999999999,-0.631448,-0.70902,-1.079365,-0.8732540000000001,-0.654338 -179,1,-0.5465300000000001,-1.551449,-0.612433,-0.544747,-0.708765,-1.271103,-1.1735149999999999,-1.137522,-1.808284,-0.596042,-0.37413,-1.449681,-0.43987200000000004,-0.41549200000000003,-0.638374,-1.2621870000000002,-0.999455,-0.91876,-1.2618340000000001,-0.20804899999999998,-0.41965,-0.411772,-0.538623,-0.408067,0.49769399999999997,-1.06828,-0.85259,-0.7117979999999999,-1.197745,-0.167907 -220,1,-0.192419,-1.51888,-0.224919,-0.30655,-0.05561,-0.043670999999999995,-0.461946,-0.518408,-0.842468,0.179225,-0.13555799999999998,-1.42641,-0.168428,-0.244562,0.007097,-0.32655300000000004,-0.626736,-0.60072,-1.648834,0.091065,-0.703695,-1.4210909999999999,-0.7301989999999999,-0.504655,-0.969366,-0.476634,-0.5097470000000001,-0.8332930000000001,-0.8272450000000001,-0.40091 -50,1,-0.681134,0.006966,-0.723236,-0.640026,-1.046301,-1.069447,-1.0408629999999999,-1.179395,-0.756726,-1.0149860000000002,-0.6723439999999999,0.5376340000000001,-0.710491,-0.64558,-0.710963,-1.036286,-0.9068370000000001,-0.974217,-1.155957,-0.555362,0.00371,-0.012434,-0.114381,-0.261096,-0.394676,-0.878591,-0.670227,-0.705796,0.425918,-0.766301 -401,1,-0.5113260000000001,-0.901702,-0.584434,-0.511699,0.220556,-0.6155609999999999,-0.579486,-0.698082,-0.713046,-0.627629,-0.624062,-1.95,-0.652001,-0.603488,-0.543724,-0.983979,-0.78769,-0.79856,-0.768957,-1.0472709999999998,-0.552108,-0.311801,-0.602482,-0.470552,-0.288671,-0.751051,-0.60126,-0.967441,-1.16021,-0.733015 -368,0,2.998723,0.12421300000000002,2.74741,3.977131,0.172337,-0.581845,0.066741,1.026178,-0.632157,-1.053223,2.153597,-0.47463999999999995,2.015476,2.534411,-0.179357,-0.35479,0.351548,0.9216409999999999,-0.345447,-1.737644,2.893965,-0.300915,2.4092849999999997,4.0430980000000005,-0.49101300000000003,-0.80247,-0.362529,0.028201999999999998,-0.958009,-0.75344 -29,0,0.774656,-1.002666,0.8232440000000001,0.608971,-0.301091,0.171344,-0.11172699999999999,0.47193,-0.234183,-0.263547,0.977778,-0.9865950000000001,0.94865,0.8538309999999999,0.15013900000000002,0.21527,0.124931,0.7895760000000001,-0.265127,-0.185367,0.7042579999999999,-0.715493,0.8855799999999999,0.45682,-0.47134499999999996,0.27116799999999996,0.072159,0.282871,-0.15647,-0.020011 -78,0,1.4704540000000001,0.984029,1.877663,1.305104,1.382207,2.303684,2.379147,2.073768,4.10794,0.869706,1.719055,1.089149,2.130809,1.678336,2.2943540000000002,4.568425,3.5982629999999998,2.8755349999999997,3.9955239999999996,2.639918,1.90035,1.212248,2.862733,1.673514,1.1130520000000002,2.396089,2.559931,2.2926490000000004,7.071917,0.8291629999999999 -204,1,-0.26904,-0.168905,-0.333935,-0.356299,0.448503,-0.10474100000000001,-0.024412,-0.19956300000000002,0.183204,0.196958,-0.47069399999999995,-0.160486,-0.44811000000000006,-0.49199899999999996,0.234114,0.027651,-0.109847,-0.27623200000000003,0.41394899999999996,0.132176,-0.032743,-0.313616,-0.182696,-0.22105300000000003,-0.029327,-0.355912,-0.161929,-0.23133299999999998,-0.329612,-0.07901799999999999 -214,0,0.12234600000000001,1.49373,0.230506,-0.121794,1.03152,0.9646100000000001,0.49036199999999996,0.95309,2.949609,1.0337370000000001,0.01781,1.051916,0.037111,-0.12567799999999998,-0.123136,0.497647,0.285006,0.404988,1.545741,0.217232,0.056044000000000004,1.139674,0.33065100000000003,-0.205432,1.279726,0.64954,0.16533,0.725704,2.960091,0.368075 -127,0,1.253017,0.008594,1.219396,1.155681,-1.326851,-0.177261,0.232735,0.109537,-0.09667200000000001,-1.027177,1.383918,-0.08834700000000001,1.294648,1.374017,-1.009858,-0.455991,0.049099000000000004,0.189608,0.49061899999999997,-1.7518200000000002,1.0265600000000001,0.227059,1.1632930000000001,0.8997139999999999,-0.871028,0.102382,0.282043,0.29422600000000004,-0.029337000000000002,-0.419444 -46,1,-1.512777,-0.6053270000000001,-1.489328,-1.1222219999999998,-0.11698,-0.754239,-0.9757610000000001,-1.354653,0.330422,-0.546168,-1.6845709999999998,-0.57005,-1.658278,-1.288347,-0.737294,-0.8511299999999999,-0.9155,-1.109197,-0.15559800000000001,0.316465,-0.8982319999999999,-0.472008,-0.877224,-0.706961,0.642366,-0.50402,-0.530967,-0.9536530000000001,0.6293300000000001,-0.458783 -554,1,-0.49268900000000004,1.6386610000000001,-0.548691,-0.5008,-0.423831,-0.586935,-0.135715,-0.7564,-0.855411,-0.638713,-0.354249,2.241047,-0.390031,-0.39985,-1.076753,-0.873682,-0.337092,-0.657467,-0.89674,-0.810531,-0.698642,0.25971700000000003,-0.675251,-0.517196,0.45702600000000004,-0.22066,0.23496,-0.6774100000000001,-0.43495,-0.375945 -449,0,1.948814,1.041024,1.8151130000000002,2.006511,0.19425499999999998,0.355188,0.8046,1.7266009999999998,-1.02366,-0.5373020000000001,1.980348,0.28631100000000004,1.9001439999999998,2.073666,0.034138999999999996,0.249383,0.858769,1.717385,-0.940551,-0.877158,0.9352469999999999,0.261531,0.829642,0.914235,-0.5246810000000001,-0.264254,-0.00145,0.464546,-1.241333,-0.601383 -564,0,1.901185,0.1177,1.752563,2.015301,0.378365,-0.273318,0.664512,1.629151,-1.360158,-0.709091,2.110995,0.7214729999999999,2.0607860000000002,2.343856,1.041842,0.21906,1.947285,2.320965,-0.312589,-0.931027,2.78208,0.071025,2.3795830000000002,2.604187,1.086384,0.191805,0.6660010000000001,2.0671779999999997,-1.138416,0.16798 -297,0,-0.602442,-0.37246,-0.66009,-0.574808,-0.818354,-1.110223,-1.012222,-0.654838,-1.492816,-0.8199219999999999,-0.6723439999999999,-0.267531,-0.698958,-0.636479,0.236249,-0.856626,-0.777772,-0.355161,-0.70324,0.010262,0.8655889999999999,1.6114030000000001,0.62965,0.193019,-0.481679,-0.8622719999999999,-0.6357430000000001,0.409395,-0.580244,-0.39221 -491,1,0.7353109999999999,-1.181794,0.590915,0.579086,-1.4794,-0.9828680000000001,-0.80305,-0.475012,-1.808284,-1.398463,1.057302,-1.410121,0.932174,0.9590620000000001,-1.279575,-0.799203,-0.556804,-0.184147,-2.159966,-1.469717,0.282341,-0.309987,0.14699500000000001,0.233502,-0.8906959999999999,-0.9615309999999999,-0.6752,-0.707094,-0.910789,-0.9402959999999999 -492,0,1.089422,0.062333000000000006,1.076424,0.9587950000000001,-0.064377,-0.137184,-0.085341,0.5221779999999999,0.5666180000000001,-0.42646999999999996,1.102744,0.295619,1.088697,1.0014399999999999,0.266139,0.46543,0.354059,0.740309,1.1112790000000001,-0.287435,1.261881,0.129084,1.231112,1.086949,0.318687,0.08505599999999999,0.181577,0.759768,1.020413,0.076065 -181,0,2.155897,1.270634,2.062335,2.124291,0.733436,3.207003,1.9468900000000002,2.675218,1.936879,2.4634650000000002,1.9775080000000003,1.6941869999999999,2.089619,1.866047,1.2624549999999999,3.389643,2.007548,2.5969599999999997,2.129892,1.58522,0.810729,-0.823628,0.766278,0.904775,-0.929364,1.235822,0.22633899999999998,0.628378,-0.310239,0.5674130000000001 -427,1,-0.726692,1.036139,-0.702088,-0.6874899999999999,-0.090679,-0.538588,-0.381348,-0.605352,0.103933,-0.405966,-0.944998,0.626063,-0.954749,-0.8389790000000001,-0.594251,-0.889033,-0.661136,-0.8996719999999999,0.746185,-0.429195,-0.35179699999999997,0.7332609999999999,-0.30991799999999997,-0.44305,-0.166,-0.22345399999999999,-0.065774,-0.218356,-0.25454299999999996,-0.41793100000000005 -471,1,-0.552743,1.246207,-0.596349,-0.550197,-1.239179,-0.998771,-1.040815,-0.900443,-0.803642,-0.9728700000000001,-0.59282,2.059536,-0.6227560000000001,-0.58301,-0.629123,-0.840327,-0.817697,-0.6486970000000001,0.15473199999999998,-0.824707,0.725191,2.58752,0.6103430000000001,0.101712,0.15868,-0.554879,-0.57175,0.005493,-0.008754000000000001,0.033322000000000004 -515,1,-0.7867460000000001,-0.431084,-0.837316,-0.706651,0.698367,-0.616197,-0.526713,-0.444407,0.25762199999999996,-0.893071,-0.7916300000000001,-0.158159,-0.7912239999999999,-0.749959,0.607733,-0.36673,-0.574758,-0.592724,0.42125100000000004,-0.097476,-0.585313,-0.375303,-0.680696,-0.487274,0.5120279999999999,-0.506814,-0.361535,-0.11778599999999999,0.45982,-0.975096 -445,1,-0.681134,0.7625609999999999,-0.678558,-0.644597,-0.05561,-0.458434,-0.533909,0.085174,-0.488175,-0.079567,-0.607021,1.303239,-0.5914510000000001,-0.6074689999999999,0.47251899999999997,-0.230469,-0.43175699999999995,-0.159385,0.0306,0.808373,-0.515655,-0.023321,-0.495555,-0.460872,0.426358,-0.292198,-0.283616,0.787343,0.048152999999999994,-0.045732 -506,1,-0.643859,-0.245442,-0.659197,-0.6421359999999999,0.34329699999999996,-0.144817,0.390094,-0.513536,-0.310219,0.24627800000000002,-0.5416979999999999,0.174612,-0.514838,-0.573909,0.94221,0.205794,-0.088504,-0.703122,1.140487,0.870748,-0.808723,-0.763754,-0.9308350000000001,-0.610703,-0.256337,-0.195509,0.25750700000000004,-0.30854499999999996,-0.8889950000000001,0.006845 -93,1,-0.242119,0.042792000000000004,-0.288065,-0.318504,0.067131,-0.5036,-0.643293,-0.5404869999999999,-0.36036999999999997,-0.99282,-0.19236,-0.230298,-0.221152,-0.28381,0.41558599999999996,-0.43002799999999997,-0.615938,-0.544747,-0.633872,-0.8076949999999999,-0.397634,0.283304,-0.379718,-0.332602,-0.385675,-0.590648,-0.436801,-0.394192,-0.206112,-0.748145 -453,1,-0.097161,-1.424431,-0.123945,-0.22973000000000002,0.102199,-0.677266,-0.647131,-0.11734000000000001,-0.47685,-0.32395100000000004,0.114375,-1.2355909999999999,0.077889,-0.030400999999999997,0.9635600000000001,-0.22591999999999998,-0.24920799999999999,0.4135,-0.59006,-0.22505999999999998,-0.357933,-0.8991040000000001,-0.35793600000000003,-0.32204099999999997,-0.302672,-0.724783,-0.522015,-0.070746,0.184972,-0.08771799999999999 -357,1,-0.240048,-0.015832,-0.31338299999999997,-0.32729400000000003,-0.748217,-0.976252,-1.052282,-0.899073,-0.871589,-0.710199,-0.073075,-0.716655,-0.142066,-0.174028,-0.6355270000000001,-0.936601,-0.9262969999999999,-0.723241,-1.415174,-0.56245,-0.544529,0.26516,-0.5589189999999999,-0.43116899999999997,-0.467679,-0.9802540000000001,-0.883294,-0.933377,-0.617779,-0.646017 -544,1,-0.252473,-0.15099300000000002,-0.241004,-0.33749,-0.261639,-0.321664,-0.645212,-0.702802,-1.054398,0.053985000000000005,-0.073075,0.328198,-0.09057899999999999,-0.19934100000000002,-0.041296,-0.048155,-0.651846,-0.65076,-0.699589,0.578721,-0.480646,-0.308173,-0.391104,-0.378805,-0.24766999999999997,-0.21004099999999998,-0.190445,-0.443666,-0.683161,-0.074101 -26,0,0.279729,1.226666,0.450921,0.028683999999999998,0.8824780000000001,2.6083950000000002,1.351518,2.367641,2.20543,2.413591,0.128576,0.521345,0.224115,-0.028694,0.643316,1.56272,0.674211,1.003666,1.607807,0.913276,-0.5438069999999999,-0.42392799999999997,-0.374272,-0.424349,-0.863028,0.283464,-0.168561,0.279627,-0.726749,-0.031737 -356,1,-0.43056400000000006,-0.134708,-0.388443,-0.50871,0.084665,0.073378,-0.07142799999999999,0.170443,0.34336500000000003,-0.042993,-0.30596599999999996,-0.162813,-0.28334899999999996,-0.406391,0.8425790000000001,0.493857,0.090781,0.183417,0.815553,0.313629,-0.34133,0.5318689999999999,-0.136658,-0.412908,0.255351,0.7736149999999999,0.637486,1.1296059999999999,0.975614,0.726279 -227,1,0.029158999999999997,-1.0368629999999999,0.20667800000000003,-0.127243,-0.822738,0.6897979999999999,0.326287,0.354685,0.086138,-0.018055,0.247862,-0.87955,0.225762,0.084216,-0.9002629999999999,0.09966599999999999,-0.298172,-0.28680700000000003,0.253307,-0.5284270000000001,-0.625736,-1.3067870000000001,-0.292097,-0.45009099999999996,-0.974032,0.36841599999999997,0.150741,-0.039925999999999996,-0.803029,-0.22351 -377,1,-0.327023,1.6207479999999999,-0.30236199999999996,-0.351553,-0.9454790000000001,-0.690625,-0.9251950000000001,-0.8648129999999999,-0.334485,-0.739015,-0.18952,2.0758259999999997,-0.250397,-0.263902,-1.508016,-1.081769,-0.955299,-0.973701,-1.4261270000000001,-0.7325619999999999,-0.8527549999999999,-0.121295,-0.725744,-0.559439,-0.699688,-0.75161,-0.808558,-1.073364,-0.741279,-0.798452 -234,1,-1.192628,-1.061289,-1.236744,-0.9575049999999999,0.790423,-1.012194,-0.9622799999999999,-0.6453979999999999,-0.23256500000000002,-0.120021,-1.295188,-0.786467,-1.3081610000000001,-1.067361,-0.8340790000000001,-1.202869,-0.907465,-0.831834,-0.9515040000000001,0.174704,-0.685649,-0.701704,-0.817325,-0.609383,1.533069,-0.8427100000000001,-0.664258,-0.352504,0.39807,-0.09641799999999999 -341,1,-1.1429280000000002,-0.42457,-1.072624,-0.9263899999999999,-0.39753,0.5555720000000001,0.776774,-0.508663,0.131435,0.792678,-1.284111,-0.57005,-1.2492590000000001,-1.064801,-0.821981,-0.228573,-0.057492999999999995,-0.670622,0.819204,1.198214,-0.796813,-0.497046,-0.711388,-0.621924,-0.36234099999999997,0.515964,0.609634,-0.533043,-0.368357,0.089304 -129,0,1.317213,1.286918,1.234289,1.245335,-0.213419,0.8386549999999999,1.415804,0.892184,0.653979,0.039023,1.60829,1.356762,1.5829790000000001,1.527599,0.36577,1.033974,2.080367,1.7019090000000001,1.42526,-0.23640100000000003,0.325291,-0.032392000000000004,-0.050026999999999995,0.505884,-0.669353,0.352767,0.519116,-0.22160100000000002,-0.576612,-0.21594499999999997 -289,1,-0.809525,0.07536,-0.8331459999999999,-0.740579,-0.9016430000000001,-0.999916,-0.944625,-0.800557,0.592503,-0.776144,-0.7831100000000001,-0.093001,-0.815526,-0.7363069999999999,-0.656877,-1.028326,-0.813679,-0.7013159999999999,0.735232,-0.46038199999999996,-0.503745,1.373724,-0.45149700000000004,-0.502675,-0.167666,-0.644302,-0.60126,-0.303679,1.648811,-0.327908 -539,1,-1.572003,1.0117120000000002,-1.571835,-1.154919,1.193713,0.33165100000000003,0.321969,-0.983733,-0.179178,1.2554,-1.827998,1.4312280000000002,-1.797089,-1.3779370000000002,-0.688901,0.29486599999999996,0.046713,-0.90999,0.8228549999999999,2.085634,-0.669769,0.475624,-0.703468,-0.629405,2.809788,2.1848259999999997,2.0101869999999997,0.299092,0.061472000000000006,1.420748 -145,1,-0.64593,-1.492825,-0.6255390000000001,-0.6521560000000001,0.43973599999999996,-0.016316999999999998,-0.8539040000000001,-0.8250719999999999,-0.281099,1.089153,-0.632582,-1.079677,-0.570856,-0.631644,1.340737,0.478696,-0.649083,-0.48722600000000005,0.669515,2.210384,-0.032382,-1.021573,0.0767,-0.336782,1.043049,1.225762,-0.132751,-0.11292,1.298894,2.128835 -138,0,0.47231599999999996,-0.691634,0.42113599999999995,0.159648,0.382749,-0.240875,0.303738,0.793211,0.830317,-0.691358,0.233661,-0.400174,0.201048,0.066014,1.4474850000000001,0.495752,0.817337,0.9626530000000001,0.530779,-0.090388,3.215185,0.426637,2.7488759999999997,1.35449,0.9863799999999999,0.520995,1.123237,2.630045,3.7701059999999997,-0.343416 -410,1,-0.6666380000000001,1.73311,-0.660984,-0.631588,0.56686,-0.585662,-0.43699899999999997,-0.420653,0.116875,-0.359971,-0.78595,-0.400174,-0.802345,-0.7255,-0.553687,-0.9705239999999999,-0.765468,-0.720146,-0.768957,-0.519922,-0.770826,0.613514,-0.7460399999999999,-0.5869409999999999,-0.5500149999999999,-0.868028,-0.6718850000000001,-0.960952,-0.863568,-0.807909 -56,0,2.0440720000000003,0.401048,1.871706,2.222734,1.316453,0.616006,0.555129,1.438819,1.029304,-0.055738,1.443561,-0.167467,1.381147,1.413834,0.636199,0.42373599999999995,0.54615,1.058091,0.384741,-0.45187700000000003,1.163349,-0.043278,0.975676,1.367691,-0.19433399999999998,-0.135149,-0.081026,0.581337,-0.24485700000000002,-0.44894799999999996 -216,1,-0.625221,0.23169,-0.627326,-0.614185,0.35644699999999996,0.320201,0.040354,-0.012276,0.802814,-0.016393,-0.635423,-0.21866300000000002,-0.6033970000000001,-0.633351,-0.194302,0.209585,-0.281725,-0.45137299999999997,0.567289,0.048537000000000004,-0.392942,0.628029,-0.385658,-0.415328,0.6103649999999999,0.9189280000000001,0.9140159999999999,0.9819950000000001,0.894492,0.370723 -281,1,-0.612797,-1.207849,-0.672005,-0.6097899999999999,-1.261097,-1.076762,-0.9827170000000001,-0.482778,0.32395100000000004,-0.945716,-0.678025,-1.226283,-0.730262,-0.647287,-1.297366,-1.154922,-0.8330139999999999,-0.549132,1.0565149999999999,-0.23640100000000003,0.5656640000000001,0.092797,0.421242,-0.05516,0.33102,-0.618034,-0.555503,1.080943,1.049472,-0.677034 -16,0,0.5799989999999999,0.84724,0.480707,0.452516,0.615079,-0.42726400000000003,0.092168,0.704897,0.20747100000000002,-0.098963,0.156977,0.195555,0.11413699999999999,0.084216,0.164372,-0.612909,-0.186433,0.094686,-0.8237209999999999,-0.507163,0.243723,0.041996,0.16283599999999998,0.111393,-0.441011,-0.774525,-0.395023,-0.114542,-0.7800239999999999,-0.6467729999999999 -444,0,0.8512770000000001,-0.5955560000000001,0.775587,0.7232350000000001,-0.266022,0.078468,0.752307,0.59222,-0.095054,-0.093975,1.108425,-0.567723,1.051626,0.95309,-0.49035,0.357406,0.253619,0.35133600000000004,-0.334494,-0.7084630000000001,-0.384641,-1.136239,-0.467833,-0.100484,-0.974699,-0.552085,-0.071079,-0.331417,-0.919264,-0.514007 -547,1,-1.1263610000000002,-0.592299,-1.077688,-0.91971,0.601928,-0.188711,-0.45043199999999994,-0.47623000000000004,-0.339339,0.600939,-1.098366,-0.6305529999999999,-1.075848,-0.950184,-0.540166,-0.44878999999999997,-0.5677270000000001,-0.632962,-0.5206930000000001,0.615579,-1.0494569999999999,-0.351717,-0.92935,-0.7263,1.076384,0.299672,-0.191108,-0.134007,0.269727,0.792473 -270,1,-0.281464,-0.818652,-0.381891,-0.344521,-2.0470740000000003,-1.297121,-1.120358,-1.23756,-0.716282,-1.260478,0.046211,-0.574704,-0.068748,-0.06339199999999999,-2.282296,-1.470464,-1.023849,-1.100607,-1.108494,-1.281175,-0.992432,-0.901826,-1.001031,-0.650526,-1.183041,-1.216611,-0.8974850000000001,-1.328195,-0.6274649999999999,-0.913062 -44,0,-0.008116,0.686025,-0.052459000000000006,-0.245902,0.786039,0.866009,0.48268599999999995,0.701851,1.281677,0.677967,-0.271884,0.586503,-0.269756,-0.350931,0.055489,0.006804000000000001,-0.077958,0.09287999999999999,-0.23956999999999998,-0.145674,-0.762886,-1.096868,-0.758416,-0.56868,-1.230376,-0.65045,-0.5760609999999999,-0.801825,-1.139627,-0.7848350000000001 -34,0,0.8160729999999999,0.257745,0.757716,0.66874,0.536175,2.0746740000000004,1.224383,1.093175,2.215137,2.180844,0.5687979999999999,-0.32803499999999997,0.619129,0.433188,0.5436840000000001,0.9771200000000001,0.58507,0.73773,0.6804680000000001,0.333476,-0.256875,-0.9636950000000001,-0.33813499999999996,-0.11676500000000001,-0.952032,0.178951,-0.174198,-0.44269200000000003,-0.42526400000000003,0.008357999999999999 -371,1,-0.014328,-1.619844,-0.082245,-0.108082,-0.866574,-0.512506,-0.652408,-0.499832,-0.669366,-0.902492,0.301824,-1.414775,0.234,0.16186,-1.190618,-0.66332,-0.688883,-0.5764729999999999,-0.330843,-1.043018,-0.818829,-1.459374,-0.756436,-0.49761400000000006,-0.676354,-0.594002,-0.543235,-0.428418,-0.493068,-0.767057 -310,1,-0.757754,0.142126,-0.7845949999999999,-0.698741,-0.441366,-0.925997,-0.926107,-0.8709040000000001,0.948415,-0.796094,-0.689385,-0.041805,-0.7265550000000001,-0.671746,-0.585,-0.9818950000000001,-0.916128,-0.9657049999999999,0.45410900000000004,-0.215137,-0.884517,0.386721,-0.869798,-0.639305,-0.325673,-0.914808,-0.712005,-0.6744899999999999,0.356903,-0.8071520000000001 -538,1,-1.489377,0.8537540000000001,-1.492009,-1.112026,-0.296707,-1.08694,-1.305831,-1.745063,0.254387,0.855298,-1.817206,1.442863,-1.8119169999999998,-1.3540459999999999,-1.094545,-1.0529629999999999,-1.114873,-1.2618200000000002,0.21314699999999998,1.425031,-0.099152,0.44478,-0.185171,-0.466372,1.8730830000000003,-0.88228,-1.057501,-1.9134470000000001,1.0022520000000001,1.163916 -72,0,1.4601,1.326001,1.320668,1.407063,1.145493,3.086136,1.844223,1.146468,0.666921,2.768252,0.872693,1.2171379999999998,0.9156979999999999,0.780737,0.764297,1.490705,1.009428,0.787254,0.42125100000000004,0.293783,0.669609,-0.31905900000000004,0.41530100000000003,0.6409739999999999,-0.407009,1.714795,0.35233600000000004,-0.085345,-0.638362,0.9471780000000001 -73,0,0.062292999999999994,-0.784455,0.090513,-0.11986,0.382749,0.635726,0.027401,0.36077600000000004,-0.504352,1.055903,-0.092956,-0.814392,-0.06339299999999999,-0.20133099999999998,0.308838,0.44837299999999997,-0.136966,0.045676999999999995,-0.546249,0.40577399999999997,-0.45646400000000004,-1.08199,-0.45001199999999997,-0.373745,-0.774691,-0.269842,-0.47426899999999994,-0.420145,-1.010073,-0.25150100000000003 -279,1,-0.266969,-0.641152,-0.264832,-0.370187,-0.607942,-0.520776,-0.6073109999999999,-0.223469,0.087755,-0.821585,-0.07875499999999999,-0.956343,-0.122707,-0.191946,-0.08541900000000001,-0.520426,-0.5525359999999999,-0.304605,1.089373,-0.604978,-0.567628,-0.539501,-0.512881,-0.46043199999999995,-0.935364,-0.562704,-0.574403,-0.22160100000000002,-0.637151,-0.832873 -196,0,0.025018000000000002,1.356941,0.129234,-0.129529,1.8117990000000002,0.36854699999999996,0.521546,0.802347,0.289978,0.5200319999999999,-0.101476,0.698202,-0.055154999999999996,-0.18767999999999999,1.682331,0.42373599999999995,0.623991,0.421496,0.081713,0.846648,0.772111,1.624104,1.009833,0.206,2.253098,0.44722,0.489274,1.428072,0.76857,0.193323 -74,1,-0.44713,-0.40177199999999996,-0.522778,-0.473728,-0.6473939999999999,-0.445075,-0.48881199999999997,-0.426439,-0.457437,-0.435337,-0.516137,-0.644516,-0.526371,-0.523284,-0.330228,-0.6832189999999999,-0.691519,-0.67578,-0.334494,-0.518504,-0.558244,-0.348088,-0.557434,-0.454491,-0.729023,-0.407331,-0.452053,-0.621447,-0.809083,-0.563936 -23,0,2.671532,1.614234,2.404872,3.048953,0.338913,0.036482,0.207788,1.3139610000000002,-0.127409,-0.481332,1.9973889999999999,0.872732,1.863073,2.130548,-0.148044,-0.040575,0.262407,0.964717,-0.15559800000000001,-1.4201,1.03414,-0.163025,0.71133,1.180456,-0.7710239999999999,-0.720312,-0.488858,-0.22971100000000003,-1.175951,-0.6838420000000001 -334,1,-0.604513,0.453158,-0.677068,-0.591156,-0.44574899999999995,-1.041647,-1.132208,-1.138587,-0.560975,-0.658108,-0.518977,-0.062749,-0.58033,-0.541771,-0.9415389999999999,-1.181075,-1.017496,-1.041668,-0.995315,-0.474559,-0.7982560000000001,0.5717840000000001,-0.8252450000000001,-0.596182,0.280018,-0.952701,-0.825269,-0.8555159999999999,-0.139519,-0.594953 -400,0,0.9382520000000001,0.34242399999999995,1.261096,0.74433,2.4079669999999997,2.146558,3.028252,1.245441,0.556912,1.986889,1.074343,0.40266399999999997,1.3358379999999999,0.9644659999999999,1.8958270000000002,2.9044849999999998,2.888907,1.8282990000000001,1.100326,1.184038,-0.007839,-0.802218,0.127194,0.025806,0.039342,0.654012,0.9866299999999999,-0.20862399999999998,-0.560872,0.49327600000000005 -431,1,-0.701842,-0.450625,-0.525756,-0.641257,0.553709,0.05492999999999999,-0.15298599999999998,-0.622863,-0.557739,0.5344399999999999,-0.490575,-0.374576,-0.432457,-0.532101,0.643316,0.516599,-0.142993,-0.539846,-0.002259,1.1656090000000001,-0.8246040000000001,0.44115200000000004,-0.32774000000000003,-0.547998,0.9863799999999999,0.417599,0.554262,-0.020461,0.16075599999999998,0.8359719999999999 -254,0,1.9529560000000001,-0.180304,1.6632049999999998,1.9186159999999999,0.759738,0.393357,0.76526,1.298734,0.773694,0.30779,1.511725,0.00939,1.4223370000000002,1.462184,0.508101,0.27402,0.616458,0.954141,-0.13004200000000002,-0.8970049999999999,0.688377,-1.057134,0.460844,0.674637,-0.797359,-0.41795,-0.146014,0.141749,-0.8345090000000001,-0.470887 -48,1,-0.519609,-0.8105100000000001,-0.517714,-0.523828,0.746587,-0.245964,0.157414,-0.748025,-0.248743,-0.051859,-0.5899800000000001,-1.084331,-0.5737399999999999,-0.584717,0.47963500000000003,-0.254348,-0.287249,-0.552743,-0.498787,-0.335633,-0.5109630000000001,-0.884408,-0.50397,-0.45031099999999996,-0.5176810000000001,-0.626418,-0.287595,-0.995341,-0.7582300000000001,-0.519303 -130,1,-0.606584,-1.281128,-0.47303500000000004,-0.5895739999999999,0.45288599999999996,0.02694,-0.8301559999999999,-0.498461,0.9192950000000001,0.46905,-0.550218,-1.396158,-0.5309020000000001,-0.56623,0.7287140000000001,-0.17532,-0.756428,-0.518437,0.24965700000000002,0.27110100000000004,-0.738704,-0.726742,-0.442092,-0.552179,-0.08932999999999999,-0.051314,-0.719962,-0.600522,0.735879,0.131668 -555,1,-1.12429,1.5035,-1.1226639999999999,-0.919359,0.264392,-0.529682,-0.346326,-0.355331,-1.091607,-0.061834,-1.0898450000000002,1.936202,-1.0832620000000002,-0.948477,-0.43128299999999997,-0.526112,-0.3617,-0.55558,-0.798164,-0.21655500000000003,-0.668686,1.854525,-0.707428,-0.56934,1.669741,0.105176,0.535363,0.878181,-0.255754,0.432378 -520,1,-1.1802030000000001,-1.276243,-1.174194,-0.9738540000000001,2.3071450000000002,-0.283496,-0.8262700000000001,-0.639307,1.262264,0.325523,-1.372439,-1.254207,-1.318458,-1.129362,2.89926,0.34414,-0.696541,-0.637347,1.407006,2.007666,-0.18541300000000002,-0.157582,-0.236654,-0.45559099999999997,2.806454,-0.004367,-0.329041,0.649465,2.352277,0.040131 -122,0,2.019222,-0.274754,2.193393,2.096165,1.632072,1.082296,1.478172,1.677876,0.519703,-0.213673,2.874993,0.21184499999999998,3.057588,3.145893,3.440117,3.455973,4.243589,3.9279300000000004,3.079138,0.846648,3.983947,3.4529620000000003,3.435978,4.238914,5.429894,4.056567,3.179967,1.0420129999999999,3.018209,2.299805 -496,1,-0.391218,-0.574386,-0.35657300000000003,-0.43399899999999997,0.9175469999999999,0.8265680000000001,0.412642,0.397319,0.532645,1.0725280000000001,-0.419572,-0.26055,-0.382205,-0.481476,0.79988,0.550711,-0.108341,0.046967,-0.622919,0.814044,-0.623571,-1.058948,-0.5792149999999999,-0.482653,-0.44567799999999996,-0.025605000000000003,-0.183482,-0.23944400000000002,-0.35746,-0.08771799999999999 -225,1,0.103709,-1.429316,0.09349099999999999,-0.012979,-0.11698,-0.647368,-0.522875,-0.08993200000000001,0.260858,-1.2870780000000002,0.060412,-1.354271,0.022283,-0.038932999999999995,0.192127,-0.532555,-0.396226,-0.074523,0.96159,-1.179108,0.42165699999999995,-0.734362,0.444013,0.174978,0.016008,-0.625859,-0.274995,0.172569,0.25640799999999997,-0.8820459999999999 -326,1,-0.15307300000000001,-1.2501879999999999,-0.263939,-0.22973000000000002,-0.187118,-0.9120020000000001,-1.051226,-0.848216,-1.295447,-0.726824,-0.004911,-1.491568,-0.07986900000000001,-0.10918299999999999,-0.232731,-0.971851,-0.893278,-0.55687,-1.601372,-0.8218719999999999,-0.534423,-0.24648499999999998,-0.647529,-0.36120399999999997,-0.11633099999999999,-0.791292,-0.867942,-0.838484,-0.772759,-0.49887700000000007 -106,1,-0.648001,0.583433,-0.647878,-0.630885,1.5970030000000002,0.074651,0.072498,0.109537,-0.153294,0.389251,-0.706426,-0.22331700000000002,-0.691956,-0.689379,1.269571,-0.050051,-0.22723600000000002,-0.36289899999999997,-0.038768000000000004,0.340564,-0.357933,0.7985770000000001,-0.35199600000000003,-0.433809,0.49969399999999997,-0.132913,-0.081026,0.354244,-0.592352,0.017058 -381,1,-0.8654370000000001,-0.7893399999999999,-0.82004,-0.762026,-1.002465,-0.356652,-0.560775,-0.716658,0.48734700000000003,-0.613775,-0.876835,-1.014519,-0.8773110000000001,-0.802575,-1.173538,-0.635841,-0.669674,-0.726852,0.698723,-0.04786,-0.869719,-0.337202,-0.7846529999999999,-0.630505,-0.581683,-0.36038400000000004,-0.486205,-0.784468,-0.21095500000000003,-0.701242 -415,1,-0.6666380000000001,0.24960300000000002,-0.660388,-0.628776,0.448503,-0.22624299999999997,-0.7473989999999999,-0.486889,0.281889,-0.578309,-0.635423,0.43756999999999996,-0.6417039999999999,-0.6288,0.097477,-0.43855600000000006,-0.794093,-0.699769,0.757138,0.014515,-0.470901,-0.025135,-0.46337799999999996,-0.457791,0.9513790000000001,0.282346,-0.517041,-0.40879099999999996,0.24672199999999997,-0.57604 -71,1,-1.353531,-1.629614,-1.3314629999999998,-1.048038,-0.5115029999999999,-0.067845,-0.617866,-1.016318,-1.046309,1.355149,-1.488033,-1.0820040000000002,-1.366651,-1.168611,0.104593,0.924055,-0.034392,-0.521016,0.329977,3.82787,0.43681499999999995,-0.661607,0.14947,-0.327761,3.389811,3.811771,0.8228340000000001,0.951175,0.589374,6.859624 -558,1,-0.163427,0.259374,-0.040545,-0.25855900000000004,-1.304933,0.39971799999999996,0.45102200000000003,-0.062524,-1.0398379999999998,-0.216444,0.131416,0.788958,0.1821,0.006288,-0.827674,0.543131,0.177034,-0.298156,-1.3056450000000002,-0.188203,-0.648835,-0.197498,-0.317839,-0.457571,-0.9330309999999999,1.168754,1.1235680000000001,0.6916399999999999,-0.503965,0.231148 -21,1,-1.2506110000000001,-1.631243,-1.254913,-0.9944219999999999,0.001377,-0.887193,-0.880434,-0.796903,-0.729224,-0.344455,-1.31308,-1.593959,-1.302806,-1.083572,0.42981899999999995,-0.7470859999999999,-0.743748,-0.726337,0.012345,0.8863409999999999,-0.461517,-0.435539,-0.473774,-0.542058,0.855042,-0.6236229999999999,-0.399334,0.391552,-0.03297,-0.31277699999999997 -316,1,-0.708054,-1.499339,-0.7643409999999999,-0.646003,-1.414523,-1.278291,-1.108365,-1.463066,-0.983215,-1.306473,-0.5530579999999999,-1.21232,-0.60628,-0.550303,-1.353587,-1.368694,-0.973881,-1.131534,-0.506089,-0.8941690000000001,-0.699725,-1.11991,-0.706933,-0.5394180000000001,-0.566016,-1.101646,-0.6851470000000001,-1.094126,-0.09350900000000001,-1.0760129999999999 -463,1,-0.724621,-0.269869,-0.732172,-0.677646,0.08028099999999999,-0.46734,-0.413491,-0.48308199999999996,0.500289,-0.29458,-0.717787,-0.216335,-0.745091,-0.688811,-0.802766,-0.867807,-0.6921470000000001,-0.803461,-1.079287,-0.596472,-0.8069189999999999,-0.818729,-0.773762,-0.603883,-0.11066400000000001,-0.47328000000000003,-0.36783499999999997,-0.697362,0.084477,-0.388428 -454,1,-0.399502,-0.574386,-0.465887,-0.43435100000000004,-0.43259899999999996,-0.652457,-0.400538,-0.24509099999999998,0.597356,-0.589947,-0.42809200000000003,-0.497911,-0.46746899999999997,-0.460714,-0.7493920000000001,-0.948351,-0.742492,-0.67578,-0.04607,-0.643253,-0.8516729999999999,-0.9968969999999999,-0.866333,-0.594422,-1.051036,-0.9467209999999999,-0.641048,-0.796147,-0.540288,-0.815852 -413,1,0.101638,0.9563450000000001,0.087534,-0.023702,-1.0813700000000002,0.510406,0.188118,0.24657600000000002,0.424254,0.474591,0.24502100000000002,0.656315,0.229057,0.110382,-0.7977850000000001,-0.034888999999999996,-0.253727,-0.262045,0.483317,-0.519922,-0.31245700000000004,0.216173,-0.275266,-0.260216,-0.864028,0.14541700000000002,0.04066,0.02658,-0.179475,0.08325199999999999 -330,0,0.515803,-0.60207,0.5075149999999999,0.332978,0.487955,1.2311530000000002,1.0718219999999998,1.271326,0.19129300000000002,0.404213,0.540396,-0.87955,0.5697,0.39337,-0.10321,0.6208319999999999,0.396746,0.554335,-0.108136,-0.430613,-0.24568600000000002,-0.8513870000000001,-0.117352,-0.15548800000000002,-0.40067600000000003,0.389654,0.174283,0.453192,-0.711009,-0.254905 -468,0,1.097705,0.5199239999999999,1.082381,0.978132,-0.5115029999999999,1.426448,1.1720899999999999,1.294166,-0.9702729999999999,2.13097,0.986298,0.940217,1.1134110000000002,0.9260709999999999,-0.246964,1.8204599999999997,1.566869,1.322737,-0.422117,1.54411,1.8902439999999998,0.45022299999999993,1.4528860000000001,1.420495,-0.091663,2.502279,1.127879,1.835218,-0.46158699999999997,2.838813 -322,1,-0.46162600000000004,-0.748629,-0.43073900000000004,-0.49412,0.978917,-0.19825299999999998,-0.44659399999999994,0.013609,-0.839233,0.08778899999999999,-0.359929,-1.389177,-0.37685100000000005,-0.42686899999999994,1.212639,-0.303242,-0.637784,-0.384824,-0.9807110000000001,0.27818899999999996,-0.663994,-0.317244,-0.619808,-0.5229159999999999,-0.377008,-0.29722800000000005,-0.426854,-0.273508,-1.031867,-0.260201 -307,1,-1.360572,-0.913101,-1.380908,-1.0461040000000001,-1.479838,-1.284653,-1.235211,-1.533565,0.145995,-0.327276,-1.456224,-1.1378540000000001,-1.4667430000000001,-1.162069,-1.8723830000000001,-1.386888,-1.0686579999999999,-1.1722629999999998,-0.08623,0.784274,-0.832183,0.159928,-0.8524719999999999,-0.67211,0.11601199999999999,-1.151053,-0.93545,-1.350255,0.7831,-0.621052 -373,0,1.8846189999999998,-0.40828600000000004,1.7734130000000001,1.8729099999999999,1.044671,0.32592600000000005,0.689459,1.470795,-0.342574,-0.74234,1.849702,-0.45136899999999996,1.764216,1.934305,-0.135234,0.061763,0.802271,1.0444200000000001,-0.8784850000000001,-1.13658,0.752621,-1.014859,0.6202439999999999,0.807087,-0.276671,-0.36485500000000004,-0.168561,0.084976,-0.942269,-0.788239 -304,1,-0.743259,-0.6623220000000001,-0.731874,-0.686963,-0.787669,-0.47942700000000005,-0.717654,-0.90623,-1.120727,-0.41926599999999997,-0.757549,-0.26287699999999997,-0.7570359999999999,-0.716114,-0.557245,-0.519289,-0.6950350000000001,-0.874394,-1.462636,-0.052113,-0.279252,-0.2864,-0.19358599999999998,-0.38298499999999996,-0.129665,0.058228999999999996,-0.321083,-0.647077,-0.194004,-0.14521199999999998 -529,1,-0.583805,-1.61333,-0.60588,-0.5813119999999999,0.864944,-0.5793010000000001,-0.527672,-0.61936,-0.19373800000000002,-0.189844,-0.5843,-1.3612520000000001,-0.58239,-0.596377,0.970677,-0.270077,-0.640169,-0.540104,-0.564504,0.46531300000000003,-0.555357,-1.293361,-0.5703050000000001,-0.47957299999999997,0.09534400000000001,-0.779555,-0.461337,-0.618041,-0.111671,-0.590414 -40,0,-0.07024,0.744648,-0.141817,-0.162929,-1.0068489999999999,-0.317847,-0.30554699999999996,-0.051864999999999994,0.150849,-0.691912,-0.19520099999999999,0.53298,-0.23845100000000002,-0.261342,-1.048999,-0.8344520000000001,-0.724413,-0.7379439999999999,-0.10083400000000001,-0.9820610000000001,-0.601554,-0.7082350000000001,-0.640599,-0.43578999999999996,-1.25371,-0.808059,-0.596618,-0.797283,-0.816347,-0.948996 -115,1,-0.538247,0.076989,-0.587413,-0.523125,0.772888,-0.091382,-0.584763,-0.641591,-0.748637,0.081139,-0.624062,0.521345,-0.635937,-0.6151479999999999,0.093918,-0.489914,-0.6970430000000001,-0.743876,-0.451325,-0.121575,-0.336999,-0.533695,-0.428726,-0.342062,0.254017,-0.022811,-0.44906899999999994,-0.6626489999999999,-0.9398479999999999,0.02311 -2,0,1.51187,-0.023974000000000002,1.347475,1.456285,0.527407,1.082932,0.854974,1.955,1.152255,0.20139100000000001,1.579888,0.456187,1.566503,1.558884,0.94221,1.052926,1.363478,2.0372310000000002,0.939685,-0.398008,1.228676,-0.780083,0.850928,1.1813360000000002,-0.297005,0.814974,0.21307600000000002,1.424827,0.23703600000000002,0.293559 -39,0,-0.15307300000000001,0.05581900000000001,0.001155,-0.24643,1.255083,1.070209,1.107324,1.6931029999999998,-0.151676,1.283108,-0.18384,0.356123,-0.147009,-0.27215,0.37288699999999997,0.400995,0.219721,0.14111500000000002,-0.334494,0.19738599999999998,-0.693589,-1.134788,-0.653965,-0.480013,-0.558016,-0.172595,-0.046543,0.133639,-0.8199799999999999,-0.22994 diff --git a/fml_manager/Examples/Homo_LR/homo_lr.ipynb b/fml_manager/Examples/Homo_LR/homo_lr.ipynb deleted file mode 100644 index 830837407..000000000 --- a/fml_manager/Examples/Homo_LR/homo_lr.ipynb +++ /dev/null @@ -1,300 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "# Homo LR Demo" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "In this demo, we use the homo_logistic_regression(https://github.com/FederatedAI/FATE/tree/master/examples/federatedml-1.x-examples/homo_logistic_regression example. It is:\n", - "* Homogeneous federated machine learning example, which both parties share same attributes but different samples;\n", - "* Use breast cancer data original from Kaggle: https://www.kaggle.com/uciml/breast-cancer-wisconsin-data\n", - "* For easy to demo, both party we use the same FATE cluster: 10000. But the underlayer is the same, each side threat the collborated party go through the network to another party. \n", - "\n", - "Step 0. Prepare the libary to manage federated machine learning." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "import json\n", - "import time\n", - "import os\n", - "import requests\n", - "\n", - "from fml_manager import *\n", - "\n", - "manager = FMLManager()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## For more details about the FMLManager, please refer to this [document](https://kubefate.readthedocs.io/README.html)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "Step 1. Upload the guest, host and test data. Because we use same cluster for this demo, we load all data in same NOTEBOOK. If we use another party for host, the host data should load in the NOTEBOOK of that party." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = manager.load_data(url='./data/breast_homo_guest.csv', namespace='homo_breast_guest', table_name='homo_breast_guest', work_mode=1, head=1, partition=10)\n", - "output = json.loads(response.content)\n", - "print(output)\n", - "guest_job_id = output['jobId']\n", - "guest_query_condition = QueryCondition(job_id=guest_job_id)\n", - "\n", - "response = manager.load_data(url='./data/breast_homo_host.csv', namespace='homo_breast_host', table_name='homo_breast_host', work_mode=1, head=1, partition=10)\n", - "output = json.loads(response.content)\n", - "host_job_id = output['jobId']\n", - "host_query_condition = QueryCondition(job_id=host_job_id)\n", - "\n", - "response = manager.load_data(url='./data/breast_homo_test.csv', namespace='homo_breast_test', table_name='homo_breast_test', work_mode=1, head=1, partition=10)\n", - "output = json.loads(response.content)\n", - "test_job_id = output['jobId']\n", - "test_query_condition = QueryCondition(job_id=test_job_id)\n", - "\n", - "\n", - "manager.query_job_status(guest_query_condition)\n", - "manager.query_job_status(host_query_condition)\n", - "manager.query_job_status(host_query_condition)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "Step 2. Create the steps DSL and configuration of each step for training." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "# dsl\n", - "data_io = ComponentBuilder(name='dataio_0',\n", - " module='DataIO')\\\n", - " .add_input_data('args.train_data')\\\n", - " .add_output_data('train')\\\n", - " .add_output_model('dataio').build()\n", - " \n", - "\n", - "homo_lr = ComponentBuilder(name='homo_lr_0',\n", - " module='HomoLR')\\\n", - " .add_input_train_data('dataio_0.train')\\\n", - " .add_output_data('train')\\\n", - " .add_output_model('homolr').build()\n", - "\n", - "evaluation = ComponentBuilder(name='evaluation_0',\n", - " module='Evaluation',\n", - " need_deploy=False)\\\n", - " .add_input_data('homo_lr_0.train')\\\n", - " .add_output_data('evaluate').build()\n", - "\n", - "pipeline = Pipeline(\n", - " data_io, \n", - " homo_lr, \n", - " evaluation\n", - ")\n", - "\n", - "# Configuration\n", - "initiator = Initiator(role='guest', party_id=10000)\n", - "\n", - "job_parameters = JobParametersBuilder()\\\n", - " .with_work_mode(1).build()\n", - "\n", - "role = RoleBuilder()\\\n", - " .add_guest(party_id=10000)\\\n", - " .add_host(party_id=10000)\\\n", - " .add_arbiter(party_id=10000).build()\n", - "\n", - "eval_config = {\n", - " 'need_run': [False]\n", - " }\n", - "\n", - "role_parameters = RoleParametersBuilder()\\\n", - " .add_guest_train_data(namespace='homo_breast_guest', name='homo_breast_guest')\\\n", - " .add_host_train_data(namespace='homo_breast_host', name='homo_breast_host')\\\n", - " .add_host_module_config(module='evaluation_0', config=eval_config).build()\n", - "\n", - "\n", - "homo_lr_params = {\n", - " 'penalty': 'L2',\n", - " 'optimizer': 'sgd',\n", - " 'eps': 1e-5,\n", - " 'alpha': 0.01,\n", - " 'max_iter': 10,\n", - " 'converge_func': 'diff',\n", - " 'batch_size': 500,\n", - " 'learning_rate': 0.15,\n", - " 'decay': 1,\n", - " 'decay_sqrt': True,\n", - " 'init_param': {\n", - " 'init_method': 'zeros'\n", - " },\n", - " 'encrypt_param': {\n", - " 'method': 'Paillier'\n", - " },\n", - " 'cv_param': {\n", - " 'n_splits': 4,\n", - " 'shuffle': True,\n", - " 'random_seed': 33,\n", - " 'need_cv': False\n", - " }\n", - " }\n", - "dotaio_config = {\n", - " 'with_label': True,\n", - " 'label_name': 'y',\n", - " 'label_type': 'int',\n", - " 'output_format': 'dense'\n", - " }\n", - "\n", - "algorithm_parameters = AlgorithmParametersBuilder()\\\n", - " .add_module_config(module='homo_lr_0', config=homo_lr_params)\\\n", - " .add_module_config(module='dataio_0', config=dotaio_config).build()\n", - "\n", - "config = Config(\n", - " initiator,\n", - " job_parameters,\n", - " role,\n", - " role_parameters,\n", - " algorithm_parameters\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "Step 3. Submit the training job to GUEST cluster. And it will notify and bring up the HOST cluster and train together. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = manager.submit_job(pipeline.to_dict(),config.to_dict())\n", - "manager.prettify(response, verbose=True)\n", - "stdout = json.loads(response.content)\n", - "job_id = stdout['jobId']\n", - "query_condition = QueryCondition(job_id)\n", - "\n", - "model_id, model_version = '', ''\n", - "manager.query_job_status(query_condition, max_tries=20)\n", - "\n", - "manager.prettify(response, verbose=True)\n", - "output = json.loads(response.content)\n", - "model_id, model_version = output['data']['model_info']['model_id'], output['data']['model_info']['model_version']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "response = manager.model_output(role='guest', party_id='10000', model_id=model_id, model_version=model_version, model_component='homo_lr_0.homolr:HomoLogisticRegression')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "And we can try offline prediction feature. Prediction also need both parts participant." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "is_vertical = False\n", - "initiator_party_role = 'guest'\n", - "initiator_party_id = '10000'\n", - "work_mode = 1\n", - "federated_roles = {\n", - " 'guest': [10000],\n", - " 'host': [10000],\n", - " 'arbiter': [10000]\n", - "}\n", - "guest_data_name = 'homo_breast_test'\n", - "guest_data_namespace = 'homo_breast_test'\n", - "host_data_name = 'homo_breast_test'\n", - "host_data_namespace = 'homo_breast_test'\n", - "\n", - "response = manager.offline_predict_on_dataset(is_vertical, initiator_party_role, initiator_party_id, work_mode, model_id, model_version, federated_roles, guest_data_name, guest_data_namespace, host_data_name, host_data_namespace)\n", - "print(response.text)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Click [here](/fateboard-10000/) to view jobs in FATE Board" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.9" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/fml_manager/Examples/Model_Management/model_management_tutorial.ipynb b/fml_manager/Examples/Model_Management/model_management_tutorial.ipynb deleted file mode 100644 index 7076743cb..000000000 --- a/fml_manager/Examples/Model_Management/model_management_tutorial.ipynb +++ /dev/null @@ -1,580 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "# Model Management Demo" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "Before managing models, we need to prepare a model by training. In this example, we use the homo_logistic_regression(https://github.com/FederatedAI/FATE/tree/master/examples/federatedml-1.x-examples/homo_logistic_regression example. " - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "pycharm": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'data': {'board_url': 'http://fateboard:8080/index.html#/dashboard?job_id=2020042011073298433874&role=local&party_id=0', 'job_dsl_path': '/data/projects/fate/python/jobs/2020042011073298433874/job_dsl.json', 'job_runtime_conf_path': '/data/projects/fate/python/jobs/2020042011073298433874/job_runtime_conf.json', 'logs_directory': '/data/projects/fate/python/logs/2020042011073298433874', 'namespace': 'homo_breast_guest', 'table_name': 'homo_breast_guest'}, 'jobId': '2020042011073298433874', 'retcode': 0, 'retmsg': 'success'}\n", - "running,running,running\n", - "running,running,running\n", - "running,running,running\n", - "running,success,success\n", - "running,success,success\n", - "running,success,success\n", - "success,success,success\n", - "Success\n", - "Success!\n", - "{\n", - " 'data': {\n", - " 'board_url': 'http://fateboard:8080/index.html#/dashboard?job_id=2020042011074074717177&role=guest&party_id=10000',\n", - " 'job_dsl_path': '/data/projects/fate/python/jobs/2020042011074074717177/job_dsl.json',\n", - " 'job_runtime_conf_path': '/data/projects/fate/python/jobs/2020042011074074717177/job_runtime_conf.json',\n", - " 'logs_directory': '/data/projects/fate/python/logs/2020042011074074717177',\n", - " 'model_info': {\n", - " 'model_id': 'arbiter-10000#guest-10000#host-10000#model',\n", - " 'model_version': '2020042011074074717177'\n", - " }\n", - " },\n", - " 'jobId': '2020042011074074717177',\n", - " 'retcode': 0,\n", - " 'retmsg': 'success'\n", - "}\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "running\n", - "success\n", - "Success\n", - "Success!\n", - "{\n", - " 'data': {\n", - " 'board_url': 'http://fateboard:8080/index.html#/dashboard?job_id=2020042011074074717177&role=guest&party_id=10000',\n", - " 'job_dsl_path': '/data/projects/fate/python/jobs/2020042011074074717177/job_dsl.json',\n", - " 'job_runtime_conf_path': '/data/projects/fate/python/jobs/2020042011074074717177/job_runtime_conf.json',\n", - " 'logs_directory': '/data/projects/fate/python/logs/2020042011074074717177',\n", - " 'model_info': {\n", - " 'model_id': 'arbiter-10000#guest-10000#host-10000#model',\n", - " 'model_version': '2020042011074074717177'\n", - " }\n", - " },\n", - " 'jobId': '2020042011074074717177',\n", - " 'retcode': 0,\n", - " 'retmsg': 'success'\n", - "}\n" - ] - } - ], - "source": [ - "import fml_manager\n", - "import json, time, requests\n", - "import os\n", - "from fml_manager import *\n", - "\n", - "manager = fml_manager.FMLManager()\n", - "response = manager.load_data('examples/data/breast_homo_guest.csv', 'homo_breast_guest', 'homo_breast_guest', 1, 1, 10, api_version='1.4')\n", - "output = json.loads(response.content)\n", - "print(output)\n", - "guest_job_id = output['jobId']\n", - "guest_query_condition = {\n", - " 'job_id':guest_job_id\n", - "}\n", - "response = manager.load_data('examples/data/breast_homo_host.csv', 'homo_breast_host', 'homo_breast_host', 1, 1, 10, api_version='1.4')\n", - "output = json.loads(response.content)\n", - "host_job_id = output['jobId']\n", - "host_query_condition = {\n", - " 'job_id':host_job_id\n", - "}\n", - "response = manager.load_data('examples/data/breast_homo_test.csv', 'homo_breast_test', 'homo_breast_test', 1, 1, 10, api_version='1.4')\n", - "output = json.loads(response.content)\n", - "test_job_id = output['jobId']\n", - "test_query_condition = {\n", - " 'job_id':test_job_id\n", - "}\n", - "\n", - "for i in range(500):\n", - " time.sleep(1)\n", - " guest_status = manager.query_job(guest_query_condition).json()['data'][0]['f_status']\n", - " host_status = manager.query_job(host_query_condition).json()['data'][0]['f_status']\n", - " test_status = manager.query_job(host_query_condition).json()['data'][0]['f_status']\n", - " \n", - " print('{},{},{}'.format(guest_status, host_status, test_status))\n", - " \n", - " if guest_status == 'failed' or host_status == 'failed' or test_status == 'failed':\n", - " print('Failed')\n", - " raise Exception('Failed to run the jobs')\n", - " if guest_status == 'success' and host_status == 'success' and test_status == 'success':\n", - " print('Success') \n", - " break\n", - "\n", - "\n", - "# dsl\n", - "data_io = ComponentBuilder()\\\n", - " .with_name('dataio_0')\\\n", - " .with_module('DataIO')\\\n", - " .add_input_data('args.train_data')\\\n", - " .add_output_data('train')\\\n", - " .add_output_model('dataio').build()\n", - " \n", - "\n", - "homo_lr = ComponentBuilder()\\\n", - " .with_name('homo_lr_0')\\\n", - " .with_module('HomoLR')\\\n", - " .add_input_train_data('dataio_0.train')\\\n", - " .add_output_data('train')\\\n", - " .add_output_model('homolr').build()\n", - "\n", - "evaluation = ComponentBuilder()\\\n", - " .with_name('evaluation_0')\\\n", - " .with_module('Evaluation')\\\n", - " .add_input_data('homo_lr_0.train')\\\n", - " .ad_output_data('evaluate')\\\n", - " .with_need_deploy(False).build()\n", - "\n", - "pipeline = Pipline()\n", - " data_io, \n", - " hetero_lr, \n", - " evaluation\n", - ")\n", - "\n", - "# Configuration\n", - "initiator = Initiator(role='guest', party_id=10000)\n", - "\n", - "job_parameters = JobParameters(work_mode=1)\n", - "\n", - "role = RoleBuilder()\\\n", - " .add_guest(10000)\\\n", - " .add_host(10000)\\\n", - " .add_arbiter(10000).build()\n", - "\n", - "eval_config = {\n", - " 'need_run': [False]\n", - " }\n", - "\n", - "role_parameters = RoleParametersBuilder()\\\n", - " .add_guest_train_data(namespace='homo_breast_guest', name='homo_breast_guest')\\\n", - " .add_host_train_data(namespace='homo_breast_host', name='homo_breast_host')\\\n", - " .add_host_module_config(module='evalution_0', config=eval_config).build()\n", - "\n", - "homo_lr_params = {\n", - " 'penalty': 'L2',\n", - " 'optimizer': 'sgd',\n", - " 'eps': 1e-5,\n", - " 'alpha': 0.01,\n", - " 'max_iter': 10,\n", - " 'converge_func': 'diff',\n", - " 'batch_size': 500,\n", - " 'learning_rate': 0.15,\n", - " 'decay': 1,\n", - " 'decay_sqrt': True,\n", - " 'init_param': {\n", - " 'init_method': 'zeros'\n", - " },\n", - " 'encrypt_param': {\n", - " 'method': 'Paillier'\n", - " },\n", - " 'cv_param': {\n", - " 'n_splits': 4,\n", - " 'shuffle': True,\n", - " 'random_seed': 33,\n", - " 'need_cv': False\n", - " }\n", - " }\n", - "\n", - "dotaio_config = {\n", - " 'with_label': True,\n", - " 'label_name': 'y',\n", - " 'label_type': 'int',\n", - " 'output_format': 'dense'\n", - " },\n", - "\n", - "algorithm_parameters = AlgorithmParametersBuilder()\\\n", - " .add_module_config(module='homo_lr_0', config=homo_lr_params)\n", - " .add_model_config(model='dataio_0', config=dotaio_config).build()\n", - "\n", - "config = ConfigBuilder()\\\n", - " .with_initiator(initiator)\\\n", - " .with_job_parameters(job_parameters)\\\n", - " .with_role(role)\\\n", - " .with_role_parameters(role_parameters)\\\n", - " .with_algorithm_parameters(algorithm_parameters).build()\n", - "\n", - "\n", - "response = manager.submit_job(pipeline.to_dict(),config.to_dict())\n", - "manager.prettify(response, True)\n", - "stdout = json.loads(response.content)\n", - "jobId = stdout['jobId']\n", - "query_condition = {\n", - " 'job_id':jobId\n", - "}\n", - "\n", - "model_id, model_version = '', ''\n", - "for i in range(500):\n", - " time.sleep(1)\n", - " job_detail = manager.query_job(query_condition).json()\n", - " final_status = job_detail['data'][0]['f_status']\n", - " print(final_status)\n", - " \n", - " if final_status == 'failed':\n", - " print('Failed')\n", - " manager.prettify(job_detail, True)\n", - " response = manager.fetch_job_log(jobId)\n", - " raise Exception('Failed to run the job')\n", - " if final_status == 'success':\n", - " print('Success')\n", - " manager.prettify(response, True)\n", - " output = json.loads(response.content)\n", - " model_id, model_version = output['data']['model_info']['model_id'], output['data']['model_info']['model_version']\n", - " break" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "Print existed model, the API is: ```print_model_version(self, role, party_id, model_id)```" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "pycharm": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Success!\n", - "{\n", - " 'data': [\n", - " {\n", - " 'commitId': '2020042011074074717177',\n", - " 'log': '[AUTO] save model at 2020-04-20 11:09:40.165231.',\n", - " 'name': '2020042011074074717177',\n", - " 'namespace': 'guest#10000#arbiter-10000#guest-10000#host-10000#model',\n", - " 'parent': '2020042011022342527372',\n", - " 'repeatCommit': true,\n", - " 'tag': null\n", - " },\n", - " {\n", - " 'commitId': '2020042011022342527372',\n", - " 'log': '[AUTO] save model at 2020-04-20 11:06:17.494397.',\n", - " 'name': '2020042011022342527372',\n", - " 'namespace': 'guest#10000#arbiter-10000#guest-10000#host-10000#model',\n", - " 'parent': '2020042010084278627567',\n", - " 'repeatCommit': true,\n", - " 'tag': null\n", - " },\n", - " {\n", - " 'commitId': '2020042010084278627567',\n", - " 'log': '[AUTO] save model at 2020-04-20 10:13:33.310625.',\n", - " 'name': '2020042010084278627567',\n", - " 'namespace': 'guest#10000#arbiter-10000#guest-10000#host-10000#model',\n", - " 'parent': '2020042009122786739056',\n", - " 'repeatCommit': true,\n", - " 'tag': null\n", - " },\n", - " {\n", - " 'commitId': '2020042009122786739056',\n", - " 'log': '[AUTO] save model at 2020-04-20 10:04:07.545335.',\n", - " 'name': '2020042009122786739056',\n", - " 'namespace': 'guest#10000#arbiter-10000#guest-10000#host-10000#model',\n", - " 'parent': '2020042009081081759152',\n", - " 'repeatCommit': true,\n", - " 'tag': null\n", - " },\n", - " {\n", - " 'commitId': '2020042009081081759152',\n", - " 'log': '[AUTO] save model at 2020-04-20 09:10:29.357328.',\n", - " 'name': '2020042009081081759152',\n", - " 'namespace': 'guest#10000#arbiter-10000#guest-10000#host-10000#model',\n", - " 'parent': '2020042008513api_version='1.4'681344',\n", - " 'repeatCommit': true,\n", - " 'tag': null\n", - " },\n", - " {\n", - " 'commitId': '2020042008513api_version='1.4'681344',\n", - " 'log': '[AUTO] save model at 2020-04-20 08:5api_version='1.4'5.427548.',\n", - " 'name': '2020042008513api_version='1.4'681344',\n", - " 'namespace': 'guest#10000#arbiter-10000#guest-10000#host-10000#model',\n", - " 'parent': '2020042008495252468240',\n", - " 'tag': null\n", - " },\n", - " {\n", - " 'commitId': '2020042008495252468240',\n", - " 'log': '[AUTO] save model at 2020-04-20 08:49:56.606507.',\n", - " 'name': '2020042008495252468240',\n", - " 'namespace': 'guest#10000#arbiter-10000#guest-10000#host-10000#model',\n", - " 'parent': '2020042008494205128236',\n", - " 'tag': null\n", - " },\n", - " {\n", - " 'commitId': '2020042008494205128236',\n", - " 'log': '[AUTO] save model at 2020-04-20 08:49:46.874495.',\n", - " 'name': '2020042008494205128236',\n", - " 'namespace': 'guest#10000#arbiter-10000#guest-10000#host-10000#model',\n", - " 'parent': '2020042008480590014232',\n", - " 'tag': null\n", - " },\n", - " {\n", - " 'commitId': '2020042008480590014232',\n", - " 'log': '[AUTO] save model at 2020-04-20 08:48:10.868260.',\n", - " 'name': '2020042008480590014232',\n", - " 'namespace': 'guest#10000#arbiter-10000#guest-10000#host-10000#model',\n", - " 'parent': '2020042008464736494228',\n", - " 'tag': null\n", - " },\n", - " {\n", - " 'commitId': '2020042008464736494228',\n", - " 'log': '[AUTO] save model at 2020-04-20 08:46:52.258443.',\n", - " 'name': '2020042008464736494228',\n", - " 'namespace': 'guest#10000#arbiter-10000#guest-10000#host-10000#model',\n", - " 'parent': '2020042008460920911924',\n", - " 'tag': null\n", - " }\n", - " ],\n", - " 'retcode': 0,\n", - " 'retmsg': 'success'\n", - "}\n" - ] - } - ], - "source": [ - "if model_id != '':\n", - " response = manager.print_model_version('guest','10000', model_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "Output the model, the API is ```model_output(self, role, party_id, model_id, model_version，model_component)```. The ```model_component``` is what you defined in training conf. The output is base64 encoding, which need decoding and parsing back." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "pycharm": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'metadata': 'CgJMMhHxaOOItfjkPhl7FK5H4XqEPyIDc2dkMPQDOTMzMzMzM8M/QApKBGRpZmZQAlgB', 'parameters': 'CAoSUP4QBQQ+79o/Y+ScSvKa1j+RTreM/XfUPyk/T2KsNtM/ik0jT2po0j+lMOUXo9zRPwWjqz1MetE/3VWUfhkz0T8KB2etSv7QP52mdO1A1tA/Ig0KAngwEbFDSoWA/sG/Ig0KAngxEcob/dudLr6/Ig0KAngyEWYZ+hcRpMG/Ig0KAngzEfmv/7Rc+r+/Ig0KAng0EbDD/W9RW7m/Ig0KAng1EbyOMlppJri/Ig0KAng2EbMEbULVqru/Ig0KAng3ERQQVXbn+MG/Ig0KAng4EW6/Nt4yu7W/Ig0KAng5EQZZbnGtrKW/Ig4KA3gxMBEsGFvBSI3AvyIOCgN4MTERZZM+KKAGur8iDgoDeDEyEc6PglYzgMC/Ig4KA3gxMxHa0QRjolK+vyIOCgN4MTQR9tqVWW+Fr78iDgoDeDE1EUAimmbYbLO/Ig4KA3gyMBG6wwtUoXG0vyIOCgN4MTYRH2mQvHuRur8iDgoDeDIxEdeP/oYJfnY/Ig4KA3gxNxFSVtQreM/AvyIOCgN4MjIRzAmY7no3sr8iDgoDeDIzEcwFGjerVbK/Ig4KA3gxOBHDXhlmJQ2rvyIOCgN4MjQRoPtqyzc7iD8iDgoDeDE5EXaSYAuFHaM/Ig4KA3gyNRGNsnK5wSqBvyIOCgN4MjYRSLDRsrf0Gj8iDgoDeDI3EUcfkMZaT6a/Ig4KA3gyOBHYUPahnWidPyIOCgN4MjkRXdrDc3/ooD8pCQy8iYha4T8yAngwMgJ4MTICeDIyAngzMgJ4NDICeDUyAng2MgJ4NzICeDgyAng5MgN4MTAyA3gxMTIDeDEyMgN4MTMyA3gxNDIDeDE1MgN4MTYyA3gxNzIDeDE4MgN4MTkyA3gyMDIDeDIxMgN4MjIyA3gyMzIDeDI0MgN4MjUyA3gyNjIDeDI3MgN4MjgyA3gyOQ=='}\n" - ] - } - ], - "source": [ - "response = manager.model_output('guest','10000', model_id, model_version, 'homo_lr_0.homolr:HomoLogisticRegression')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "And we can try offline prediction feature." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "pycharm": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'data':{'board_url':'http://fateboard:8080/index.html#/dashboard?job_id=2020042011api_version='1.4'807243678&role=guest&party_id=10000','job_dsl_path':'/data/projects/fate/python/jobs/2020042011103807243678/job_dsl.json','job_runtime_conf_path':'/data/projects/fate/python/jobs/2020042011103807243678/job_runtime_conf.json','logs_directory':'/data/projects/fate/python/logs/2020042011103807243678','model_info':{'model_id':'arbiter-10000#guest-10000#host-10000#model','model_version':'2020042011074074717177'}},'jobId':'2020042011103807243678','retcode':0,'retmsg':'success'}\n", - "\n" - ] - } - ], - "source": [ - "is_vertical = False\n", - "initiator_party_role = 'guest'\n", - "initiator_party_id = '10000'\n", - "work_mode = 1\n", - "federated_roles = {\n", - " 'guest': [10000],\n", - " 'host': [10000],\n", - " 'arbiter': [10000]\n", - "}\n", - "guest_data_name = 'homo_breast_test'\n", - "guest_data_namespace = 'homo_breast_test'\n", - "host_data_name = 'homo_breast_test'\n", - "host_data_namespace = 'homo_breast_test'\n", - "\n", - "response = manager.offline_predict_on_dataset(is_vertical, initiator_party_role, initiator_party_id, work_mode, model_id, model_version, federated_roles, guest_data_name, guest_data_namespace, host_data_name, host_data_namespace)\n", - "print(response.text)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "The result can be checked in FATE-Board." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} \ No newline at end of file diff --git a/fml_manager/Examples/Pipeline/demo/__init__.py b/fml_manager/Examples/Pipeline/demo/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/fml_manager/Examples/Pipeline/demo/pipeline-mini-demo.py b/fml_manager/Examples/Pipeline/demo/pipeline-mini-demo.py deleted file mode 100644 index 6e53adbd0..000000000 --- a/fml_manager/Examples/Pipeline/demo/pipeline-mini-demo.py +++ /dev/null @@ -1,128 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -from pipeline.backend.config import Backend, WorkMode -from pipeline.backend.pipeline import PipeLine -from pipeline.component import DataIO -from pipeline.component import HeteroLR -from pipeline.component import Intersection -from pipeline.component import Reader -from pipeline.interface import Data -from pipeline.interface import Model -from pipeline.runtime.entity import JobParameters - - -def main(): - # parties config - guest = 9999 - host = 10000 - arbiter = 10000 - # 0 for eggroll, 1 for spark - backend = Backend.EGGROLL - # 0 for standalone, 1 for cluster - work_mode = WorkMode.STANDALONE - # use the work mode below for cluster deployment - # work_mode = WorkMode.CLUSTER - - # specify input data name & namespace in database - guest_train_data = {"name": "breast_hetero_guest", "namespace": "experiment"} - host_train_data = {"name": "breast_hetero_host", "namespace": "experiment"} - - guest_eval_data = {"name": "breast_hetero_guest", "namespace": "experiment"} - host_eval_data = {"name": "breast_hetero_host", "namespace": "experiment"} - - # initialize pipeline - pipeline = PipeLine() - # set job initiator - pipeline.set_initiator(role="guest", party_id=guest) - # set participants information - pipeline.set_roles(guest=guest, host=host, arbiter=arbiter) - - # define Reader components to read in data - reader_0 = Reader(name="reader_0") - # configure Reader for guest - reader_0.get_party_instance(role="guest", party_id=guest).component_param(table=guest_train_data) - # configure Reader for host - reader_0.get_party_instance(role="host", party_id=host).component_param(table=host_train_data) - - reader_1 = Reader(name="reader_1") - reader_1.get_party_instance(role="guest", party_id=guest).component_param(table=guest_eval_data) - reader_1.get_party_instance(role="host", party_id=host).component_param(table=host_eval_data) - - # define DataIO components - dataio_0 = DataIO(name="dataio_0") - dataio_1 = DataIO(name="dataio_1") - - # get DataIO party instance of guest - dataio_0_guest_party_instance = dataio_0.get_party_instance(role="guest", party_id=guest) - # configure DataIO for guest - dataio_0_guest_party_instance.component_param(with_label=True, output_format="dense") - # get and configure DataIO party instance of host - dataio_0.get_party_instance(role="host", party_id=host).component_param(with_label=False) - - # define Intersection components - intersection_0 = Intersection(name="intersection_0") - intersection_1 = Intersection(name="intersection_1") - - # define HeteroLR component - hetero_lr_0 = HeteroLR(name="hetero_lr_0", early_stop="weight_diff", learning_rate=0.15, optimizer="rmsprop", - max_iter=10, early_stopping_rounds=2, validation_freqs=1) - - # add components to pipeline, in order of task execution - pipeline.add_component(reader_0) - pipeline.add_component(reader_1) - pipeline.add_component(dataio_0, data=Data(data=reader_0.output.data)) - # set dataio_1 to replicate model from dataio_0 - pipeline.add_component(dataio_1, data=Data(data=reader_1.output.data), model=Model(dataio_0.output.model)) - # set data input sources of intersection components - pipeline.add_component(intersection_0, data=Data(data=dataio_0.output.data)) - pipeline.add_component(intersection_1, data=Data(data=dataio_1.output.data)) - # set train & validate data of hetero_lr_0 component - pipeline.add_component(hetero_lr_0, data=Data(train_data=intersection_0.output.data, validate_data=intersection_1.output.data)) - - # compile pipeline once finished adding modules, this step will form conf and dsl files for running job - pipeline.compile() - - # fit model - job_parameters = JobParameters(backend=backend, work_mode=work_mode) - pipeline.fit(job_parameters) - # query component summary - import json - print(json.dumps(pipeline.get_component("hetero_lr_0").get_summary(), indent=4)) - - # predict - # deploy required components - pipeline.deploy_component([dataio_0, intersection_0, hetero_lr_0]) - - # initiate predict pipeline - predict_pipeline = PipeLine() - - reader_2 = Reader(name="reader_2") - reader_2.get_party_instance(role="guest", party_id=guest).component_param(table=guest_eval_data) - reader_2.get_party_instance(role="host", party_id=host).component_param(table=host_eval_data) - # add data reader onto predict pipeline - predict_pipeline.add_component(reader_2) - # add selected components from train pipeline onto predict pipeline - # specify data source - predict_pipeline.add_component(pipeline, - data=Data(predict_input={pipeline.dataio_0.input.data: reader_2.output.data})) - # run predict model - predict_pipeline.predict(job_parameters) - - -if __name__ == "__main__": - main() diff --git a/fml_manager/Examples/Pipeline/demo/pipeline-upload-spark.py b/fml_manager/Examples/Pipeline/demo/pipeline-upload-spark.py deleted file mode 100644 index 230c63a4b..000000000 --- a/fml_manager/Examples/Pipeline/demo/pipeline-upload-spark.py +++ /dev/null @@ -1,68 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os - -from pipeline.backend.config import Backend, WorkMode -from pipeline.backend.pipeline import PipeLine - -# path to data -# default fate installation path -DATA_BASE = "/data/projects/fate" - -# site-package ver -# import site -# DATA_BASE = site.getsitepackages()[0] - - -def main(): - # parties config - guest = 9999 - # 0 for eggroll, 1 for spark - backend = Backend.SPARK - # 0 for standalone, 1 for cluster - # work_mode = WorkMode.STANDALONE - work_mode = WorkMode.CLUSTER - - # partition for data storage - partition = 4 - - dense_data = {"name": "breast_hetero_guest", "namespace": "experiment"} - - tag_data = {"name": "tag_value_1", "namespace": "experiment"} - - pipeline_upload = PipeLine().set_initiator(role="guest", party_id=guest).set_roles(guest=guest) - # add upload data info - # csv file name from python path & file name - pipeline_upload.add_upload_data(file=os.path.join(DATA_BASE, "examples/data/breast_hetero_guest.csv"), - table_name=dense_data["name"], # table name - namespace=dense_data["namespace"], # namespace - head=1, partition=partition, # data info - id_delimiter=",") # id delimiter, needed for spark - - pipeline_upload.add_upload_data(file=os.path.join(DATA_BASE, "examples/data/tag_value_1000_140.csv"), - table_name=tag_data["name"], - namespace=tag_data["namespace"], - head=0, partition=partition, - id_delimiter=",") - # upload all data - pipeline_upload.upload(work_mode=work_mode, backend=backend, drop=1) - import json - print(json.dumps(pipeline_upload._upload_conf(), indent=4)) - - -if __name__ == "__main__": - main() diff --git a/fml_manager/Examples/Pipeline/demo/pipeline-upload.py b/fml_manager/Examples/Pipeline/demo/pipeline-upload.py deleted file mode 100644 index 088e52e2a..000000000 --- a/fml_manager/Examples/Pipeline/demo/pipeline-upload.py +++ /dev/null @@ -1,66 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os - -from pipeline.backend.config import Backend, WorkMode -from pipeline.backend.pipeline import PipeLine - -# path to data -# default fate installation path -DATA_BASE = "/data/projects/fate" - -# site-package ver -# import site -# DATA_BASE = site.getsitepackages()[0] - - -def main(): - # parties config - guest = 9999 - # 0 for eggroll, 1 for spark - backend = Backend.EGGROLL - # 0 for standalone, 1 for cluster - work_mode = WorkMode.STANDALONE - # use the work mode below for cluster deployment - # work_mode = WorkMode.CLUSTER - - # partition for data storage - partition = 4 - - dense_data = {"name": "breast_hetero_guest", "namespace": f"experiment"} - - tag_data = {"name": "tag_value_1", "namespace": f"experiment"} - - pipeline_upload = PipeLine().set_initiator(role="guest", party_id=guest).set_roles(guest=guest) - # add upload data info - # original csv file path - pipeline_upload.add_upload_data(file=os.path.join(DATA_BASE, "examples/data/breast_hetero_guest.csv"), - table_name=dense_data["name"], # table name - namespace=dense_data["namespace"], # namespace - head=1, partition=partition) # data info - - pipeline_upload.add_upload_data(file=os.path.join(DATA_BASE, "examples/data/tag_value_1000_140.csv"), - table_name=tag_data["name"], - namespace=tag_data["namespace"], - head=0, partition=partition) - - # upload all data - pipeline_upload.upload(work_mode=work_mode, backend=backend, drop=1) - - -if __name__ == "__main__": - main() diff --git a/fml_manager/Examples/Toy_Example/toy_example_submit_job.ipynb b/fml_manager/Examples/Toy_Example/toy_example_submit_job.ipynb deleted file mode 100644 index 614c4215d..000000000 --- a/fml_manager/Examples/Toy_Example/toy_example_submit_job.ipynb +++ /dev/null @@ -1,184 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "# Submit Training Job Demo" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "import json\n", - "import time\n", - "import os\n", - "import requests\n", - "\n", - "import fml_manager\n", - "from fml_manager import *" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## For more details about the FMLManager, please refer to this [document](https://kubefate.readthedocs.io/README.html)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "The DSL and Config file can be presented in JOSN format. \n", - "Submitting a job with JSON defined in line, can use ```submit_job(self,dsl, config)```. \n", - "Note: the parameters are dict, the JSON string have to transform to dict with ```json.loads```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "# dsl\n", - "secure_add_example = Component(name='secure_add_example_0',\n", - " module='SecureAddExample')\n", - "\n", - "pipeline = Pipeline(\n", - " secure_add_example\n", - ")\n", - "\n", - "# Configuration\n", - "initiator = Initiator(role='guest',\n", - " party_id=9999)\n", - "\n", - "job_parameters = JobParameters(work_mode=1)\n", - "\n", - "role = RoleBuilder()\\\n", - " .add_guest(9999)\\\n", - " .add_host(9999).build()\n", - "\n", - "secure_add_example_guest_config = {\n", - " \"seed\": [\n", - " 123\n", - " ]\n", - "}\n", - "\n", - "secure_add_example_host_config = {\n", - " \"seed\": [\n", - " 321\n", - " ]\n", - "}\n", - "role_parameters = RoleParametersBuilder()\\\n", - " .add_host_module_config(module='secure_add_example_0', config=secure_add_example_host_config)\\\n", - " .add_guest_module_config(module='secure_add_example_0', config=secure_add_example_guest_config).build()\n", - "\n", - "secure_add_example = {\n", - " \"partition\": 10,\n", - " \"data_num\": 1000\n", - "}\n", - "\n", - "algorithm_parameters = AlgorithmParametersBuilder()\\\n", - " .add_module_config(module='secure_add_example_0', config=secure_add_example).build()\n", - "\n", - "config = Config(\n", - " initiator,\n", - " job_parameters,\n", - " role,\n", - " role_parameters,\n", - " algorithm_parameters\n", - ")\n", - "\n", - "manager = fml_manager.FMLManager()\n", - "response = manager.submit_job(pipeline.to_dict(), config.to_dict())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "manager.prettify(response, verbose=True)\n", - "stdout = json.loads(response.content)\n", - "job_id = stdout['jobId']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "query_condition = QueryCondition(job_id)\n", - "\n", - "job_status = manager.query_job(query_condition)\n", - "manager.prettify(job_status, verbose=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "manager.query_job_status(query_condition, max_tries=20)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "We can also fetch the logs of job submitted, and save it to working folder." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "response = manager.fetch_job_log(job_id)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.8" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/fml_manager/LICENSE b/fml_manager/LICENSE deleted file mode 100644 index c2fc91acd..000000000 --- a/fml_manager/LICENSE +++ /dev/null @@ -1,11 +0,0 @@ -Copyright 2019-2020 VMware, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -you may obtain a copy of the License at -http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/fml_manager/Makefile b/fml_manager/Makefile deleted file mode 100644 index d239afdb1..000000000 --- a/fml_manager/Makefile +++ /dev/null @@ -1,12 +0,0 @@ -NAME ?= federatedai/client -VERSION ?= v1.5.0 -IMG ?= ${NAME}:${VERSION:v%=%-release} - -docker-build: - docker build . -t ${IMG} - -docker-push: - docker push ${IMG} - -docker-save: docker-build - docker save -o client-${VERSION}.docker ${IMG} \ No newline at end of file diff --git a/fml_manager/Pipfile b/fml_manager/Pipfile deleted file mode 100644 index 6f56bf3ca..000000000 --- a/fml_manager/Pipfile +++ /dev/null @@ -1,16 +0,0 @@ -[[source]] -name = "pypi" -url = "https://pypi.org/simple" -verify_ssl = true - -[dev-packages] - -[packages] -tar = "*" -requests = "*" -setuptools = {index = "https://pypi.tuna.tsinghua.edu.cn/simple",version = "*"} -wheel = {index = "https://pypi.tuna.tsinghua.edu.cn/simple",version = "*"} -pep8 = {index = "https://pypi.tuna.tsinghua.edu.cn/simple",version = "*"} - -[requires] -python_version = "3.6" diff --git a/fml_manager/README.md b/fml_manager/README.md deleted file mode 100644 index 8a2f2b373..000000000 --- a/fml_manager/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# FML_Manager -This is a client for FATE cluster - -# Compatible KubeFATE Versions -FML Manager version | KubeFATE version 1.3.x | KubeFATE version 1.4.x ----|---|--- -0.2.0 | incompatible | compatible -0.3.0 | incompatible | compatible -0.4.0 | incompatible | compatible - -- incompatible: the combination does not work at all -- compatible: all Kubeflow features have been tested and verified for the Kubernetes version -- no known issues: the combination has not been fully tested but there are no repoted issues diff --git a/fml_manager/fml_manager/fml_cluster_manager.py b/fml_manager/fml_manager/fml_cluster_manager.py deleted file mode 100644 index be26d9a52..000000000 --- a/fml_manager/fml_manager/fml_cluster_manager.py +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright 2019-2020 VMware, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# you may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import subprocess -import json -import tempfile - -from fml_manager.utils.fate_builders import RouteTable - - -class ClusterManager: - """ClusterManager is used to manage FATE cluster""" - - def __init__(self, cluster_namespace, cluster_name=None): - """ Init the ClusterManager instance - - :param cluster_namespace: Which namespace the cluster is belonged to - :type cluster_namespace: string - :param cluster_name: The name of fate cluster, no need to provide in most cases - :type cluster_name: string - - """ - self.namespace = cluster_namespace - self.name = cluster_name - - # get configmap in dict - def fetch_config_map(self, component): - args = "kubectl get configmap {} -n {} -o json".format( - component, self.namespace).split(" ") - try: - data, err = subprocess.Popen( - args, stdout=subprocess.PIPE).communicate() - return json.loads(data) - except Exception as error: - print(error) - - # get route table in dict - def fetch_route_table(self, configmap): - route_table_json = json.loads(configmap["data"]["route_table.json"]) - return route_table_json - - def update_config_map(self, configmap, route_table): - configmap["data"]["route_table.json"] = json.dumps(route_table) - - def patch_config_map(self, configmap, component): - args = "kubectl patch configmap {} -n {} --patch".format( - component, self.namespace).split(" ") - args.append(json.dumps(configmap)) - try: - data, err = subprocess.Popen( - args, stdout=subprocess.PIPE).communicate() - print(data) - except Exception as error: - print(error) - - def get_route_table(self): - """ Fetch route table from configmap - - :rtype: RouteTable - - """ - # get configmap in dict - configmap = self.fetch_config_map("rollsite-config") - - # get route table in dict - route_table = self.fetch_route_table(configmap) - - route_table_ins = RouteTable().from_dict(route_table) - return route_table_ins - - def set_route_table(self, route_table): - """ Patch configmap with route table - - :param route_table: route table to patch configmap - :type route_table: RouteTable - - """ - # get configmap in dict - configmap = self.fetch_config_map("rollsite-config") - - # paste update to the configmap - self.update_config_map(configmap, route_table.to_dict()) - - # patch config - self.patch_config_map(configmap, "rollsite-config") - - def get_entry_point(self): - """ Return the entrypoint of FATE cluster - - :rtype: string - - """ - get_address = "kubectl get nodes -o jsonpath=\'{$.items[0].status.addresses[?(@.type==\'InternalIP\')].address}\'" - get_port = "kubectl get service rollsite -n {0} -o jsonpath=\'{{.spec.ports[0].nodePort}}\'".format( - self.namespace) - - ip = "" - port = "" - - try: - ip, err = subprocess.Popen(get_address.split( - " "), stdout=subprocess.PIPE).communicate() - port, err = subprocess.Popen(get_port.split( - " "), stdout=subprocess.PIPE).communicate() - except Exception as error: - print(error) - - if ip == "" or port == "": - raise(Exception("Unable to get entrypoint")) - - return "{}:{}".format(ip.decode("utf-8").replace("'", ""), port.decode("utf-8").replace("'", "")) diff --git a/fml_manager/fml_manager/fml_manager.py b/fml_manager/fml_manager/fml_manager.py deleted file mode 100644 index eabd313a6..000000000 --- a/fml_manager/fml_manager/fml_manager.py +++ /dev/null @@ -1,715 +0,0 @@ -# Copyright 2019-2020 VMware, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# you may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import tarfile -import requests -import base64 -import random -import time -import subprocess -import tempfile - -import pandas as pd -from contextlib import closing -from fml_manager.utils import file_utils -from fml_manager.utils.core import get_lan_ip - -cFateFlowHostEnv = "FATE_FLOW_HOST" -cFateServingHostEnv = "FATE_SERVING_HOST" - -cFateFlowServieName = "fateflow" -cFateServingServieName = "fateserving" -cFateFlowServiePort = 9380 -cFateServingServiePort = 8059 - -cFateClusterCR = "fatecluster" - - -class FMLManager: - """FMLManager is used to communicate with FATE cluster""" - - def __init__(self, server_conf=None, log_path="./"): - """ Init the FMLManager with config and log path - - :param server_conf: Path to config file, default=None - :type server_conf: string - :param log_path: Path to log file, default=./ - :type log_path: string - - """ - - #: Url of FATE Flow service - self.server_url = None - - #: Url of FATE Serving service - self.serving_url = None - self.log_path = log_path - - if server_conf is not None: - self._init_from_config(server_conf) - elif os.getenv(cFateFlowHostEnv) is not None and os.getenv(cFateFlowHostEnv) != "": - self._init_from_env() - else: - self._init_from_kube_api() - - # if the server url is still None, the initialization is failed - if self.server_url is None: - raise Exception( - "Unable to find fate_flow url, failed to initialize the FML Manager") - if self.serving_url is None: - print("Unable to find fate_serving url but it's ok to continue") - - def _init_from_config(self, server_conf): - self.server_conf = file_utils.load_json_conf(server_conf) - self.ip = self.server_conf.get("servers").get("fateflow").get("host") - self.serving_url = self.server_conf.get("servings") - self.http_port = self.server_conf.get( - "servers").get("fateflow").get("http.port") - self.server_url = "http://{}:{}/{}".format( - self.ip, self.http_port, "v1") - - def _init_from_env(self): - server_host = os.getenv(cFateFlowHostEnv) - serving_host = os.getenv(cFateServingHostEnv, "") - - self.server_url = "http://{}/{}".format(server_host, "v1") - self.serving_url = "http://{}".format(serving_host) - - def _init_from_kube_api(self): - args = "kubectl get {} -A -o json".format(cFateClusterCR).split(" ") - try: - data, err = subprocess.Popen( - args, stdout=subprocess.PIPE).communicate() - data_json = json.loads(data) - if len(data_json["items"]) != 0: - # fetch the first fatecluster by default - fate_cluster_namespace = data_json["items"][0]["metadata"]["namespace"] - self.server_url = "http://{}.{}:{}/{}".format( - cFateFlowServieName, fate_cluster_namespace, cFateFlowServiePort, "v1") - self.serving_url = "http://{}.{}:{}/{}".format( - cFateServingServieName, fate_cluster_namespace, cFateServingServiePort, "v1") - except Exception as e: - print(e) - - # Job management - - def submit_job(self, dsl, config): - """ Submit job to FATE cluster - - :param dsl: DSL definition - :type dsl: Pipline - :param config: Config definition - :type config: Config - - :returns: response - :rtype: dict - - """ - post_data = {'job_dsl': dsl, - 'job_runtime_conf': config} - response = requests.post( - "/".join([self.server_url, "job", "submit"]), json=post_data) - - return self.prettify(response) - - def submit_job_by_files(self, dsl_path, config_path): - """ Submit job with file to FATE cluster - - :param dsl_path: DSL definition path - :type dsl_path: string - :param config_path: Config definition path - :type config_path: string - - :returns: response - :rtype: dict - - """ - - config_data = {} - if config_path: - config_path = os.path.abspath(config_path) - with open(config_path, 'r') as f: - config_data = json.load(f) - else: - raise Exception('Conf cannot be null.') - dsl_data = {} - if dsl_path: - dsl_path = os.path.abspath(dsl_path) - with open(dsl_path, 'r') as f: - dsl_data = json.load(f) - else: - raise Exception('DSL_path cannot be null.') - - return self.submit_job(dsl_data, config_data) - - def query_job_status(self, query_conditions, max_tries=200): - """ Fetch status of job - - :param query_conditions: Condition of the job - :type query_conditions: dict - - :returns: response - :rtype: dict - - """ - job_status = "failed" - for i in range(max_tries): - time.sleep(1) - try: - guest_status = self.query_job(query_conditions).json()[ - "data"][0]["f_status"] - except Exception as e: - print("Failed to fetch status: ", e) - - print("Status: %s" % guest_status) - if guest_status == "failed": - job_status = "failed" - raise Exception("Failed to upload data.") - if guest_status == "success": - job_status = "success" - break - return job_status - - def query_job(self, query_conditions): - """ Fetch job - - :param query_conditions: Condition of the job - :type query_conditions: QueryCondition - - :returns: response - :rtype: dict - - """ - - response = requests.post( - "/".join([self.server_url, "job", "query"]), json=query_conditions.to_dict()) - return self.prettify(response) - - def query_job_conf(self, query_conditions): - """ Fetch config of job - - :param query_conditions: Condition of the job - :type query_conditions: dict - - :returns: response - :rtype: dict - - """ - - response = requests.post( - "/".join([self.server_url, "job", "config"]), json=query_conditions) - return self.prettify(response) - - def stop_job(self, job_id): - """ Stop job - - :param job_id: job id of data - :type job_id: string - - :returns: response - :rtype: dict - - """ - post_data = { - 'job_id': job_id - } - response = requests.post( - "/".join([self.server_url, "job", "stop"]), json=post_data) - return self.prettify(response) - - def update_job(self, job_id, role, party_id, notes): - post_data = { - "job_id": job_id, - "role": role, - "party_id": party_id, - "notes": notes - } - response = requests.post( - "/".join([self.server_url, "job", "update"]), json=post_data) - return self.prettify(response) - - def fetch_job_log(self, job_id): - """ Fetch the log of job - - :param job_id: The UUID of job - :type job_id: string - - :returns: response - :rtype: dict - - """ - data = { - "job_id": job_id - } - - tar_file_name = 'job_{}_log.tar.gz'.format(job_id) - extract_dir = os.path.join(self.log_path, 'job_{}_log'.format(job_id)) - with closing(requests.get("/".join([self.server_url, "job", "log"]), json=data, - stream=True)) as response: - if response.status_code == 200: - self.__download_from_request( - http_response=response, tar_file_name=tar_file_name, extract_dir=extract_dir) - response = {'retcode': 0, - 'directory': extract_dir, - 'retmsg': 'download successfully, please check {} directory, file name is {}'.format(extract_dir, tar_file_name)} - - return self.prettify(response, True) - else: - return self.prettify(response, True) - - # Data management - def load_data(self, url, namespace, table_name, work_mode, head, partition, drop="1", api_version="1.4"): - """ Upload data to FATE cluster - - :param url: URL of data to upload - :type url: string - - :param namespace: Namespace of the data in FATE cluster - :type namespace: string - - :param table_name: Table name of the data in FATE cluster - :type table_name: string - - :param work_mode: The work mode of upload - :type work_mode: int - - :param head: Head included flag. '1': with head, '0': without head. - :type head: int - - :param partition: Partitions of the upload data - :type partition: int - - :param drop: Flag to overwrite data with same identifier - :type drop: string - - :returns: response - :rtype: dict - - """ - if api_version == "1.4": - temp_file = None - if url.startswith("http://") or url.startswith("https://"): - downloader = HttpDownloader(url) - temp_file = downloader.download_to( - file_utils.get_project_base_directory()) - url = temp_file - - post_data = { - "namespace": namespace, - "table_name": table_name, - "work_mode": work_mode, - "head": head, - "partition": partition, - "drop": drop - } - data_files = { - "file": open(url, "rb") - } - response = requests.post( - "/".join([self.server_url, "data", "upload"]), params=post_data, files=data_files) - - if temp_file is not None and os.path.exists(temp_file): - print("Delete temp file...") - os.remove(temp_file) - else: - post_data = { - "file": url, - "namespace": namespace, - "table_name": table_name, - "work_mode": work_mode, - "head": head, - "partition": partition - } - response = requests.post( - "/".join([self.server_url, "data", "upload"]), json=post_data) - - return self.prettify(response) - - def query_data(self, job_id, limit): - """ Query data of job - """ - post_data = { - "job_id": job_id, - "limit": limit - } - - response = requests.post( - "/".join([self.server_url, "data", "upload", "history"]), json=post_data) - - return self.prettify(response) - - # The data is download to fateflow. FATE not ready to download to local. - def download_data(self, namespace, table_name, filename, work_mode, delimitor, output_folder="./"): - """ Download data to local - """ - - DEFAULT_DATA_FOLDER = "/data/projects/fate/python/download_dir" - output_path = "{}/{}".format(DEFAULT_DATA_FOLDER, filename) - post_data = { - "namespace": namespace, - "table_name": table_name, - "work_mode": work_mode, - "delimitor": delimitor, - "output_path": output_path - } - response = requests.post( - "/".join([self.server_url, "data", "download"]), json=post_data) - - if response.status_code == 200: - output = json.loads(response.content) - job_id = output["jobId"] - query_condition = { - "job_id": job_id - } - for i in range(500): - time.sleep(1) - status = self.query_job(query_condition).json()[ - "data"][0]["f_status"] - if status == "failed": - print("Failed") - print(self.query_job(query_condition).json()) - raise Exception("Failed to download data.") - if status == "success": - return self.prettify(response, True) - response = { - 'retcode': 1, - 'retmsg': 'Download failed' - } - - return self.prettify(response, True) - - # Model management - def load_model(self, initiator_party_id, federated_roles, work_mode, model_id, model_version): - post_data = { - "initiator": { - "party_id": initiator_party_id, - "role": "guest" - }, - "role": federated_roles, - "job_parameters": { - "work_mode": work_mode, - "model_id": model_id, - "model_version": model_version - } - } - response = requests.post( - "/".join([self.server_url, "model", "load"]), json=post_data) - - return self.prettify(response) - - def bind_model(self, service_id, initiator_party_id, federated_roles, work_mode, model_id, model_version): - if self.serving_url == nil: - raise Exception( - 'Federated Serving is not deployed or not correctly configured yet. ') - post_data = { - "service_id": service_id, - "initiator": { - "party_id": initiator_party_id, - "role": "guest" - }, - "role": federated_roles, - "job_parameters": { - "work_mode": work_mode, - "model_id": model_id, - "model_version": model_version - }, - "servings": self.serving_url - } - - response = requests.post( - "/".join([self.server_url, "model", "bind"]), json=post_data) - return self.prettify(response) - - def print_model_version(self, role, party_id, model_id, api_version="1.4"): - """ Print model version - """ - - action = "version" - if api_version == "1.4": - action = "version_history" - - namespace = "#".join([role, str(party_id), model_id]) - post_data = { - "namespace": namespace - } - - response = requests.post( - "/".join([self.server_url, "model", action]), json=post_data) - - return self.prettify(response, True) - - def model_output(self, role, party_id, model_id, model_version, model_component): - """ Output the model - """ - - namespace = "#".join([role, str(party_id), model_id]) - post_data = { - "name": model_version, - "namespace": namespace - } - response = requests.post( - "/".join([self.server_url, "model", "transfer"]), json=post_data) - model = json.loads(response.content) - if model["data"] != "": - en_model_metadata = model["data"]["%sMeta" % model_component] - en_model_parameters = model["data"]["%sParam" % model_component] - - model = { - "metadata": en_model_metadata, - "parameters": en_model_parameters - } - - return self.prettify(model, True) - - def offline_predict_on_dataset(self, is_vertical, initiator_party_role, initiator_party_id, work_mode, model_id, model_version, federated_roles, guest_data_name="", guest_data_namespace="", host_data_name="", host_data_namespace=""): - if is_vertical: - print("This API is not support vertical federated machine learning yet. ") - return - - # For predict job, dsl is empty dict. - dsl = {} - - config = { - "initiator": { - "role": initiator_party_role, - "party_id": initiator_party_id - }, - "job_parameters": { - "work_mode": work_mode, - "job_type": "predict", - "model_id": model_id, - "model_version": model_version - }, - "role": federated_roles, - "role_parameters": {} - } - - if guest_data_name != "" or guest_data_namespace != "": - if initiator_party_role != "guest": - raise Exception("Initiator not has data sets.") - - guest_parameters = { - "args": { - "data": { - "eval_data": [{"name": guest_data_name, "namespace": guest_data_namespace}] - } - } - } - - config["role_parameters"]["guest"] = guest_parameters - - if host_data_name != "" or host_data_namespace != "": - host_parameters = { - "args": { - "data": { - "eval_data": [{"name": host_data_name, "namespace": host_data_namespace}] - } - } - } - config["role_parameters"]["host"] = guest_parameters - - return self.submit_job(dsl, config) - - # Task - def query_task(self, query_conditions): - """ Query task - """ - - response = requests.post( - "/".join([self.server_url, "job", "task", "query"]), json=query_conditions) - return self.prettify(response) - - # Tracking - def track_job_data(self, job_id, role, party_id): - """ Track job data - """ - - post_data = { - "job_id": job_id, - "role": role, - "party_id": party_id - } - - response = requests.post( - "/".join([self.server_url, "tracking", "job", "data_view"]), json=post_data) - return self.prettify(response, True) - - def track_component_all_metric(self, job_id, role, party_id, component_name): - """ Track output all metric of component - """ - post_data = { - "job_id": job_id, - "role": role, - "party_id": party_id, - "component_name": component_name - } - - response = requests.post( - "/".join([self.server_url, "tracking", "component", "metric", "all"]), json=post_data) - return self.prettify(response, True) - - def track_component_metric_type(self, job_id, role, party_id, component_name): - """ Track output metric type of component - """ - post_data = { - "job_id": job_id, - "role": role, - "party_id": party_id, - "component_name": component_name - } - - response = requests.post( - "/".join([self.server_url, "tracking", "component", "metrics"]), json=post_data) - return self.prettify(response, True) - - """ - metric_name and metric_namespace can be found in API track_component_metric_type - e.g. response = manager.track_component_metric_type(jobId, "guest", "10000", "homo_lr_0") - { - "data": { - "train": [ - "loss" - ] - }, - "retcode": 0, - "retmsg": "success" - } - - The metric_name is "loss" and metric_namespace is "train" - """ - - def track_component_metric_data(self, job_id, role, party_id, component_name, metric_name, metric_namespace): - """ Track output metric data of component - """ - post_data = { - "job_id": job_id, - "role": role, - "party_id": party_id, - "component_name": component_name, - "metric_name": metric_name, - "metric_namespace": metric_namespace - } - - response = requests.post( - "/".join([self.server_url, "tracking", "component", "metric_data"]), json=post_data) - return self.prettify(response, True) - - def track_component_parameters(self, job_id, role, party_id, component_name): - """ Track output parameter of component - """ - post_data = { - "job_id": job_id, - "role": role, - "party_id": party_id, - "component_name": component_name - } - - response = requests.post( - "/".join([self.server_url, "tracking", "component", "parameters"]), json=post_data) - return self.prettify(response, True) - - def track_component_output_model(self, job_id, role, party_id, component_name): - """ Track output model of component - """ - post_data = { - "job_id": job_id, - "role": role, - "party_id": party_id, - "component_name": component_name - } - - response = requests.post( - "/".join([self.server_url, "tracking", "component", "output", "model"]), json=post_data) - return self.prettify(response, True) - - def track_component_output_data(self, job_id, role, party_id, component_name): - """ Track output data of component - - :rtype: pandas.DataFrame - """ - post_data = { - "job_id": job_id, - "role": role, - "party_id": party_id, - "component_name": component_name - } - - response = requests.post( - "/".join([self.server_url, "tracking", "component", "output", "data"]), json=post_data) - - result = response.json() - data = result['data'] - header = result['meta']['header'] - - return pd.DataFrame(data, columns=header) - - # Utils - def prettify(self, response, verbose=False): - if verbose: - if isinstance(response, requests.Response): - if response.status_code == 200: - print("Success!") - print(json.dumps(response.json(), indent=4, ensure_ascii=False)) - else: - print(response) - - return response - - def __download_data_from_request(self, http_response, output): - with open(output, 'wb') as fw: - for chunk in http_response.iter_content(1024): - if chunk: - fw.write(chunk) - - def __download_from_request(self, http_response, tar_file_name, extract_dir): - with open(tar_file_name, 'wb') as fw: - for chunk in http_response.iter_content(1024): - if chunk: - fw.write(chunk) - tar = tarfile.open(tar_file_name, "r:gz") - file_names = tar.getnames() - for file_name in file_names: - tar.extract(file_name, extract_dir) - tar.close() - os.remove(tar_file_name) - - -class HttpDownloader: - def __init__(self, url): - self.url = url - - def download_to(self, path_to_save): - r = requests.get(self.url, allow_redirects=True) - filename = self.__get_filename_from_cd( - r.headers.get('content-disposition')) - temp_file_to_write = os.path.join( - file_utils.get_project_base_directory(), filename) - open(temp_file_to_write, 'wb').write(r.content) - - return temp_file_to_write - - def __get_filename_from_cd(self, cd): - """ - Get filename from content-disposition - """ - if not cd: - # just return file name - fname = self.url.split('/')[-1] - if len(fname) == 0: - return None - return fname - fname = re.findall('filename=(.+)', cd) - if len(fname) == 0: - return None - return fname[0] diff --git a/fml_manager/fml_manager/requirements.txt b/fml_manager/fml_manager/requirements.txt deleted file mode 100644 index b312b8334..000000000 --- a/fml_manager/fml_manager/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -cachetools==3.0.0 -requests==2.20.0 \ No newline at end of file diff --git a/fml_manager/fml_manager/utils/__init__.py b/fml_manager/fml_manager/utils/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/fml_manager/fml_manager/utils/core.py b/fml_manager/fml_manager/utils/core.py deleted file mode 100644 index cd3b14000..000000000 --- a/fml_manager/fml_manager/utils/core.py +++ /dev/null @@ -1,117 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import base64 -import json -import os -import pickle -import socket -import time -import uuid - - -def get_fate_uuid(): - return uuid.uuid1().hex - - -def get_commit_id(): - # the model may be larger, SHA1 is not used - return get_fate_uuid() - - -def string_to_bytes(string): - return string if isinstance(string, bytes) else string.encode(encoding="utf-8") - - -def bytes_to_string(byte): - return byte.decode(encoding="utf-8") - - -def json_dumps(src, byte=False): - if byte: - return string_to_bytes(json.dumps(src)) - else: - return json.dumps(src) - - -def json_loads(src): - if isinstance(src, bytes): - return json.loads(bytes_to_string(src)) - else: - return json.loads(src) - - -def current_timestamp(): - return int(time.time() * 1000) - - -def timestamp_to_date(timestamp, format_string="%Y-%m-%d %H:%M:%S"): - timestamp = int(timestamp) / 1000 - time_array = time.localtime(timestamp) - str_date = time.strftime(format_string, time_array) - return str_date - - -def base64_encode(src): - return bytes_to_string(base64.b64encode(src.encode("utf-8"))) - - -def base64_decode(src): - return bytes_to_string(base64.b64decode(src)) - - -def serialize_b64(src, to_str=False): - dest = base64.b64encode(pickle.dumps(src)) - if not to_str: - return dest - else: - return bytes_to_string(dest) - - -def deserialize_b64(src): - return pickle.loads(base64.b64decode(string_to_bytes(src) if isinstance(src, str) else src)) - - -def get_lan_ip(): - if os.name != "nt": - import fcntl - import struct - - def get_interface_ip(ifname): - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - return socket.inet_ntoa( - fcntl.ioctl(s.fileno(), 0x8915, struct.pack('256s', string_to_bytes(ifname[:15])))[20:24]) - - ip = socket.gethostbyname(socket.getfqdn()) - if ip.startswith("127.") and os.name != "nt": - interfaces = [ - "bond1", - "eth0", - "eth1", - "eth2", - "wlan0", - "wlan1", - "wifi0", - "ath0", - "ath1", - "ppp0", - ] - for ifname in interfaces: - try: - ip = get_interface_ip(ifname) - break - except IOError as e: - pass - return ip or '' diff --git a/fml_manager/fml_manager/utils/fate_builders.py b/fml_manager/fml_manager/utils/fate_builders.py deleted file mode 100644 index d2120398c..000000000 --- a/fml_manager/fml_manager/utils/fate_builders.py +++ /dev/null @@ -1,1027 +0,0 @@ -# Copyright 2019-2020 VMware, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# you may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -from enum import Enum - -"""Module to Build Complex Data Structural in FATE""" - - -class PartyType(Enum): - - """PartyType is used to identified the type of party""" - - #: For normal party - NORMAL = 1 - - #: For exchange - EXCHANGE = 2 - - -class Party(): - """Party is used to define route table record""" - - def __init__(self, p_id='', ip=None, port=None, p_type=PartyType.NORMAL) -> None: - """ Return default party instance""" - - #: ID of the party - if p_type is PartyType.EXCHANGE: - self._id = 'default' - else: - self._id = p_id - - #: IP address of the party - self._ip = ip - - #: Service port of the party - self._port = port - - #: Type of the party - self._type = p_type - - def set_id(self, party_id) -> None: - """ Set party ID - - :param party_id: The ID of party - :type party_id: string - - """ - - self._id = party_id - - def set_ip(self, party_ip) -> None: - """ Set IP address of party - - :param party_ip: The IP address of party - :type party_ip: string - - """ - - self._ip = party_ip - - def set_port(self, party_port) -> None: - """ Set service port of the party - - :param party_port: The service port of party - :type party_port: int - - """ - - self._port = party_port - - def set_type(self, party_type) -> None: - """ Set party type - - :param party_id: The type of party - :type party_type: PartyType - - """ - - self._type = party_type - - def get_id(self) -> str: - """ Get party ID - - :rtype: string - - """ - - return self._id - - def to_entry_point(self) -> dict: - """ Get entrypoint of party - - :rtype: dict - - """ - return {'default': [{'ip': self._ip, 'port': self._port}]} - - -class PartyBuilder(): - """PartyBuilder is used to construct party instance""" - - def __init__(self) -> None: - """ Get default party instance - """ - - self.reset() - - def reset(self) -> None: - """ Reset the previous instance - """ - - self._party = Party() - - def with_id(self, party_id): - """ Update ID - - :param party_id: The ID of party - :type party_id: string - - """ - - self._party.set_id(party_id) - return self - - def with_ip(self, party_ip): - """ Update IP - - :param party_ip: The IP of party - :type party_ip: string - - """ - - self._party.set_ip(party_ip) - return self - - def with_port(self, party_port): - """ Update port - - :param party_port: The port of party - :type party_port: int - - """ - - self._party.set_port(party_port) - return self - - def with_type(self, party_type): - """ Update type - - :param party_type: The type of party - :type party_type: PartyType - - """ - - self._party.set_type(party_type) - return self - - def build(self): - """ Return party instance with config - - :rtype: Party - - """ - party = self._party - self.reset() - - # reset id to 'default' if party is exchange - if party._type == PartyType.EXCHANGE: - party._id = 'default' - - return party - - -class RouteTable(): - """RouteTable is used to communicate with other parties""" - - def __init__(self) -> None: - """ Return instance with empty route table - """ - - #: The underlying route table config file - self._route_table = None - - def add_parties(self, *parties) -> None: - """ Append parties to route table - - :param parties: A list of party instance - :type parties: list - - """ - - for party in parties: - self._route_table['route_table'][party.get_id()] = party.to_entry_point( - ) - - def update_parties(self, *parties) -> None: - """ Update parties of route table - - :param parties: A list of party instance - :type parties: list - - """ - - for party in parties: - self._route_table['route_table'][party.get_id()] = party.to_entry_point( - ) - - def remove_parties(self, *party_ids) -> None: - """ Remove parties from route table - - :param parties: A list of party ID - :type parties: list - - """ - - for party_id in party_ids: - if(self._route_table['route_table'].get(party_id) != None): - self._route_table['route_table'].pop(party_id) - - def get_parties(self) -> dict: - """ List all parties - - :rtype: dict - - """ - return self._route_table['route_table'] - - def from_dict(self, route_table): - """ Load route table config from dict - - :param route_table: The underlying route table - :type route_table: dict - :rtype: RouteTable - - """ - self._route_table = route_table - return self - - def to_dict(self) -> dict: - """ Return underlying route table - - :rtype: dict - - """ - return self._route_table - - -class QueryCondition(): - """QueryCondition is context for job query""" - - def __init__(self, job_id): - """ Init QueryCondition with job id - - :param job_id: The uuid of job - :type job_id: string - - """ - self._job_id = job_id - - def get_job_id(self): - """ Fetch the job id - - :rtype: dict - - """ - return {'job_id': self._job_id} - - def set_job_id(self, job_id): - """ Set job id - - :param job_id: The uuid of job - :type job_id: string - - """ - self._job_id = job_id - - def __str__(self): - - return self.get_job_id().__str__() - - def to_dict(self): - return {'job_id': self._job_id} - - -class Component(): - # TODO: add setter/getter - """Component is used to describe steps in a pipline""" - - def __init__(self, name='', module='', need_deploy=True): - """ Init an empty component - """ - self._name = name - self._module = module - - #: Need deploy of data io - self._need_deploy = need_deploy - - #: The input part contains two sub structures, for more details please refer to `DSL definition `_ - #: They should be list type - self._input_data = [] - self._input_train_data = [] - self._input_eval_data = [] - self._input_model = [] - self._input_isometric_model = [] - - #: The output part also contains two sub structures, for more details please refer to `DSL definition `_ - #: They should be list type - self._output_data = [] - self._output_model = [] - - def to_dict(self): - """ Convert Component to dictionary - - :rtype: dict - - """ - name = self._name - body = {} - module = {'module': self._module} - inputs = {'input': {}} - outputs = {'output': {}} - need_deploy = {'need_deploy': self._need_deploy} - - # check all input - if len(self._input_data) != 0: - inputs['input']['data'] = { - 'data': self._input_data - } - elif len(self._input_train_data) != 0: - inputs['input']['data'] = { - 'train_data': self._input_train_data - } - elif len(self._input_eval_data) != 0: - inputs['input']['data'] = { - 'eval_data': self._input_eval_data - } - - if len(self._input_model) != 0: - inputs['input']['model'] = self._input_model - elif len(self._input_isometric_model) != 0: - inputs['input']['isometric_model'] = self._input_isometric_model - - if len(self._output_data) != 0: - outputs['output']['data'] = self._output_data - if len(self._output_model) != 0: - outputs['output']['model'] = self._output_model - - body.update(module) - if inputs != {'input': {}}: - body.update(inputs) - if outputs != {'output': {}}: - body.update(outputs) - body.update(need_deploy) - - return {name: body} - - -class ComponentBuilder(): - """ComponentBuilder is used to build Component instance""" - - def __init__(self, name='', module='', need_deploy=True): - """ Init ComponentBuilder instance - """ - - self.reset(name, module, need_deploy) - - def reset(self, name, module, need_deploy): - self._component = Component(name, module, need_deploy) - - def with_need_deploy(self, need_deploy): - """ Set 'need_deploy' for DataIO module - - :param need_deploy: Value of need_deploy - :type need_deploy: bool - - """ - self._component._need_deploy = need_deploy - return self - - def with_name(self, name): - """ Set component name - - :param name: name of the component - :type name: string - - """ - self._component._name = name - return self - - def with_module(self, module): - """ Set component module - - :param module: The available module in FATE, for more details please refer to `Modules ` - :type module: string - """ - self._component._module = module - return self - - def add_input_data(self, *data): - """ Set input data - """ - self._component._input_data.extend([d for d in data]) - return self - - def add_input_train_data(self, *train_data): - """ Set input data for training - """ - self._component._input_train_data.extend([d for d in train_data]) - return self - - def add_input_eval_data(self, *eval_data): - """ Set input data for evaluation - """ - self._component._input_eval_data.extend([d for d in eval_data]) - return self - - def add_input_model(self, *model): - """ Set input model - """ - self._component._input_model.extend([d for d in module]) - return self - - def add_input_isometric_model(self, *isometric_model): - """ Set input isometric model - """ - self._component._input_isometric_model.extend( - [d for d in isometric_model]) - return self - - def add_output_data(self, *data): - """ Set output data - """ - self._component._output_data.extend([d for d in data]) - return self - - def add_output_model(self, *model): - """ Set output model - """ - - self._component._output_model.extend([d for d in model]) - return self - - def build(self): - component = self._component - self.reset('', '', True) - return component - - -class Pipeline(): - """Pipline is used to described pipline in FATE""" - - def __init__(self, *components): - self._name = 'components' - self._components = {} - - for component in components: - self._components.update(component.to_dict()) - - def to_dict(self): - return {self._name: self._components} - - -class PipelineBuilder(): - """PiplineBuilder is used to build pipline instance""" - - def __init__(self): - self.reset() - - def reset(self): - self._pipline = Pipeline() - - def with_components(self, *components): - for component in components: - self._pipline._components.update(component.to_dict()) - return self - - def build(self): - return self._pipline - - -class Initiator(): - """Define initiator in configuration of job""" - - def __init__(self, role=None, party_id=None): - self._role = role - self._party_id = party_id - - def to_dict(self) -> dict: - """ Return dictionary - - :rtype: dict - - """ - name = 'initiator' - body = { - 'role': self._role, - 'party_id': self._party_id - } - - return {name: body} - - -class InitiatorBuilder(object): - """Build Initiator instance""" - - def __init__(self): - self.reset() - - def reset(self): - self._initiator = Initiator() - - def with_role(self, role: str) -> object: - """ - - :param role: - :return: - """ - self._initiator._role = role - return self - - def with_party_id(self, part_id: int) -> object: - """ - - :param part_id: - :return: - """ - self._initiator._party_id = part_id - return self - - def build(self) -> object: - """ - - :return: - """ - initiator = self._initiator - self.reset() - return initiator - - -class JobParameters(): - """Define job parameters in configuration of job""" - - def __init__(self, work_mode=1, job_type=None, model_id=None, model_version=None): - self._work_mode = 1 - self._job_type = job_type - self._model_id = model_id - self._model_version = model_version - - def to_dict(self): - """ Return dictionary - - :rtype: dict - - """ - - name = 'job_parameters' - body = { - 'work_mode': self._work_mode - } - - if self._job_type is not None: - body['job_type'] = self._job_type - - if self._model_id is not None: - body['model_id'] = self._model_id - - if self._model_version is not None: - body['model_version'] = self._model_version - - return {name: body} - - -class JobParametersBuilder(object): - """Build JobParameters instance""" - - def __init__(self): - self.reset() - - def reset(self): - self._job_parameters = JobParameters() - - def with_work_mode(self, work_mode: int) -> object: - """ - - :param work_mode: - :type work_mode: int - :return: - - """ - self._job_parameters._work_mode = work_mode - return self - - def with_job_type(self, job_type: str) -> object: - """ - - :param job_type: - :type job_type: string - :return: - - """ - - self._job_parameters._job_type = job_type - return self - - def with_model_id(self, model_id: str) -> object: - """ - - :param model_id: id of model - - """ - self._job_parameters._model_id = model_id - return self - - def with_model_version(self, model_version: str) -> object: - """ - :param model_version: version of model - - """ - self._job_parameters._model_version = model_version - return self - - def build(self) -> dict: - """ - - :return: - """ - job_parameters = self._job_parameters - self.reset() - return job_parameters - - -class Role(): - """Define role in configuration of job""" - - def __init__(self): - self._guest = [] - self._host = [] - self._arbiter = [] - - def to_dict(self): - name = 'role' - body = { - 'guest': self._guest, - 'host': self._host, - 'arbiter': self._arbiter - } - - return {name: body} - - -class RoleBuilder(): - """Build Role instance""" - - def __init__(self): - self.reset() - - def reset(self): - self._role = Role() - - def add_guest(self, party_id=''): - self._role._guest.append(party_id) - return self - - def add_host(self, party_id=''): - self._role._host.append(party_id) - return self - - def add_arbiter(self, party_id=''): - self._role._arbiter.append(party_id) - return self - - def with_guests(self, *guests) -> object: - """ - - :param guest: A list of guests - :return: - """ - - self._role._guest.extend([d for d in guests]) - return self - - def with_hosts(self, *hosts) -> object: - """ - - :param host: A list of hosts - :return: - """ - self._role._host.extend([d for d in hosts]) - return self - - def with_arbiters(self, *arbiters) -> object: - """ - - :param arbiter: A list of arbiters - :return: - """ - self._role._arbiter.extend([d for d in arbiters]) - return self - - def build(self) -> object: - """ - - :return: - """ - role = self._role - self.reset() - return role - - -class RoleParameters(): - """Define role_parameters in configuration of job""" - - def __init__(self): - self._guest_data = [] - self._guest_module_config = [] - self._host_data = [] - self._host_module_config = [] - - def to_dict(self): - name = 'role_parameters' - body = { - 'guest': { - 'args': { - 'data': {} - } - }, - 'host': { - 'args': { - 'data': {} - } - } - } - - for guest_data in self._guest_data: - key = list(guest_data.keys())[0] - value = guest_data[key] - - if body['guest']['args']['data'].get(key) != None: - body['guest']['args']['data'][key].extend(value) - else: - body['guest']['args']['data'].update(guest_data) - - for host_data in self._host_data: - key = list(host_data.keys())[0] - value = host_data[key] - - if body['host']['args']['data'].get(key) != None: - body['host']['args']['data'][key].extend(value) - else: - body['host']['args']['data'].update(host_data) - - for guest_module_config in self._guest_module_config: - - body['guest'].update(guest_module_config) - - for host_module_config in self._host_module_config: - - body['host'].update(host_module_config) - - if body['guest']['args']['data'] == {}: - body['guest'].pop('args') - - if body['host']['args']['data'] == {}: - body['host'].pop('args') - - return {name: body} - - -class RoleParametersBuilder(object): - """Build RoleParameters instance""" - - def __init__(self): - self.reset() - - def reset(self): - self._role_parameters = RoleParameters() - - def _set_data(self, namespace='', name='', data_type='train_data', role='guest'): - - # simple check - # if len(namespaces) != len(names): - # raise Exception("The number of namespaces and name is not matched") - - key = data_type - body = [{'namespace': namespace, - 'name': name}] - - # for ns, n in zip(namespaces, names): - # body.append({'namespace': namespace, 'name': name}) - - if role == 'guest': - self._role_parameters._guest_data.append({key: body}) - - elif role == 'host': - self._role_parameters._host_data.append({key: body}) - - def add_guest_train_data(self, namespace='', name='') -> object: - """ - - :param namespaces: The namespace of train data - :type namespaces: string - :param name: The name of train data - :type name: string - - :return: - """ - self._set_data(namespace, name, 'train_data', 'guest') - - return self - - def add_guest_eval_data(self, namespace='', name='') -> object: - """ - - :param namespaces: The namespace of train data - :type namespaces: string - :param name: The name of train data - :type name: string - - :return: - """ - self._set_data(namespace, name, 'eval_data', 'guest') - - return self - - def add_host_train_data(self, namespace='', name='') -> object: - """ - - :param namespaces: The namespace of train data - :type namespaces: string - :param name: The name of train data - :type name: string - - :return: - """ - self._set_data(namespace, name, 'train_data', 'host') - - return self - - def add_host_eval_data(self, namespace='', name='') -> object: - """ - - :param namespaces: The namespace of train data - :type namespaces: string - :param name: The name of train data - :type name: string - - :return: - """ - self._set_data(namespace, name, 'eval_data', 'host') - - return self - - def _set_config(self, module='', config={}, role='guest'): - # simple check - # if len(modules) != len(configs): - # raise Exception("The number of modules and configs in not matched") - - # for m, c in zip(modules, configs): - key = module - body = config - - if role == 'guest': - self._role_parameters._guest_module_config.append({key: body}) - elif role == 'host': - self._role_parameters._host_module_config.append({key: body}) - - def add_guest_module_config(self, module='', config={}): - """ Set guest module config - - :param modules: The modules - :type modules: str - :param configs: The configs - :type configs: dict - """ - - self._set_config(module, config, 'guest') - return self - - def add_host_module_config(self, module='', config={}): - """ Set guest module config - - :param modules: The modules - :type modules: str - :param configs: The configs - :type configs: dict - """ - - self._set_config(module, config, 'host') - return self - - def build(self) -> object: - """ - - :return: - """ - - role_parameters = self._role_parameters - self.reset() - return role_parameters - - -class AlgorithmParameters(): - """Class to define algorithm_parameters in configuration of job""" - - def __init__(self): - self._parameters_list = [] - - def to_dict(self): - name = 'algorithm_parameters' - body = {} - - for parameters in self._parameters_list: - body.update(parameters) - - return {name: body} - - -class AlgorithmParametersBuilder(): - """Class to build AlgorithmParameters instance""" - - def __init__(self): - self.reset() - - def reset(self): - self._algorithm_parameters = AlgorithmParameters() - - def add_module_config(self, module='', config={}): - """ - - :param modules: The available module in FATE, for more details please refer to `Modules ` - :type modules: list - - :param configs: A list of configs, the available module config in FATE, for more details please refer to `parameters ` - - :type confgis: list - """ - # if len(modules) != len(configs): - # raise Exception("The number of modules and configs is not matched") - - # for m, c in zip(modules, configs): - # self._algorithm_parameters._parameters_list.append({m: c}) - self._algorithm_parameters._parameters_list.append({module: config}) - - return self - - def build(self): - algorithm_parameters = self._algorithm_parameters - self.reset() - return algorithm_parameters - - -class Config(object): - """Define configuration of job""" - - def __init__(self, initiator=None, job_parameters=None, - role=None, role_parameters=None, algorithm_parameters=None): - self._initiator = initiator - self._job_parameters = job_parameters - self._role = role - self._role_parameters = role_parameters - self._algorithm_parameters = algorithm_parameters - - def to_dict(self) -> dict: - - result = {} - - if self._initiator is not None: - result.update(self._initiator.to_dict()) - - if self._job_parameters is not None: - result.update(self._job_parameters.to_dict()) - - if self._role is not None: - result.update(self._role.to_dict()) - - if self._role_parameters is not None: - result.update(self._role_parameters.to_dict()) - - if self._algorithm_parameters is not None: - result.update(self._algorithm_parameters.to_dict()) - - return result - - -class ConfigBuilder(object): - """Class to build Config instance""" - - def __init__(self): - self.reset() - - def reset(self): - self._config = Config() - - def with_initiator(self, initiator: Initiator) -> object: - self._config._initiator = initiator - return self - - def with_job_parameters(self, job_parameters: JobParameters) -> object: - self._config._job_parameters = job_parameters - return self - - def with_role(self, role: Role) -> object: - self._config._role = role - return self - - def with_role_parameters(self, role_parameters: RoleParameters) -> object: - self._config._role_parameters = role_parameters - return self - - def with_algorithm_parameters(self, algorithm_parameters: AlgorithmParameters) -> object: - self._config._algorithm_parameters = algorithm_parameters - return self - - def build(self) -> object: - config = self._config - self.reset() - return config diff --git a/fml_manager/fml_manager/utils/file_utils.py b/fml_manager/fml_manager/utils/file_utils.py deleted file mode 100644 index 910b1589c..000000000 --- a/fml_manager/fml_manager/utils/file_utils.py +++ /dev/null @@ -1,61 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import json -import os - -from cachetools import LRUCache -from cachetools import cached - -PROJECT_BASE = None - - -def get_project_base_directory(): - global PROJECT_BASE - if PROJECT_BASE is None: - PROJECT_BASE = os.path.abspath( - os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, os.pardir)) - return PROJECT_BASE - - -@cached(cache=LRUCache(maxsize=10)) -def load_json_conf(conf_path): - if os.path.isabs(conf_path): - json_conf_path = conf_path - else: - json_conf_path = os.path.join(get_project_base_directory(), conf_path) - try: - with open(json_conf_path) as f: - return json.load(f) - except: - raise EnvironmentError("loading json file config from '{}' failed!".format(json_conf_path)) - - -def dump_json_conf(config_data, conf_path): - if os.path.isabs(conf_path): - json_conf_path = conf_path - else: - json_conf_path = os.path.join(get_project_base_directory(), conf_path) - try: - with open(json_conf_path, "w") as f: - json.dump(config_data, f, indent=4) - except: - raise EnvironmentError("loading json file config from '{}' failed!".format(json_conf_path)) - - -if __name__ == "__main__": - print(get_project_base_directory()) - print(load_json_conf('federatedml/transfer_variable/definition/transfer_conf.json')) diff --git a/fml_manager/setup.py b/fml_manager/setup.py deleted file mode 100644 index fc206391f..000000000 --- a/fml_manager/setup.py +++ /dev/null @@ -1,27 +0,0 @@ -import setuptools - -with open("README.md", "r") as fh: - long_description = fh.read() - -setuptools.setup( - name="fml_manager", - version="0.5.0", - author="Layne Peng/Jiahao Chen", - author_email="jiahaoc@vmware.com", - description="Python client for FATE cluster", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://not-decided.yet", - packages=setuptools.find_packages(), - install_requires=[ - 'cachetools==3.0.0', - 'requests>=2.21.0', - 'pandas>=1.1.0' - ], - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - ], - python_requires='>=3.6', -) diff --git a/fml_manager/source/README.md b/fml_manager/source/README.md deleted file mode 100644 index 68ab54b5e..000000000 --- a/fml_manager/source/README.md +++ /dev/null @@ -1,16 +0,0 @@ -FML_Manager -=============== -This is a client for FATE cluster - -Compatible KubeFATE Versions -============= - - |FML Manager version | KubeFATE version 1.3.x | KubeFATE version 1.4.x| - |---|---|---| - |0.2.0 | incompatible | compatible| - |0.3.0 | incompatible | compatible| - |0.4.0 | incompatible | compatible| - -- incompatible: the combination does not work at all -- compatible: all Kubeflow features have been tested and verified for the Kubernetes version -- no known issues: the combination has not been fully tested but there are no repoted issues diff --git a/fml_manager/source/conf.py b/fml_manager/source/conf.py deleted file mode 100644 index 4b66a0813..000000000 --- a/fml_manager/source/conf.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -__mtime__ = '2020-07-30' -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) - -import os -import sys -sys.path.insert(0, os.path.abspath('../../fml_manager')) -print(sys.path) -# -- Project information ----------------------------------------------------- - -project = 'KubeFATE' -copyright = '2020, vmware' -author = 'vmware' - -# The full version, including alpha/beta/rc tags -release = '1.4.2' -from recommonmark.parser import CommonMarkParser -source_parsers = { - '.md': CommonMarkParser, -} -source_suffix = ['.rst', '.md'] - -# -- General configuration --------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.napoleon", - 'sphinx.ext.doctest', - 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', - 'sphinx.ext.mathjax', -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = 'en_US' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [] - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -import sphinx_rtd_theme -html_theme = "sphinx_rtd_theme" -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - -# html_theme = 'alabaster' - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] -master_doc = 'index' diff --git a/fml_manager/source/fml_manager.rst b/fml_manager/source/fml_manager.rst deleted file mode 100644 index da3b3b78b..000000000 --- a/fml_manager/source/fml_manager.rst +++ /dev/null @@ -1,38 +0,0 @@ -fml\_manager package -==================== - -Subpackages ------------ - -.. toctree:: - :maxdepth: 4 - - fml_manager.utils - -Submodules ----------- - -fml\_manager.fml\_cluster\_manager module ------------------------------------------ - -.. automodule:: fml_manager.fml_cluster_manager - :members: - :undoc-members: - :show-inheritance: - -fml\_manager.fml\_manager module --------------------------------- - -.. automodule:: fml_manager.fml_manager - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: fml_manager - :members: - :undoc-members: - :show-inheritance: diff --git a/fml_manager/source/fml_manager.utils.rst b/fml_manager/source/fml_manager.utils.rst deleted file mode 100644 index 179341bf3..000000000 --- a/fml_manager/source/fml_manager.utils.rst +++ /dev/null @@ -1,38 +0,0 @@ -fml\_manager.utils package -========================== - -Submodules ----------- - -fml\_manager.utils.core module ------------------------------- - -.. automodule:: fml_manager.utils.core - :members: - :undoc-members: - :show-inheritance: - -fml\_manager.utils.fate\_builders module ----------------------------------------- - -.. automodule:: fml_manager.utils.fate_builders - :members: - :undoc-members: - :show-inheritance: - -fml\_manager.utils.file\_utils module -------------------------------------- - -.. automodule:: fml_manager.utils.file_utils - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: fml_manager.utils - :members: - :undoc-members: - :show-inheritance: diff --git a/fml_manager/source/index.rst b/fml_manager/source/index.rst deleted file mode 100644 index 5d79697b6..000000000 --- a/fml_manager/source/index.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. KubeFATE documentation master file, created by - sphinx-quickstart on Wed Jul 15 11:22:08 2020. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to KubeFATE's documentation! -==================================== - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - README.md - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/fml_manager/source/modules.rst b/fml_manager/source/modules.rst deleted file mode 100644 index 4769d5d96..000000000 --- a/fml_manager/source/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -fml_manager -=========== - -.. toctree:: - :maxdepth: 4 - - fml_manager/fml_manager diff --git a/fml_manager/tests/test_builder.py b/fml_manager/tests/test_builder.py deleted file mode 100644 index 31715c05a..000000000 --- a/fml_manager/tests/test_builder.py +++ /dev/null @@ -1,60 +0,0 @@ -import pprint -import json - -from fml_manager import * - -# dsl -secure_add_example = Component(name='secure_add_example_0', - module='SecureAddExample') - -dsl = Pipeline( - secure_add_example -) - - -# Configuration -initiator = Initiator(role='guest', - party_id=9999) - - -job_parameters = JobParameters(work_mode=1) - -role = RoleBuilder()\ - .add_guest(9999)\ - .add_host(9999).build() - -secure_add_example_guest_config = { - "seed": [ - 123 - ] -} - - -secure_add_example_host_config = { - "seed": [ - 321 - ] -} -role_parameters = RoleParametersBuilder()\ - .add_host_module_config(module='secure_add_example_0', config=secure_add_example_host_config)\ - .add_guest_module_config(module='secure_add_example_0', config=secure_add_example_guest_config).build() - -secure_add_example = { - "partition": 10, - "data_num": 1000 -} - - -algorithm_parameters = AlgorithmParametersBuilder()\ - .add_module_config(module='secure_add_example_0', config=secure_add_example).build() - -config = Config( - initiator, - job_parameters, - role, - role_parameters, - algorithm_parameters -) - -pprint.pprint(config.to_dict()) -pprint.pprint(dsl.to_dict()) diff --git a/fml_manager/tests/test_clustermanager.py b/fml_manager/tests/test_clustermanager.py deleted file mode 100644 index 7d54cae1a..000000000 --- a/fml_manager/tests/test_clustermanager.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2019-2020 VMware, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# you may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# To add a new cell, type '# %%' -# To add a new markdown cell, type '# %% [markdown]' - -# %% -from fml_manager import ClusterManager -from fml_manager import Party, PartyType -# For more details about the FMLManager, please refer to https://kubefate.readthedocs.io/README.html - - -# %% -cluster_manager = ClusterManager(cluster_namespace='fate-10000', cluster_name='fate') - - -# %% -# get route table -route_table = cluster_manager.get_route_table() - - -# %% -# show all party -print(route_table.get_parties()) - - -# %% -# delete route table party -party_id_1 = '9999' -party_id_2 = '8888' -route_table.remove_parties(party_id_1, party_id_2) -print(route_table.get_parties()) - - -# %% -# define normal party -party = Party(p_id='9999', ip='192.168.2.2', port=30010) - - -# %% -# append normal party to route table -route_table.add_parties(party) -print(route_table.get_parties()) - - -# %% -# define exchange -party = Party(ip='192.168.2.2', port=30009, p_type=PartyType.EXCHANGE) - - -# %% -# append exchange to route table -route_table.add_parties(party) -print(route_table.get_parties()) - - -# %% -# update route table of configmap -cluster_manager.set_route_table(route_table) - - -# %% -# get entrypoint -print(cluster_manager.get_entry_point()) - diff --git a/fml_manager/tests/test_hetero_lr.py b/fml_manager/tests/test_hetero_lr.py deleted file mode 100644 index ebcb857c8..000000000 --- a/fml_manager/tests/test_hetero_lr.py +++ /dev/null @@ -1,239 +0,0 @@ -import pprint -import json - -from fml_manager import * - -# hetero lr -pipeline_str = ''' -{ - "components" : { - "dataio_0": { - "module": "DataIO", - "input": { - "data": { - "data": [ - "args.train_data" - ] - } - }, - "output": { - "data": ["train"], - "model": ["dataio"] - }, - "need_deploy": true - }, - "hetero_feature_binning_0": { - "module": "HeteroFeatureBinning", - "input": { - "data": { - "data": [ - "dataio_0.train" - ] - } - }, - "output": { - "data": ["train"], - "model": ["hetero_feature_binning"] - } - }, - "hetero_feature_selection_0": { - "module": "HeteroFeatureSelection", - "input": { - "data": {"data": [ - "hetero_feature_binning_0.train" - ] - }, - "isometric_model": [ - "hetero_feature_binning_0.hetero_feature_binning" - ] - }, - "output": { - "data": ["train"], - "model": ["selected"] - } - }, - "hetero_lr_0": { - "module": "HeteroLR", - "input": { - "data": { - "train_data": ["hetero_feature_selection_0.train"] - } - }, - "output": { - "data": ["train"], - "model": ["hetero_lr"] - } - }, - "evaluation_0": { - "module": "Evaluation", - "input": { - "data": { - "data": ["hetero_lr_0.train"] - } - }, - "output": { - "data": ["evaluate"] - } - } - } -} -''' - -# Pipeline -data_io = ComponentBuilder(name='dataio_0', - module='DataIO')\ - .add_input_data('args.train_data')\ - .add_output_data('train')\ - .add_output_model('dataio').build() - -hetero_feature_binning = ComponentBuilder(name='hetero_feature_binning_0', - module='HeteroFeatureBinning')\ - .add_input_data('dataio_0.train')\ - .add_output_data('train')\ - .add_output_model('hetero_feature_binning').build() - -hetero_feature_selection = ComponentBuilder(name='hetero_feature_selection_0', - module='HeteroFeatureSelection')\ - .add_input_data('hetero_feature_binning_0.train')\ - .add_input_isometric_model('hetero_feature_binning_0.hetero_feature_binning')\ - .add_output_data('train')\ - .add_output_model('selected').build() - -hetero_lr = ComponentBuilder(name='hetero_lr_0', - module='HeteroLR')\ - .add_input_train_data('hetero_feature_selection_0.train')\ - .add_output_data('train')\ - .add_output_model('hetero_lr').build() - -evaluation = ComponentBuilder(name='evaluation_0', - module='Evaluation', - need_deploy=False)\ - .add_input_data('hetero_lr_0.train')\ - .add_output_data('evaluate').build() -pipeline = Pipeline( - data_io, - hetero_feature_selection, - hetero_feature_binning, - hetero_lr, - evaluation) - -lho = pipeline.to_dict() -rho = json.loads(pipeline_str) - -pprint.pprint(lho) -print('------') -pprint.pprint(rho) - -# config -config_str = ''' -{ - "initiator": { - "role": "guest", - "party_id": 10000 - }, - "job_parameters": { - "work_mode": 1 - }, - "role": { - "guest": [10000], - "host": [9999], - "arbiter": [9999] - }, - "role_parameters": { - "guest": { - "args": { - "data": { - "train_data": [{"name": "breast_b", "namespace": "fate_flow_test_breast"}] - } - }, - "dataio_0":{ - "with_label": [true], - "label_name": ["y"], - "label_type": ["int"], - "output_format": ["dense"] - } - }, - "host": { - "args": { - "data": { - "train_data": [{"name": "breast_a", "namespace": "fate_flow_test_breast"}] - } - }, - "dataio_0":{ - "with_label": [false], - "output_format": ["dense"] - } - } - }, - "algorithm_parameters": { - "hetero_lr_0": { - "penalty": "L2", - "optimizer": "rmsprop", - "eps": 1e-5, - "alpha": 0.01, - "max_iter": 3, - "converge_func": "diff", - "batch_size": 320, - "learning_rate": 0.15, - "init_param": { - "init_method": "random_uniform" - } - } - } -} -''' - -# Configuration -initiator = Initiator(role='guest', party_id=10000) - -job_parameters = JobParameters(work_mode=1) - -role = RoleBuilder()\ - .add_guest(10000)\ - .add_host(9999)\ - .add_arbiter(9999).build() - -guest_data_io_config = { - 'with_label': [True], - 'label_name': ['y'], - 'label_type': ['int'], - 'output_format': ['dense'] -} - -host_data_io_config = { - 'with_label': [False], - 'output_format': ['dense'] -} - -role_parameters = RoleParametersBuilder()\ - .add_guest_train_data(namespace='fate_flow_test_breast', name='breast_b')\ - .add_guest_module_config(module='dataio_0', config=guest_data_io_config)\ - .add_host_train_data(namespace='fate_flow_test_breast', name='breast_a')\ - .add_host_module_config(module='dataio_0', config=host_data_io_config).build() - -hetero_lr_params = { - "penalty": "L2", - "optimizer": "rmsprop", - "eps": 1e-5, - "alpha": 0.01, - "max_iter": 3, - "converge_func": "diff", - "batch_size": 320, - "learning_rate": 0.15, - "init_param": { - "init_method": "random_uniform" - } -} - -algorithm_parameters = AlgorithmParametersBuilder()\ - .add_module_config(module='hetero_lr_0', config=hetero_lr_params).build() - -config = Config(initiator, job_parameters, role, - role_parameters, algorithm_parameters) - -lho = config.to_dict() -rho = json.loads(config_str) - -pprint.pprint(lho) -print('------') -pprint.pprint(rho) diff --git a/fml_manager/tests/test_homo_lr.py b/fml_manager/tests/test_homo_lr.py deleted file mode 100644 index 96bfa0e24..000000000 --- a/fml_manager/tests/test_homo_lr.py +++ /dev/null @@ -1,99 +0,0 @@ - -from fml_manager import * - -# dsl -data_io = ComponentBuilder()\ - .with_name('dataio_0')\ - .with_module('DataIO')\ - .add_input_data('args.train_data')\ - .add_output_data('train')\ - .add_output_model('dataio').build() - - -homo_lr = ComponentBuilder()\ - .with_name('homo_lr_0')\ - .with_module('HomoLR')\ - .add_input_train_data('dataio_0.train')\ - .add_output_data('train')\ - .add_output_model('homolr').build() - -evaluation = ComponentBuilder()\ - .with_name('evaluation_0')\ - .with_module('Evaluation')\ - .add_input_data('homo_lr_0.train')\ - .add_output_data('evaluate').build() - -pipeline = Pipeline( - data_io, - homo_lr, - evaluation -) - -# Configuration -initiator = Initiator(role='guest', party_id=10000) - -job_parameters = JobParametersBuilder()\ - .with_work_mode(1).build() - -role = RoleBuilder()\ - .add_guest(party_id=10000)\ - .add_host(party_id=10000)\ - .add_host(party_id=9999)\ - .add_arbiter(party_id=10000).build() - -eval_config = { - 'need_run': [False] -} - -role_parameters = RoleParametersBuilder()\ - .add_guest_train_data(namespace='homo_breast_guest', name='homo_breast_guest')\ - .add_host_train_data(namespace='homo_breast_host', name='homo_breast_host_a')\ - .add_host_train_data(namespace='homo_breast_host', name='homo_breast_host_b')\ - .add_host_module_config(module='evaluation_0', config=eval_config).build() - - -homo_lr_params = { - 'penalty': 'L2', - 'optimizer': 'sgd', - 'eps': 1e-5, - 'alpha': 0.01, - 'max_iter': 10, - 'converge_func': 'diff', - 'batch_size': 500, - 'learning_rate': 0.15, - 'decay': 1, - 'decay_sqrt': True, - 'init_param': { - 'init_method': 'zeros' - }, - 'encrypt_param': { - 'method': 'Paillier' - }, - 'cv_param': { - 'n_splits': 4, - 'shuffle': True, - 'random_seed': 33, - 'need_cv': False - } -} -dotaio_config = { - 'with_label': True, - 'label_name': 'y', - 'label_type': 'int', - 'output_format': 'dense' -} - -algorithm_parameters = AlgorithmParametersBuilder()\ - .add_module_config(module='homo_lr_0', config=homo_lr_params)\ - .add_module_config(module='dataio_0', config=dotaio_config).build() - -config = Config( - initiator, - job_parameters, - role, - role_parameters, - algorithm_parameters -) - -print(pipeline.to_dict()) -print(config.to_dict()) diff --git a/helm-charts/FATE-Exchange/Chart.yaml b/helm-charts/FATE-Exchange/Chart.yaml index 26ce21b4e..6eddb332c 100644 --- a/helm-charts/FATE-Exchange/Chart.yaml +++ b/helm-charts/FATE-Exchange/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: v1.8.0 +appVersion: v1.9.0 description: A Helm chart for fate exchange name: fate-exchange -version: v1.8.0 +version: v1.9.0 diff --git a/helm-charts/FATE-Exchange/templates/rollsite-module.yaml b/helm-charts/FATE-Exchange/templates/rollsite-module.yaml index f4523b365..a874f140c 100644 --- a/helm-charts/FATE-Exchange/templates/rollsite-module.yaml +++ b/helm-charts/FATE-Exchange/templates/rollsite-module.yaml @@ -34,6 +34,17 @@ data: eggroll.rollsite.push.batches.per.stream=10 eggroll.rollsite.adapter.sendbuf.size=100000 + {{- if .Values.modules.rollsite.enableTLS }} + eggroll.core.security.secure.cluster.enabled=true + eggroll.core.security.secure.client.auth.enabled=true + eggroll.core.security.ca.crt.path=conf/cert/ca.pem + eggroll.core.security.crt.path=conf/cert/server.crt + eggroll.core.security.key.path=conf/cert/server.key + eggroll.core.security.client.ca.crt.path=conf/cert/ca.pem + eggroll.core.security.client.crt.path=conf/cert/client.crt + eggroll.core.security.client.key.path=conf/cert/client.key + {{- end }} + route_table.json: | { "route_table": { @@ -133,6 +144,10 @@ spec: - mountPath: /data/projects/fate/eggroll/conf/eggroll.properties name: rollsite-confs subPath: eggroll.properties + {{- if .Values.modules.rollsite.enableTLS }} + - mountPath: /data/projects/fate/eggroll/conf/cert/ + name: eggroll-certs + {{- end }} {{- with .Values.modules.rollsite.nodeSelector }} nodeSelector: {{ toYaml . | indent 8 }} @@ -155,6 +170,11 @@ spec: - name: rollsite-confs configMap: name: rollsite-config + {{- if .Values.modules.rollsite.enableTLS }} + - name: eggroll-certs + secret: + secretName: eggroll-certs + {{- end}} --- apiVersion: v1 kind: Service diff --git a/helm-charts/FATE-Exchange/values-template-example.yaml b/helm-charts/FATE-Exchange/values-template-example.yaml index 7e4d068c3..5d6494c43 100644 --- a/helm-charts/FATE-Exchange/values-template-example.yaml +++ b/helm-charts/FATE-Exchange/values-template-example.yaml @@ -1,11 +1,9 @@ name: fate-exchange namespace: fate-exchange chartName: fate-exchange -chartVersion: v1.8.0 +chartVersion: v1.9.0 partyId: 1 registry: "" -imageTag: "1.8.0-release" -pullPolicy: imagePullSecrets: - name: myregistrykey persistence: false @@ -22,6 +20,7 @@ rollsite: type: NodePort nodePort: 30001 loadBalancerIP: 192.168.0.1 + enableTLS: false partyList: - partyId: 10000 partyIp: 192.168.10.1 diff --git a/helm-charts/FATE-Exchange/values-template.yaml b/helm-charts/FATE-Exchange/values-template.yaml index e89ae3c3d..8decf69e8 100644 --- a/helm-charts/FATE-Exchange/values-template.yaml +++ b/helm-charts/FATE-Exchange/values-template.yaml @@ -4,7 +4,6 @@ partyName: {{ .name }} image: registry: {{ .registry | default "federatedai" }} isThridParty: {{ empty .registry | ternary "false" "true" }} - tag: {{ .imageTag | default "1.8.0-release" }} pullPolicy: {{ .pullPolicy | default "IfNotPresent" }} {{- with .imagePullSecrets }} imagePullSecrets: @@ -50,6 +49,7 @@ modules: {{ toYaml . | indent 6 }} {{- end }} type: {{ .type }} + enableTLS: {{ .enableTLS | default false }} nodePort: {{ .nodePort }} partyList: {{- range .partyList }} diff --git a/helm-charts/FATE-Exchange/values.yaml b/helm-charts/FATE-Exchange/values.yaml index 0ba13c2d4..6bacfe002 100644 --- a/helm-charts/FATE-Exchange/values.yaml +++ b/helm-charts/FATE-Exchange/values.yaml @@ -4,7 +4,7 @@ partyName: fate-exchange image: registry: federatedai isThridParty: - tag: 1.8.0-release + tag: 1.9.0-release pullPolicy: IfNotPresent imagePullSecrets: # - name: @@ -29,7 +29,8 @@ modules: ip: rollsite type: ClusterIP nodePort: 30001 - loadBalancerIP: + loadBalancerIP: + enableTLS: false nodeSelector: tolerations: affinity: diff --git a/helm-charts/FATE-Serving/Chart.yaml b/helm-charts/FATE-Serving/Chart.yaml index 7f7e3b758..bb8c6674a 100644 --- a/helm-charts/FATE-Serving/Chart.yaml +++ b/helm-charts/FATE-Serving/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 -appVersion: v2.1.5 +appVersion: v2.1.6 description: A Helm chart for FATE-Serving name: fate-serving -version: v2.1.5 +version: v2.1.6 sources: - https://github.com/FederatedAI/KubeFATE - https://github.com/FederatedAI/FATE-Serving \ No newline at end of file diff --git a/helm-charts/FATE-Serving/templates/serving-server-module.yaml b/helm-charts/FATE-Serving/templates/serving-server-module.yaml index caf8ad117..c427a42fe 100644 --- a/helm-charts/FATE-Serving/templates/serving-server-module.yaml +++ b/helm-charts/FATE-Serving/templates/serving-server-module.yaml @@ -108,7 +108,10 @@ spec: subPath: serving-server.properties - name: data mountPath: /root/.fate - subPath: {{ .Values.servingServer.subPath }} + subPath: cache + - name: data + mountPath: /data/projects/fate-serving/serving-server/.fate + subPath: model_cache {{- with .Values.servingServer.nodeSelector }} nodeSelector: {{ toYaml . | indent 8 }} diff --git a/helm-charts/FATE-Serving/values-template-example.yaml b/helm-charts/FATE-Serving/values-template-example.yaml index 533a60374..fda57e60a 100644 --- a/helm-charts/FATE-Serving/values-template-example.yaml +++ b/helm-charts/FATE-Serving/values-template-example.yaml @@ -1,11 +1,10 @@ name: fate-serving-10000 namespace: fate-serving-10000 chartName: fate-serving -chartVersion: v2.1.5 +chartVersion: v2.1.6 partyId: 10000 registry: "" -imageTag: "2.1.5-release" -pullPolicy: +pullPolicy: imagePullSecrets: - name: myregistrykey persistence: false diff --git a/helm-charts/FATE-Serving/values-template.yaml b/helm-charts/FATE-Serving/values-template.yaml index 66a5cce3a..6e9b94fee 100644 --- a/helm-charts/FATE-Serving/values-template.yaml +++ b/helm-charts/FATE-Serving/values-template.yaml @@ -6,7 +6,6 @@ partyName: {{ .name }} image: registry: {{ .registry | default "federatedai" }} isThridParty: {{ empty .registry | ternary "false" "true" }} - tag: {{ .imageTag | default "2.1.5-release" }} pullPolicy: {{ .pullPolicy | default "IfNotPresent" }} {{- with .imagePullSecrets }} imagePullSecrets: @@ -71,6 +70,7 @@ servingProxy: type: {{ .type }} nodePort: {{ .nodePort }} loadBalancerIP: {{ .loadBalancerIP }} + {{- with .partyList }} partyList: {{- range . }} diff --git a/helm-charts/FATE-Serving/values.yaml b/helm-charts/FATE-Serving/values.yaml index 433cd948e..f5e120598 100644 --- a/helm-charts/FATE-Serving/values.yaml +++ b/helm-charts/FATE-Serving/values.yaml @@ -5,7 +5,7 @@ partyName: fate-serving-9999 image: registry: federatedai isThridParty: - tag: 2.1.5-release + tag: 2.1.6-release pullPolicy: IfNotPresent imagePullSecrets: # - name: diff --git a/helm-charts/FATE/Chart.yaml b/helm-charts/FATE/Chart.yaml index 6f323acda..5e410fd96 100644 --- a/helm-charts/FATE/Chart.yaml +++ b/helm-charts/FATE/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 -appVersion: v1.8.0 +appVersion: v1.9.0 description: A Helm chart for fate-training name: fate -version: v1.8.0 +version: v1.9.0 home: https://fate.fedai.org icon: https://aisp-1251170195.cos.ap-hongkong.myqcloud.com/wp-content/uploads/sites/12/2019/09/logo.png sources: diff --git a/helm-charts/FATE/templates/_helpers.tpl b/helm-charts/FATE/templates/_helpers.tpl index 8b6bae344..80247f6a3 100644 --- a/helm-charts/FATE/templates/_helpers.tpl +++ b/helm-charts/FATE/templates/_helpers.tpl @@ -35,4 +35,8 @@ Create the name of the controller service account to use {{- else -}} {{ default "default" }} {{- end -}} +{{- end -}} + +{{- define "images.registry" -}} +{{- .Values.image.registry | default "federatedai" -}} {{- end -}} \ No newline at end of file diff --git a/fml_manager/tests/test_fmlmananger.py b/helm-charts/FATE/templates/backends/eggroll/_helpers.tpl similarity index 65% rename from fml_manager/tests/test_fmlmananger.py rename to helm-charts/FATE/templates/backends/eggroll/_helpers.tpl index 867e675b2..c35a3cc4d 100644 --- a/fml_manager/tests/test_fmlmananger.py +++ b/helm-charts/FATE/templates/backends/eggroll/_helpers.tpl @@ -1,8 +1,8 @@ -# Copyright 2019-2020 VMware, Inc. -# + +# Copyright 2019-2022 VMware, Inc. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# you may obtain a copy of the License at +# You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -10,9 +10,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from fml_manager import FMLManager +{{/* Images Suffix */}} -if __name__ == '__main__': - fml_manager = FMLManager() - print(fml_manager.server_url) - print(fml_manager.serving_url) \ No newline at end of file +{{- define "images.eggroll.suffix" -}} +{{- if eq .Values.algorithm "NN" -}} +-nn +{{- end -}} +{{- if eq .Values.device "IPCL" -}} +-ipcl +{{- end -}} +{{- end -}} diff --git a/helm-charts/FATE/templates/backends/eggroll/clustermanager/deployment.yaml b/helm-charts/FATE/templates/backends/eggroll/clustermanager/deployment.yaml index e708c4942..6b21c3754 100644 --- a/helm-charts/FATE/templates/backends/eggroll/clustermanager/deployment.yaml +++ b/helm-charts/FATE/templates/backends/eggroll/clustermanager/deployment.yaml @@ -35,8 +35,15 @@ spec: - env: - name: PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION value: python - image: {{ .Values.image.registry }}/eggroll:{{ .Values.image.tag }} + image: {{ .Values.image.registry }}/eggroll{{ include "images.eggroll.suffix" . }}:{{ .Values.image.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.modules.clustermanager.resources}} + resources: + {{- range $key, $val := .Values.modules.clustermanager.resources }} + {{ $key }}: +{{ toYaml $val | indent 14 }} + {{- end }} + {{- end }} name: clustermanager command: - bash diff --git a/helm-charts/FATE/templates/backends/eggroll/clustermanager/configmap.yaml b/helm-charts/FATE/templates/backends/eggroll/configmap.yaml similarity index 87% rename from helm-charts/FATE/templates/backends/eggroll/clustermanager/configmap.yaml rename to helm-charts/FATE/templates/backends/eggroll/configmap.yaml index f3ab2929b..b8b96b619 100644 --- a/helm-charts/FATE/templates/backends/eggroll/clustermanager/configmap.yaml +++ b/helm-charts/FATE/templates/backends/eggroll/configmap.yaml @@ -63,7 +63,7 @@ data: hadoop.dfs.namenode.rpc-address.nn2= # session - eggroll.session.processors.per.node=4 + eggroll.session.processors.per.node={{ .Values.modules.nodemanager.sessionProcessorsPerNode | default 2 }} eggroll.session.start.timeout.ms=180000 # rollpair @@ -83,7 +83,6 @@ data: eggroll.rollsite.push.long.retry=2 eggroll.rollsite.push.batches.per.stream=10 eggroll.rollsite.adapter.sendbuf.size=100000 - # polling # {{ .Values.modules.rollsite.polling.enabled }} # {{ .Values.modules.rollsite.polling.type }} @@ -96,4 +95,15 @@ data: eggroll.rollsite.polling.server.enabled=true eggroll.rollsite.polling.concurrency= {{ .Values.modules.rollsite.polling.concurrency | default 50 }} {{- end }} + {{- if .Values.modules.rollsite.enableTLS }} + cert_configs: | + eggroll.core.security.secure.cluster.enabled=true + eggroll.core.security.secure.client.auth.enabled=true + eggroll.core.security.ca.crt.path=conf/cert/ca.pem + eggroll.core.security.crt.path=conf/cert/server.crt + eggroll.core.security.key.path=conf/cert/server.key + eggroll.core.security.client.ca.crt.path=conf/cert/ca.pem + eggroll.core.security.client.crt.path=conf/cert/client.crt + eggroll.core.security.client.key.path=conf/cert/client.key + {{- end }} {{ end }} \ No newline at end of file diff --git a/helm-charts/FATE/templates/backends/eggroll/lb-rollsite/deployment.yaml b/helm-charts/FATE/templates/backends/eggroll/lb-rollsite/deployment.yaml index 7f15293c0..5764d6b76 100644 --- a/helm-charts/FATE/templates/backends/eggroll/lb-rollsite/deployment.yaml +++ b/helm-charts/FATE/templates/backends/eggroll/lb-rollsite/deployment.yaml @@ -52,8 +52,15 @@ spec: env: - name: PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION value: python - image: {{ .Values.image.registry }}/eggroll:{{ .Values.image.tag }} + image: {{ .Values.image.registry }}/eggroll{{ include "images.eggroll.suffix" . }}:{{ .Values.image.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.modules.lbrollsite.resources}} + resources: + {{- range $key, $val := .Values.modules.lbrollsite.resources }} + {{ $key }}: +{{ toYaml $val | indent 14 }} + {{- end }} + {{- end }} command: - bash - -c diff --git a/helm-charts/FATE/templates/backends/eggroll/nodemanager/configmap.yaml b/helm-charts/FATE/templates/backends/eggroll/nodemanager/configmap.yaml index b1d479f0b..3893c6869 100644 --- a/helm-charts/FATE/templates/backends/eggroll/nodemanager/configmap.yaml +++ b/helm-charts/FATE/templates/backends/eggroll/nodemanager/configmap.yaml @@ -10,79 +10,14 @@ # limitations under the License. {{- if .Values.modules.nodemanager.include }} -{{- $root := . -}} -{{- range $nodemanager := .Values.modules.nodemanager.list }} ---- kind: ConfigMap apiVersion: v1 metadata: - name: {{ $nodemanager.name }}-config + name: fluentd-config labels: - fateMoudle: {{ $nodemanager.name }} -{{ include "fate.labels" $root | indent 4 }} + fateMoudle: nodemanager +{{ include "fate.labels" . | indent 4 }} data: - eggroll.properties: | - [eggroll] - eggroll.resourcemanager.clustermanager.jdbc.driver.class.name=com.mysql.cj.jdbc.Driver - eggroll.resourcemanager.clustermanager.jdbc.url=jdbc:mysql://{{ $root.Values.modules.mysql.ip }}:{{ $root.Values.modules.mysql.port }}/{{ $root.Values.modules.mysql.mysql_database }}?useSSL=false&serverTimezone=UTC&characterEncoding=utf8&allowPublicKeyRetrieval=true - eggroll.resourcemanager.clustermanager.jdbc.username={{ $root.Values.modules.mysql.mysql_user }} - eggroll.resourcemanager.clustermanager.jdbc.password={{ $root.Values.modules.mysql.mysql_password }} - - eggroll.data.dir=data/ - eggroll.logs.dir=logs/ - eggroll.resourcemanager.clustermanager.host=clustermanager - eggroll.resourcemanager.clustermanager.port=4670 - eggroll.resourcemanager.nodemanager.port=4671 - eggroll.resourcemanager.process.tag={{ $root.Values.partyId }} - - eggroll.bootstrap.root.script=bin/eggroll_boot.sh - - eggroll.resourcemanager.bootstrap.egg_pair.exepath=bin/roll_pair/egg_pair_bootstrap.sh - eggroll.resourcemanager.bootstrap.egg_pair.venv= - eggroll.resourcemanager.bootstrap.egg_pair.pythonpath=/data/projects/fate/python:/data/projects/fate/eggroll/python - eggroll.resourcemanager.bootstrap.egg_pair.filepath=python/eggroll/roll_pair/egg_pair.py - eggroll.resourcemanager.bootstrap.egg_pair.ld_library_path= - - eggroll.resourcemanager.bootstrap.egg_frame.exepath=bin/roll_pair/roll_pair_master_bootstrap.sh - eggroll.resourcemanager.bootstrap.egg_frame.javahome=/usr/lib/jvm/java-1.8.0-openjdk - eggroll.resourcemanager.bootstrap.egg_frame.classpath=conf/:lib/* - eggroll.resourcemanager.bootstrap.egg_frame.mainclass=com.webank.eggroll.rollframe.EggFrameBootstrap - eggroll.resourcemanager.bootstrap.egg_frame.jvm.options= - - eggroll.core.grpc.channel.keepalive.timeout.sec=20 - - # roll_frame - arrow.enable_unsafe_memory_access=true - - # hadoop - hadoop.fs.defaultFS=file:/// - - # hadoop HA mode - hadoop.dfs.nameservices= - hadoop.dfs.namenode.rpc-address.nn1= - hadoop.dfs.namenode.rpc-address.nn2= - - # session - eggroll.session.processors.per.node={{ $nodemanager.sessionProcessorsPerNode }} - eggroll.session.start.timeout.ms=180000 - - # rollpair - eggroll.rollpair.transferpair.sendbuf.size=250000 - - # rollsite - eggroll.rollsite.coordinator={{ $root.Values.partyName }} - eggroll.rollsite.host=rollsite - eggroll.rollsite.port=9370 - eggroll.rollsite.party.id={{ $root.Values.partyId }} - eggroll.rollsite.route.table.path=conf/route_table/route_table.json - eggroll.rollsite.route.table.key= - eggroll.rollsite.route.table.whitelist=127.0.0.1 - eggroll.rollsite.jvm.options=-XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:logs/eggroll/rollsite.gc.log - - eggroll.rollsite.push.max.retry=3 - eggroll.rollsite.push.long.retry=2 - eggroll.rollsite.push.batches.per.stream=10 - eggroll.rollsite.adapter.sendbuf.size=100000 fluent.conf: | @type tail @@ -100,6 +35,4 @@ data: @type stdout - -{{- end }} -{{ end }} \ No newline at end of file +{{- end }} \ No newline at end of file diff --git a/helm-charts/FATE/templates/backends/eggroll/nodemanager/persistentvolumeclaim.yaml b/helm-charts/FATE/templates/backends/eggroll/nodemanager/persistentvolumeclaim.yaml deleted file mode 100644 index 82838f51e..000000000 --- a/helm-charts/FATE/templates/backends/eggroll/nodemanager/persistentvolumeclaim.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2019-2022 VMware, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.modules.nodemanager.include }} -{{- $root := . -}} -{{- range $nodemanager := .Values.modules.nodemanager.list }} ---- -{{- if and $root.Values.persistence.enabled (not $nodemanager.existingClaim) }} -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: {{ $nodemanager.name }}-data - labels: - fateMoudle: {{ $nodemanager.name }} -{{ include "fate.labels" $root | indent 4 }} -spec: - accessModes: - - {{ $nodemanager.accessMode }} - resources: - requests: - storage: {{ $nodemanager.size }} - {{- if $nodemanager.storageClass }} - {{- if eq "-" $nodemanager.storageClass }} - storageClassName: "" - {{- else }} - storageClassName: {{ $nodemanager.storageClass }} - {{- end }} - {{- end }} -{{- end }} - -{{- end }} -{{- end }} \ No newline at end of file diff --git a/helm-charts/FATE/templates/backends/eggroll/nodemanager/service.yaml b/helm-charts/FATE/templates/backends/eggroll/nodemanager/service.yaml index a49d3d83d..1a6812611 100644 --- a/helm-charts/FATE/templates/backends/eggroll/nodemanager/service.yaml +++ b/helm-charts/FATE/templates/backends/eggroll/nodemanager/service.yaml @@ -10,16 +10,13 @@ # limitations under the License. {{- if .Values.modules.nodemanager.include }} -{{- $root := . -}} -{{- range $nodemanager := .Values.modules.nodemanager.list }} ---- apiVersion: v1 kind: Service metadata: labels: - fateMoudle: {{ $nodemanager.name }} -{{ include "fate.labels" $root | indent 4 }} - name: {{ $nodemanager.name }} + fateMoudle: nodemanager +{{ include "fate.labels" . | indent 4 }} + name: nodemanager spec: ports: - name: "tcp-nodemanager" @@ -28,8 +25,6 @@ spec: protocol: TCP clusterIP: None selector: - fateMoudle: {{ $nodemanager.name }} -{{ include "fate.matchLabels" $root | indent 4 }} - -{{- end }} + fateMoudle: nodemanager +{{ include "fate.matchLabels" . | indent 4 }} {{- end }} \ No newline at end of file diff --git a/helm-charts/FATE/templates/backends/eggroll/nodemanager/deployment.yaml b/helm-charts/FATE/templates/backends/eggroll/nodemanager/statefulSet.yaml similarity index 64% rename from helm-charts/FATE/templates/backends/eggroll/nodemanager/deployment.yaml rename to helm-charts/FATE/templates/backends/eggroll/nodemanager/statefulSet.yaml index 6fe7b84a2..b98f3202b 100644 --- a/helm-charts/FATE/templates/backends/eggroll/nodemanager/deployment.yaml +++ b/helm-charts/FATE/templates/backends/eggroll/nodemanager/statefulSet.yaml @@ -10,36 +10,33 @@ # limitations under the License. {{- if .Values.modules.nodemanager.include }} -{{- $root := . -}} -{{- range $nodemanager := .Values.modules.nodemanager.list }} --- apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: - name: {{ $nodemanager.name }} + name: nodemanager labels: - fateMoudle: {{ $nodemanager.name }} + fateMoudle: nodemanager app: nodemanager -{{ include "fate.labels" $root | indent 4 }} +{{ include "fate.labels" . | indent 4 }} spec: - replicas: 1 - strategy: - type: Recreate + replicas: {{ .Values.modules.nodemanager.replicas }} + serviceName: nodemanager selector: matchLabels: - fateMoudle: {{ $nodemanager.name }} -{{ include "fate.matchLabels" $root | indent 6 }} + fateMoudle: nodemanager +{{ include "fate.matchLabels" . | indent 6 }} template: metadata: labels: - fateMoudle: {{ $nodemanager.name }} + fateMoudle: nodemanager app: nodemanager -{{ include "fate.labels" $root | indent 8 }} +{{ include "fate.labels" . | indent 8 }} spec: containers: - - name: {{ $nodemanager.name }}-eggrollpair - {{- if $root.Values.image.isThridParty }} - image: {{ $root.Values.image.registry }}/fluentd:v1.12 + - name: nodemanager-eggrollpair + {{- if .Values.image.isThridParty }} + image: {{ .Values.image.registry }}/fluentd:v1.12 {{- else }} image: fluent/fluentd:v1.12 {{- end }} @@ -49,9 +46,16 @@ spec: - name: fluentd-conf subPath: fluent.conf mountPath: /fluentd/etc/fluent.conf - - image: {{ $root.Values.image.registry }}/eggroll:{{ $root.Values.image.tag }} - imagePullPolicy: {{ $root.Values.image.pullPolicy }} - name: {{ $nodemanager.name }} + - image: {{ .Values.image.registry }}/eggroll{{ include "images.eggroll.suffix" . }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.modules.nodemanager.resources}} + resources: + {{- range $key, $val := .Values.modules.nodemanager.resources }} + {{ $key }}: +{{ toYaml $val | indent 14 }} + {{- end }} + {{- end }} + name: nodemanager command: - bash - -c @@ -92,50 +96,62 @@ spec: - name: eggroll-log mountPath: /data/projects/fate/eggroll/logs/ - mountPath: /data/projects/fate/eggroll/conf/eggroll.properties - name: nodemanager-confs + name: eggroll-confs subPath: eggroll.properties - mountPath: /data/projects/fate/conf/ name: python-confs - name: data-dir mountPath: /data/projects/fate/eggroll/data - subPath: {{ $nodemanager.subPath }} - {{- with $nodemanager.nodeSelector }} + subPath: {{ .Values.modules.nodemanager.subPath }} + {{- with .Values.modules.nodemanager.nodeSelector }} nodeSelector: {{ toYaml . | indent 8 }} {{- end }} - {{- with $nodemanager.tolerations }} + {{- with .Values.modules.nodemanager.tolerations }} tolerations: {{ toYaml . | indent 8 }} {{- end }} - {{- with $nodemanager.affinity }} + {{- with .Values.modules.nodemanager.affinity }} affinity: {{ toYaml . | indent 8 }} {{- end }} - {{- with $root.Values.image.imagePullSecrets }} + {{- with .Values.image.imagePullSecrets }} imagePullSecrets: {{ toYaml . | indent 6 }} {{- end }} - serviceAccountName: {{ template "serviceAccountName" $root }} + serviceAccountName: {{ template "serviceAccountName" . }} restartPolicy: Always volumes: - name: eggroll-log emptyDir: {} - name: fluentd-conf configMap: - name: {{ $nodemanager.name }}-config - - name: nodemanager-confs + name: fluentd-config + - name: eggroll-confs configMap: - name: {{ $nodemanager.name }}-config + name: eggroll-config - name: python-confs configMap: name: python-config - {{- if not $root.Values.persistence.enabled }} + {{- if not .Values.persistence.enabled }} - name: data-dir emptyDir: {} - {{- else }} + {{- else if and .Values.persistence.enabled (.Values.modules.nodemanager.existingClaim) }} - name: data-dir persistentVolumeClaim: - claimName: {{ $nodemanager.existingClaim | default (printf "%s-data" $nodemanager.name) }} + claimName: {{ .Values.modules.nodemanager.existingClaim }} + {{- else }} + volumeClaimTemplates: + - metadata: + name: data-dir + labels: + fateMoudle: nodemanager +{{ include "fate.labels" . | indent 10 }} + spec: + accessModes: [ {{ .Values.modules.nodemanager.accessMode | quote }} ] + storageClassName: {{ .Values.modules.nodemanager.storageClass }} + resources: + requests: + storage: {{ .Values.modules.nodemanager.size }} {{- end }} -{{- end }} {{- end }} \ No newline at end of file diff --git a/helm-charts/FATE/templates/backends/eggroll/rollsite/deployment.yaml b/helm-charts/FATE/templates/backends/eggroll/rollsite/deployment.yaml index 0dc2210b8..5c9d0127b 100644 --- a/helm-charts/FATE/templates/backends/eggroll/rollsite/deployment.yaml +++ b/helm-charts/FATE/templates/backends/eggroll/rollsite/deployment.yaml @@ -40,8 +40,15 @@ spec: env: - name: PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION value: python - image: {{ .Values.image.registry }}/eggroll:{{ .Values.image.tag }} + image: {{ .Values.image.registry }}/eggroll{{ include "images.eggroll.suffix" . }}:{{ .Values.image.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.modules.rollsite.resources}} + resources: + {{- range $key, $val := .Values.modules.rollsite.resources }} + {{ $key }}: +{{ toYaml $val | indent 14 }} + {{- end }} + {{- end }} command: - bash - -c @@ -52,8 +59,12 @@ spec: ln -sf /dev/stdout /data/projects/fate/eggroll/logs/eggroll/eggroll-audit.log touch /data/projects/fate/eggroll/logs/eggroll/eggroll.jvm.log ln -sf /dev/stdout /data/projects/fate/eggroll/logs/eggroll/eggroll.jvm.log - touch /data/projects/fate/eggroll/logs/eggroll/eggroll.jvm.err.log + touch /data/projects/fate/eggroll/logs/eggroll/eggroll.jvm.err.log ln -sf /dev/stderr /data/projects/fate/eggroll/logs/eggroll/eggroll.jvm.err.log + cp /data/projects/fate/eggroll/conf/temp_eggroll.properties /data/projects/fate/eggroll/conf/eggroll.properties + {{- if .Values.modules.rollsite.enableTLS }} + cat /data/projects/fate/eggroll/conf/cert_configs >> /data/projects/fate/eggroll/conf/eggroll.properties + {{- end}} java -Dlog4j.configurationFile=$${EGGROLL_HOME}/conf/log4j2.properties -cp $${EGGROLL_HOME}/lib/*:$${EGGROLL_HOME}/conf/ com.webank.eggroll.rollsite.EggSiteBootstrap -c $${EGGROLL_HOME}/conf/eggroll.properties ports: - containerPort: 9370 @@ -81,9 +92,16 @@ spec: volumeMounts: - mountPath: /data/projects/fate/eggroll/conf/route_table/ name: rollsite-confs - - mountPath: /data/projects/fate/eggroll/conf/eggroll.properties + - mountPath: /data/projects/fate/eggroll/conf/temp_eggroll.properties name: eggroll-confs subPath: eggroll.properties + {{- if .Values.modules.rollsite.enableTLS }} + - mountPath: /data/projects/fate/eggroll/conf/cert_configs + name: eggroll-confs + subPath: cert_configs + - mountPath: /data/projects/fate/eggroll/conf/cert/ + name: eggroll-certs + {{- end }} {{- with .Values.modules.rollsite.nodeSelector }} nodeSelector: {{ toYaml . | indent 8 }} @@ -109,4 +127,9 @@ spec: - name: eggroll-confs configMap: name: eggroll-config + {{- if .Values.modules.rollsite.enableTLS }} + - name: eggroll-certs + secret: + secretName: eggroll-certs + {{- end}} {{ end }} \ No newline at end of file diff --git a/fml_manager/fml_manager/__init__.py b/helm-charts/FATE/templates/backends/spark/_helpers.tpl similarity index 65% rename from fml_manager/fml_manager/__init__.py rename to helm-charts/FATE/templates/backends/spark/_helpers.tpl index 5848217e2..6df82b044 100644 --- a/fml_manager/fml_manager/__init__.py +++ b/helm-charts/FATE/templates/backends/spark/_helpers.tpl @@ -1,8 +1,8 @@ -# Copyright 2019-2020 VMware, Inc. -# + +# Copyright 2019-2022 VMware, Inc. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# you may obtain a copy of the License at +# You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -10,6 +10,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from fml_manager.fml_manager import FMLManager -from fml_manager.fml_cluster_manager import ClusterManager -from fml_manager.utils.fate_builders import * +{{/* Images Suffix */}} + +{{- define "images.spark-worker.suffix" -}} +{{- if eq .Values.algorithm "NN" -}} +-nn +{{- end -}} +{{- if eq .Values.device "IPCL" -}} +-ipcl +{{- end -}} +{{- end -}} diff --git a/helm-charts/FATE/templates/backends/spark/hdfs/configmap.yaml b/helm-charts/FATE/templates/backends/spark/hdfs/configmap.yaml index 2468b224d..5ebef6a69 100644 --- a/helm-charts/FATE/templates/backends/spark/hdfs/configmap.yaml +++ b/helm-charts/FATE/templates/backends/spark/hdfs/configmap.yaml @@ -34,6 +34,7 @@ metadata: fateMoudle: namenode {{ include "fate.labels" . | indent 4 }} data: + MULTIHOMED_NETWORK: "0" CORE_CONF_fs_defaultFS: "hdfs://namenode:9000" CORE_CONF_hadoop_http_staticuser_user: "root" CORE_CONF_hadoop_proxyuser_hue_hosts: "*" @@ -42,6 +43,12 @@ data: HDFS_CONF_dfs_webhdfs_enabled: 'true' HDFS_CONF_dfs_permissions_enabled: 'false' HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check: 'false' + HDFS_CONF_dfs_client_use_datanode_hostname: "false" + HDFS_CONF_dfs_datanode_use_datanode_hostname: "false" + HDFS_CONF_dfs_namenode_rpc___bind___host: "0.0.0.0" + HDFS_CONF_dfs_namenode_servicerpc___bind___host: "0.0.0.0" + HDFS_CONF_dfs_namenode_http___bind___host: "0.0.0.0" + HDFS_CONF_dfs_namenode_https___bind___host: "0.0.0.0" YARN_CONF_yarn_log___aggregation___enable: 'true' YARN_CONF_yarn_log_server_url: "http://historyserver:8188/applicationhistory/logs/" YARN_CONF_yarn_resourcemanager_recovery_enabled: 'true' @@ -83,6 +90,7 @@ metadata: fateMoudle: datanode {{ include "fate.labels" . | indent 4 }} data: + MULTIHOMED_NETWORK: "0" CORE_CONF_fs_defaultFS: "hdfs://namenode:9000" CORE_CONF_hadoop_http_staticuser_user: "root" CORE_CONF_hadoop_proxyuser_hue_hosts: "*" @@ -91,6 +99,12 @@ data: HDFS_CONF_dfs_webhdfs_enabled: 'true' HDFS_CONF_dfs_permissions_enabled: 'false' HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check: 'false' + HDFS_CONF_dfs_client_use_datanode_hostname: "false" + HDFS_CONF_dfs_datanode_use_datanode_hostname: "false" + HDFS_CONF_dfs_namenode_rpc___bind___host: "0.0.0.0" + HDFS_CONF_dfs_namenode_servicerpc___bind___host: "0.0.0.0" + HDFS_CONF_dfs_namenode_http___bind___host: "0.0.0.0" + HDFS_CONF_dfs_namenode_https___bind___host: "0.0.0.0" YARN_CONF_yarn_log___aggregation___enable: 'true' YARN_CONF_yarn_log_server_url: "http://historyserver:8188/applicationhistory/logs/" YARN_CONF_yarn_resourcemanager_recovery_enabled: 'true' diff --git a/helm-charts/FATE/templates/backends/spark/hdfs/datanode.yaml b/helm-charts/FATE/templates/backends/spark/hdfs/datanode.yaml new file mode 100644 index 000000000..38af1162e --- /dev/null +++ b/helm-charts/FATE/templates/backends/spark/hdfs/datanode.yaml @@ -0,0 +1,120 @@ +# Copyright 2019-2022 VMware, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.modules.hdfs.include }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: datanode + labels: + fateMoudle: datanode +{{ include "fate.labels" . | indent 4 }} +spec: + serviceName: datanode + replicas: {{ .Values.modules.hdfs.datanode.replicas | default 3}} + selector: + matchLabels: + fateMoudle: datanode +{{ include "fate.matchLabels" . | indent 6 }} + template: + metadata: + labels: + fateMoudle: datanode +{{ include "fate.labels" . | indent 8 }} + spec: + containers: + - name: datanode + image: {{ .Values.image.registry }}/hadoop-datanode:2.0.0-hadoop3.2.1-java8 + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: SERVICE_PRECONDITION + value: "namenode:9000" + envFrom: + - configMapRef: + name: datanode-env + ports: + - containerPort: 9000 + - containerPort: 9864 + livenessProbe: + exec: + command: + - /bin/bash + - -c + - curl -f localhost:9864 + initialDelaySeconds: 1 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + exec: + command: + - /bin/bash + - -c + - curl -f localhost:9864 + initialDelaySeconds: 1 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + exec: + command: + - /bin/bash + - -c + - curl -f localhost:9864 + failureThreshold: 12 + periodSeconds: 10 + volumeMounts: + - name: dfs + mountPath: /hadoop/dfs/data + {{- with .Values.modules.hdfs.datanode.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.modules.hdfs.datanode.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.modules.hdfs.datanode.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.image.imagePullSecrets }} + imagePullSecrets: +{{ toYaml . | indent 6 }} + {{- end }} + serviceAccountName: {{ template "serviceAccountName" . }} + restartPolicy: Always + {{- if not .Values.persistence.enabled }} + volumes: + - name: dfs + emptyDir: {} + {{- else if and .Values.persistence.enabled (.Values.modules.hdfs.datanode.existingClaim) }} + volumes: + - name: dfs + persistentVolumeClaim: + claimName: {{ .Values.modules.hdfs.datanode.existingClaim | default "datanode-data" }} + {{- else }} + volumeClaimTemplates: + - metadata: + name: dfs + labels: + fateMoudle: datanode +{{ include "fate.labels" . | indent 10 }} + spec: + accessModes: [{{ .Values.modules.hdfs.datanode.accessMode | quote }}] + storageClassName: {{ .Values.modules.hdfs.datanode.storageClass }} + resources: + requests: + storage: {{ .Values.modules.hdfs.datanode.size }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm-charts/FATE/templates/backends/spark/hdfs/deployment.yaml b/helm-charts/FATE/templates/backends/spark/hdfs/namenode.yaml similarity index 53% rename from helm-charts/FATE/templates/backends/spark/hdfs/deployment.yaml rename to helm-charts/FATE/templates/backends/spark/hdfs/namenode.yaml index 938bed3c7..ec5c473fd 100644 --- a/helm-charts/FATE/templates/backends/spark/hdfs/deployment.yaml +++ b/helm-charts/FATE/templates/backends/spark/hdfs/namenode.yaml @@ -9,18 +9,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -{{ if .Values.modules.hdfs.include }} +{{- if .Values.modules.hdfs.include }} apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: namenode labels: fateMoudle: namenode {{ include "fate.labels" . | indent 4 }} spec: + serviceName: namenode replicas: 1 - strategy: - type: Recreate selector: matchLabels: fateMoudle: namenode @@ -42,7 +41,7 @@ spec: {{ end }} containers: - name: namenode - image: {{ .Values.image.registry }}/hadoop-namenode:2.0.0-hadoop2.7.4-java8 + image: {{ .Values.image.registry }}/hadoop-namenode:2.0.0-hadoop3.2.1-java8 imagePullPolicy: {{ .Values.image.pullPolicy }} env: - name: CLUSTER_NAME @@ -53,13 +52,12 @@ spec: ports: - containerPort: 9000 - containerPort: 9870 - - containerPort: 50070 livenessProbe: exec: command: - /bin/bash - -c - - curl -f localhost:50070 + - curl -f localhost:9870 initialDelaySeconds: 30 periodSeconds: 10 timeoutSeconds: 5 @@ -70,7 +68,7 @@ spec: command: - /bin/bash - -c - - curl -f localhost:50070 + - curl -f localhost:9870 initialDelaySeconds: 5 periodSeconds: 10 timeoutSeconds: 1 @@ -81,7 +79,7 @@ spec: command: - /bin/bash - -c - - curl -f localhost:50070 + - curl -f localhost:9870 failureThreshold: 12 periodSeconds: 10 volumeMounts: @@ -112,106 +110,25 @@ spec: - name: namenode-confs configMap: name: namenode-config + {{- if not .Values.persistence.enabled }} + - name: dfs + emptyDir: {} + {{- else if and .Values.persistence.enabled (.Values.modules.hdfs.namenode.existingClaim) }} - name: dfs - {{ if .Values.persistence.enabled }} persistentVolumeClaim: claimName: {{ .Values.modules.hdfs.namenode.existingClaim | default "namenode-data" }} - {{ else }} - emptyDir: {} - {{ end }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: datanode - labels: - fateMoudle: datanode -{{ include "fate.labels" . | indent 4 }} -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - fateMoudle: datanode -{{ include "fate.matchLabels" . | indent 6 }} - template: - metadata: - labels: - fateMoudle: datanode -{{ include "fate.labels" . | indent 8 }} - spec: - containers: - - name: datanode - image: {{ .Values.image.registry }}/hadoop-datanode:2.0.0-hadoop2.7.4-java8 - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - - name: SERVICE_PRECONDITION - value: "namenode:9000" - envFrom: - - configMapRef: - name: datanode-env - ports: - - containerPort: 9000 - - containerPort: 9870 - - containerPort: 50075 - livenessProbe: - exec: - command: - - /bin/bash - - -c - - curl -f localhost:50075 - initialDelaySeconds: 1 - periodSeconds: 10 - timeoutSeconds: 3 - successThreshold: 1 - failureThreshold: 3 - readinessProbe: - exec: - command: - - /bin/bash - - -c - - curl -f localhost:50075 - initialDelaySeconds: 1 - periodSeconds: 10 - timeoutSeconds: 3 - successThreshold: 1 - failureThreshold: 3 - startupProbe: - exec: - command: - - /bin/bash - - -c - - curl -f localhost:50075 - failureThreshold: 12 - periodSeconds: 10 - volumeMounts: - - name: dfs - mountPath: /hadoop/dfs/data - {{- with .Values.modules.hdfs.datanode.nodeSelector }} - nodeSelector: -{{ toYaml . | indent 8 }} - {{- end }} - {{- with .Values.modules.hdfs.datanode.tolerations }} - tolerations: -{{ toYaml . | indent 8 }} - {{- end }} - {{- with .Values.modules.hdfs.datanode.affinity }} - affinity: -{{ toYaml . | indent 8 }} - {{- end }} - {{- with .Values.image.imagePullSecrets }} - imagePullSecrets: -{{ toYaml . | indent 6 }} + {{- else }} + volumeClaimTemplates: + - metadata: + name: dfs + labels: + fateMoudle: namenode +{{ include "fate.labels" . | indent 10 }} + spec: + accessModes: [{{ .Values.modules.hdfs.namenode.accessMode | quote }}] + storageClassName: {{ .Values.modules.hdfs.namenode.storageClass }} + resources: + requests: + storage: {{ .Values.modules.hdfs.namenode.size }} {{- end }} - serviceAccountName: {{ template "serviceAccountName" . }} - restartPolicy: Always - volumes: - - name: dfs - {{ if .Values.persistence.enabled }} - persistentVolumeClaim: - claimName: {{ .Values.modules.hdfs.datanode.existingClaim | default "datanode-data" }} - {{ else }} - emptyDir: {} - {{ end }} -{{ end }} \ No newline at end of file +{{- end }} \ No newline at end of file diff --git a/helm-charts/FATE/templates/backends/spark/hdfs/persistentvolumeclaim.yaml b/helm-charts/FATE/templates/backends/spark/hdfs/persistentvolumeclaim.yaml deleted file mode 100644 index 5284bf503..000000000 --- a/helm-charts/FATE/templates/backends/spark/hdfs/persistentvolumeclaim.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2019-2022 VMware, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{ if .Values.modules.hdfs.include }} -{{ if and .Values.persistence.enabled (not .Values.modules.hdfs.namenode.existingClaim) }} -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: namenode-data - labels: - fateMoudle: namenode -{{ include "fate.labels" . | indent 4 }} -spec: - accessModes: - - {{ .Values.modules.hdfs.namenode.accessMode }} - resources: - requests: - storage: {{ .Values.modules.hdfs.namenode.size }} - {{ if .Values.modules.hdfs.namenode.storageClass }} - {{ if eq "-" .Values.modules.hdfs.namenode.storageClass }} - storageClassName: "" - {{ else }} - storageClassName: {{ .Values.modules.hdfs.namenode.storageClass }} - {{ end }} - {{ end }} -{{ end }} ---- -{{ if and .Values.persistence.enabled (not .Values.modules.hdfs.datanode.existingClaim) }} -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: datanode-data - labels: - fateMoudle: datanode -{{ include "fate.labels" . | indent 4 }} -spec: - accessModes: - - {{ .Values.modules.hdfs.datanode.accessMode }} - resources: - requests: - storage: {{ .Values.modules.hdfs.datanode.size }} - {{ if .Values.modules.hdfs.datanode.storageClass }} - {{ if eq "-" .Values.modules.hdfs.datanode.storageClass }} - storageClassName: "" - {{ else }} - storageClassName: {{ .Values.modules.hdfs.datanode.storageClass }} - {{ end }} - {{ end }} -{{ end }} - -{{ end }} \ No newline at end of file diff --git a/helm-charts/FATE/templates/backends/spark/hdfs/service.yaml b/helm-charts/FATE/templates/backends/spark/hdfs/service.yaml index 9922b02e6..cee82da18 100644 --- a/helm-charts/FATE/templates/backends/spark/hdfs/service.yaml +++ b/helm-charts/FATE/templates/backends/spark/hdfs/service.yaml @@ -23,13 +23,9 @@ spec: port: 9000 targetPort: 9000 protocol: TCP - - name: "tcp-9870" - port: 9870 - targetPort: 9870 - protocol: TCP - - name: "tcp-50075" - port: 50075 - targetPort: 50075 + - name: "tcp-9864" + port: 9864 + targetPort: 9864 protocol: TCP type: {{ .Values.modules.hdfs.datanode.type }} selector: @@ -56,10 +52,6 @@ spec: port: 9870 targetPort: 9870 protocol: TCP - - name: "tcp-50070" - port: 50070 - targetPort: 50070 - protocol: TCP type: {{ .Values.modules.hdfs.namenode.type }} selector: fateMoudle: namenode diff --git a/helm-charts/FATE/templates/backends/spark/pulsar/configmap.yaml b/helm-charts/FATE/templates/backends/spark/pulsar/configmap.yaml index 62541c95d..523ea104a 100644 --- a/helm-charts/FATE/templates/backends/spark/pulsar/configmap.yaml +++ b/helm-charts/FATE/templates/backends/spark/pulsar/configmap.yaml @@ -125,7 +125,11 @@ data: backlogQuotaCheckIntervalInSeconds=60 # Default per-topic backlog quota limit + {{- if .Values.modules.pulsar.confs }} + backlogQuotaDefaultLimitGB={{ .Values.modules.pulsar.confs.backlogQuotaDefaultLimitGB | default 10 }} + {{- else }} backlogQuotaDefaultLimitGB=10 + {{- end }} # Default ttl for namespaces if ttl is not already configured at namespace policies. (disable default-ttl with value 0) ttlDurationDefaultInSeconds=0 @@ -134,7 +138,11 @@ data: brokerDeleteInactiveTopicsEnabled=true # How often to check for inactive topics + {{- if .Values.modules.pulsar.confs }} + brokerDeleteInactiveTopicsFrequencySeconds={{ .Values.modules.pulsar.confs.brokerDeleteInactiveTopicsFrequencySeconds | default 60 }} + {{- else }} brokerDeleteInactiveTopicsFrequencySeconds=60 + {{- end }} # Max pending publish requests per connection to avoid keeping large number of pending # requests in memory. Default: 1000 diff --git a/helm-charts/FATE/templates/backends/spark/pulsar/persistentvolumeclaim.yaml b/helm-charts/FATE/templates/backends/spark/pulsar/persistentvolumeclaim.yaml deleted file mode 100644 index 5ac735ef8..000000000 --- a/helm-charts/FATE/templates/backends/spark/pulsar/persistentvolumeclaim.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2019-2022 VMware, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{ if .Values.modules.pulsar.include }} -{{- if and .Values.persistence.enabled (not .Values.modules.pulsar.existingClaim) }} -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: pulsar-data - labels: - fateMoudle: pulsar -{{ include "fate.labels" . | indent 4 }} -spec: - accessModes: - - {{ .Values.modules.pulsar.accessMode }} - resources: - requests: - storage: {{ .Values.modules.pulsar.size }} - {{ if .Values.modules.pulsar.storageClass }} - {{ if eq "-" .Values.modules.pulsar.storageClass }} - storageClassName: "" - {{ else }} - storageClassName: {{ .Values.modules.pulsar.storageClass }} - {{ end }} - {{ end }} -{{- end }} -{{ end }} \ No newline at end of file diff --git a/helm-charts/FATE/templates/backends/spark/pulsar/deployment.yaml b/helm-charts/FATE/templates/backends/spark/pulsar/statefulSet.yaml similarity index 76% rename from helm-charts/FATE/templates/backends/spark/pulsar/deployment.yaml rename to helm-charts/FATE/templates/backends/spark/pulsar/statefulSet.yaml index 9f3a0a06c..f1e5a99be 100644 --- a/helm-charts/FATE/templates/backends/spark/pulsar/deployment.yaml +++ b/helm-charts/FATE/templates/backends/spark/pulsar/statefulSet.yaml @@ -9,18 +9,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -{{ if .Values.modules.pulsar.include }} +{{- if .Values.modules.pulsar.include }} apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: pulsar labels: fateMoudle: pulsar {{ include "fate.labels" . | indent 4 }} spec: + serviceName: pulsar replicas: 1 - strategy: - type: Recreate selector: matchLabels: fateMoudle: pulsar @@ -33,13 +32,23 @@ spec: spec: containers: - name: pulsar - image: {{ .Values.image.registry }}/pulsar:2.7.0 + image: {{ .Values.image.registry }}/pulsar:2.10.1 imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.modules.pulsar.resources}} + resources: + {{- range $key, $val := .Values.modules.pulsar.resources }} + {{ $key }}: +{{ toYaml $val | indent 14 }} + {{- end }} + {{- end }} command: - /bin/bash - -c - "bin/pulsar standalone -nss" + {{- with .Values.modules.pulsar.env }} env: +{{ toYaml . | indent 12 }} + {{- end }} ports: - containerPort: 6650 - containerPort: 6651 @@ -115,11 +124,25 @@ spec: secret: secretName: pulsar-cert {{- end }} + {{- if not .Values.persistence.enabled }} + - name: pulsar-data + emptyDir: {} + {{- else if and .Values.persistence.enabled (.Values.modules.pulsar.existingClaim) }} - name: pulsar-data - {{ if .Values.persistence.enabled }} persistentVolumeClaim: claimName: {{ .Values.modules.pulsar.existingClaim | default "pulsar-data" }} - {{ else }} - emptyDir: {} - {{ end }} -{{ end }} \ No newline at end of file + {{- else }} + volumeClaimTemplates: + - metadata: + name: pulsar-data + labels: + fateMoudle: pulsar +{{ include "fate.labels" . | indent 10 }} + spec: + accessModes: [{{ .Values.modules.pulsar.accessMode | quote }}] + storageClassName: {{ .Values.modules.pulsar.storageClass }} + resources: + requests: + storage: {{ .Values.modules.pulsar.size }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm-charts/FATE/templates/backends/spark/rabbitmq/deployment.yaml b/helm-charts/FATE/templates/backends/spark/rabbitmq/deployment.yaml index cfed8b0d9..2ed1ba22a 100644 --- a/helm-charts/FATE/templates/backends/spark/rabbitmq/deployment.yaml +++ b/helm-charts/FATE/templates/backends/spark/rabbitmq/deployment.yaml @@ -35,6 +35,13 @@ spec: - name: rabbitmq image: {{ .Values.image.registry }}/rabbitmq:3.8.3-management imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.modules.rabbitmq.resources }} + resources: + {{- range $key, $val := .Values.modules.rabbitmq.resources }} + {{ $key }}: +{{ toYaml $val | indent 14 }} + {{- end }} + {{- end }} env: - name: RABBITMQ_DEFAULT_USER value: fate diff --git a/helm-charts/FATE/templates/backends/spark/spark/configmap.yaml b/helm-charts/FATE/templates/backends/spark/spark/configmap.yaml index 8ec16b28f..5203bc046 100644 --- a/helm-charts/FATE/templates/backends/spark/spark/configmap.yaml +++ b/helm-charts/FATE/templates/backends/spark/spark/configmap.yaml @@ -22,6 +22,26 @@ data: use_registry: false use_deserialize_safe_module: false dependent_distribution: false + encrypt_password: false + encrypt_module: fate_arch.common.encrypt_utils#pwdecrypt + private_key: + party_id: + hook_module: + client_authentication: fate_flow.hook.flow.client_authentication + site_authentication: fate_flow.hook.flow.site_authentication + permission: fate_flow.hook.flow.permission + hook_server_name: + authentication: + client: + switch: false + http_app_key: + http_secret_key: + site: + switch: false + permission: + switch: false + component: false + dataset: false fateflow: # you must set real ip address, 127.0.0.1 and 0.0.0.0 is not supported host: 127.0.0.1 @@ -108,6 +128,9 @@ data: password: fate # default conf/rabbitmq_route_table.yaml route_table: + # mode: replication / client, default: replication + mode: replication + max_message_size: 1048576 pulsar: host: 192.168.0.5 port: 6650 @@ -119,6 +142,9 @@ data: topic_ttl: 5 # default conf/pulsar_route_table.yaml route_table: + # mode: replication / client, default: replication + mode: replication + max_message_size: 1048576 nginx: host: 127.0.0.1 http_port: 9300 @@ -127,11 +153,6 @@ data: fateboard: host: 127.0.0.1 port: 8080 - - # on API `/model/load` and `/model/load/do` - # automatic upload models to the model store if it exists locally but does not exist in the model storage - # or download models from the model store if it does not exist locally but exists in the model storage - # this config will not affect API `/model/store` or `/model/restore` enable_model_store: false model_store_address: # use mysql as the model store engine @@ -144,21 +165,13 @@ data: # other optional configs send to the engine # max_connections: 10 # stale_timeout: 10 - # use redis as the model store engine - # storage: redis - # host: 127.0.0.1 - # port: 6379 - # db: 0 - # password: - # the expiry time of keys, in seconds. defaults None (no expiry time) - # ex: # use tencent cos as model store engine storage: tencent_cos Region: SecretId: SecretKey: Bucket: - + servings: hosts: - 127.0.0.1:8000 @@ -166,4 +179,4 @@ data: host: 127.0.0.1 port: 8001 federatedId: 0 -{{ end }} \ No newline at end of file + {{ end }} \ No newline at end of file diff --git a/helm-charts/FATE/templates/backends/spark/spark/deployment.yaml b/helm-charts/FATE/templates/backends/spark/spark/deployment.yaml index c0e314e27..c38db9a6d 100644 --- a/helm-charts/FATE/templates/backends/spark/spark/deployment.yaml +++ b/helm-charts/FATE/templates/backends/spark/spark/deployment.yaml @@ -35,7 +35,7 @@ spec: - name: spark-master image: {{ if .Values.modules.spark.master.Image }}{{ .Values.modules.spark.master.Image }}{{ else }}{{ .Values.image.registry }}/spark-master{{ end }}:{{ default .Values.image.tag .Values.modules.spark.master.ImageTag }} imagePullPolicy: {{ .Values.image.pullPolicy }} - {{- if .Values.modules.spark.master.resources}} + {{- if .Values.modules.spark.master.resources }} resources: {{- range $key, $val := .Values.modules.spark.master.resources }} {{ $key }}: @@ -121,7 +121,7 @@ spec: spec: containers: - name: spark-worker - image: {{ if .Values.modules.spark.worker.Image }}{{ .Values.modules.spark.worker.Image }}{{ else }}{{ .Values.image.registry }}/spark-worker{{ end }}:{{ default .Values.image.tag .Values.modules.spark.worker.ImageTag }} + image: {{ if .Values.modules.spark.worker.Image }}{{ .Values.modules.spark.worker.Image }}{{ else }}{{ .Values.image.registry }}/spark-worker{{ include "images.spark-worker.suffix" . }}{{ end }}:{{ default .Values.image.tag .Values.modules.spark.worker.ImageTag }} imagePullPolicy: {{ .Values.image.pullPolicy }} {{- if .Values.modules.spark.worker.resources }} resources: diff --git a/helm-charts/FATE/templates/backends/spark/spark/service.yaml b/helm-charts/FATE/templates/backends/spark/spark/service.yaml index 8430c771b..2a150d35e 100644 --- a/helm-charts/FATE/templates/backends/spark/spark/service.yaml +++ b/helm-charts/FATE/templates/backends/spark/spark/service.yaml @@ -50,13 +50,13 @@ spec: protocol: TCP port: 8080 targetPort: 8080 + {{- if eq .Values.modules.spark.master.type "NodePort" "LoadBalancer" }} + nodePort: {{ .Values.modules.spark.master.nodePort }} + {{- end }} - name: master protocol: TCP port: 7077 targetPort: 7077 - {{- if eq .Values.modules.spark.master.type "NodePort" "LoadBalancer" }} - nodePort: {{ .Values.modules.spark.master.nodePort }} - {{- end }} - name: master-rest protocol: TCP port: 6066 diff --git a/helm-charts/FATE/templates/core/_helpers.tpl b/helm-charts/FATE/templates/core/_helpers.tpl new file mode 100644 index 000000000..306c582a9 --- /dev/null +++ b/helm-charts/FATE/templates/core/_helpers.tpl @@ -0,0 +1,24 @@ + +# Copyright 2019-2022 VMware, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{/* Images Suffix */}} +{{- define "images.fateflow.suffix" -}} +{{- if or (eq .Values.computing "Spark") (eq .Values.computing "Spark_local") -}} +-spark +{{- end -}} +{{- if eq .Values.algorithm "NN" -}} +-nn +{{- end -}} +{{- if eq .Values.device "IPCL" -}} +-ipcl +{{- end -}} +{{- end -}} diff --git a/helm-charts/FATE/templates/core/client/ingress.yaml b/helm-charts/FATE/templates/core/client/ingress.yaml index d15ca015c..95da397fb 100644 --- a/helm-charts/FATE/templates/core/client/ingress.yaml +++ b/helm-charts/FATE/templates/core/client/ingress.yaml @@ -8,8 +8,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -{{ if .Values.modules.client.include }} +{{- if not .Values.istio.enabled }} +{{- if .Values.modules.client.include }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -41,4 +41,5 @@ spec: {{ toYaml .Values.ingress.client.tls | indent 4 }} {{- end }} -{{ end }} \ No newline at end of file +{{- end }} +{{- end }} \ No newline at end of file diff --git a/helm-charts/FATE/templates/core/client/persistentvolumeclaim.yaml b/helm-charts/FATE/templates/core/client/persistentvolumeclaim.yaml deleted file mode 100644 index d13d640d7..000000000 --- a/helm-charts/FATE/templates/core/client/persistentvolumeclaim.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2019-2022 VMware, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{ if .Values.modules.client.include }} -{{- if and .Values.persistence.enabled (not .Values.modules.client.existingClaim) }} -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: client-data - labels: - fateMoudle: client -{{ include "fate.labels" . | indent 4 }} -spec: - accessModes: - - {{ .Values.modules.client.accessMode }} - resources: - requests: - storage: {{ .Values.modules.client.size }} - {{- if .Values.modules.client.storageClass }} - {{- if eq "-" .Values.modules.client.storageClass }} - storageClassName: "" - {{- else }} - storageClassName: {{ .Values.modules.client.storageClass }} - {{- end }} - {{- end }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/helm-charts/FATE/templates/core/client/deployment.yaml b/helm-charts/FATE/templates/core/client/statefulSet.yaml similarity index 85% rename from helm-charts/FATE/templates/core/client/deployment.yaml rename to helm-charts/FATE/templates/core/client/statefulSet.yaml index fcc546d4c..bb0ef7dbf 100644 --- a/helm-charts/FATE/templates/core/client/deployment.yaml +++ b/helm-charts/FATE/templates/core/client/statefulSet.yaml @@ -9,18 +9,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -{{ if .Values.modules.client.include }} +{{- if .Values.modules.client.include }} apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: client labels: fateMoudle: client {{ include "fate.labels" . | indent 4 }} spec: + serviceName: notebook replicas: 1 - strategy: - type: Recreate selector: matchLabels: fateMoudle: client @@ -105,9 +104,22 @@ spec: {{- if not .Values.persistence.enabled }} - name: persistence emptyDir: {} - {{- else }} + {{- else if and .Values.persistence.enabled (.Values.modules.client.existingClaim) }} - name: persistence persistentVolumeClaim: claimName: {{ .Values.modules.client.existingClaim | default "client-data" }} + {{- else }} + volumeClaimTemplates: + - metadata: + name: persistence + labels: + fateMoudle: client +{{ include "fate.labels" . | indent 10 }} + spec: + accessModes: [{{ .Values.modules.client.accessMode | quote }}] + storageClassName: {{ .Values.modules.client.storageClass }} + resources: + requests: + storage: {{ .Values.modules.client.size }} {{- end }} -{{ end }} \ No newline at end of file +{{- end }} \ No newline at end of file diff --git a/helm-charts/FATE/templates/core/fateboard/configmap.yaml b/helm-charts/FATE/templates/core/fateboard/configmap.yaml index 023372cd8..29ec7bc01 100644 --- a/helm-charts/FATE/templates/core/fateboard/configmap.yaml +++ b/helm-charts/FATE/templates/core/fateboard/configmap.yaml @@ -20,7 +20,10 @@ metadata: data: application.properties: | server.port=8080 - fateflow.url=http://{{ if .Values.istio.enabled }}127.0.0.1{{ else }}fateflow{{ end }}:9380 + fateflow.url=http://fateflow:9380 + #priority is higher than {fateflow.url}, split by ; + #below config can support configuring more than one fate flow for this fate board + fateflow.url-list= fateflow.http_app_key= fateflow.http_secret_key= spring.http.encoding.charset=UTF-8 @@ -39,4 +42,6 @@ data: server.servlet.session.timeout=4h server.servlet.session.cookie.max-age=4h management.endpoints.web.exposure.exclude=* + feign.client.config.default.connectTimeout=10000 + feign.client.config.default.readTimeout=10000 {{- end }} \ No newline at end of file diff --git a/helm-charts/FATE/templates/core/fateboard/ingress.yaml b/helm-charts/FATE/templates/core/fateboard/ingress.yaml index a24e57787..a86d62980 100644 --- a/helm-charts/FATE/templates/core/fateboard/ingress.yaml +++ b/helm-charts/FATE/templates/core/fateboard/ingress.yaml @@ -8,8 +8,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -{{ if .Values.modules.fateboard.include }} +{{- if not .Values.istio.enabled }} +{{- if .Values.modules.fateboard.include }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -41,4 +41,5 @@ spec: {{ toYaml .Values.ingress.fateboard.tls | indent 4 }} {{- end }} -{{ end }} \ No newline at end of file +{{- end }} +{{- end }} \ No newline at end of file diff --git a/helm-charts/FATE/templates/core/fateboard/istio.yaml b/helm-charts/FATE/templates/core/fateboard/istio.yaml index 5158000dd..93584adf6 100644 --- a/helm-charts/FATE/templates/core/fateboard/istio.yaml +++ b/helm-charts/FATE/templates/core/fateboard/istio.yaml @@ -23,7 +23,7 @@ spec: - {{ .name }} {{- end }} gateways: - - fate-9999-gateway + - {{ .Values.partyName }}-gateway http: - route: - destination: diff --git a/helm-charts/FATE/templates/core/fateflow/configmap.yaml b/helm-charts/FATE/templates/core/fateflow/configmap.yaml index 41aec9ecc..e516e7680 100644 --- a/helm-charts/FATE/templates/core/fateflow/configmap.yaml +++ b/helm-charts/FATE/templates/core/fateflow/configmap.yaml @@ -19,7 +19,7 @@ metadata: {{ include "fate.labels" . | indent 4 }} data: spark-defaults.conf: | - spark.master {{ if eq .Values.modules.python.backend "spark_local_pulsar" }}local[*]{{ else }}{{ .Values.modules.python.spark.master | default "spark://spark-master:7077"}}{{ end }} + spark.master {{ if eq .Values.computing "Spark_local" }}local[*]{{ else }}{{ .Values.modules.python.spark.master | default "spark://spark-master:7077"}}{{ end }} spark.driver.host {{ .Values.modules.python.spark.driverHost | default "fateflow" }} {{- if .Values.modules.python.spark.driverStartPort }} spark.driver.port {{ .Values.modules.python.spark.driverStartPort }} @@ -48,13 +48,28 @@ data: encrypt_password: false encrypt_module: fate_arch.common.encrypt_utils#pwdecrypt private_key: + party_id: + hook_module: + client_authentication: fate_flow.hook.flow.client_authentication + site_authentication: fate_flow.hook.flow.site_authentication + permission: fate_flow.hook.flow.permission + hook_server_name: + authentication: + client: + switch: false + http_app_key: + http_secret_key: + site: + switch: false + permission: + switch: false + component: false + dataset: false fateflow: # you must set real ip address, 127.0.0.1 and 0.0.0.0 is not supported - host: {{ if .Values.istio.enabled }}127.0.0.1{{ else }}fateflow{{ end }} + host: fateflow http_port: 9380 grpc_port: 9360 - # http_app_key: - # http_secret_key: # support rollsite/nginx/fateflow as a coordination proxy # rollsite support fate on eggroll, use grpc protocol # nginx support fate on eggroll and fate on spark, use http or grpc protocol, default is http @@ -67,7 +82,7 @@ data: # host: xx # http_port: xx # grpc_port: xx - {{- if eq .Values.modules.python.backend "spark_pulsar" "spark_rabbitmq" "spark_local_pulsar" }} + {{- if eq .Values.computing "Spark" "Spark_local" }} proxy: nginx {{- else }} proxy: rollsite @@ -92,23 +107,13 @@ data: # password: fate # engine services default_engines: - {{- if eq .Values.modules.python.backend "spark_pulsar" }} - computing: spark - federation: pulsar - storage: hdfs - {{- else if eq .Values.modules.python.backend "spark_rabbitmq" }} - computing: spark - federation: rabbitmq - storage: hdfs - {{- else if eq .Values.modules.python.backend "spark_local_pulsar" }} - computing: spark - federation: pulsar - storage: localfs + {{- if eq .Values.computing "Spark_local" }} + computing: "spark" {{- else }} - computing: eggroll - federation: eggroll - storage: eggroll + computing: {{ .Values.computing | lower }} {{- end }} + federation: {{ .Values.federation | lower }} + storage: {{ .Values.storage | lower }} fate_on_standalone: standalone: cores_per_node: 20 @@ -123,7 +128,7 @@ data: fate_on_spark: spark: # default use SPARK_HOME environment variable - home: /data/projects/spark-2.4.1-bin-hadoop2.7/ + home: /data/projects/spark-3.1.2-bin-hadoop3.2/ cores_per_node: {{ .Values.modules.python.spark.cores_per_node }} nodes: {{ .Values.modules.python.spark.nodes }} linkis_spark: @@ -154,6 +159,9 @@ data: password: {{ .Values.modules.python.rabbitmq.password }} # default conf/rabbitmq_route_table.yaml route_table: conf/rabbitmq_route_table/rabbitmq_route_table.yaml + # mode: replication / client, default: replication + mode: replication + max_message_size: 1048576 pulsar: host: {{ .Values.modules.python.pulsar.host }} port: {{ .Values.modules.python.pulsar.port }} @@ -161,6 +169,9 @@ data: topic_ttl: 3 # default conf/pulsar_route_table.yaml route_table: conf/pulsar_route_table/pulsar_route_table.yaml + # mode: replication / client, default: replication + mode: replication + max_message_size: 1048576 nginx: host: {{ .Values.modules.python.nginx.host }} http_port: {{ .Values.modules.python.nginx.http_port }} @@ -168,10 +179,6 @@ data: fateboard: host: fateboard port: 8080 - # on API `/model/load` and `/model/load/do` - # automatic upload models to the model store if it exists locally but does not exist in the model storage - # or download models from the model store if it does not exist locally but exists in the model storage - # this config will not affect API `/model/store` or `/model/restore` enable_model_store: false model_store_address: storage: mysql @@ -243,9 +250,11 @@ data: auto_retry_delay: 1 #seconds # It can also be specified in the job configuration using the federated_status_collect_type parameter federated_status_collect_type: PUSH + detect_connect_max_retry_count: 3 + detect_connect_long_retry_count: 2 # upload - upload_max_bytes: 4194304000 # bytes + upload_max_bytes: 104857600 # bytes #component output output_data_summary_count_limit: 100 diff --git a/helm-charts/FATE/templates/core/fateflow/persistentvolumeclaim.yaml b/helm-charts/FATE/templates/core/fateflow/persistentvolumeclaim.yaml deleted file mode 100644 index 3b8d39b86..000000000 --- a/helm-charts/FATE/templates/core/fateflow/persistentvolumeclaim.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2019-2022 VMware, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -{{- if and .Values.persistence.enabled (not .Values.modules.python.existingClaim) }} -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: python-data - labels: - fateMoudle: python -{{ include "fate.labels" . | indent 4 }} -spec: - accessModes: - - {{ .Values.modules.python.accessMode }} - resources: - requests: - storage: {{ .Values.modules.python.size }} - {{- if .Values.modules.python.storageClass }} - {{- if eq "-" .Values.modules.python.storageClass }} - storageClassName: "" - {{- else }} - storageClassName: {{ .Values.modules.python.storageClass }} - {{- end }} - {{- end }} -{{- end }} \ No newline at end of file diff --git a/helm-charts/FATE/templates/core/istio.yaml b/helm-charts/FATE/templates/core/istio.yaml index 8aa0f853c..9d71951d2 100644 --- a/helm-charts/FATE/templates/core/istio.yaml +++ b/helm-charts/FATE/templates/core/istio.yaml @@ -26,4 +26,26 @@ spec: protocol: HTTP hosts: - "*" + - port: + number: 443 + name: fateboard-https + protocol: HTTPS + tls: + mode: SIMPLE + credentialName: fateboard-credential # must be the same as secret + hosts: + {{- range .Values.ingress.fateboard.hosts }} + - {{ .name }} + {{- end }} + - port: + number: 443 + name: client-https + protocol: HTTPS + tls: + mode: SIMPLE + credentialName: client-credential # must be the same as secret + hosts: + {{- range .Values.ingress.client.hosts }} + - {{ .name }} + {{- end }} {{ end }} \ No newline at end of file diff --git a/helm-charts/FATE/templates/core/mysql/configmap.yaml b/helm-charts/FATE/templates/core/mysql/configmap.yaml index db11b54bb..e21f5e64b 100644 --- a/helm-charts/FATE/templates/core/mysql/configmap.yaml +++ b/helm-charts/FATE/templates/core/mysql/configmap.yaml @@ -18,7 +18,7 @@ metadata: fateMoudle: mysql {{ include "fate.labels" . | indent 4 }} data: - {{- if eq .Values.modules.python.backend "spark" }} + {{- if eq .Values.computing "Spark" }} {{- else }} create-eggroll-meta-tables.sql: | -- create database if not exists, default database is eggroll_meta @@ -147,9 +147,9 @@ data: CREATE INDEX `idx_session_processor_si` ON `session_processor` (`session_id`(767)); - INSERT INTO server_node (host, port, node_type, status) values ('clustermanager', '9460', 'CLUSTER_MANAGER', 'HEALTHY'); - {{- range $nodemanager := .Values.modules.nodemanager.list }} - INSERT INTO server_node (host, port, node_type, status) values ('{{ $nodemanager.name }}', '9461', 'NODE_MANAGER', 'HEALTHY'); + INSERT INTO server_node (host, port, node_type, status) values ('clustermanager', '4670', 'CLUSTER_MANAGER', 'HEALTHY'); + {{- range .Values.modules.nodemanager.replicas | int | until }} + INSERT INTO server_node (host, port, node_type, status) values ('nodemanager-{{ . }}.nodemanager', '4671', 'NODE_MANAGER', 'HEALTHY'); {{- end }} show tables; select * from server_node; diff --git a/helm-charts/FATE/templates/core/mysql/persistentvolumeclaim.yaml b/helm-charts/FATE/templates/core/mysql/persistentvolumeclaim.yaml deleted file mode 100644 index 7299ff18d..000000000 --- a/helm-charts/FATE/templates/core/mysql/persistentvolumeclaim.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2019-2022 VMware, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{ if .Values.modules.mysql.include }} -{{- if and .Values.persistence.enabled (not .Values.modules.mysql.existingClaim) }} -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: mysql-data - labels: - fateMoudle: mysql -{{ include "fate.labels" . | indent 4 }} -spec: - accessModes: - - {{ .Values.modules.mysql.accessMode }} - resources: - requests: - storage: {{ .Values.modules.mysql.size }} - {{- if .Values.modules.mysql.storageClass }} - {{- if eq "-" .Values.modules.mysql.storageClass }} - storageClassName: "" - {{- else }} - storageClassName: {{ .Values.modules.mysql.storageClass }} - {{- end }} - {{- end }} -{{- end }} -{{ end }} \ No newline at end of file diff --git a/helm-charts/FATE/templates/core/mysql/deployment.yaml b/helm-charts/FATE/templates/core/mysql/statefulSet.yaml similarity index 86% rename from helm-charts/FATE/templates/core/mysql/deployment.yaml rename to helm-charts/FATE/templates/core/mysql/statefulSet.yaml index a0b1c98fe..10a71c72d 100644 --- a/helm-charts/FATE/templates/core/mysql/deployment.yaml +++ b/helm-charts/FATE/templates/core/mysql/statefulSet.yaml @@ -9,18 +9,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -{{ if .Values.modules.mysql.include }} +{{- if .Values.modules.mysql.include }} apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: mysql labels: fateMoudle: mysql {{ include "fate.labels" . | indent 4 }} spec: + serviceName: mysql replicas: 1 - strategy: - type: Recreate selector: matchLabels: fateMoudle: mysql @@ -109,10 +108,22 @@ spec: {{- if not .Values.persistence.enabled }} - name: data emptyDir: {} - {{- else }} + {{- else if and .Values.persistence.enabled (.Values.modules.mysql.existingClaim) }} - name: data persistentVolumeClaim: claimName: {{ .Values.modules.mysql.existingClaim | default "mysql-data" }} + {{- else }} + volumeClaimTemplates: + - metadata: + name: data + labels: + fateMoudle: mysql +{{ include "fate.labels" . | indent 10 }} + spec: + accessModes: [{{ .Values.modules.mysql.accessMode | quote }}] + storageClassName: {{ .Values.modules.mysql.storageClass }} + resources: + requests: + storage: {{ .Values.modules.mysql.size }} {{- end }} ---- -{{ end }} \ No newline at end of file +{{- end }} \ No newline at end of file diff --git a/helm-charts/FATE/templates/core/python-spark.yaml b/helm-charts/FATE/templates/core/python-spark.yaml index 4bb5ec9e5..2664b9a30 100644 --- a/helm-charts/FATE/templates/core/python-spark.yaml +++ b/helm-charts/FATE/templates/core/python-spark.yaml @@ -9,24 +9,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -{{ if .Values.modules.python.include }} +{{- if .Values.modules.python.include }} apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: python labels: fateMoudle: python {{ include "fate.labels" . | indent 4 }} spec: + serviceName: fateflow replicas: 1 - strategy: - type: Recreate selector: matchLabels: fateMoudle: python {{ include "fate.matchLabels" . | indent 6 }} template: metadata: + annotations: + {{- if .Values.istio.enabled }} + sidecar.istio.io/rewriteAppHTTPProbers: "false" + {{- end }} labels: fateMoudle: python {{ include "fate.labels" . | indent 8 }} @@ -75,12 +78,8 @@ spec: checkMySQL {{- end }} containers: - - name: python - {{- if eq .Values.modules.python.backend "spark_pulsar" "spark_rabbitmq" "spark_local_pulsar" }} - image: {{ .Values.image.registry }}/python-spark:{{ .Values.image.tag }} - {{- else }} - image: {{ .Values.image.registry }}/{{ if .Values.modules.python.enabledNN }}python-nn{{ else }}python{{ end }}:{{ .Values.image.tag }} - {{- end }} + - name: fateflow + image: {{ .Values.image.registry }}/fateflow{{ include "images.fateflow.suffix" . }}:{{ .Values.image.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} {{- if .Values.modules.python.resources}} resources: @@ -102,7 +101,7 @@ spec: value: "$PYTHONPATH:/data/projects/fate/fate/python:/data/projects/fate/eggroll/python:/data/projects/fate/fateflow/python:/data/projects/fate/fate/python/fate_client" - name: FATE_PROJECT_BASE value: "/data/projects/fate" - {{- if eq .Values.modules.python.backend "spark_pulsar" "spark_rabbitmq" }} + {{- if eq .Values.computing "Spark" }} - name: FATE_FLOW_UPLOAD_MAX_NUM value: "1000000" - name: FATE_FLOW_UPLOAD_MAX_BYTES @@ -121,18 +120,13 @@ spec: mkdir -p /data/projects/fate/conf/ cp /data/projects/fate/conf-tmp/transfer_conf.yaml /data/projects/fate/conf/transfer_conf.yaml cp /data/projects/fate/conf-tmp/service_conf.yaml /data/projects/fate/conf/service_conf.yaml - cp /data/projects/fate/conf-tmp/component_registry.json /data/projects/fate/conf/component_registry.json - cp /data/projects/fate/conf-tmp/job_default_config.yaml /data/projects/fate/conf/job_default_config.yaml + cp /data/projects/fate/conf-tmp/component_registry.json /data/projects/fate/fateflow/conf/component_registry.json + cp /data/projects/fate/conf-tmp/job_default_config.yaml /data/projects/fate/fateflow/conf/job_default_config.yaml # fix fateflow conf must use IP sed -i "s/host: fateflow/host: ${POD_IP}/g" /data/projects/fate/conf/service_conf.yaml - cp /data/projects/spark-2.4.1-bin-hadoop2.7/conf/spark-defaults-template.conf /data/projects/spark-2.4.1-bin-hadoop2.7/conf/spark-defaults.conf - sed -i "s/fateflow/${POD_IP}/g" /data/projects/spark-2.4.1-bin-hadoop2.7/conf/spark-defaults.conf - - # logs - mkdir -p /data/projects/fate/fateflow/logs/fate_flow/ - touch /data/projects/fate/fateflow/logs/fate_flow/INFO.log - ln -sf /dev/stdout /data/projects/fate/fateflow/logs/fate_flow/INFO.log + cp /data/projects/spark-3.1.2-bin-hadoop3.2/conf/spark-defaults-template.conf /data/projects/spark-3.1.2-bin-hadoop3.2/conf/spark-defaults.conf + sed -i "s/fateflow/${POD_IP}/g" /data/projects/spark-3.1.2-bin-hadoop3.2/conf/spark-defaults.conf sleep 5 && python fateflow/python/fate_flow/fate_flow_server.py livenessProbe: @@ -157,7 +151,7 @@ spec: failureThreshold: 12 periodSeconds: 10 volumeMounts: - {{- if eq .Values.modules.python.backend "eggroll" }} + {{- if eq .Values.computing "Eggroll" }} - mountPath: /data/projects/fate/eggroll/conf/eggroll.properties name: eggroll-confs subPath: eggroll.properties @@ -167,22 +161,21 @@ spec: subPath: logs - mountPath: /data/projects/fate/conf-tmp/ name: python-confs - - mountPath: /data/projects/spark-2.4.1-bin-hadoop2.7/conf/spark-defaults-template.conf + - mountPath: /data/projects/spark-3.1.2-bin-hadoop3.2/conf/spark-defaults-template.conf name: python-confs subPath: spark-defaults.conf - {{- if eq .Values.modules.python.backend "spark_rabbitmq" }} + {{- if eq .Values.federation "RabbitMQ" }} - mountPath: /data/projects/fate/conf/rabbitmq_route_table name: rabbitmq-route-table {{- end }} - {{- if eq .Values.modules.python.backend "spark_pulsar" "spark_local_pulsar" }} + {{- if eq .Values.federation "Pulsar" }} - mountPath: /data/projects/fate/conf/pulsar_route_table name: pulsar-route-table {{- end }} - # TODO add mountPath of job and model-cache - - mountPath: /data/projects/fate/jobs + - mountPath: /data/projects/fate/fateflow/jobs name: python-data subPath: jobs - - mountPath: /data/projects/fate/model_local_cache + - mountPath: /data/projects/fate/fateflow/model_local_cache name: python-data subPath: model-local-cache {{- if .Values.modules.fateboard.include }} @@ -255,7 +248,7 @@ spec: {{- end }} restartPolicy: Always volumes: - {{- if eq .Values.modules.python.backend "eggroll" }} + {{- if eq .Values.computing "Eggroll" }} - name: eggroll-confs configMap: name: eggroll-config @@ -263,12 +256,12 @@ spec: - name: python-confs configMap: name: python-config - {{- if eq .Values.modules.python.backend "spark_rabbitmq" }} + {{- if eq .Values.federation "RabbitMQ" }} - name: rabbitmq-route-table configMap: name: rabbitmq-route-table {{- end }} - {{- if eq .Values.modules.python.backend "spark_pulsar" "spark_local_pulsar" }} + {{- if eq .Values.federation "Pulsar" }} - name: pulsar-route-table configMap: name: pulsar-route-table @@ -281,10 +274,22 @@ spec: {{- if not .Values.persistence.enabled }} - name: python-data emptyDir: {} - {{- else }} + {{- else if and .Values.persistence.enabled (.Values.modules.python.existingClaim) }} - name: python-data persistentVolumeClaim: claimName: {{ .Values.modules.python.existingClaim | default "python-data" }} + {{- else }} + volumeClaimTemplates: + - metadata: + name: python-data + labels: + fateMoudle: python +{{ include "fate.labels" . | indent 10 }} + spec: + accessModes: [{{ .Values.modules.python.accessMode | quote }}] + storageClassName: {{ .Values.modules.python.storageClass }} + resources: + requests: + storage: {{ .Values.modules.mysql.size }} {{- end }} - -{{ end }} +{{- end }} diff --git a/helm-charts/FATE/values-template-example.yaml b/helm-charts/FATE/values-template-example.yaml index 4362a48d9..d3f57bde0 100644 --- a/helm-charts/FATE/values-template-example.yaml +++ b/helm-charts/FATE/values-template-example.yaml @@ -1,10 +1,9 @@ name: fate-9999 namespace: fate-9999 chartName: fate -chartVersion: v1.8.0 +chartVersion: v1.9.0 partyId: 9999 registry: "" -imageTag: "1.8.0-release" pullPolicy: imagePullSecrets: - name: myregistrykey @@ -28,8 +27,19 @@ modules: # - nginx # - rabbitmq -# backend could be eggroll, spark_rabbitmq and spark_pulsar spark_local_pulsar -backend: eggroll +# Computing : [Eggroll, Spark, Spark_local] +computing: Eggroll +# Federation: [Eggroll(computing: Eggroll), Pulsar/RabbitMQ(computing: Spark/Spark_local)] +federation: Eggroll +# Storage: [Eggroll(computing: Eggroll), HDFS(computing: Spark), LocalFS(computing: Spark_local)] +storage: Eggroll +# Algorithm: [Basic, NN] +algorithm: Basic +# Device: [IPCL, CPU] +device: CPU + +skippedKeys: + - route_table # ingress: # fateboard: @@ -65,6 +75,7 @@ backend: eggroll # - partyId: 10000 # partyIp: 192.168.0.1 # partyPort: 30101 + # enableTLS: true # nodeSelector: # - disktype: ssd # tolerations: @@ -82,6 +93,13 @@ backend: eggroll # clientList: # - partID: 9999 # concurrency: 49 + # resources: + # requests: + # cpu: "1" + # memory: "1Gi" + # limits: + # cpu: "4" + # memory: "1Gi" # lbrollsite: @@ -96,46 +114,45 @@ backend: eggroll # nodeSelector: # tolerations: # affinity: + # resources: + # requests: + # cpu: "1" + # memory: "1Gi" + # limits: + # cpu: "1" + # memory: "1Gi" # nodemanager: - # count: 3 - # sessionProcessorsPerNode: 4 - # storageClass: "nodemanagers" - # accessMode: ReadWriteOnce - # size: 2Gi - # list: - # - name: nodemanager - # nodeSelector: - # kubernetes.io/hostname: localhost.localdomain - # tolerations: - # - key: "swapenable" - # operator: "Equal" - # value: "true" - # effect: "PreferNoSchedule" - # affinity: - # nodeAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # nodeSelectorTerms: - # - matchExpressions: - # - key: kubernetes.io/hostname - # operator: In - # values: - # - localhost.localdomain - # podAntiAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # - labelSelector: - # matchExpressions: - # - key: app - # operator: In - # values: - # - nodemanager - # topologyKey: kubernetes.io/hostname - # sessionProcessorsPerNode: 4 - # subPath: "nodemanager" - # existingClaim: "" - # storageClass: "nodemanager" - # accessMode: ReadWriteOnce - # size: 1Gi +# replicas: 2 +# sessionProcessorsPerNode: 4 +# nodeSelector: +# tolerations: +# affinity: +# subPath: "nodemanager" +# storageClass: "local-scsi" +# accessMode: ReadWriteOnce +# size: 1Gi +# existingClaim: "nodemanager" +# resources: +# requests: +# cpu: "2" +# memory: "4Gi" +# limits: +# cpu: "4" +# memory: "8Gi" + +# clustermanager: +# nodeSelector: +# tolerations: +# affinity: + # resources: + # requests: + # cpu: "1" + # memory: "1Gi" + # limits: + # cpu: "1" + # memory: "1Gi" + # python: # type: NodePort @@ -152,6 +169,13 @@ backend: eggroll # storageClass: "python" # accessMode: ReadWriteMany # size: 1Gi + # resources: + # requests: + # cpu: "2" + # memory: "4Gi" + # limits: + # cpu: "4" + # memory: "8Gi" # clustermanager: # cores_per_node: 16 # nodes: 2 @@ -268,11 +292,20 @@ backend: eggroll # affinity: # type: ClusterIP # nodePort: 30900 + # existingClaim: "" + # storageClass: "namenode" + # accessMode: ReadWriteOnce + # size: 1Gi # datanode: + # replicas: # nodeSelector: # tolerations: # affinity: # type: ClusterIP + # existingClaim: "" + # storageClass: "datanode" + # accessMode: + # size: # nginx: # nodeSelector: # tolerations: @@ -313,28 +346,52 @@ backend: eggroll # 10000: # host: 192.168.0.1 # port: 30104 + # resources: + # requests: + # cpu: "2" + # memory: "4Gi" + # limits: + # cpu: "4" + # memory: "8Gi" # pulsar: - # nodeSelector: - # tolerations: - # affinity: - # type: ClusterIP - # httpNodePort: 30094 - # httpsNodePort: 30099 - # loadBalancerIP: - # publicLB: - # enabled: false - # exchange: - # ip: 192.168.10.1 - # port: 30000 - # domain: fate.org - # route_table: - # 9999: - # host: pulsar - # port: 6650 - # sslPort:6651 - # 10000: - # host: 192.168.10.1 - # port: 30105 - # sslPort: 30109 - # proxy: "" +# nodeSelector: +# tolerations: +# affinity: +# env: +# - name: PULSAR_MEM +# value: "-Xms4g -Xmx4g -XX:MaxDirectMemorySize=8g" +# confs: +# brokerDeleteInactiveTopicsFrequencySeconds: 60 +# backlogQuotaDefaultLimitGB: 10 +# type: ClusterIP +# httpNodePort: 30094 +# httpsNodePort: 30099 +# loadBalancerIP: +# storageClass: "pulsar" +# existingClaim: +# accessMode: ReadWriteOnce +# size: 1Gi +# publicLB: +# enabled: false +# exchange: +# ip: 192.168.10.1 +# port: 30000 +# domain: fate.org +# route_table: +# 9999: +# host: pulsar +# port: 6650 +# sslPort: 6651 +# 10000: +# host: 192.168.10.1 +# port: 30105 +# sslPort: 30109 +# proxy: "" +# resources: +# requests: +# cpu: "2" +# memory: "4Gi" +# limits: +# cpu: "4" +# memory: "8Gi" diff --git a/helm-charts/FATE/values-template.yaml b/helm-charts/FATE/values-template.yaml index aba25def6..cdd68942e 100644 --- a/helm-charts/FATE/values-template.yaml +++ b/helm-charts/FATE/values-template.yaml @@ -2,7 +2,6 @@ image: registry: {{ .registry | default "federatedai" }} isThridParty: {{ empty .registry | ternary "false" "true" }} - tag: {{ .imageTag | default "1.8.0-release" }} pullPolicy: {{ .pullPolicy | default "IfNotPresent" }} {{- with .imagePullSecrets }} imagePullSecrets: @@ -12,6 +11,12 @@ image: partyId: {{ .partyId | int64 | toString }} partyName: {{ .name }} +computing: {{ .computing }} +federation: {{ .federation }} +storage: {{ .storage }} +algorithm: {{ .algorithm }} +device: {{ .device }} + {{- $partyId := (.partyId | int64 | toString) }} {{- with .ingress }} @@ -147,6 +152,7 @@ modules: type: {{ .type | default "ClusterIP" }} nodePort: {{ .nodePort }} loadBalancerIP: {{ .loadBalancerIP }} + enableTLS: {{ .enableTLS | default false}} {{- with .nodeSelector }} nodeSelector: {{ toYaml . | indent 6 }} @@ -174,6 +180,10 @@ modules: {{- end }} concurrency: {{ .concurrency }} {{- end }} + {{- with .resources }} + resources: +{{ toYaml . | indent 6 }} + {{- end }} {{- end }} @@ -195,6 +205,10 @@ modules: {{- end }} {{- with .affinity }} affinity: +{{ toYaml . | indent 6 }} + {{- end }} + {{- with .resources }} + resources: {{ toYaml . | indent 6 }} {{- end }} {{- end }} @@ -202,13 +216,12 @@ modules: python: include: {{ has "python" .modules }} - backend: {{ default "eggroll" .backend }} {{- with .python }} {{- with .resources }} resources: {{ toYaml . | indent 6 }} {{- end }} - logLevel: {{ .logLevel }} + logLevel: {{ .logLevel | default "INFO" }} type: {{ .type | default "ClusterIP" }} httpNodePort: {{ .httpNodePort }} grpcNodePort: {{ .grpcNodePort }} @@ -226,7 +239,6 @@ modules: affinity: {{ toYaml . | indent 6 }} {{- end }} - enabledNN: {{ .enabledNN | default false }} existingClaim: {{ .existingClaim }} claimName: {{ .claimName | default "python-data" }} storageClass: {{ .storageClass | default "python" }} @@ -275,7 +287,8 @@ modules: {{- with .clustermanager }} ip: clustermanager type: "ClusterIP" - {{- with .nodeSelector }} + enableTLS: {{ .enableTLS | default false }} + {{- with .nodeSelector }} nodeSelector: {{ toYaml . | indent 6 }} {{- end }} @@ -285,6 +298,10 @@ modules: {{- end }} {{- with .affinity }} affinity: +{{ toYaml . | indent 6 }} + {{- end }} + {{- with .resources }} + resources: {{ toYaml . | indent 6 }} {{- end }} {{- end }} @@ -293,28 +310,27 @@ modules: nodemanager: include: {{ has "nodemanager" .modules }} {{- with .nodemanager }} - list: - {{- $nodemanager := . }} - {{- range .count | int | until }} - - name: nodemanager-{{ . }} - {{- with $nodemanager.nodeSelector }} - nodeSelector: -{{ toYaml . | indent 8 }} - {{- end}} - {{- with $nodemanager.tolerations }} - tolerations: -{{ toYaml . | indent 8 }} - {{- end}} - {{- with $nodemanager.affinity }} - affinity: -{{ toYaml . | indent 8 }} - {{- end}} - sessionProcessorsPerNode: {{ $nodemanager.sessionProcessorsPerNode }} - subPath: "nodemanager-{{ . }}" - existingClaim: "" - storageClass: "{{ $nodemanager.storageClass }}" - accessMode: {{ $nodemanager.accessMode }} - size: {{ $nodemanager.size }} + sessionProcessorsPerNode: {{ .sessionProcessorsPerNode }} + replicas: {{ .replicas | default 2 }} + subPath: {{ .subPath }} + storageClass: {{ .storageClass | default "client" }} + accessMode: {{ .accessMode | default "ReadWriteOnce" }} + size: {{ .size | default "1Gi" }} + {{- with .nodeSelector }} + nodeSelector: +{{ toYaml . | indent 6 }} + {{- end }} + {{- with .tolerations }} + tolerations: +{{ toYaml . | indent 6 }} + {{- end }} + {{- with .affinity }} + affinity: +{{ toYaml . | indent 6 }} + {{- end }} + {{- with .resources }} + resources: +{{ toYaml . | indent 6 }} {{- end }} {{- end }} @@ -413,6 +429,7 @@ modules: {{ toYaml . | indent 8 }} {{- end }} type: {{ .master.type }} + nodePort: {{ .master.nodePort }} {{- end }} {{- if .worker }} worker: @@ -461,8 +478,9 @@ modules: existingClaim: {{ .namenode.existingClaim }} storageClass: {{ .namenode.storageClass | default "" }} accessMode: {{ .namenode.accessMode | default "ReadWriteOnce" }} - size: {{ .namenode.size }} + size: {{ .namenode.size | default "1Gi" }} datanode: + replicas: {{ .datanode.replicas | default 3 }} {{- with .datanode.nodeSelector }} nodeSelector: {{ toYaml . | indent 8 }} @@ -479,7 +497,7 @@ modules: existingClaim: {{ .datanode.existingClaim }} storageClass: {{ .datanode.storageClass | default "" }} accessMode: {{ .datanode.accessMode | default "ReadWriteOnce" }} - size: {{ .datanode.size }} + size: {{ .datanode.size | default "1Gi" }} {{- end }} @@ -519,6 +537,10 @@ modules: rabbitmq: include: {{ has "rabbitmq" .modules }} {{- with .rabbitmq }} + {{- with .resources }} + resources: +{{ toYaml . | indent 6 }} + {{- end }} {{- with .nodeSelector }} nodeSelector: {{ toYaml . | indent 6 }} @@ -549,6 +571,18 @@ modules: pulsar: include: {{ has "pulsar" .modules }} {{- with .pulsar }} + {{- with .resources }} + resources: +{{ toYaml . | indent 6 }} + {{- end }} + {{- with .env }} + env: +{{ toYaml . | indent 6 }} + {{- end }} + {{- with .confs }} + confs: +{{ toYaml . | indent 6 }} + {{- end }} {{- with .nodeSelector }} nodeSelector: {{ toYaml . | indent 6 }} @@ -583,7 +617,7 @@ modules: existingClaim: {{ .existingClaim }} storageClass: {{ .storageClass | default "" }} accessMode: {{ .accessMode | default "ReadWriteOnce" }} - size: {{ .size }} + size: {{ .size | default "1Gi" }} {{- end }} externalMysqlIp: {{ .externalMysqlIp }} diff --git a/helm-charts/FATE/values.yaml b/helm-charts/FATE/values.yaml index 519711c70..a4c1a0dc3 100644 --- a/helm-charts/FATE/values.yaml +++ b/helm-charts/FATE/values.yaml @@ -2,7 +2,7 @@ image: registry: federatedai isThridParty: - tag: 1.8.0-release + tag: 1.9.0-release pullPolicy: IfNotPresent imagePullSecrets: # - name: @@ -10,6 +10,17 @@ image: partyId: 9999 partyName: fate-9999 +# Computing : Eggroll, Spark, Spark_local +computing: Eggroll +# Federation: Eggroll(computing: Eggroll), Pulsar/RabbitMQ(computing: Spark/Spark_local) +federation: Eggroll +# Storage: Eggroll(computing: Eggroll), HDFS(computing: Spark), LocalFS(computing: Spark_local) +storage: Eggroll +# Algorithm: Basic, NN +algorithm: Basic +# Device: CPU, IPCL +device: IPCL + istio: enabled: false @@ -79,7 +90,8 @@ modules: ip: rollsite type: ClusterIP nodePort: 30091 - loadBalancerIP: + loadBalancerIP: + enableTLS: false nodeSelector: tolerations: affinity: @@ -116,13 +128,11 @@ modules: nodeSelector: tolerations: affinity: - backend: eggroll - enabledNN: false logLevel: INFO # subPath: "" - existingClaim: "" + existingClaim: claimName: python-data - storageClass: "python" + storageClass: accessMode: ReadWriteOnce size: 1Gi clustermanager: @@ -163,8 +173,8 @@ modules: tolerations: affinity: subPath: "client" - existingClaim: "" - storageClass: "nodemanager-0" + existingClaim: + storageClass: accessMode: ReadWriteOnce size: 1Gi clustermanager: @@ -176,27 +186,20 @@ modules: affinity: nodemanager: include: true - list: - - name: nodemanager-0 - nodeSelector: - tolerations: - affinity: - sessionProcessorsPerNode: 2 - subPath: "nodemanager-0" - existingClaim: "" - storageClass: "nodemanager-0" - accessMode: ReadWriteOnce - size: 1Gi - - name: nodemanager-1 - nodeSelector: - tolerations: - affinity: - sessionProcessorsPerNode: 2 - subPath: "nodemanager-1" - existingClaim: "" - storageClass: "nodemanager-1" - accessMode: ReadWriteOnce - size: 1Gi + replicas: 2 + nodeSelector: + tolerations: + affinity: + sessionProcessorsPerNode: 2 + subPath: "nodemanager" + storageClass: + accessMode: ReadWriteOnce + size: 1Gi + existingClaim: + resources: + requests: + cpu: "2" + memory: "4Gi" client: include: true @@ -206,8 +209,8 @@ modules: tolerations: affinity: subPath: "client" - existingClaim: "" - storageClass: "client" + existingClaim: + storageClass: accessMode: ReadWriteOnce size: 1Gi @@ -223,9 +226,9 @@ modules: user: fate password: fate_dev subPath: "mysql" - existingClaim: "" + existingClaim: claimName: mysql-data - storageClass: "mysql" + storageClass: accessMode: ReadWriteOnce size: 1Gi serving: @@ -257,17 +260,14 @@ modules: Image: "" ImageTag: "" replicas: 2 - resources: - requests: - cpu: "2" - memory: "4Gi" - limits: - cpu: "4" - memory: "8Gi" nodeSelector: tolerations: affinity: type: ClusterIP + resources: + requests: + cpu: "2" + memory: "4Gi" hdfs: include: true namenode: @@ -276,11 +276,20 @@ modules: affinity: type: ClusterIP nodePort: 30900 + existingClaim: + storageClass: + accessMode: ReadWriteOnce + size: 1Gi datanode: + replicas: 3 nodeSelector: tolerations: affinity: type: ClusterIP + existingClaim: + storageClass: + accessMode: ReadWriteOnce + size: 1Gi nginx: include: true nodeSelector: @@ -325,11 +334,17 @@ modules: include: true nodeSelector: tolerations: + env: + confs: affinity: type: ClusterIP httpNodePort: 30094 httpsNodePort: 30099 - loadBalancerIP: + loadBalancerIP: + existingClaim: + accessMode: ReadWriteOnce + storageClass: + size: 1Gi publicLB: enabled: false # exchange: diff --git a/helm-charts/Images_list.md b/helm-charts/Images_list.md index d27510697..ce3246e4a 100644 --- a/helm-charts/Images_list.md +++ b/helm-charts/Images_list.md @@ -11,8 +11,8 @@ - federatedai/python-spark:${version-tag} - federatedai/spark-master:${version-tag} - federatedai/spark-worker:${version-tag} -- federatedai/hadoop-datanode:2.0.0-hadoop2.7.4-java8 -- federatedai/hadoop-namenode:2.0.0-hadoop2.7.4-java8 +- federatedai/hadoop-datanode:2.0.0-hadoop3.2.1-java8 +- federatedai/hadoop-namenode:2.0.0-hadoop3.2.1-java8 - nginx:1.17 - federatedai/nginx:${version-tag} - federatedai/rabbitmq:3.8.3-management diff --git a/helm-charts/UpgradeManager/Chart.yaml b/helm-charts/UpgradeManager/Chart.yaml new file mode 100644 index 000000000..cde2358fa --- /dev/null +++ b/helm-charts/UpgradeManager/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +appVersion: v1.0.0 +description: A Helm chart for a K8s job, which is used for upgrading the framworks +name: fate-upgrade-manager +version: v1.0.0 +sources: + - https://github.com/FederatedAI/KubeFATE \ No newline at end of file diff --git a/helm-charts/UpgradeManager/templates/fate-mysql-upgrade-job.yaml b/helm-charts/UpgradeManager/templates/fate-mysql-upgrade-job.yaml new file mode 100644 index 000000000..144fd5243 --- /dev/null +++ b/helm-charts/UpgradeManager/templates/fate-mysql-upgrade-job.yaml @@ -0,0 +1,27 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: fate-mysql-upgrade-job +spec: + backoffLimit: 0 + template: + spec: + serviceAccountName: fum + containers: + - name: fate-mysql-upgrader + image: federatedai/fate-upgrade-manager:latest + imagePullPolicy: IfNotPresent + command: + - "/bin/bash" + - "-c" + - | + python shutdown-flow.py {{ .Release.Namespace }} + retVal=$? + if [ $retVal -ne 0 ]; then + echo "failed to terminate the flow's pod" + else + {{- with .Values }} + python upgrade-mysql.py {{ .username }} {{ .password }} {{ .start }} {{ .target }} + {{- end }} + fi + restartPolicy: Never diff --git a/helm-charts/UpgradeManager/templates/rbac.yaml b/helm-charts/UpgradeManager/templates/rbac.yaml new file mode 100644 index 000000000..a815ef73a --- /dev/null +++ b/helm-charts/UpgradeManager/templates/rbac.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fum + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: fum + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: fum-role +subjects: + - kind: ServiceAccount + name: fum + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: fum-role + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: + - apps + resources: + - deployments + - deployments/status + - statefulsets + verbs: + - update + - patch + - list + - get \ No newline at end of file diff --git a/helm-charts/UpgradeManager/values-template.yaml b/helm-charts/UpgradeManager/values-template.yaml new file mode 100644 index 000000000..3c320ff97 --- /dev/null +++ b/helm-charts/UpgradeManager/values-template.yaml @@ -0,0 +1,4 @@ +username: {{ .username }} +password: {{ .password }} +start: {{ .start }} +target: {{ .target }} \ No newline at end of file diff --git a/helm-charts/UpgradeManager/values.yaml b/helm-charts/UpgradeManager/values.yaml new file mode 100644 index 000000000..960804cec --- /dev/null +++ b/helm-charts/UpgradeManager/values.yaml @@ -0,0 +1,4 @@ +username: fate +password: fate_dev +start: 1.9.0 +target: 1.9.0 \ No newline at end of file diff --git a/k8s-deploy/Makefile b/k8s-deploy/Makefile index 8b166d04c..0b65dd552 100644 --- a/k8s-deploy/Makefile +++ b/k8s-deploy/Makefile @@ -1,5 +1,5 @@ NAME ?= federatedai/kubefate -VERSION ?= v1.4.4 +VERSION ?= v1.4.5 IMG ?= ${NAME}:${VERSION} ifeq (,$(shell go env GOBIN)) diff --git a/k8s-deploy/README.md b/k8s-deploy/README.md index 5e95b694c..68dcfe6ee 100644 --- a/k8s-deploy/README.md +++ b/k8s-deploy/README.md @@ -1,9 +1,11 @@ # Kubernetes Deployment -We recommend using [Kubernetes](https://kubernetes.io/) as an underlying infrastructure to create and manage the FATE clusters in a production environment. KubeFATE supports deploying multiple FATE clusters in an instance of Kubernetes with different namespaces for the purposes of development, testing and production. Considering the different IT designs and standards in each company, the actual deployment should be customized. KubeFATE is flexibile for the FATE configuration. -If you focus on how to quickly use KubeFATE, please jump to [Use Scenarios](#use-scenarios). +This document introduces the architecture, the user cases and the usage of KubeFATE. If you only care about how to use it, then you can jump to [Usage](#usage). + +We recommend using [Kubernetes](https://kubernetes.io/) as an underlying infrastructure to create and manage the FATE clusters in a production environment. KubeFATE supports deploying multiple FATE clusters in an instance of Kubernetes with different namespaces for the purposes of development, testing and production. Considering the different IT designs and standards in each company, the actual deployment should be customized. KubeFATE is flexible for the FATE configuration. ## High-level architecture of multiple federated learning parties + The high-level architecture of a multi-party federated learning deployment (e.g. two parties) is shown as follows:
@@ -14,36 +16,45 @@ The high-level architecture of a multi-party federated learning deployment (e.g. * Kubernetes: Container orchestration engine. KubeFATE is responsible for: + * Day 1 initialization: Provision a FATE cluster on Kubernetes * Day 2 operations: Provides RESTful APIs to manage FATE clusters ## High-level architecture of KubeFATE -The high-level architecture of KubeFATE is shwon as follows: + +The high-level architecture of KubeFATE is shown as follows:
The numbers depicted in the diagram: + 1. Accepting external API calls of Authentication & authorization -2. Rendering templates via Helm; +2. Rendering templates via Helm 3. Storing jobs and configuration of a FATE deployment 4. KubeFATE is running as a service of Kubernetes There are two parts of KubeFATE: + * The KubeFATE CLI. KubeFATE CLI is an executable helps to quickly initialize and manage a FATE cluster in an interactive mode. It does not rely on Kubernetes. Eventually, KubeFATE CLI calls KubeFATE Service for operations with a KubeFATE user token. * The KubeFATE Service. The KubeFATE service provides RESTful APIs for managing FATE clusters. The KubeFATE service is deployed in Kubernetes, and exposes APIs via [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/). For the authentication and authorization, the KubeFATE service implements [JWT](https://jwt.io/introduction/), and neutral to other security solutions which can be added to Kubernetes ingress. KubeFATE is designed to handle different versions FATE. Normally, KubeFATE CLI and KubeFATE service can work with several FATE releases. -## User scenarios +## Real-world story + Suppose in an organization, there are two roles: + * System Admin: who is responsible for the infrastructure management as well as Kubernetes administration -* ML Infrastructure Operators: who is responsible for managing the machine learning cluster like FATE +* ML Infrastructure Operators: who is responsible for managing the machine learning cluster such as FATE
-### Initializing a FATE deployment + +## Usage + +### Prerequisite Recommended version of dependent software: @@ -51,13 +62,16 @@ Kubernetes: [v1.23.5](https://github.com/kubernetes/kubernetes/releases/tag/v1.2 Ingress-nginx: [v1.1.3](https://github.com/kubernetes/ingress-nginx/releases/tag/controller-v1.1.3) -#### Creating role, namespace and other resource in Kubernetes +### Creating role, namespace and other resource in Kubernetes + The example yaml can be found in [rbac-config.yaml](./rbac-config.yaml). In this example, we create a kube-fate namespace for KubeFATE service. Resource constraints can be applied to kube-fate namespace, refer to [Kubernetes Namespace](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/), [Configure Memory and CPU Quotas for Namespace](https://kubernetes.io/docs/tasks/administer-cluster/manage-resources/quota-memory-cpu-namespace/). Run the following command to create the namespace: + ``` -$ kubectl apply -f ./rbac-config.yaml +kubectl apply -f ./rbac-config.yaml ``` + Note that, the default username and password of KubeFATE service can be set in `rbac-config.yaml` Secret->kubefate-secret->stringData : ``` @@ -66,15 +80,19 @@ stringData: kubefatePassword: admin ``` -#### Preparing domain name and deploying KubeFATE in Kubernetes +### Preparing domain name and deploying KubeFATE in Kubernetes + Because KubeFATE service exposes RESTful APIs for external access, system admin needs to prepare a domain name for KubeFATE service. In our example, the domain name is `example.com` . Moreover, system admin should create a namespace (e.g. fate-9999) for FATE deployment. + ``` -$ kubectl apply -f ./kubefate.yaml -$ kubectl create namespace fate-9999 +kubectl apply -f ./kubefate.yaml +kubectl create namespace fate-9999 ``` + For more about the configuration of KubeFATE service, please refer to: [KubeFATE service Configuration Guild](../docs/configurations/kubefate_service_configuration.md). -#### Preparing cluster configuration and deploying FATE +### Preparing cluster configuration and deploying FATE + After the system admin deployed the KubeFATE service and prepared the namespace for FATE. The ML Infrastructure Operator is able to start the deployment of FATE. The `config.yaml` for `kubefate` CLI is required. It contains the username and password of KubeFATE access, and the KubeFATE service URL: ``` @@ -85,6 +103,7 @@ user: password: admin serviceurl: example.com +safeconnect: false ``` |Name |Type |Description | @@ -92,165 +111,161 @@ serviceurl: example.com |log |scalars |The log level of command line. | |user |mappings|User name and password when logging into KubeFATE service. | |serviceurl |scalars |KubeFATE service's ingress domain name, defined in kubefate.yaml. | +|safeconnect|scalars |Whether use HTTPS to connect the KubeFATE service URL. You can refer to: [kubefate_service_tls_enable](../docs/configurations/kubefate_service_tls_enable.md) | -Create a `cluster.yaml` for FATE cluster configuration. The details of configuration can be found here: [FATE Cluster Configuration Guide](../docs/configurations/FATE_cluster_configuration.md). +Create a `cluster.yaml` for FATE cluster configuration. The details of configuration can be found here: [FATE Cluster Configuration Guide](../docs/configurations/FATE_cluster_configuration.md). **NOTE:** For Chinese user, specifying a local image registry in `cluster.yaml` can accelerate the download of images. The details are as follows: -``` + +```yaml registry: "hub.c.163.com/federatedai" ``` Next, install the FATE cluster, -``` +```bash $ kubefate cluster install -f ./cluster.yaml create job success, job id=d92d7a56-7002-46a4-9363-da9c7346e05a ``` + *NOTE: If you want to deploy **FATE on Spark**, you can use `cluster-spark.yaml`.* -#### Checking the status of "Installing Cluster" job +Deploying FATE with KubeFATE can support many different engine combinations. For more details on the different types of FATE see: [Introduction to FATE Engine Architecture](../docs/Introduction_to_Engine_Architecture.md). + +**If you have resource requirements (CPU and memory) for the components, please make sure to check [the example](../helm-charts/FATE/values-template-example.yaml), search for "resources" to know how to define the resource requirements.** + +We support such definition for: + +1. Eggroll components: cluster manager, node manager and rollsite. +2. Spark components: master and worker. +3. Rabbitmq. +4. Pulsar. + +### Checking the status of "Installing Cluster" job + After the above command has finished, a job is created for installing a FATE cluster. Run the command `kubefate job describe` to check the status of the job, until the "Status" turns to `Success`. ```bash $ kubefate job describe d92d7a56-7002-46a4-9363-da9c7346e05a -UUID d92d7a56-7002-46a4-9363-da9c7346e05a -StartTime 2022-04-12 07:34:09 -EndTime 2022-04-12 07:48:14 -Duration 14m -Status Success -Creator admin -ClusterId 24bb75ff-f636-4c64-8c04-1b9073f89a2f -States - update job status to Running - - create Cluster in DB Success - - helm install Success - - checkout Cluster status [794] - - job run Success - -SubJobs nodemanager-0 ModuleStatus: Available, SubJobStatus: Success, Duration: 13m, StartTime: - 2022-04-12 07:34:09, EndTime: 2022-04-12 07:47:26 - nodemanager-1 ModuleStatus: Available, SubJobStatus: Success, Duration: 13m, StartTime: - 2022-04-12 07:34:09, EndTime: 2022-04-12 07:47:18 - python ModuleStatus: Available, SubJobStatus: Success, Duration: 14m, StartTime: - 2022-04-12 07:34:09, EndTime: 2022-04-12 07:48:14 - rollsite ModuleStatus: Available, SubJobStatus: Success, Duration: 13m, StartTime: - 2022-04-12 07:34:09, EndTime: 2022-04-12 07:47:24 - client ModuleStatus: Available, SubJobStatus: Success, Duration: 11m, StartTime: - 2022-04-12 07:34:09, EndTime: 2022-04-12 07:45:22 - clustermanager ModuleStatus: Available, SubJobStatus: Success, Duration: 13m, StartTime: - 2022-04-12 07:34:09, EndTime: 2022-04-12 07:47:11 - mysql ModuleStatus: Available, SubJobStatus: Success, Duration: 13m, StartTime: - 2022-04-12 07:34:09, EndTime: 2022-04-12 07:47:11 +UUID d92d7a56-7002-46a4-9363-da9c7346e05a +StartTime 2022-04-12 07:34:09 +EndTime 2022-04-12 07:48:14 +Duration 14m +Status Success +Creator admin +ClusterId 24bb75ff-f636-4c64-8c04-1b9073f89a2f +States - update job status to Running + - create Cluster in DB Success + - helm install Success + - checkout Cluster status [794] + - job run Success + +SubJobs nodemanager-0 ModuleStatus: Available, SubJobStatus: Success, Duration: 13m, StartTime: + 2022-04-12 07:34:09, EndTime: 2022-04-12 07:47:26 + nodemanager-1 ModuleStatus: Available, SubJobStatus: Success, Duration: 13m, StartTime: + 2022-04-12 07:34:09, EndTime: 2022-04-12 07:47:18 + python ModuleStatus: Available, SubJobStatus: Success, Duration: 14m, StartTime: + 2022-04-12 07:34:09, EndTime: 2022-04-12 07:48:14 + rollsite ModuleStatus: Available, SubJobStatus: Success, Duration: 13m, StartTime: + 2022-04-12 07:34:09, EndTime: 2022-04-12 07:47:24 + client ModuleStatus: Available, SubJobStatus: Success, Duration: 11m, StartTime: + 2022-04-12 07:34:09, EndTime: 2022-04-12 07:45:22 + clustermanager ModuleStatus: Available, SubJobStatus: Success, Duration: 13m, StartTime: + 2022-04-12 07:34:09, EndTime: 2022-04-12 07:47:11 + mysql ModuleStatus: Available, SubJobStatus: Success, Duration: 13m, StartTime: + 2022-04-12 07:34:09, EndTime: 2022-04-12 07:47:11 ``` -#### Describing the cluster and finding FATE access information + +### Describing the cluster and finding FATE access information + After the `installing cluster` job succeeded, use `kubefate cluster describe` to check the FATE access information: + ```bash $ kubefate cluster describe 24bb75ff-f636-4c64-8c04-1b9073f89a2f -UUID 24bb75ff-f636-4c64-8c04-1b9073f89a2f -Name fate-9999 -NameSpace fate-9999 -ChartName fate -ChartVersion v1.8.0 -Revision 1 -Age 44h -Status Running -Spec backend: eggroll - chartName: fate - chartVersion: v1.8.0 - imagePullSecrets: - - name: myregistrykey - imageTag: 1.8.0-release - ingress: - client: - hosts: - - name: party9999.notebook.example.com - fateboard: - hosts: - - name: party9999.fateboard.example.com - ingressClassName: nginx - istio: - enabled: false - modules: - - rollsite - - clustermanager - - nodemanager - - mysql - - python - - fateboard - - client - name: fate-9999 - namespace: fate-9999 - partyId: 9999 - persistence: false - podSecurityPolicy: - enabled: false - pullPolicy: null - python: - grpcNodePort: 30092 - httpNodePort: 30097 - logLevel: INFO - type: NodePort - registry: "" - rollsite: - nodePort: 30091 - partyList: - - partyId: 10000 - partyIp: 192.168.10.1 - partyPort: 30101 - type: NodePort - servingIp: 192.168.9.2 - servingPort: 30095 - -Info dashboard: - - party9999.notebook.example.com - - party9999.fateboard.example.com - ip: 192.168.9.1 - port: 30091 - status: - containers: - client: Running - clustermanager: Running - fateboard: Running - mysql: Running - nodemanager-0: Running - nodemanager-0-eggrollpair: Running - nodemanager-1: Running - nodemanager-1-eggrollpair: Running - python: Running - rollsite: Running - deployments: - client: Available - clustermanager: Available - mysql: Available - nodemanager-0: Available - nodemanager-1: Available - python: Available - rollsite: Available +UUID 24bb75ff-f636-4c64-8c04-1b9073f89a2f +Name fate-9999 +NameSpace fate-9999 +ChartName fate +ChartVersion v1.9.0 +Revision 1 +Age 15m +Status Running +Spec algorithm: Basic + chartName: fate + chartVersion: v1.9.0 + computing: Eggroll + device: CPU + federation: Eggroll + imagePullSecrets: + - name: myregistrykey + ingressClassName: nginx + istio: + enabled: false + modules: + - rollsite + - clustermanager + - nodemanager + - mysql + - python + - fateboard + - client + name: fate-9999 + namespace: fate-9999 + partyId: 9999 + persistence: false + podSecurityPolicy: + enabled: false + pullPolicy: null + registry: "" + storage: Eggroll + +Info dashboard: + - party9999.notebook.example.com + - party9999.fateboard.example.com + ip: 192.168.9.1 + status: + containers: + client: Running + clustermanager: Running + fateboard: Running + fateflow: Running + mysql: Running + nodemanager: Running + nodemanager-eggrollpair: Running + rollsite: Running + deployments: + clustermanager: Available + rollsite: Available ``` -#### Access the UI of FATEBoard and Notebook +### Access the UI of FATEBoard and Notebook If the components of fateboard and client are installed, you can use the information `party9999.fateboard.example.com` and `party9999.notebook.example.com` obtained in the previous step to access FATEBoard and Notebook UI, and configure the resolution of these two domain names It can be opened in the browser. -##### FATEBoard +#### FATEBoard - http://party9999.fateboard.example.com + Access to FATEBoard UI requires a login user name and password, which can be found in `cluster.yaml` [Configuration](../docs/configurations/FATE_cluster_configuration.md#fateboard mappings). ![fate_board](../docs/tutorials/images/tkg_fate_board.png) -##### Notebook +#### Notebook - http://party9999.fateboard.example.com + ![notebook](../docs/tutorials/images/tkg_notebook.png) ### Other user scenarios + #### [Manage FATE and FATE-Serving Version](../docs/Manage_FATE_and_FATE-Serving_Version.md) + #### [Update and Delete a FATE Cluster](../docs/Update_and_Delete_a_FATE_Cluster.md) + #### [KubeFATE Examples](examples) -#### [KubeFATE Command Line User Guide](../docs/KubeFATE_command_line_user_guide.md) +#### [KubeFATE Command Line User Guide](../docs/KubeFATE_CLI_user_guide.md) ## KubeFATE service RESTful APIs reference -#### [API Reference](docs/KubeFATE_API_Reference_Swagger.md) + +**[API Reference](docs/KubeFATE_API_Reference_Swagger.md)** diff --git a/k8s-deploy/README_zh.md b/k8s-deploy/README_zh.md index 9ae44c30f..8b7cefbf5 100644 --- a/k8s-deploy/README_zh.md +++ b/k8s-deploy/README_zh.md @@ -1,9 +1,11 @@ # Kubernetes部署方案 -我们推荐在生产环境中使用[Kubernetes](https://kubernetes.io/) 作为底层的基础平台支撑FATE集群的创建、管理。KubeFATE目前支持在一个Kubernetes平台上以不同的namespace部署多个FATE集群，用于开发、测试和产品等场景。考虑到不同企业的IT设计和标准各异，KubeFATE提供灵活的机制支持定制化安装、配置FATE。KubeFATE的设计架构分离了主程序服务与FATE的具体配置。 -如果您想快速了解如何使用KubeFATE，可以跳转到[用户场景](#use-scenarios)章节。 +这篇文档介绍了KubeFATE的架构，用户的实际使用场景以及KubeFATE的使用指南。如果您只关心使用的话可以跳转到 ***[Kubefate 使用指南](#kubefate-使用指南)*** 。 + +我们推荐在生产环境中使用[Kubernetes](https://kubernetes.io/) 作为底层的基础平台支撑FATE集群的创建、管理。KubeFATE目前支持在一个Kubernetes平台上以不同的namespace部署多个FATE集群，用于开发、测试和产品等场景。考虑到不同企业的IT设计和标准各异，KubeFATE提供灵活的机制支持定制化安装、配置FATE。KubeFATE的设计架构分离了主程序服务与FATE的具体配置。 ## 多方联邦学习高层架构 + 抽象从高层去表现多方（譬如两方）联邦学习的机构，如下图：
@@ -14,51 +16,62 @@ * Kubernetes：编排引擎 KubeFATE负责： + * Day 1初始化： 可执行二进制文件负责部署、配置FATE集群； * Day 2运维管理：提供可执行二进制工具进行日常运维，且提供RESTful APIs以供与FATE-Cloud或者企业管理系统进行整合。 ## KubeFATE架构图 + KubeFATE的架构可表现如下图：
图中所标为： -1. 请求进行授权认证； -2. 渲染Helm模板； -3. 持久化任务及配置FATE的部署规范； + +1. 请求进行授权认证 +2. 渲染Helm模板 +3. 持久化任务及配置FATE的部署规范 4. KubeFATE作为Kubernetes的应用部署提供服务 所以，KubeFATE由两大部分构成： + * KubeFATE命令行工具。KubeFATE命令行为可执行二进制，用作快速初始化、部署、管理FATE集群。它设计为可以在Kubernetes集群外使用，并使用独立的授权机制。在底层实现中，命令行会调用KubeFATE服务实行Kuberentes的操作； * KubeFATE服务。KubeFATE服务向外提供RESTful APIs供管理FATE集群。它的API服务建立在Kubernetes的[Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/)技术上，可以轻易与各种管理平台、网络设备进行整合。KubeFATE实现了独立的授权认证机制，基于[JWT](https://jwt.io/introduction/)。 KubeFATE在设计上分离了服务与FATE集群的配置(基于Helm Chart)，譬如版本升级等。所以在大多版本下，KubeFATE可以实现FATE集群的独立升级，往往只需要配置一行修改。 -## 用户场景 +## 用户应用场景 + 假设在一个组织内部，有两个不同角色： + * 系统管理员： 负责基础设施的管理，包括Kubernetes的管理； * 机器学习的系统运维： 负责管理机器学习相关服务，譬如FATE的管理。 +
-### 初始化新的FATE集群 + +## KubeFATE 使用指南 + +### 需要满足的依赖 依赖软件的建议版本： Kubernetes: [v1.23.5](https://github.com/kubernetes/kubernetes/releases/tag/v1.23.5) - Ingress-nginx: [v1.1.3](https://github.com/kubernetes/ingress-nginx/releases/tag/controller-v1.1.3) -#### 创建Kubernetes服务账号、namespace等 +### 创建Kubernetes服务账号、namespace等 + 我们的发布包提供了[rbac-config.yaml](./rbac-config.yaml)作为样例。但是现实使用请与系统管理员确认权限问题，以及商量资源配额。更多请参考[Kubernetes Namespace](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/)和[Configure Memory and CPU Quotas for Namespace](https://kubernetes.io/docs/tasks/administer-cluster/manage-resources/quota-memory-cpu-namespace/)。 -``` +```bash kubectl apply -f ./rbac-config.yaml ``` 请务必注意，默认的用户名密码，设置在```Secret->kubefate-secret->stringData```: -``` + +```yaml stringData: kubefateUsername: admin kubefatePassword: admin @@ -66,20 +79,22 @@ stringData: 具体参考：[Overview of Cloud Native Security](https://kubernetes.io/docs/concepts/security/)。 -#### 准备域名及部署KubeFATE服务 +### 准备域名及部署KubeFATE服务 + 因为KubeFATE服务向外暴露RESTful APIs，所以系统管理员需要为此准备域名的DNS解析。在我们的配置样例里，使用```example.com```作为域名。同时，系统管理员需要为每个FATE集群创建一个namespace，譬如party_id为9999的FATE集群，创建一个fate-9999的namespace，然后进行配额控制。准备好后，把这些信息反馈给机器学习的系统运维。 -``` +```bash kubectl apply -f ./kubefate.yaml kubectl create namespace fate-9999 ``` 具体创建KubeFATE服务的配置，可以参考： [KubeFATE service Configuration Guild](../docs/configurations/kubefate_service_configuration.md)。 -#### 准备FATE的安装配置文件并部署FATE +### 准备FATE的安装配置文件并部署FATE + 当系统管理员成功部署了KubeFATE服务，并准备好新的FATE集群namespace，机器学习的系统运维可以根据这些信息开始部署FATE集群。`config.yaml`是使用KubeFATE命令行的配置文件，包含KubeFATE服务的访问用户名密码，以及log配置，域名。 -``` +```yaml log: level: info user: @@ -87,6 +102,7 @@ user: password: admin serviceurl: example.com +safeconnect: false ``` |名字 |种类 |描述 | @@ -94,168 +110,161 @@ serviceurl: example.com |log |scalars |命令行的log级别.debug级别包含REST通信信息 | |user |mappings|KubeFATE的认证用户名、密码 | |serviceurl |scalars |KubeFATE服务使用的域名 | +|safeconnect|scalars |是否使用https访问KubeFATE服务URL。你可以参考 [kubefate_service_tls_enable](../docs/configurations/kubefate_service_tls_enable.md) | 我们的配置样例里还包含了`cluster.yaml`，是FATE集群的部署计划。更多自定义内容参考：[FATE Cluster Configuration Guild](../docs/configurations/FATE_cluster_configuration.md) **注意:** 对于中国的用户，可以通过在`cluster.yaml`修改`registry`来加速镜像的下载，具体设置内容如下： -``` + +```yaml registry: "hub.c.163.com/federatedai" ``` 准备好后，通过```kubefate cluster install```命令进行部署FATE集群： -``` +```bash $ kubefate cluster install -f ./cluster.yaml create job success, job id=d92d7a56-7002-46a4-9363-da9c7346e05a ``` *如果想要部署使用 **Spark** 计算引擎的FATE集群，可以使用`cluster-spark.yaml`。* -#### 检查安装集群任务的状态 +使用KubeFATE部署FATE可以支持不同类型的引擎。关于不同类型的FATE的更多细节查看: [不同类型FATE的架构介绍](../docs/Introduction_to_Engine_Architecture_zh.md)。 + +**如果某个部件对计算机性能（CPU和内存）有所要求，可以查看[示例文件](../helm-charts/FATE/values-template-example.yaml), 在其中搜索"resources"关键字了解如何进行资源配置.** + +我们支持定义如下部件的资源需求: + +1. Eggroll部件：包括cluster manager，node manager和rollsite。 +2. Spark components：包括master和worker。 +3. Rabbitmq。 +4. Pulsar。 + +### 检查安装集群任务的状态 + 上面的命令会创建一个安装FATE集群的任务，用于异步部署。使用```kubefate job describe```命令可以检查任务的状态，直到看到结果为`install success` ```bash $ kubefate job describe d92d7a56-7002-46a4-9363-da9c7346e05a -UUID d92d7a56-7002-46a4-9363-da9c7346e05a -StartTime 2022-04-12 07:34:09 -EndTime 2022-04-12 07:48:14 -Duration 14m -Status Success -Creator admin -ClusterId 24bb75ff-f636-4c64-8c04-1b9073f89a2f -States - update job status to Running - - create Cluster in DB Success - - helm install Success - - checkout Cluster status [794] - - job run Success - -SubJobs nodemanager-0 ModuleStatus: Available, SubJobStatus: Success, Duration: 13m, StartTime: - 2022-04-12 07:34:09, EndTime: 2022-04-12 07:47:26 - nodemanager-1 ModuleStatus: Available, SubJobStatus: Success, Duration: 13m, StartTime: - 2022-04-12 07:34:09, EndTime: 2022-04-12 07:47:18 - python ModuleStatus: Available, SubJobStatus: Success, Duration: 14m, StartTime: - 2022-04-12 07:34:09, EndTime: 2022-04-12 07:48:14 - rollsite ModuleStatus: Available, SubJobStatus: Success, Duration: 13m, StartTime: - 2022-04-12 07:34:09, EndTime: 2022-04-12 07:47:24 - client ModuleStatus: Available, SubJobStatus: Success, Duration: 11m, StartTime: - 2022-04-12 07:34:09, EndTime: 2022-04-12 07:45:22 - clustermanager ModuleStatus: Available, SubJobStatus: Success, Duration: 13m, StartTime: - 2022-04-12 07:34:09, EndTime: 2022-04-12 07:47:11 - mysql ModuleStatus: Available, SubJobStatus: Success, Duration: 13m, StartTime: - 2022-04-12 07:34:09, EndTime: 2022-04-12 07:47:11 +UUID d92d7a56-7002-46a4-9363-da9c7346e05a +StartTime 2022-04-12 07:34:09 +EndTime 2022-04-12 07:48:14 +Duration 14m +Status Success +Creator admin +ClusterId 24bb75ff-f636-4c64-8c04-1b9073f89a2f +States - update job status to Running + - create Cluster in DB Success + - helm install Success + - checkout Cluster status [794] + - job run Success + +SubJobs nodemanager-0 ModuleStatus: Available, SubJobStatus: Success, Duration: 13m, StartTime: + 2022-04-12 07:34:09, EndTime: 2022-04-12 07:47:26 + nodemanager-1 ModuleStatus: Available, SubJobStatus: Success, Duration: 13m, StartTime: + 2022-04-12 07:34:09, EndTime: 2022-04-12 07:47:18 + python ModuleStatus: Available, SubJobStatus: Success, Duration: 14m, StartTime: + 2022-04-12 07:34:09, EndTime: 2022-04-12 07:48:14 + rollsite ModuleStatus: Available, SubJobStatus: Success, Duration: 13m, StartTime: + 2022-04-12 07:34:09, EndTime: 2022-04-12 07:47:24 + client ModuleStatus: Available, SubJobStatus: Success, Duration: 11m, StartTime: + 2022-04-12 07:34:09, EndTime: 2022-04-12 07:45:22 + clustermanager ModuleStatus: Available, SubJobStatus: Success, Duration: 13m, StartTime: + 2022-04-12 07:34:09, EndTime: 2022-04-12 07:47:11 + mysql ModuleStatus: Available, SubJobStatus: Success, Duration: 13m, StartTime: + 2022-04-12 07:34:09, EndTime: 2022-04-12 07:47:11 ``` -#### 描述集群的情况，以及FATE集群访问信息 +### 描述集群的情况，以及FATE集群访问信息 + 当我们看到上面命令的返回结果出现`install success`，就可以使用```kubefate cluster describe```查看FATE集群的访问信息 + ```bash $ kubefate cluster describe 24bb75ff-f636-4c64-8c04-1b9073f89a2f -UUID 24bb75ff-f636-4c64-8c04-1b9073f89a2f -Name fate-9999 -NameSpace fate-9999 -ChartName fate -ChartVersion v1.8.0 -Revision 1 -Age 44h -Status Running -Spec backend: eggroll - chartName: fate - chartVersion: v1.8.0 - imagePullSecrets: - - name: myregistrykey - imageTag: 1.8.0-release - ingress: - client: - hosts: - - name: party9999.notebook.example.com - fateboard: - hosts: - - name: party9999.fateboard.example.com - ingressClassName: nginx - istio: - enabled: false - modules: - - rollsite - - clustermanager - - nodemanager - - mysql - - python - - fateboard - - client - name: fate-9999 - namespace: fate-9999 - partyId: 9999 - persistence: false - podSecurityPolicy: - enabled: false - pullPolicy: null - python: - grpcNodePort: 30092 - httpNodePort: 30097 - logLevel: INFO - type: NodePort - registry: "" - rollsite: - nodePort: 30091 - partyList: - - partyId: 10000 - partyIp: 192.168.10.1 - partyPort: 30101 - type: NodePort - servingIp: 192.168.9.2 - servingPort: 30095 - -Info dashboard: - - party9999.notebook.example.com - - party9999.fateboard.example.com - ip: 192.168.9.1 - port: 30091 - status: - containers: - client: Running - clustermanager: Running - fateboard: Running - mysql: Running - nodemanager-0: Running - nodemanager-0-eggrollpair: Running - nodemanager-1: Running - nodemanager-1-eggrollpair: Running - python: Running - rollsite: Running - deployments: - client: Available - clustermanager: Available - mysql: Available - nodemanager-0: Available - nodemanager-1: Available - python: Available - rollsite: Available +UUID 24bb75ff-f636-4c64-8c04-1b9073f89a2f +Name fate-9999 +NameSpace fate-9999 +ChartName fate +ChartVersion v1.9.0 +Revision 1 +Age 15m +Status Running +Spec algorithm: Basic + chartName: fate + chartVersion: v1.9.0 + computing: Eggroll + device: CPU + federation: Eggroll + imagePullSecrets: + - name: myregistrykey + ingressClassName: nginx + istio: + enabled: false + modules: + - rollsite + - clustermanager + - nodemanager + - mysql + - python + - fateboard + - client + name: fate-9999 + namespace: fate-9999 + partyId: 9999 + persistence: false + podSecurityPolicy: + enabled: false + pullPolicy: null + registry: "" + storage: Eggroll + +Info dashboard: + - party9999.notebook.example.com + - party9999.fateboard.example.com + ip: 192.168.9.1 + status: + containers: + client: Running + clustermanager: Running + fateboard: Running + fateflow: Running + mysql: Running + nodemanager: Running + nodemanager-eggrollpair: Running + rollsite: Running + deployments: + clustermanager: Available + rollsite: Available ``` -#### 访问 FATEBoard 和 Notebook UI +### 访问 FATEBoard 和 Notebook UI 假如安装了fateboard 和client 的组件就可以通过上一步得到的信息`party9999.fateboard.example.com` 和`party9999.notebook.example.com`来访问 FATEBoard 和 Notebook UI，配置下这两个域名的解析就可以在浏览器打开。 -##### FATEBoard +#### FATEBoard - http://party9999.fateboard.example.com + 访问FATEBoard UI需要登录用户名和密码，这个可以在`cluster.yaml`里[配置](../docs/configurations/FATE_cluster_configuration.md#fateboard mappings)。 ![fate_board](../docs/tutorials/images/tkg_fate_board.png) -##### Notebook +#### Notebook - http://party9999.fateboard.example.com + ![notebook](../docs/tutorials/images/tkg_notebook.png) ### 其他常用用户场景 + #### [Manage FATE and FATE-Serving Version](../docs/Manage_FATE_and_FATE-Serving_Version.md) + #### [Update and Delete a FATE Cluster](../docs/Update_and_Delete_a_FATE_Cluster.md) + #### [KubeFATE Examples](examples) -#### [KubeFATE Command Line User Guide](../docs/KubeFATE_command_line_user_guide.md) +#### [KubeFATE Command Line User Guide](../docs/KubeFATE_CLI_user_guide.md) ## KubeFATE服务的API参考 -#### [API Reference](docs/KubeFATE_API_Reference_Swagger.md) +**[API Reference](docs/KubeFATE_API_Reference_Swagger.md)** diff --git a/k8s-deploy/cluster-serving.yaml b/k8s-deploy/cluster-serving.yaml index ec6263979..fad021c20 100644 --- a/k8s-deploy/cluster-serving.yaml +++ b/k8s-deploy/cluster-serving.yaml @@ -1,11 +1,10 @@ name: fate-serving-9999 namespace: fate-serving-9999 chartName: fate-serving -chartVersion: v2.1.5 +chartVersion: v2.1.6 partyId: 9999 registry: "" -imageTag: "2.1.5-release" -pullPolicy: +pullPolicy: imagePullSecrets: - name: myregistrykey persistence: false diff --git a/k8s-deploy/cluster-spark-pulsar.yaml b/k8s-deploy/cluster-spark-pulsar.yaml index 2df64d707..cb1c2b231 100644 --- a/k8s-deploy/cluster-spark-pulsar.yaml +++ b/k8s-deploy/cluster-spark-pulsar.yaml @@ -1,11 +1,10 @@ name: fate-9999 namespace: fate-9999 chartName: fate -chartVersion: v1.8.0 +chartVersion: v1.9.0 partyId: 9999 registry: "" -imageTag: "1.8.0-release" -pullPolicy: +pullPolicy: imagePullSecrets: - name: myregistrykey persistence: false @@ -24,9 +23,21 @@ modules: - nginx - pulsar -# backend could be eggroll, spark_rabbitmq and spark_pulsar spark_local_pulsar -backend: spark_pulsar +# Computing : Eggroll, Spark, Spark_local +computing: Spark +# Federation: Eggroll(computing: Eggroll), Pulsar/RabbitMQ(computing: Spark/Spark_local) +federation: Pulsar +# Storage: Eggroll(computing: Eggroll), HDFS(computing: Spark), LocalFS(computing: Spark_local) +storage: HDFS +# Algorithm: Basic, NN +algorithm: Basic +# Device: CPU, IPCL +device: CPU +# you can customize some keys which will be ignored in yaml validation +skippedKeys: + - route_table + # Specify domain name for service # ingress: # fateboard: @@ -38,50 +49,50 @@ backend: spark_pulsar # spark: # hosts: # - name: party9999.spark.example.com - # rabbitmq: - # hosts: - # - name: party9999.rabbitmq.example.com # pulsar: # hosts: # - name: party9999.pulsar.example.com -# Specify rollsite properties -# rollsite: - # type: NodePort - # nodePort: 30091 - # exchange: - # ip: 192.168.0.1 - # port: 30000 - # partyList: - # - partyId: 10000 - # partyIp: 192.168.0.1 - # partyPort: 30101 - # nodeSelector: - # Specify the fateflow service's properties # python: # type: NodePort - # nodePort: 30102 - # nodeSelector: + # httpNodePort: 30097 + # grpcNodePort: 30092 + # loadBalancerIP: + # serviceAccountName: "" + # nodeSelector: + # tolerations: + # affinity: + # enabledNN: false # logLevel: INFO - # spark: + # existingClaim: "" + # storageClass: "python" + # accessMode: ReadWriteMany + # size: 1Gi + # resources: + # requests: + # cpu: "2" + # memory: "4Gi" + # limits: + # cpu: "4" + # memory: "8Gi" + # spark: # cores_per_node: 20 # nodes: 2 # master: spark://spark-master:7077 - # Only for using external Spark - ## driverHost: - ## driverHostType: - ## portMaxRetries: - ## driverStartPort: - ## blockManagerStartPort: - ## pysparkPython: + # driverHost: + # driverHostType: + # portMaxRetries: + # driverStartPort: + # blockManagerStartPort: + # pysparkPython: # hdfs: # name_node: hdfs://namenode:9000 # path_prefix: # pulsar: # host: pulsar - # port: 6650 # mng_port: 8080 + # port: 6650 # nginx: # host: nginx # http_port: 9300 @@ -116,7 +127,7 @@ backend: spark_pulsar # spark: # master: # Image: "federatedai/spark-master" - # ImageTag: "1.6.0-release" + # ImageTag: "1.6.1-release" # replicas: 1 # resources: # requests: @@ -125,11 +136,14 @@ backend: spark_pulsar # limits: # cpu: "1" # memory: "2Gi" - # nodeSelector: + # nodeSelector: + # tolerations: + # affinity: # type: ClusterIP + # nodePort: 30977 # worker: # Image: "federatedai/spark-worker" - # ImageTag: "1.6.0-release" + # ImageTag: "1.6.1-release" # replicas: 2 # resources: # requests: @@ -138,40 +152,113 @@ backend: spark_pulsar # limits: # cpu: "4" # memory: "8Gi" - # nodeSelector: + # nodeSelector: + # tolerations: + # affinity: # type: ClusterIP # hdfs: # namenode: - # nodeSelector: + # nodeSelector: + # tolerations: + # affinity: # type: ClusterIP + # nodePort: 30900 + # existingClaim: "" + # storageClass: "namenode" + # accessMode: ReadWriteOnce + # size: 1Gi # datanode: - # nodeSelector: + # replicas: + # nodeSelector: + # tolerations: + # affinity: # type: ClusterIP + # existingClaim: "" + # storageClass: "datanode" + # accessMode: + # size: # nginx: - # nodeSelector: + # nodeSelector: + # tolerations: + # affinity: # type: ClusterIP + # loadBalancerIP: # httpNodePort: 30093 # grpcNodePort: 30098 - # route_table: - # 10000: - # proxy: - # - host: 192.168.0.1 + # exchange: + # ip: 192.168.10.1 + # httpPort: 30003 + # grpcPort: 30008 + # route_table: + # 10000: + # proxy: + # - host: 192.168.0.1 # http_port: 30103 - # grpc_port: 30108 - # fateflow: + # grpc_port: 30108 + # fateflow: # - host: 192.168.0.1 # http_port: 30107 # grpc_port: 30102 -# pulsar: +# rabbitmq: + # nodeSelector: + # tolerations: + # affinity: # type: ClusterIP # nodePort: 30094 + # loadBalancerIP: + # default_user: fate + # default_pass: fate + # user: fate + # password: fate # route_table: + # 9999: + # host: rabbitmq + # port: 5672 # 10000: # host: 192.168.0.1 - # port: 6650 - # sslPort:6651 - # proxy: "" + # port: 30104 + # resources: + # requests: + # cpu: "2" + # memory: "4Gi" + # limits: + # cpu: "4" + # memory: "8Gi" + +# pulsar: + # nodeSelector: + # tolerations: + # affinity: + # type: ClusterIP + # httpNodePort: 30094 + # httpsNodePort: 30099 + # loadBalancerIP: + # storageClass: "pulsar" + # existingClaim: + # accessMode: ReadWriteOnce + # size: 1Gi + # env: + # confs: + # publicLB: + # enabled: false + # exchange: + # ip: 192.168.10.1 + # port: 30000 + # domain: fate.org + # route_table: # 9999: # host: pulsar # port: 6650 - # sslPort:6651 + # sslPort: 6651 + # 10000: + # host: 192.168.10.1 + # port: 30105 + # sslPort: 30109 + # proxy: "" + # resources: + # requests: + # cpu: "2" + # memory: "4Gi" + # limits: + # cpu: "4" + # memory: "8Gi" diff --git a/k8s-deploy/cluster-spark-rabbitmq.yaml b/k8s-deploy/cluster-spark-rabbitmq.yaml index 9b42eac54..0897e0dca 100644 --- a/k8s-deploy/cluster-spark-rabbitmq.yaml +++ b/k8s-deploy/cluster-spark-rabbitmq.yaml @@ -1,11 +1,10 @@ name: fate-9999 namespace: fate-9999 chartName: fate -chartVersion: v1.8.0 +chartVersion: v1.9.0 partyId: 9999 registry: "" -imageTag: "1.8.0-release" -pullPolicy: +pullPolicy: imagePullSecrets: - name: myregistrykey persistence: false @@ -24,9 +23,21 @@ modules: - nginx - rabbitmq -# backend could be eggroll, spark_rabbitmq and spark_pulsar spark_local_pulsar -backend: spark_rabbitmq +# Computing : Eggroll, Spark, Spark_local +computing: Spark +# Federation: Eggroll(computing: Eggroll), Pulsar/RabbitMQ(computing: Spark/Spark_local) +federation: RabbitMQ +# Storage: Eggroll(computing: Eggroll), HDFS(computing: Spark), LocalFS(computing: Spark_local) +storage: HDFS +# Algorithm: Basic, NN +algorithm: Basic +# Device: CPU, IPCL +device: CPU +# you can customize some keys which will be ignored in yaml validation +skippedKeys: + - route_table + # Specify domain name for service # ingress: # fateboard: @@ -41,22 +52,6 @@ backend: spark_rabbitmq # rabbitmq: # hosts: # - name: party9999.rabbitmq.example.com - # pulsar: - # hosts: - # - name: party9999.pulsar.example.com - -# Specify rollsite properties -# rollsite: - # type: NodePort - # nodePort: 30091 - # exchange: - # ip: 192.168.0.1 - # port: 30000 - # partyList: - # - partyId: 10000 - # partyIp: 192.168.0.1 - # partyPort: 30101 - # nodeSelector: # Specify the fateflow service's properties # python: @@ -78,18 +73,22 @@ backend: spark_rabbitmq # hdfs: # name_node: hdfs://namenode:9000 # path_prefix: - # pulsar: - # host: pulsar - # port: 6650 - # mng_port: 8080 # nginx: # host: nginx # http_port: 9300 # grpc_port: 9310 + # rabbitmq: + # host: rabbitmq + # mng_port: 15672 + # port: 5672 + # user: fate + # password: fate # Specify the mysql properties # mysql: - # nodeSelector: + # nodeSelector: + # tolerations: + # affinity: # ip: mysql # port: 3306 # database: eggroll_meta @@ -116,7 +115,7 @@ backend: spark_rabbitmq # spark: # master: # Image: "federatedai/spark-master" - # ImageTag: "1.6.0-release" + # ImageTag: "1.6.1-release" # replicas: 1 # resources: # requests: @@ -125,11 +124,14 @@ backend: spark_rabbitmq # limits: # cpu: "1" # memory: "2Gi" - # nodeSelector: + # nodeSelector: + # tolerations: + # affinity: # type: ClusterIP + # nodePort: 30977 # worker: # Image: "federatedai/spark-worker" - # ImageTag: "1.6.0-release" + # ImageTag: "1.6.1-release" # replicas: 2 # resources: # requests: @@ -138,40 +140,75 @@ backend: spark_rabbitmq # limits: # cpu: "4" # memory: "8Gi" - # nodeSelector: + # nodeSelector: + # tolerations: + # affinity: # type: ClusterIP # hdfs: # namenode: - # nodeSelector: + # nodeSelector: + # tolerations: + # affinity: # type: ClusterIP + # nodePort: 30900 + # existingClaim: "" + # storageClass: "namenode" + # accessMode: ReadWriteOnce + # size: 1Gi # datanode: - # nodeSelector: + # replicas: + # nodeSelector: + # tolerations: + # affinity: # type: ClusterIP + # existingClaim: "" + # storageClass: "datanode" + # accessMode: + # size: # nginx: - # nodeSelector: + # nodeSelector: + # tolerations: + # affinity: # type: ClusterIP + # loadBalancerIP: # httpNodePort: 30093 # grpcNodePort: 30098 - # route_table: - # 10000: - # proxy: - # - host: 192.168.0.1 + # exchange: + # ip: 192.168.10.1 + # httpPort: 30003 + # grpcPort: 30008 + # route_table: + # 10000: + # proxy: + # - host: 192.168.0.1 # http_port: 30103 - # grpc_port: 30108 - # fateflow: + # grpc_port: 30108 + # fateflow: # - host: 192.168.0.1 # http_port: 30107 # grpc_port: 30102 -# pulsar: +# rabbitmq: + # nodeSelector: + # tolerations: + # affinity: # type: ClusterIP # nodePort: 30094 + # loadBalancerIP: + # default_user: fate + # default_pass: fate + # user: fate + # password: fate # route_table: + # 9999: + # host: rabbitmq + # port: 5672 # 10000: # host: 192.168.0.1 - # port: 6650 - # sslPort:6651 - # proxy: "" - # 9999: - # host: pulsar - # port: 6650 - # sslPort:6651 + # port: 30104 + # resources: + # requests: + # cpu: "2" + # memory: "4Gi" + # limits: + # cpu: "4" + # memory: "8Gi" diff --git a/k8s-deploy/cluster-spark-slim.yaml b/k8s-deploy/cluster-spark-slim.yaml index 57cd4a953..67b5029ff 100644 --- a/k8s-deploy/cluster-spark-slim.yaml +++ b/k8s-deploy/cluster-spark-slim.yaml @@ -1,11 +1,10 @@ name: fate-9999 namespace: fate-9999 chartName: fate -chartVersion: v1.8.0 +chartVersion: v1.9.0 partyId: 9999 registry: "" -imageTag: "1.8.0-release" -pullPolicy: +pullPolicy: imagePullSecrets: - name: myregistrykey persistence: false @@ -22,9 +21,21 @@ modules: - nginx - pulsar -# backend could be eggroll, spark_rabbitmq and spark_pulsar spark_local_pulsar -backend: spark_local_pulsar +# Computing : Eggroll, Spark, Spark_local +computing: Spark_local +# Federation: Eggroll(computing: Eggroll), Pulsar/RabbitMQ(computing: Spark/Spark_local) +federation: Pulsar +# Storage: Eggroll(computing: Eggroll), HDFS(computing: Spark), LocalFS(computing: Spark_local) +storage: LocalFS +# Algorithm: Basic, NN +algorithm: Basic +# Device: CPU, IPCL +device: CPU +# you can customize some keys which will be ignored in yaml validation +skippedKeys: + - route_table + # Specify domain name for service # ingress: # fateboard: @@ -36,33 +47,17 @@ backend: spark_local_pulsar # spark: # hosts: # - name: party9999.spark.example.com - # rabbitmq: - # hosts: - # - name: party9999.rabbitmq.example.com - # pulsar: + # pulsar: # hosts: # - name: party9999.pulsar.example.com -# Specify rollsite properties -# rollsite: - # type: NodePort - # nodePort: 30091 - # exchange: - # ip: 192.168.0.1 - # port: 30000 - # partyList: - # - partyId: 10000 - # partyIp: 192.168.0.1 - # partyPort: 30101 - # nodeSelector: - # Specify the fateflow service's properties # python: # type: NodePort # nodePort: 30102 - # nodeSelector: + # nodeSelector: # logLevel: INFO - # spark: + # spark: # cores_per_node: 20 # nodes: 2 # master: spark://spark-master:7077 @@ -87,7 +82,9 @@ backend: spark_local_pulsar # Specify the mysql properties # mysql: - # nodeSelector: + # nodeSelector: + # tolerations: + # affinity: # ip: mysql # port: 3306 # database: eggroll_meta @@ -110,41 +107,6 @@ backend: spark_local_pulsar # servingIp: 192.168.0.1 # servingPort: 30095 -# FATE on spark configuration -# spark: - # master: - # Image: "federatedai/spark-master" - # ImageTag: "1.6.0-release" - # replicas: 1 - # resources: - # requests: - # cpu: "1" - # memory: "2Gi" - # limits: - # cpu: "1" - # memory: "2Gi" - # nodeSelector: - # type: ClusterIP - # worker: - # Image: "federatedai/spark-worker" - # ImageTag: "1.6.0-release" - # replicas: 2 - # resources: - # requests: - # cpu: "2" - # memory: "4Gi" - # limits: - # cpu: "4" - # memory: "8Gi" - # nodeSelector: - # type: ClusterIP -# hdfs: - # namenode: - # nodeSelector: - # type: ClusterIP - # datanode: - # nodeSelector: - # type: ClusterIP # nginx: # nodeSelector: # type: ClusterIP @@ -173,3 +135,10 @@ backend: spark_local_pulsar # host: pulsar # port: 6650 # sslPort:6651 + # resources: + # requests: + # cpu: "2" + # memory: "4Gi" + # limits: + # cpu: "4" + # memory: "8Gi" diff --git a/k8s-deploy/cluster.yaml b/k8s-deploy/cluster.yaml index f19220a52..5bf2c8a9f 100644 --- a/k8s-deploy/cluster.yaml +++ b/k8s-deploy/cluster.yaml @@ -1,11 +1,10 @@ name: fate-9999 namespace: fate-9999 chartName: fate -chartVersion: v1.8.0 +chartVersion: v1.9.0 partyId: 9999 registry: "" -imageTag: "1.8.0-release" -pullPolicy: +pullPolicy: imagePullSecrets: - name: myregistrykey persistence: false @@ -23,9 +22,20 @@ modules: - fateboard - client - -# backend could be eggroll, spark_rabbitmq and spark_pulsar spark_local_pulsar -backend: eggroll +# Computing : Eggroll, Spark, Spark_local +computing: Eggroll +# Federation: Eggroll(computing: Eggroll), Pulsar/RabbitMQ(computing: Spark/Spark_local) +federation: Eggroll +# Storage: Eggroll(computing: Eggroll), HDFS(computing: Spark), LocalFS(computing: Spark_local) +storage: Eggroll +# Algorithm: Basic, NN +algorithm: Basic +# Device: CPU, IPCL +device: CPU + +# you can customize some keys which will be ignored in yaml validation +skippedKeys: + - route_table # Specify domain name for service # ingress: @@ -46,9 +56,10 @@ backend: eggroll # - name: party9999.pulsar.example.com # Specify rollsite properties -# rollsite: +# rollsite: # type: NodePort # nodePort: 30091 + # loadBalancerIP: # exchange: # ip: 192.168.0.1 # port: 30000 @@ -56,30 +67,64 @@ backend: eggroll # - partyId: 10000 # partyIp: 192.168.0.1 # partyPort: 30101 - # nodeSelector: + # nodeSelector: + # tolerations: + # affinity: + # polling: + # enabled: true + # type: server + # # server: + # # ip: 192.168.100.1 + # # port: 9370 + # clientList: + # - partID: 9999 + # concurrency: 49 + # resources: + # requests: + # cpu: "1" + # memory: "1Gi" + # limits: + # cpu: "4" + # memory: "1Gi" + +# Specify clustermanager properties +# clustermanager: +# nodeSelector: +# tolerations: +# affinity: + # resources: + # requests: + # cpu: "1" + # memory: "1Gi" + # limits: + # cpu: "1" + # memory: "1Gi" # Specify nodemanager properties # nodemanager: - # count: 3 - # sessionProcessorsPerNode: 4 - # storageClass: "nodemanagers" - # accessMode: ReadWriteOnce - # size: 2Gi - # list: - # - name: nodemanager - # nodeSelector: - # sessionProcessorsPerNode: 4 - # subPath: "nodemanager" - # existingClaim: "" - # storageClass: "nodemanager" - # accessMode: ReadWriteOnce - # size: 1Gi +# replicas: 2 +# sessionProcessorsPerNode: 4 +# nodeSelector: +# tolerations: +# affinity: +# subPath: "nodemanager" +# storageClass: "local-scsi" +# accessMode: ReadWriteOnce +# size: 1Gi + # resources: + # requests: + # cpu: "2" + # memory: "4Gi" + # limits: + # cpu: "4" + # memory: "8Gi" # Specify the fateflow service's properties # python: # type: NodePort # httpNodePort: 30097 # grpcNodePort: 30092 + # loadBalancerIP: # serviceAccountName: "" # nodeSelector: # tolerations: @@ -90,36 +135,16 @@ backend: eggroll # storageClass: "python" # accessMode: ReadWriteMany # size: 1Gi + # resources: + # requests: + # cpu: "2" + # memory: "4Gi" + # limits: + # cpu: "4" + # memory: "8Gi" # clustermanager: # cores_per_node: 16 # nodes: 2 - # spark: - # cores_per_node: 20 - # nodes: 2 - # master: spark://spark-master:7077 - # driverHost: - # driverHostType: - # portMaxRetries: - # driverStartPort: - # blockManagerStartPort: - # pysparkPython: - # hdfs: - # name_node: hdfs://namenode:9000 - # path_prefix: - # rabbitmq: - # host: rabbitmq - # mng_port: 15672 - # port: 5672 - # user: fate - # password: fate - # pulsar: - # host: pulsar - # mng_port: 8080 - # port: 6650 - # nginx: - # host: nginx - # http_port: 9300 - # grpc_port: 9310 # Specify the fateboard service's properties # fateboard: @@ -138,7 +163,9 @@ backend: eggroll # Specify the mysql properties # mysql: - # nodeSelector: + # nodeSelector: + # tolerations: + # affinity: # ip: mysql # port: 3306 # database: eggroll_meta diff --git a/k8s-deploy/config.yaml b/k8s-deploy/config.yaml index fbda7a140..0d81aaf4a 100644 --- a/k8s-deploy/config.yaml +++ b/k8s-deploy/config.yaml @@ -7,4 +7,5 @@ user: username: admin password: admin -serviceurl: example.com \ No newline at end of file +serviceurl: example.com +safeconnect: false \ No newline at end of file diff --git a/k8s-deploy/docs/docs.go b/k8s-deploy/docs/docs.go index 71c406446..c680f6981 100644 --- a/k8s-deploy/docs/docs.go +++ b/k8s-deploy/docs/docs.go @@ -40,6 +40,7 @@ const docTemplate = `{ "parameters": [ { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -105,6 +106,7 @@ const docTemplate = `{ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -175,6 +177,7 @@ const docTemplate = `{ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -243,6 +246,7 @@ const docTemplate = `{ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -309,6 +313,7 @@ const docTemplate = `{ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -385,6 +390,7 @@ const docTemplate = `{ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -462,6 +468,7 @@ const docTemplate = `{ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -541,6 +548,7 @@ const docTemplate = `{ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -615,6 +623,7 @@ const docTemplate = `{ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -678,6 +687,7 @@ const docTemplate = `{ "parameters": [ { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -745,6 +755,7 @@ const docTemplate = `{ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -821,6 +832,7 @@ const docTemplate = `{ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -889,6 +901,7 @@ const docTemplate = `{ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -999,6 +1012,7 @@ const docTemplate = `{ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -1118,6 +1132,7 @@ const docTemplate = `{ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -1169,6 +1184,7 @@ const docTemplate = `{ "parameters": [ { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -1232,6 +1248,7 @@ const docTemplate = `{ "parameters": [ { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -1305,6 +1322,7 @@ const docTemplate = `{ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -1375,6 +1393,7 @@ const docTemplate = `{ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -1477,6 +1496,7 @@ const docTemplate = `{ "parameters": [ { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -1523,6 +1543,7 @@ const docTemplate = `{ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -1591,6 +1612,7 @@ const docTemplate = `{ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -1625,6 +1647,78 @@ const docTemplate = `{ } } }, + "/valueTemplateExample/{chartName}/{chartVersion}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "produces": [ + "application/json" + ], + "tags": [ + "Chart" + ], + "summary": "return valueTemplateExample string by chart name and version", + "parameters": [ + { + "type": "string", + "description": "chart name", + "name": "chartName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "chart version", + "name": "chartVersion", + "in": "path", + "required": true + }, + { + "type": "string", + "default": "Bearer \u003cToken\u003e", + "description": "Authentication header", + "name": "Authorization", + "in": "header", + "required": true + } + ], + "responses": { + "200": { + "description": "Success", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/api.JSONResult" + }, + { + "type": "object", + "properties": { + "data": { + "type": "string" + } + } + } + ] + } + }, + "401": { + "description": "Unauthorized operation", + "schema": { + "$ref": "#/definitions/api.JSONERRORResult" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/api.JSONERRORResult" + } + } + } + } + }, "/version": { "get": { "security": [ @@ -1827,6 +1921,9 @@ const docTemplate = `{ "modules.ClusterArgs": { "type": "object", "properties": { + "Keep_upgrade_job": { + "type": "boolean" + }, "chart_name": { "type": "string" }, @@ -1883,6 +1980,9 @@ const docTemplate = `{ "values_template": { "type": "string" }, + "values_template_example": { + "type": "string" + }, "version": { "type": "string" } diff --git a/k8s-deploy/docs/swagger.json b/k8s-deploy/docs/swagger.json index d08ef1b26..04d8c53f2 100644 --- a/k8s-deploy/docs/swagger.json +++ b/k8s-deploy/docs/swagger.json @@ -32,6 +32,7 @@ "parameters": [ { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -97,6 +98,7 @@ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -167,6 +169,7 @@ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -235,6 +238,7 @@ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -301,6 +305,7 @@ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -377,6 +382,7 @@ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -454,6 +460,7 @@ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -533,6 +540,7 @@ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -607,6 +615,7 @@ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -670,6 +679,7 @@ "parameters": [ { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -737,6 +747,7 @@ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -813,6 +824,7 @@ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -881,6 +893,7 @@ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -991,6 +1004,7 @@ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -1110,6 +1124,7 @@ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -1161,6 +1176,7 @@ "parameters": [ { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -1224,6 +1240,7 @@ "parameters": [ { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -1297,6 +1314,7 @@ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -1367,6 +1385,7 @@ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -1469,6 +1488,7 @@ "parameters": [ { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -1515,6 +1535,7 @@ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -1583,6 +1604,7 @@ }, { "type": "string", + "default": "Bearer \u003cToken\u003e", "description": "Authentication header", "name": "Authorization", "in": "header", @@ -1617,6 +1639,78 @@ } } }, + "/valueTemplateExample/{chartName}/{chartVersion}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "produces": [ + "application/json" + ], + "tags": [ + "Chart" + ], + "summary": "return valueTemplateExample string by chart name and version", + "parameters": [ + { + "type": "string", + "description": "chart name", + "name": "chartName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "chart version", + "name": "chartVersion", + "in": "path", + "required": true + }, + { + "type": "string", + "default": "Bearer \u003cToken\u003e", + "description": "Authentication header", + "name": "Authorization", + "in": "header", + "required": true + } + ], + "responses": { + "200": { + "description": "Success", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/api.JSONResult" + }, + { + "type": "object", + "properties": { + "data": { + "type": "string" + } + } + } + ] + } + }, + "401": { + "description": "Unauthorized operation", + "schema": { + "$ref": "#/definitions/api.JSONERRORResult" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/api.JSONERRORResult" + } + } + } + } + }, "/version": { "get": { "security": [ @@ -1819,6 +1913,9 @@ "modules.ClusterArgs": { "type": "object", "properties": { + "Keep_upgrade_job": { + "type": "boolean" + }, "chart_name": { "type": "string" }, @@ -1875,6 +1972,9 @@ "values_template": { "type": "string" }, + "values_template_example": { + "type": "string" + }, "version": { "type": "string" } diff --git a/k8s-deploy/docs/swagger.yaml b/k8s-deploy/docs/swagger.yaml index 407da5adb..f0304134a 100644 --- a/k8s-deploy/docs/swagger.yaml +++ b/k8s-deploy/docs/swagger.yaml @@ -113,6 +113,8 @@ definitions: type: object modules.ClusterArgs: properties: + Keep_upgrade_job: + type: boolean chart_name: type: string chart_version: @@ -150,6 +152,8 @@ definitions: type: string values_template: type: string + values_template_example: + type: string version: type: string type: object @@ -262,7 +266,8 @@ paths: /chart: get: parameters: - - description: Authentication header + - default: Bearer + description: Authentication header in: header name: Authorization required: true @@ -301,7 +306,8 @@ paths: name: file required: true type: file - - description: Authentication header + - default: Bearer + description: Authentication header in: header name: Authorization required: true @@ -343,7 +349,8 @@ paths: name: chartId required: true type: string - - description: Authentication header + - default: Bearer + description: Authentication header in: header name: Authorization required: true @@ -379,7 +386,8 @@ paths: name: chartId required: true type: string - - description: Authentication header + - default: Bearer + description: Authentication header in: header name: Authorization required: true @@ -422,7 +430,8 @@ paths: required: true schema: $ref: '#/definitions/modules.ClusterArgs' - - description: Authentication header + - default: Bearer + description: Authentication header in: header name: Authorization required: true @@ -467,7 +476,8 @@ paths: required: true schema: $ref: '#/definitions/modules.ClusterArgs' - - description: Authentication header + - default: Bearer + description: Authentication header in: header name: Authorization required: true @@ -513,7 +523,8 @@ paths: name: all required: true type: boolean - - description: Authentication header + - default: Bearer + description: Authentication header in: header name: Authorization required: true @@ -560,7 +571,8 @@ paths: name: clusterId required: true type: string - - description: Authentication header + - default: Bearer + description: Authentication header in: header name: Authorization required: true @@ -604,7 +616,8 @@ paths: name: clusterId required: true type: string - - description: Authentication header + - default: Bearer + description: Authentication header in: header name: Authorization required: true @@ -644,7 +657,8 @@ paths: /job/: get: parameters: - - description: Authentication header + - default: Bearer + description: Authentication header in: header name: Authorization required: true @@ -684,7 +698,8 @@ paths: name: jobId required: true type: string - - description: Authentication header + - default: Bearer + description: Authentication header in: header name: Authorization required: true @@ -720,7 +735,8 @@ paths: name: jobId required: true type: string - - description: Authentication header + - default: Bearer + description: Authentication header in: header name: Authorization required: true @@ -767,7 +783,8 @@ paths: name: jobStatus required: true type: string - - description: Authentication header + - default: Bearer + description: Authentication header in: header name: Authorization required: true @@ -848,7 +865,8 @@ paths: name: limit-bytes required: true type: string - - description: Authentication header + - default: Bearer + description: Authentication header in: header name: Authorization required: true @@ -925,7 +943,8 @@ paths: name: limit-bytes required: true type: string - - description: Authentication header + - default: Bearer + description: Authentication header in: header name: Authorization required: true @@ -957,7 +976,8 @@ paths: /namespace/: get: parameters: - - description: Authentication header + - default: Bearer + description: Authentication header in: header name: Authorization required: true @@ -994,7 +1014,8 @@ paths: /user: get: parameters: - - description: Authentication header + - default: Bearer + description: Authentication header in: header name: Authorization required: true @@ -1038,7 +1059,8 @@ paths: required: true schema: $ref: '#/definitions/modules.User' - - description: Authentication header + - default: Bearer + description: Authentication header in: header name: Authorization required: true @@ -1080,7 +1102,8 @@ paths: required: true schema: $ref: '#/definitions/modules.User' - - description: Authentication header + - default: Bearer + description: Authentication header in: header name: Authorization required: true @@ -1122,7 +1145,8 @@ paths: name: userId required: true type: string - - description: Authentication header + - default: Bearer + description: Authentication header in: header name: Authorization required: true @@ -1158,7 +1182,8 @@ paths: name: userId required: true type: string - - description: Authentication header + - default: Bearer + description: Authentication header in: header name: Authorization required: true @@ -1220,7 +1245,8 @@ paths: /user/logout: post: parameters: - - description: Authentication header + - default: Bearer + description: Authentication header in: header name: Authorization required: true @@ -1241,6 +1267,50 @@ paths: summary: logout tags: - User + /valueTemplateExample/{chartName}/{chartVersion}: + get: + parameters: + - description: chart name + in: path + name: chartName + required: true + type: string + - description: chart version + in: path + name: chartVersion + required: true + type: string + - default: Bearer + description: Authentication header + in: header + name: Authorization + required: true + type: string + produces: + - application/json + responses: + "200": + description: Success + schema: + allOf: + - $ref: '#/definitions/api.JSONResult' + - properties: + data: + type: string + type: object + "401": + description: Unauthorized operation + schema: + $ref: '#/definitions/api.JSONERRORResult' + "500": + description: Internal server error + schema: + $ref: '#/definitions/api.JSONERRORResult' + security: + - ApiKeyAuth: [] + summary: return valueTemplateExample string by chart name and version + tags: + - Chart /version: get: parameters: diff --git a/k8s-deploy/examples/config.sh b/k8s-deploy/examples/config.sh index 6598d50c3..fec2c829b 100644 --- a/k8s-deploy/examples/config.sh +++ b/k8s-deploy/examples/config.sh @@ -3,9 +3,7 @@ source party.config echo "FATE chartVersion: "${fate_chartVersion} -echo "FATE imageTAG: "${fate_imageTAG} echo "FATE-Serving chartVersion: "${fate_serving_chartVersion} -echo "FATE-Serving_imageTAG: "${fate_serving_imageTAG} echo "Party 9999 IP: "${party_9999_IP} echo "Party 10000 IP: "${party_10000_IP} echo "Party exchange IP: "${party_exchange_IP} @@ -27,12 +25,6 @@ $SED -i "s/chartVersion: .*/chartVersion: ${fate_chartVersion}/g" ./party-9999/c $SED -i "s/chartVersion: .*/chartVersion: ${fate_chartVersion}/g" ./party-9999/cluster-spark-pulsar.yaml $SED -i "s/chartVersion: .*/chartVersion: ${fate_chartVersion}/g" ./party-9999/cluster-spark-local-pulsar.yaml -$SED -i "s/imageTag: .*/imageTag: ${fate_imageTAG}/g" ./party-9999/cluster.yaml -$SED -i "s/imageTag: .*/imageTag: ${fate_serving_imageTAG}/g" ./party-9999/cluster-serving.yaml -$SED -i "s/imageTag: .*/imageTag: ${fate_imageTAG}/g" ./party-9999/cluster-spark-rabbitmq.yaml -$SED -i "s/imageTag: .*/imageTag: ${fate_imageTAG}/g" ./party-9999/cluster-spark-pulsar.yaml -$SED -i "s/imageTag: .*/imageTag: ${fate_imageTAG}/g" ./party-9999/cluster-spark-local-pulsar.yaml - $SED -i "s/192.168.9.1/${party_9999_IP}/g" ./party-9999/cluster.yaml $SED -i "s/192.168.9.1/${party_9999_IP}/g" ./party-9999/cluster-serving.yaml $SED -i "s/192.168.9.1/${party_9999_IP}/g" ./party-9999/cluster-spark-rabbitmq.yaml @@ -51,6 +43,13 @@ $SED -i "s/192.168.0.1/${party_exchange_IP}/g" ./party-9999/cluster-spark-rabbit $SED -i "s/192.168.0.1/${party_exchange_IP}/g" ./party-9999/cluster-spark-pulsar.yaml $SED -i "s/192.168.0.1/${party_exchange_IP}/g" ./party-9999/cluster-spark-local-pulsar.yaml + +for item in `ls party-9999` +do + $SED -i "s/algorithm: .*/algorithm: ${algorithm}/g" ./party-9999/$item + $SED -i "s/device: .*/device: ${device}/g" ./party-9999/$item +done + # 10000 config $SED -i "s/chartVersion: .*/chartVersion: ${fate_chartVersion}/g" ./party-10000/cluster.yaml @@ -59,12 +58,6 @@ $SED -i "s/chartVersion: .*/chartVersion: ${fate_chartVersion}/g" ./party-10000/ $SED -i "s/chartVersion: .*/chartVersion: ${fate_chartVersion}/g" ./party-10000/cluster-spark-pulsar.yaml $SED -i "s/chartVersion: .*/chartVersion: ${fate_chartVersion}/g" ./party-10000/cluster-spark-local-pulsar.yaml -$SED -i "s/imageTag: .*/imageTag: ${fate_imageTAG}/g" ./party-10000/cluster.yaml -$SED -i "s/imageTag: .*/imageTag: ${fate_serving_imageTAG}/g" ./party-10000/cluster-serving.yaml -$SED -i "s/imageTag: .*/imageTag: ${fate_imageTAG}/g" ./party-10000/cluster-spark-rabbitmq.yaml -$SED -i "s/imageTag: .*/imageTag: ${fate_imageTAG}/g" ./party-10000/cluster-spark-pulsar.yaml -$SED -i "s/imageTag: .*/imageTag: ${fate_imageTAG}/g" ./party-10000/cluster-spark-local-pulsar.yaml - $SED -i "s/192.168.9.1/${party_9999_IP}/g" ./party-10000/cluster.yaml $SED -i "s/192.168.9.1/${party_9999_IP}/g" ./party-10000/cluster-serving.yaml $SED -i "s/192.168.9.1/${party_9999_IP}/g" ./party-10000/cluster-spark-rabbitmq.yaml @@ -83,14 +76,17 @@ $SED -i "s/192.168.0.1/${party_exchange_IP}/g" ./party-10000/cluster-spark-rabbi $SED -i "s/192.168.0.1/${party_exchange_IP}/g" ./party-10000/cluster-spark-pulsar.yaml $SED -i "s/192.168.0.1/${party_exchange_IP}/g" ./party-10000/cluster-spark-local-pulsar.yaml +for item in `ls party-10000` +do + $SED -i "s/algorithm: .*/algorithm: ${algorithm}/g" ./party-10000/$item + $SED -i "s/device: .*/device: ${device}/g" ./party-10000/$item +done + # exchange config $SED -i "s/chartVersion: .*/chartVersion: ${chartVersion}/g" ./party-exchange/rollsite.yaml $SED -i "s/chartVersion: .*/chartVersion: ${chartVersion}/g" ./party-exchange/trafficServer.yaml -$SED -i "s/imageTag: .*/imageTag: ${fate_imageTAG}/g" ./party-exchange/rollsite.yaml -$SED -i "s/imageTag: .*/imageTag: ${fate_imageTAG}/g" ./party-exchange/trafficServer.yaml - $SED -i "s/192.168.9.1/${party_9999_IP}/g" ./party-exchange/rollsite.yaml $SED -i "s/192.168.9.1/${party_9999_IP}/g" ./party-exchange/trafficServer.yaml diff --git a/k8s-deploy/examples/party-10000/cluster-serving.yaml b/k8s-deploy/examples/party-10000/cluster-serving.yaml index 8a9a93e65..0625388f0 100644 --- a/k8s-deploy/examples/party-10000/cluster-serving.yaml +++ b/k8s-deploy/examples/party-10000/cluster-serving.yaml @@ -1,11 +1,10 @@ name: fate-serving-10000 namespace: fate-serving-10000 chartName: fate-serving -chartVersion: v2.1.5 +chartVersion: v2.1.6 partyId: 10000 registry: "" -imageTag: 2.1.5-release -pullPolicy: +pullPolicy: imagePullSecrets: - name: myregistrykey persistence: false diff --git a/k8s-deploy/examples/party-10000/cluster-spark-local-pulsar.yaml b/k8s-deploy/examples/party-10000/cluster-spark-local-pulsar.yaml index af6aac46a..88fca933f 100644 --- a/k8s-deploy/examples/party-10000/cluster-spark-local-pulsar.yaml +++ b/k8s-deploy/examples/party-10000/cluster-spark-local-pulsar.yaml @@ -1,11 +1,10 @@ name: fate-10000 namespace: fate-10000 chartName: fate -chartVersion: v1.8.0 +chartVersion: v1.9.0 partyId: 10000 registry: "" -imageTag: 1.8.0-release -pullPolicy: +pullPolicy: imagePullSecrets: - name: myregistrykey persistence: false @@ -22,7 +21,11 @@ modules: - nginx - pulsar -backend: spark_local_pulsar +computing: Spark_local +federation: Pulsar +storage: LocalFS +algorithm: Basic +device: CPU ingress: fateboard: diff --git a/k8s-deploy/examples/party-10000/cluster-spark-pulsar.yaml b/k8s-deploy/examples/party-10000/cluster-spark-pulsar.yaml index 2c5104327..9ee94ff36 100644 --- a/k8s-deploy/examples/party-10000/cluster-spark-pulsar.yaml +++ b/k8s-deploy/examples/party-10000/cluster-spark-pulsar.yaml @@ -1,11 +1,10 @@ name: fate-10000 namespace: fate-10000 chartName: fate -chartVersion: v1.8.0 +chartVersion: v1.9.0 partyId: 10000 registry: "" -imageTag: 1.8.0-release -pullPolicy: +pullPolicy: imagePullSecrets: - name: myregistrykey persistence: false @@ -24,7 +23,11 @@ modules: - nginx - pulsar -backend: spark_pulsar +computing: Spark +federation: Pulsar +storage: HDFS +algorithm: Basic +device: CPU ingress: fateboard: diff --git a/k8s-deploy/examples/party-10000/cluster-spark-rabbitmq.yaml b/k8s-deploy/examples/party-10000/cluster-spark-rabbitmq.yaml index 59db94247..2df1b4f04 100644 --- a/k8s-deploy/examples/party-10000/cluster-spark-rabbitmq.yaml +++ b/k8s-deploy/examples/party-10000/cluster-spark-rabbitmq.yaml @@ -1,11 +1,10 @@ name: fate-10000 namespace: fate-10000 chartName: fate -chartVersion: v1.8.0 +chartVersion: v1.9.0 partyId: 10000 registry: "" -imageTag: 1.8.0-release -pullPolicy: +pullPolicy: imagePullSecrets: - name: myregistrykey persistence: false @@ -24,7 +23,11 @@ modules: - nginx - rabbitmq -backend: spark_rabbitmq +computing: Spark +federation: RabbitMQ +storage: HDFS +algorithm: Basic +device: CPU ingress: fateboard: @@ -68,7 +71,5 @@ rabbitmq: host: 192.168.9.1 port: 30094 10000: - host: pulsar - port: 6650 - sslPort: 6651 - \ No newline at end of file + host: rabbitmq + port: 5672 \ No newline at end of file diff --git a/k8s-deploy/examples/party-10000/cluster.yaml b/k8s-deploy/examples/party-10000/cluster.yaml index 29d14a795..cf150c3f2 100644 --- a/k8s-deploy/examples/party-10000/cluster.yaml +++ b/k8s-deploy/examples/party-10000/cluster.yaml @@ -1,11 +1,10 @@ name: fate-10000 namespace: fate-10000 chartName: fate -chartVersion: v1.8.0 +chartVersion: v1.9.0 partyId: 10000 registry: "" -imageTag: 1.8.0-release -pullPolicy: +pullPolicy: imagePullSecrets: - name: myregistrykey persistence: false @@ -23,7 +22,11 @@ modules: - fateboard - client -backend: eggroll +computing: Eggroll +federation: Eggroll +storage: Eggroll +algorithm: Basic +device: CPU ingress: fateboard: @@ -37,9 +40,9 @@ rollsite: type: NodePort nodePort: 30101 partyList: - - partyId: 9999 - partyIp: 192.168.9.1 - partyPort: 30091 + - partyId: 9999 + partyIp: 192.168.9.1 + partyPort: 30091 python: type: NodePort diff --git a/k8s-deploy/examples/party-9999/cluster-serving.yaml b/k8s-deploy/examples/party-9999/cluster-serving.yaml index 19eacb5af..db978b513 100644 --- a/k8s-deploy/examples/party-9999/cluster-serving.yaml +++ b/k8s-deploy/examples/party-9999/cluster-serving.yaml @@ -1,11 +1,10 @@ name: fate-serving-9999 namespace: fate-serving-9999 chartName: fate-serving -chartVersion: v2.1.5 +chartVersion: v2.1.6 partyId: 9999 registry: "" -imageTag: 2.1.5-release -pullPolicy: +pullPolicy: imagePullSecrets: - name: myregistrykey persistence: false diff --git a/k8s-deploy/examples/party-9999/cluster-spark-local-pulsar.yaml b/k8s-deploy/examples/party-9999/cluster-spark-local-pulsar.yaml index 3f03d6584..7ff1d32c0 100644 --- a/k8s-deploy/examples/party-9999/cluster-spark-local-pulsar.yaml +++ b/k8s-deploy/examples/party-9999/cluster-spark-local-pulsar.yaml @@ -1,11 +1,10 @@ name: fate-9999 namespace: fate-9999 chartName: fate -chartVersion: v1.8.0 +chartVersion: v1.9.0 partyId: 9999 registry: "" -imageTag: 1.8.0-release -pullPolicy: +pullPolicy: imagePullSecrets: - name: myregistrykey persistence: false @@ -22,7 +21,11 @@ modules: - nginx - pulsar -backend: spark_local_pulsar +computing: Spark_local +federation: Pulsar +storage: LocalFS +algorithm: Basic +device: CPU ingress: fateboard: diff --git a/k8s-deploy/examples/party-9999/cluster-spark-pulsar.yaml b/k8s-deploy/examples/party-9999/cluster-spark-pulsar.yaml index 71c3c7cb5..d4232c824 100644 --- a/k8s-deploy/examples/party-9999/cluster-spark-pulsar.yaml +++ b/k8s-deploy/examples/party-9999/cluster-spark-pulsar.yaml @@ -1,11 +1,10 @@ name: fate-9999 namespace: fate-9999 chartName: fate -chartVersion: v1.8.0 +chartVersion: v1.9.0 partyId: 9999 registry: "" -imageTag: 1.8.0-release -pullPolicy: +pullPolicy: imagePullSecrets: - name: myregistrykey persistence: false @@ -24,7 +23,11 @@ modules: - nginx - pulsar -backend: spark_pulsar +computing: Spark +federation: Pulsar +storage: HDFS +algorithm: Basic +device: CPU ingress: fateboard: diff --git a/k8s-deploy/examples/party-9999/cluster-spark-rabbitmq.yaml b/k8s-deploy/examples/party-9999/cluster-spark-rabbitmq.yaml index a08f4f1a9..416e875d5 100644 --- a/k8s-deploy/examples/party-9999/cluster-spark-rabbitmq.yaml +++ b/k8s-deploy/examples/party-9999/cluster-spark-rabbitmq.yaml @@ -1,12 +1,11 @@ name: fate-9999 namespace: fate-9999 chartName: fate -chartVersion: v1.8.0 +chartVersion: v1.9.0 partyId: 9999 registry: "" -imageTag: 1.8.0-release -pullPolicy: -imagePullSecrets: +pullPolicy: +imagePullSecrets: - name: myregistrykey persistence: false istio: @@ -24,7 +23,11 @@ modules: - nginx - rabbitmq -backend: spark_rabbitmq +computing: Spark +federation: RabbitMQ +storage: HDFS +algorithm: Basic +device: CPU ingress: fateboard: diff --git a/k8s-deploy/examples/party-9999/cluster.yaml b/k8s-deploy/examples/party-9999/cluster.yaml index 3ba8312f1..630dfa581 100644 --- a/k8s-deploy/examples/party-9999/cluster.yaml +++ b/k8s-deploy/examples/party-9999/cluster.yaml @@ -1,11 +1,10 @@ name: fate-9999 namespace: fate-9999 chartName: fate -chartVersion: v1.8.0 +chartVersion: v1.9.0 partyId: 9999 registry: "" -imageTag: 1.8.0-release -pullPolicy: +pullPolicy: imagePullSecrets: - name: myregistrykey persistence: false @@ -23,7 +22,11 @@ modules: - fateboard - client -backend: eggroll +computing: Eggroll +federation: Eggroll +storage: Eggroll +algorithm: Basic +device: CPU ingress: fateboard: @@ -33,13 +36,13 @@ ingress: hosts: - name: party9999.notebook.example.com -rollsite: +rollsite: type: NodePort nodePort: 30091 partyList: - - partyId: 10000 - partyIp: 192.168.10.1 - partyPort: 30101 + - partyId: 10000 + partyIp: 192.168.10.1 + partyPort: 30101 python: type: NodePort diff --git a/k8s-deploy/examples/party-exchange/rollsite.yaml b/k8s-deploy/examples/party-exchange/rollsite.yaml index 0c045ac32..6cf57bfd4 100644 --- a/k8s-deploy/examples/party-exchange/rollsite.yaml +++ b/k8s-deploy/examples/party-exchange/rollsite.yaml @@ -1,11 +1,10 @@ name: fate-exchange namespace: fate-exchange chartName: fate-exchange -chartVersion: +chartVersion: v1.9.0 partyId: 1 registry: "" -imageTag: 1.8.0-release -pullPolicy: +pullPolicy: imagePullSecrets: - name: myregistrykey persistence: false @@ -19,6 +18,7 @@ modules: rollsite: type: NodePort nodePort: 30000 + enableTLS: false partyList: - partyId: 10000 partyIp: 192.168.10.1 diff --git a/k8s-deploy/examples/party-exchange/trafficServer.yaml b/k8s-deploy/examples/party-exchange/trafficServer.yaml index 136f5ec66..bb642f518 100644 --- a/k8s-deploy/examples/party-exchange/trafficServer.yaml +++ b/k8s-deploy/examples/party-exchange/trafficServer.yaml @@ -1,11 +1,10 @@ name: fate-exchange namespace: fate-exchange chartName: fate-exchange -chartVersion: +chartVersion: v1.9.0 partyId: 1 registry: "" -imageTag: 1.8.0-release -pullPolicy: +pullPolicy: imagePullSecrets: - name: myregistrykey persistence: false diff --git a/k8s-deploy/examples/party.config b/k8s-deploy/examples/party.config index 6de091521..19159cb94 100644 --- a/k8s-deploy/examples/party.config +++ b/k8s-deploy/examples/party.config @@ -1,7 +1,10 @@ -fate_chartVersion=v1.8.0 -fate_imageTAG=1.8.0-release -fate_serving_chartVersion=v2.1.5 -fate_serving_imageTAG=2.1.5-release +fate_chartVersion=v1.9.0 +fate_imageTAG=1.9.0-release +fate_serving_chartVersion=v2.1.6 +fate_serving_imageTAG=2.1.6-release party_9999_IP=192.168.9.1 party_10000_IP=192.168.10.1 -party_exchange_IP=192.168.0.1 \ No newline at end of file +party_exchange_IP=192.168.0.1 + +algorithm=Basic +device=CPU \ No newline at end of file diff --git a/k8s-deploy/go.mod b/k8s-deploy/go.mod index 49124c888..0786ccfa5 100644 --- a/k8s-deploy/go.mod +++ b/k8s-deploy/go.mod @@ -9,6 +9,7 @@ require ( github.com/gin-gonic/gin v1.7.7 github.com/gofrs/flock v0.8.1 github.com/gosuri/uitable v0.0.4 + github.com/hashicorp/go-version v1.6.0 github.com/json-iterator/go v1.1.12 github.com/naoina/toml v0.1.1 github.com/pkg/errors v0.9.1 @@ -34,6 +35,7 @@ require ( ) require ( + github.com/AlecAivazis/survey/v2 v2.3.5 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/BurntSushi/toml v1.0.0 // indirect github.com/KyleBanks/depth v1.2.1 // indirect @@ -60,7 +62,7 @@ require ( github.com/docker/go-units v0.4.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect - github.com/fatih/color v1.13.0 // indirect + github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-errors/errors v1.4.2 // indirect @@ -140,7 +142,7 @@ require ( github.com/spf13/cobra v1.3.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/testify v1.7.0 // indirect + github.com/stretchr/testify v1.7.0 github.com/subosito/gotenv v1.2.0 // indirect github.com/ugorji/go/codec v1.2.7 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect @@ -150,7 +152,7 @@ require ( go.starlark.net v0.0.0-20220302181546-5411bad688d1 // indirect golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect - golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 // indirect + golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect @@ -175,3 +177,8 @@ require ( sigs.k8s.io/kustomize/kyaml v0.13.3 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect ) + +require ( + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect +) diff --git a/k8s-deploy/go.sum b/k8s-deploy/go.sum index 7d50fbc35..e18a1e53d 100644 --- a/k8s-deploy/go.sum +++ b/k8s-deploy/go.sum @@ -53,6 +53,8 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg= +github.com/AlecAivazis/survey/v2 v2.3.5 h1:A8cYupsAZkjaUmhtTYv3sSqc7LO5mp1XDfqe5E/9wRQ= +github.com/AlecAivazis/survey/v2 v2.3.5/go.mod h1:4AuI9b7RjAR+G7v9+C4YSlX/YL3K3cWNXgWXOhllqvI= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v56.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= @@ -129,6 +131,8 @@ github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5 github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= +github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= @@ -356,8 +360,9 @@ github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKY github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI= +github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI= github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= @@ -716,6 +721,8 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -732,6 +739,8 @@ github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= +github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= @@ -780,6 +789,8 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/karrick/godirwalk v1.15.8/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -843,6 +854,7 @@ github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -874,6 +886,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= @@ -1598,11 +1612,12 @@ golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 h1:y/woIyUBFbpQGKS0u1aHF/40WUDnek3fPOyD08H5Vng= -golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 h1:xHms4gcpe1YE7A3yIllJXP16CMAGuqwO2lX1mTyyRRc= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/k8s-deploy/kubefate.yaml b/k8s-deploy/kubefate.yaml index d7221f682..aa672084d 100644 --- a/k8s-deploy/kubefate.yaml +++ b/k8s-deploy/kubefate.yaml @@ -19,7 +19,7 @@ spec: spec: serviceAccountName: kubefate-admin containers: - - image: federatedai/kubefate:v1.4.4 + - image: federatedai/kubefate:v1.4.5 imagePullPolicy: IfNotPresent name: kubefate env: @@ -73,36 +73,24 @@ spec: memory: 512Mi cpu: "0.5" livenessProbe: - httpGet: - path: / + tcpSocket: port: 8080 - httpHeaders: - - name: X-Custom-Header - value: livenessProbe initialDelaySeconds: 30 periodSeconds: 10 timeoutSeconds: 5 successThreshold: 1 failureThreshold: 3 readinessProbe: - httpGet: - path: / + tcpSocket: port: 8080 - httpHeaders: - - name: X-Custom-Header - value: readinessProbe initialDelaySeconds: 5 periodSeconds: 10 timeoutSeconds: 1 successThreshold: 1 failureThreshold: 3 startupProbe: - httpGet: - path: / + tcpSocket: port: 8080 - httpHeaders: - - name: X-Custom-Header - value: startupProbe failureThreshold: 30 periodSeconds: 10 restartPolicy: Always @@ -164,7 +152,7 @@ spec: - mysqladmin - ping initialDelaySeconds: 30 - periodSeconds: 10 + periodSeconds: 30 timeoutSeconds: 5 successThreshold: 1 failureThreshold: 3 @@ -173,8 +161,8 @@ spec: command: - mysqladmin - ping - initialDelaySeconds: 5 - periodSeconds: 10 + initialDelaySeconds: 30 + periodSeconds: 30 timeoutSeconds: 1 successThreshold: 1 failureThreshold: 3 diff --git a/k8s-deploy/pkg/api/chart.go b/k8s-deploy/pkg/api/chart.go index a522d497a..f7216a653 100644 --- a/k8s-deploy/pkg/api/chart.go +++ b/k8s-deploy/pkg/api/chart.go @@ -39,6 +39,7 @@ func (c *Chart) Router(r *gin.RouterGroup) { chart.GET("/", c.getChartList) chart.GET("/:chartId", c.getChart) chart.DELETE("/:chartId", c.deleteChart) + chart.GET("/valueTemplateExample/:chartName/:chartVersion", c.getValueTemplateExample) } } @@ -47,6 +48,37 @@ type HelmUUID struct { HelmUUID string } +// getValueTemplateExample return valueTemplateExample string by chart name and version +// @Summary return valueTemplateExample string by chart name and version +// @Tags Chart +// @Produce json +// @Param chartName path string true "chart name" +// @Param chartVersion path string true "chart version" +// @Success 200 {object} JSONResult{data=string} "Success" +// @Failure 401 {object} JSONERRORResult "Unauthorized operation" +// @Failure 500 {object} JSONERRORResult "Internal server error" +// @Router /valueTemplateExample/{chartName}/{chartVersion} [get] +// @Param Authorization header string true "Authentication header" default(Bearer ) +// @Security ApiKeyAuth +func (*Chart) getValueTemplateExample(c *gin.Context) { + name, version := c.Param("chartName"), c.Param("chartVersion") + if name == "" || version == "" { + err := errors.New("name or version is empty") + log.Error().Err(err).Msg("request params error") + c.JSON(500, gin.H{"error": err.Error()}) + return + } + helmChart, err := modules.GetFateChart(name, version) + if err != nil { + log.Error().Err(err).Msg("GetFateChart error") + c.JSON(500, gin.H{"error": err.Error()}) + return + } + v := helmChart.ValuesTemplateExample + log.Debug().Interface("ValuesTemplateExample", v).Msg("getValueTemplateExample Success") + c.JSON(200, gin.H{"msg": "getChartByNameVersion Success", "data": v}) +} + // createChart Upload a Chart // @Summary Upload a Chart // @Tags Chart @@ -57,7 +89,7 @@ type HelmUUID struct { // @Failure 401 {object} JSONERRORResult "Unauthorized operation" // @Failure 500 {object} JSONERRORResult "Internal server error" // @Router /chart [post] -// @Param Authorization header string true "Authentication header" +// @Param Authorization header string true "Authentication header" default(Bearer ) // @Security ApiKeyAuth func (*Chart) createChart(c *gin.Context) { // single file @@ -110,7 +142,7 @@ func (*Chart) createChart(c *gin.Context) { // @Failure 401 {object} JSONERRORResult "Unauthorized operation" // @Failure 500 {object} JSONERRORResult "Internal server error" // @Router /chart [get] -// @Param Authorization header string true "Authentication header" +// @Param Authorization header string true "Authentication header" default(Bearer ) // @Security ApiKeyAuth func (*Chart) getChartList(c *gin.Context) { @@ -134,7 +166,7 @@ func (*Chart) getChartList(c *gin.Context) { // @Failure 401 {object} JSONERRORResult "Unauthorized operation" // @Failure 500 {object} JSONERRORResult "Internal server error" // @Router /chart/{chartId} [get] -// @Param Authorization header string true "Authentication header" +// @Param Authorization header string true "Authentication header" default(Bearer ) // @Security ApiKeyAuth func (*Chart) getChart(c *gin.Context) { @@ -166,7 +198,7 @@ func (*Chart) getChart(c *gin.Context) { // @Failure 401 {object} JSONEMSGResult "Unauthorized operation" // @Failure 500 {object} JSONERRORResult "Internal server error" // @Router /chart/{chartId} [delete] -// @Param Authorization header string true "Authentication header" +// @Param Authorization header string true "Authentication header" default(Bearer ) // @Security ApiKeyAuth func (*Chart) deleteChart(c *gin.Context) { chartID := c.Param("chartId") diff --git a/k8s-deploy/pkg/api/cluster.go b/k8s-deploy/pkg/api/cluster.go index 9732f7c4a..0dda5b76b 100644 --- a/k8s-deploy/pkg/api/cluster.go +++ b/k8s-deploy/pkg/api/cluster.go @@ -54,7 +54,7 @@ func (c *Cluster) Router(r *gin.RouterGroup) { // @Failure 401 {object} JSONERRORResult "Unauthorized operation" // @Failure 500 {object} JSONERRORResult "Internal server error" // @Router /cluster [post] -// @Param Authorization header string true "Authentication header" +// @Param Authorization header string true "Authentication header" default(Bearer ) // @Security ApiKeyAuth // @Security OAuth2Application[write, admin] func (*Cluster) createCluster(c *gin.Context) { @@ -91,7 +91,7 @@ func (*Cluster) createCluster(c *gin.Context) { // @Failure 401 {object} JSONERRORResult "Unauthorized operation" // @Failure 500 {object} JSONERRORResult "Internal server error" // @Router /cluster [put] -// @Param Authorization header string true "Authentication header" +// @Param Authorization header string true "Authentication header" default(Bearer ) // @Security ApiKeyAuth // @Security OAuth2Application[write, admin] func (*Cluster) setCluster(c *gin.Context) { @@ -134,7 +134,7 @@ func (*Cluster) setCluster(c *gin.Context) { // @Failure 401 {object} JSONERRORResult "Unauthorized operation" // @Failure 500 {object} JSONERRORResult "Internal server error" // @Router /cluster/{clusterId} [get] -// @Param Authorization header string true "Authentication header" +// @Param Authorization header string true "Authentication header" default(Bearer ) // @Security ApiKeyAuth // @Security OAuth2Application[write, admin] func (*Cluster) getCluster(c *gin.Context) { @@ -202,7 +202,7 @@ func (*Cluster) getCluster(c *gin.Context) { // @Failure 401 {object} JSONERRORResult "Unauthorized operation" // @Failure 500 {object} JSONERRORResult "Internal server error" // @Router /cluster/ [get] -// @Param Authorization header string true "Authentication header" +// @Param Authorization header string true "Authentication header" default(Bearer ) // @Security ApiKeyAuth // @Security OAuth2Application[write, admin] func (*Cluster) getClusterList(c *gin.Context) { @@ -236,7 +236,7 @@ func (*Cluster) getClusterList(c *gin.Context) { // @Failure 401 {object} JSONERRORResult "Unauthorized operation" // @Failure 500 {object} JSONERRORResult "Internal server error" // @Router /cluster/{clusterId} [delete] -// @Param Authorization header string true "Authentication header" +// @Param Authorization header string true "Authentication header" default(Bearer ) // @Security ApiKeyAuth // @Security OAuth2Application[write, admin] func (*Cluster) deleteCluster(c *gin.Context) { diff --git a/k8s-deploy/pkg/api/job.go b/k8s-deploy/pkg/api/job.go index 5981806b9..2ca4b7fdf 100644 --- a/k8s-deploy/pkg/api/job.go +++ b/k8s-deploy/pkg/api/job.go @@ -50,7 +50,7 @@ func (j *Job) Router(r *gin.RouterGroup) { // @Failure 400 {object} JSONERRORResult "Bad Request" // @Failure 401 {object} JSONERRORResult "Unauthorized operation" // @Router /job/ [get] -// @Param Authorization header string true "Authentication header" +// @Param Authorization header string true "Authentication header" default(Bearer ) // @Security ApiKeyAuth func (*Job) getJobList(c *gin.Context) { @@ -74,7 +74,7 @@ func (*Job) getJobList(c *gin.Context) { // @Failure 401 {object} JSONERRORResult "Unauthorized operation" // @Failure 500 {object} JSONERRORResult "Internal server error" // @Router /job/{jobId} [get] -// @Param Authorization header string true "Authentication header" +// @Param Authorization header string true "Authentication header" default(Bearer ) // @Security ApiKeyAuth func (*Job) getJob(c *gin.Context) { @@ -106,7 +106,7 @@ func (*Job) getJob(c *gin.Context) { // @Failure 401 {object} JSONERRORResult "Unauthorized operation" // @Failure 500 {object} JSONERRORResult "Internal server error" // @Router /job/{jobId} [put] -// @Param Authorization header string true "Authentication header" +// @Param Authorization header string true "Authentication header" default(Bearer ) // @Security ApiKeyAuth func (*Job) putJob(c *gin.Context) { @@ -143,7 +143,7 @@ func (*Job) putJob(c *gin.Context) { // @Failure 401 {object} JSONERRORResult "Unauthorized operation" // @Failure 500 {object} JSONERRORResult "Internal server error" // @Router /job/{jobId} [delete] -// @Param Authorization header string true "Authentication header" +// @Param Authorization header string true "Authentication header" default(Bearer ) // @Security ApiKeyAuth func (*Job) deleteJob(c *gin.Context) { diff --git a/k8s-deploy/pkg/api/kube_log.go b/k8s-deploy/pkg/api/kube_log.go index e5ca8bff8..ee1ccb4e6 100644 --- a/k8s-deploy/pkg/api/kube_log.go +++ b/k8s-deploy/pkg/api/kube_log.go @@ -70,7 +70,7 @@ func (e *kubeLog) Router(r *gin.RouterGroup) { // @Failure 401 {object} JSONERRORResult "Unauthorized operation" // @Failure 500 {object} JSONERRORResult "Internal server error" // @Router /log/{clusterID} [get] -// @Param Authorization header string true "Authentication header" +// @Param Authorization header string true "Authentication header" default(Bearer ) // @Security ApiKeyAuth func (*kubeLog) getClusterLog(c *gin.Context) { @@ -138,7 +138,7 @@ func (*kubeLog) getClusterLog(c *gin.Context) { // @Failure 401 {object} JSONERRORResult "Unauthorized operation" // @Failure 500 {object} JSONERRORResult "Internal server error" // @Router /log/{clusterID}/ws [get] -// @Param Authorization header string true "Authentication header" +// @Param Authorization header string true "Authentication header" default(Bearer ) // @Security ApiKeyAuth func (*kubeLog) getClusterLogWs(c *gin.Context) { diff --git a/k8s-deploy/pkg/api/namespace.go b/k8s-deploy/pkg/api/namespace.go index de92de05f..ba1dba107 100644 --- a/k8s-deploy/pkg/api/namespace.go +++ b/k8s-deploy/pkg/api/namespace.go @@ -45,7 +45,7 @@ func (j *Namespace) Router(r *gin.RouterGroup) { // @Failure 401 {object} JSONERRORResult "Unauthorized operation" // @Failure 500 {object} JSONERRORResult "Internal server error" // @Router /namespace/ [get] -// @Param Authorization header string true "Authentication header" +// @Param Authorization header string true "Authentication header" default(Bearer ) // @Security ApiKeyAuth func (*Namespace) getNamespaceList(c *gin.Context) { diff --git a/k8s-deploy/pkg/api/service.go b/k8s-deploy/pkg/api/service.go index a2ec2c94e..4eedd3a2e 100644 --- a/k8s-deploy/pkg/api/service.go +++ b/k8s-deploy/pkg/api/service.go @@ -17,6 +17,7 @@ package api import ( "fmt" + "github.com/rs/zerolog" "os" "time" @@ -132,6 +133,7 @@ func Run() error { logger.SetLogger( logger.WithUTC(true), logger.WithLogger(logging.GetGinLogger), + logger.WithDefaultLevel(zerolog.DebugLevel), ), ) diff --git a/k8s-deploy/pkg/api/user.go b/k8s-deploy/pkg/api/user.go index 1da16ccfc..877062c91 100644 --- a/k8s-deploy/pkg/api/user.go +++ b/k8s-deploy/pkg/api/user.go @@ -75,7 +75,7 @@ func (authMiddleware *authMiddleware) login(c *gin.Context) { // @Success 200 {object} string // @Failure 401 {object} JSONEMSGResult // @Router /user/logout [post] -// @Param Authorization header string true "Authentication header" +// @Param Authorization header string true "Authentication header" default(Bearer ) // @Security ApiKeyAuth func (authMiddleware *authMiddleware) logout(c *gin.Context) { authMiddleware.LogoutHandler(c) @@ -116,7 +116,7 @@ func generateAdminUser() error { // @Failure 401 {object} JSONERRORResult Unauthorized operation" // @Failure 500 {object} JSONERRORResult "Internal server error" // @Router /user [post] -// @Param Authorization header string true "Authentication header" +// @Param Authorization header string true "Authentication header" default(Bearer ) // @Security ApiKeyAuth func (*User) createUser(c *gin.Context) { @@ -153,7 +153,7 @@ func (*User) createUser(c *gin.Context) { // @Failure 401 {object} JSONERRORResult "Unauthorized operation" // @Failure 500 {object} JSONERRORResult "Internal server error" // @Router /user [put] -// @Param Authorization header string true "Authentication header" +// @Param Authorization header string true "Authentication header" default(Bearer ) // @Security ApiKeyAuth func (*User) setUser(c *gin.Context) { @@ -184,7 +184,7 @@ func (*User) setUser(c *gin.Context) { // @Failure 401 {object} JSONERRORResult "Unauthorized operation" // @Failure 500 {object} JSONERRORResult "Internal server error" // @Router /user/{userId} [get] -// @Param Authorization header string true "Authentication header" +// @Param Authorization header string true "Authentication header" default(Bearer ) // @Security ApiKeyAuth func (*User) getUser(c *gin.Context) { @@ -211,7 +211,7 @@ func (*User) getUser(c *gin.Context) { // @Failure 401 {object} JSONERRORResult "Unauthorized operation" // @Failure 500 {object} JSONERRORResult "Internal server error" // @Router /user [get] -// @Param Authorization header string true "Authentication header" +// @Param Authorization header string true "Authentication header" default(Bearer ) // @Security ApiKeyAuth func (*User) getUserList(c *gin.Context) { @@ -241,7 +241,7 @@ func getUserFindByUUID(uuid string) (modules.User, error) { // @Failure 401 {object} JSONERRORResult "Unauthorized operation" // @Failure 500 {object} JSONERRORResult "Internal server error" // @Router /user/{userId} [delete] -// @Param Authorization header string true "Authentication header" +// @Param Authorization header string true "Authentication header" default(Bearer ) // @Security ApiKeyAuth func (*User) deleteUser(c *gin.Context) { diff --git a/k8s-deploy/pkg/api/version.go b/k8s-deploy/pkg/api/version.go index 2aff0369c..c0e0f4181 100644 --- a/k8s-deploy/pkg/api/version.go +++ b/k8s-deploy/pkg/api/version.go @@ -20,7 +20,7 @@ import ( ) // ServiceVersion code release version -const ServiceVersion = "v1.4.4" +const ServiceVersion = "v1.4.5" // Version API struct type Version struct { diff --git a/k8s-deploy/pkg/cli/api.go b/k8s-deploy/pkg/cli/api.go index 97e162f2a..506681fa3 100644 --- a/k8s-deploy/pkg/cli/api.go +++ b/k8s-deploy/pkg/cli/api.go @@ -16,6 +16,7 @@ package cli import ( "encoding/json" + "errors" "fmt" "os" @@ -28,6 +29,7 @@ const ( ERROR MSG JOB + VALUE ) type Item interface { @@ -161,6 +163,41 @@ func GetItem(i Item, UUID string) error { return nil } +func GetResponse(i Item, req *Request, respType int) (msg interface{}, err error) { + rep, err := Send(req) + if err != nil { + return + } + + log.Debug().Int("rep.Code", rep.Code).Msg("rep.Code") + + if rep.Code != 200 { + if rep.Code == 404 { + return nil, errors.New("get reponse failed, code 404") + } + msg, err = i.getResult(ERROR) + if err != nil { + return + } + err = json.Unmarshal(rep.Body, &msg) + if err != nil { + return + } + err = i.output(msg, ERROR) + if err != nil { + return + } + return + } + + msg, err = i.getResult(respType) + if err != nil { + return + } + err = json.Unmarshal(rep.Body, &msg) + return +} + func GetItemList(i Item) error { req := &Request{ Type: "GET", diff --git a/k8s-deploy/pkg/cli/chart.go b/k8s-deploy/pkg/cli/chart.go index 39f5f78ee..b6fc354a5 100644 --- a/k8s-deploy/pkg/cli/chart.go +++ b/k8s-deploy/pkg/cli/chart.go @@ -52,6 +52,11 @@ type ChartResultErr struct { Error string } +type ValueResult struct { + Data string + Msg string +} + func (c *Chart) getResult(Type int) (result interface{}, err error) { switch Type { case LIST: @@ -62,6 +67,8 @@ func (c *Chart) getResult(Type int) (result interface{}, err error) { result = new(ChartResultMsg) case ERROR: result = new(ChartResultErr) + case VALUE: + result = new(ValueResult) default: err = fmt.Errorf("no type %d", Type) } diff --git a/k8s-deploy/pkg/cli/chart_cli.go b/k8s-deploy/pkg/cli/chart_cli.go index ef807ef16..984e464f8 100644 --- a/k8s-deploy/pkg/cli/chart_cli.go +++ b/k8s-deploy/pkg/cli/chart_cli.go @@ -160,7 +160,12 @@ func ChartCreateCommand() *cli.Command { if serviceUrl == "" { serviceUrl = "localhost:8080/" } - Url := "http://" + serviceUrl + "/" + apiVersion + r.Path + safeconnect := viper.GetString("safeconnect") + scheme := "http://" + if safeconnect == "true" { + scheme = "https://" + } + Url := scheme + serviceUrl + "/" + apiVersion + r.Path body := bytes.NewReader(r.Body) log.Debug().Str("Type", r.Type).Str("url", Url).Msg("Request") request, err := http.NewRequest(r.Type, Url, body) diff --git a/k8s-deploy/pkg/cli/cli.go b/k8s-deploy/pkg/cli/cli.go index 08ba9e69c..e6c19e762 100644 --- a/k8s-deploy/pkg/cli/cli.go +++ b/k8s-deploy/pkg/cli/cli.go @@ -16,9 +16,10 @@ package cli import ( "fmt" - "github.com/urfave/cli/v2" // imports as package "cli" "sort" "time" + + "github.com/urfave/cli/v2" // imports as package "cli" ) const ( @@ -42,6 +43,7 @@ func initCommandLine() *cli.App { ChartCommand(), VersionCommand(), NamespaceCommand(), + SupportBundleCommand(), }, } diff --git a/k8s-deploy/pkg/cli/cluster_cli.go b/k8s-deploy/pkg/cli/cluster_cli.go index 0a786f8b2..86c2a1c55 100644 --- a/k8s-deploy/pkg/cli/cluster_cli.go +++ b/k8s-deploy/pkg/cli/cluster_cli.go @@ -1,5 +1,5 @@ /* - * Copyright 2019-2021 VMware, Inc. + * Copyright 2019-2022 VMware, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ import ( "errors" "github.com/FederatedAI/KubeFATE/k8s-deploy/pkg/modules" + "github.com/fatih/color" jsoniter "github.com/json-iterator/go" "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" @@ -153,13 +154,27 @@ func ClusterInstallCommand() *cli.Command { } chartVersion, ok := m["chartVersion"] - if !ok { - return errors.New("chartVersion not found, check your Cluster file") + if !ok || chartVersion == "" { + return errors.New("chartVersion not found, check your cluster.yaml file") } chartName, ok := m["chartName"] - if !ok { - chartName = "" + if !ok || chartName == "" { + return errors.New("chartName not found, check your cluster.yaml file") + } + + var errs []error + template, err := GetValueTemplateExample(chartName.(string), chartVersion.(string)) + errs = append(errs, err) + skippedKeys := append(getSkippedKeys(m), "route_table") + errs = append(errs, ValidateYaml(template, string(clusterConfig), skippedKeys)...) + warn := !ContainsSkipError(errs) + for _, err := range errs { + if err != nil && warn { + color.Yellow("Config Warning: %s\n", err.Error()) + } else if err != nil { + log.Debug().Str("validation", err.Error()).Msg("Skipped Error") + } } var json = jsoniter.ConfigCompatibleWithStandardLibrary @@ -199,16 +214,23 @@ func ClusterUpdateCommand() *cli.Command { Usage: "Enter your own configured Cluster.yaml file", Required: true, }, + &cli.BoolFlag{ + Name: "keepUpgradeJob", + Value: false, + Usage: "Only useful when upgrading the FML framework's version. If true, will keep the" + + "upgrade manager's job on the k8s cluster. Otherwise, will clean up the job", + }, }, Usage: "Update a Cluster", Action: func(c *cli.Context) error { - valTemValPath := c.String("file") + keepUpgradeJob := c.Bool("keepUpgradeJob") + log.Debug().Bool("keepUpgradeJob", keepUpgradeJob).Msg("install flag") + valTemValPath := c.String("file") clusterConfig, err := ioutil.ReadFile(valTemValPath) if err != nil { return err } - log.Debug().Str("yaml", string(clusterConfig)).Msg("ReadFile Success") var m map[string]interface{} @@ -246,12 +268,13 @@ func ClusterUpdateCommand() *cli.Command { cluster := new(Cluster) args := &modules.ClusterArgs{ - Name: name.(string), - Namespace: namespace.(string), - ChartVersion: chartVersion.(string), - ChartName: chartName.(string), - Cover: false, - Data: valBJ, + Name: name.(string), + Namespace: namespace.(string), + ChartVersion: chartVersion.(string), + ChartName: chartName.(string), + Cover: false, + Data: valBJ, + KeepUpgradeJob: keepUpgradeJob, } body, err := json.Marshal(args) diff --git a/k8s-deploy/pkg/cli/job_cli.go b/k8s-deploy/pkg/cli/job_cli.go index f74064677..70688bc93 100644 --- a/k8s-deploy/pkg/cli/job_cli.go +++ b/k8s-deploy/pkg/cli/job_cli.go @@ -130,7 +130,12 @@ func JobStopCommand() *cli.Command { if serviceURL == "" { serviceURL = "localhost:8080/" } - URL := "http://" + serviceURL + "/" + apiVersion + r.Path + "/" + uuid + "?jobStatus=stop" + safeconnect := viper.GetString("safeconnect") + scheme := "http://" + if safeconnect == "true" { + scheme = "https://" + } + URL := scheme + serviceURL + "/" + apiVersion + r.Path + "/" + uuid + "?jobStatus=stop" body := bytes.NewReader(r.Body) log.Debug().Str("Type", r.Type).Str("url", URL).Msg("Request") request, err := http.NewRequest(r.Type, URL, body) diff --git a/k8s-deploy/pkg/cli/kube_log_cli.go b/k8s-deploy/pkg/cli/kube_log_cli.go index cc9e79fce..ce86d17fa 100644 --- a/k8s-deploy/pkg/cli/kube_log_cli.go +++ b/k8s-deploy/pkg/cli/kube_log_cli.go @@ -135,7 +135,12 @@ func GetModuleLog(uuid, args string) (string, error) { if serviceUrl == "" { serviceUrl = "localhost:8080/" } - Url := "http://" + serviceUrl + "/" + apiVersion + r.Path + fmt.Sprintf("/%s?%s", uuid, args) + safeconnect := viper.GetString("safeconnect") + scheme := "http://" + if safeconnect == "true" { + scheme = "https://" + } + Url := scheme + serviceUrl + "/" + apiVersion + r.Path + fmt.Sprintf("/%s?%s", uuid, args) body := bytes.NewReader(r.Body) log.Debug().Str("Type", r.Type).Str("url", Url).Msg("Request") @@ -206,10 +211,15 @@ func GetModuleLogFollow(uuid, args string) error { if serviceUrl == "" { serviceUrl = "localhost:8080/" } - Url := "ws://" + serviceUrl + "/" + apiVersion + r.Path + fmt.Sprintf("/%s/ws?%s", uuid, args) + safeconnect := viper.GetString("safeconnect") + scheme := "ws://" + if safeconnect == "true" { + scheme = "wss://" + } + Url := scheme + serviceUrl + "/" + apiVersion + r.Path + fmt.Sprintf("/%s/ws?%s", uuid, args) log.Debug().Str("Url", Url).Msg("ok") - config, err := websocket.NewConfig(Url, "http://"+serviceUrl+"/") + config, err := websocket.NewConfig(Url, scheme+serviceUrl+"/") config.Header.Add("user-agent", "kubefate") token, err := getToken() if err != nil { diff --git a/k8s-deploy/pkg/cli/requst.go b/k8s-deploy/pkg/cli/requst.go index 6872c10fb..fd640b0aa 100644 --- a/k8s-deploy/pkg/cli/requst.go +++ b/k8s-deploy/pkg/cli/requst.go @@ -31,8 +31,12 @@ import ( func getToken() (string, error) { serviceurl := viper.GetString("serviceurl") - - loginUrl := "http://" + serviceurl + "/v1/user/login" + safeconnect := viper.GetString("safeconnect") + scheme := "http://" + if safeconnect == "true" { + scheme = "https://" + } + loginUrl := scheme + serviceurl + "/v1/user/login" login := map[string]string{ "username": viper.GetString("user.username"), @@ -101,7 +105,12 @@ func Send(r *Request) (*Response, error) { if serviceUrl == "" { serviceUrl = "localhost:8080/" } - Url := "http://" + serviceUrl + "/" + apiVersion + r.Path + safeconnect := viper.GetString("safeconnect") + scheme := "http://" + if safeconnect == "true" { + scheme = "https://" + } + Url := scheme + serviceUrl + "/" + apiVersion + r.Path body := bytes.NewReader(r.Body) log.Debug().Str("Type", r.Type).Str("url", Url).Str("Body", string(r.Body)).Msg("Request") request, err := http.NewRequest(r.Type, Url, body) diff --git a/k8s-deploy/pkg/cli/support_bundle.go b/k8s-deploy/pkg/cli/support_bundle.go new file mode 100644 index 000000000..70f1e3088 --- /dev/null +++ b/k8s-deploy/pkg/cli/support_bundle.go @@ -0,0 +1,154 @@ +/* + * Copyright 2019-2022 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package cli + +import ( + "errors" + "fmt" + + "github.com/AlecAivazis/survey/v2" + client "github.com/FederatedAI/KubeFATE/k8s-deploy/pkg/utils/k8sclient" + "github.com/FederatedAI/KubeFATE/k8s-deploy/pkg/utils/supportbundle" + "github.com/urfave/cli/v2" +) + +var ( + bundler *supportbundle.Bundler +) + +func SupportBundleCommand() *cli.Command { + return &cli.Command{ + Name: "supportbundle", + Flags: []cli.Flag{}, + Subcommands: []*cli.Command{ + SupportBundleCollectCommand(), + SupportBundlePackCommand(), + }, + Usage: "Collect data to perform troubleshooting", + } +} + +func SupportBundleCollectCommand() *cli.Command { + return &cli.Command{ + Name: "collect", + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "tail", + Aliases: []string{"t"}, + Value: 200, + Usage: "Specify how many rows to record.", + }, + &cli.StringFlag{ + Name: "kubeconfig", + Aliases: []string{"c"}, + Value: "", + Usage: "Specify the kubeconfig.", + }, + &cli.StringFlag{ + Name: "collectDir", + Value: "./kubefate-supportbundle", + Usage: "Specify where to save temporary data.", + }, + &cli.StringFlag{ + Name: "packDir", + Value: "./", + Usage: "Specify where to pack zip file.", + }, + }, + Usage: "Collect data", + Action: func(c *cli.Context) error { + tail := c.Int("tail") + kubeconfig := c.String("kubeconfig") + if kubeconfig == "" { + kubeconfig = client.GetKubeconfig() + } + collectDir := c.String("collectDir") + packDir := c.String("packDir") + bundler, err := supportbundle.NewBundler(kubeconfig, collectDir, packDir) + if err != nil || bundler == nil { + return errors.New("supportbundler not initialized sucessfully. " + err.Error()) + } + namespaceList, err := bundler.Client.GetNamespaceList() + if err != nil { + return err + } + var namespaces []string + namespacePrompt := &survey.MultiSelect{ + Message: "Choose namespaces to troubleShooting:", + Options: client.NamespaceListToNames(namespaceList), + Default: []string{"kube-fate"}, + } + survey.AskOne(namespacePrompt, &namespaces) + + for _, namespace := range namespaces { + err := CollectInNamespace(bundler, namespace, tail) + if err != nil { + fmt.Println(err) + } + } + fmt.Printf("You can attach additional files about this issue into %s\n", collectDir) + return nil + }, + } +} + +func SupportBundlePackCommand() *cli.Command { + return &cli.Command{ + Name: "pack", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "collectDir", + Value: "./kubefate-supportbundle", + Usage: "Specify where to save temporary data.", + }, + &cli.StringFlag{ + Name: "packDir", + Value: "./", + Usage: "Specify where to pack zip file.", + }, + }, + Usage: "Pack data", + Action: func(c *cli.Context) error { + collectDir := c.String("collectDir") + packDir := c.String("packDir") + bundler = &supportbundle.Bundler{CollectDir: collectDir, PackDir: packDir} + privacyAck := true + privacyPrompt := &survey.Confirm{ + Message: "This program collects some data, please ensure that" + + " there is no any privacy issue.\n Enter y" + + " if you're willing to share these collected data with us.", + } + survey.AskOne(privacyPrompt, &privacyAck) + if !privacyAck { + return errors.New("please acknowledge the privacy agreement") + } + return bundler.Pack() + }, + } +} + +func CollectInNamespace(b *supportbundle.Bundler, namespace string, tail int) (err error) { + podList, err := b.Client.GetPodList(namespace) + if err != nil { + return err + } + var pods []string + podsPrompt := &survey.MultiSelect{ + Message: fmt.Sprintf("In namespace %s, choose pods to troubleShooting:", namespace), + Options: client.PodListToNames(podList), + } + survey.AskOne(podsPrompt, &pods) + return b.CollectNamespace(namespace, tail, pods...) +} diff --git a/k8s-deploy/pkg/cli/validation.go b/k8s-deploy/pkg/cli/validation.go new file mode 100644 index 000000000..6bdc321cb --- /dev/null +++ b/k8s-deploy/pkg/cli/validation.go @@ -0,0 +1,608 @@ +package cli + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "reflect" + "regexp" + "strconv" + "strings" + + "sigs.k8s.io/yaml" +) + +type any interface{} +type KeyValue map[string]*TreeNode +type ListValue []*TreeNode + +type TreeNode struct { + leaf bool + lineno int + route []string + children KeyValue + value any +} + +type ValidationTree struct { + root *TreeNode + yamlMap, rawYamlMap map[string]interface{} + lines []string +} + +type ValidationManager struct { + templateTree, testTree *ValidationTree + skippedKeys []string + preprocessor []func(m *ValidationManager) []error +} + +// SkipError is a type of error shows +// that you shall skip the validation +type SkipError string + +// ConfigError is config validation error +type ConfigError string + +func (e SkipError) Error() string { return string(e) } +func (e ConfigError) Error() string { return string(e) } + +// // trimComments trims the comments started with "# ". +func trimComments(t []byte) []byte { + pattern := regexp.MustCompile(`^ *# `) + for { + ok := pattern.Match(t) + if ok { + t = bytes.Replace(t, []byte("# "), nil, 1) + } else { + break + } + } + return t +} + +// deconstructKey deconstructs the key to the original key and the lineno. +func deconstructKey(k interface{}) (string, int) { + key, lineno := "", 0 + var err error + switch k := k.(type) { + case string: + group := strings.Split(k, "__lineno__") + if len(group) == 1 { + key, lineno = k, 0 + } else { + key = group[0] + lineno, err = strconv.Atoi(group[1]) + if err != nil { + lineno = 0 + } + } + default: + key, lineno = fmt.Sprint(k), 0 + } + return key, lineno +} + +// NewTreeNode return default TreeNode. +func NewTreeNode() *TreeNode { + node := new(TreeNode) + node.leaf = false + node.children = make(map[string]*TreeNode) + return node +} + +// mapToTreeNode recursively converts the yaml map to TreeNode, +// the route is the path to the current node. +// if node is a anomymous member in one array, the current route is @ArrayItem. +// node.value depends on the type of the key (map, list or basic type). +func mapToTreeNode(body interface{}, route []string) *TreeNode { + node := NewTreeNode() + node.route = route + if body == nil { + node.leaf = true + node.value = nil + return node + } + bodyType, bodyValue := reflect.TypeOf(body), reflect.ValueOf(body) + switch bodyType.Kind() { + case reflect.Map: + for _, k := range bodyValue.MapKeys() { + v := bodyValue.MapIndex(k).Interface() + key, lineno := deconstructKey(k.Interface()) + r := append(route, key) + child := mapToTreeNode(v, r) + child.lineno = lineno + node.children[key] = child + } + node.value = node.children + case reflect.Array, reflect.Slice: + arr := ListValue{} + for i := 0; i < bodyValue.Len(); i++ { + v := bodyValue.Index(i).Interface() + arr = append(arr, mapToTreeNode(v, append(route, "@ArrayItem"))) + } + node.value = arr + default: + node.leaf = true + node.value = body + } + return node +} + +// Contains checks whether an element is in slice/array/map. +func Contains(element interface{}, set interface{}) bool { + setVal := reflect.ValueOf(set) + switch setVal.Type().Kind() { + case reflect.Slice, reflect.Array: + for i := 0; i < setVal.Len(); i++ { + if setVal.Index(i).Interface() == element { + return true + } + } + case reflect.Map: + if setVal.MapIndex(reflect.ValueOf(element)).IsValid() { + return true + } + default: + panic(fmt.Sprintf("invalid type %v ", setVal.Type().Kind())) + } + + return false +} + +// GetValueTemplateExample gets the value template example from api. +func GetValueTemplateExample(chartName, chartVersion string) (value string, err error) { + if !versionValid(chartVersion, []int{1, 9, 0}) { + err = SkipError(fmt.Sprintf("version of %s does not meet the validation"+ + " requirement that chartVersion >= %s", chartVersion, "1.9.0")) + return + } + + c := new(Chart) + req := &Request{ + Type: "GET", + Path: fmt.Sprintf("chart/valueTemplateExample/%s/%s", chartName, chartVersion), + Body: nil, + } + msg, err := GetResponse(c, req, VALUE) + if err != nil { + err = SkipError(fmt.Sprintf("get value template example for validation failed %v,"+ + ", you may need to upgrade KubeFATE service", err)) + return + } + + if result, ok := msg.(*ValueResult); ok { + value = result.Data + } + + if result, ok := msg.(*ChartResultErr); ok { + err = SkipError(fmt.Sprintf("get value template example failed\n %v", result.Error)) + } + return +} + +// compareTwoTrees recursively compares the two trees, walking through the nodes not skipped. +func compareTwoTrees(rootTemp, rootTest *TreeNode, + testLines, skippedKeys []string) (errs []error) { + valueTemp, valueTest := rootTemp.value, rootTest.value + typeTemp, typeTest := reflect.TypeOf(valueTemp), reflect.TypeOf(valueTest) + if typeTemp != typeTest && typeTest != nil { + route := strings.Join(rootTest.route, "/") + errs = append(errs, + ConfigError(fmt.Sprintf("your yaml at '%s', line %d \n '%s' may not match the type", + route, rootTest.lineno, testLines[rootTest.lineno]))) + return + } + switch valueTest := valueTest.(type) { + case KeyValue: + for k, v := range valueTest { + if Contains(k, skippedKeys) { + continue + } + if childTemp, ok := valueTemp.(KeyValue)[k]; !ok { + route := strings.Join(v.route, "/") + errs = append(errs, + ConfigError(fmt.Sprintf("your yaml at '%s', line %d \n '%s' may be redundant", + route, v.lineno, testLines[v.lineno]))) + } else { + errs = append(errs, compareTwoTrees(childTemp, v, testLines, skippedKeys)...) + } + } + case ListValue: + item := valueTemp.(ListValue)[0] + for _, v := range valueTest { + errs = append(errs, compareTwoTrees(item, v, testLines, skippedKeys)...) + } + default: + } + return +} + +func (m *ValidationManager) Validate() error { + compareTwoTrees(m.templateTree.root, m.testTree.root, m.testTree.lines, m.skippedKeys) + return nil +} + +// versionValid checks if the chart version is valid. +func versionValid(chartVersion string, startVersion []int) (valid bool) { + chartVersion = strings.TrimLeft(chartVersion, "v") + for i, v := range strings.Split(chartVersion, ".") { + if i >= len(startVersion) { + return + } + if v, err := strconv.Atoi(v); err != nil || v < startVersion[i] { + return + } else if v > startVersion[i] { + break + } + } + valid = true + return +} + +func (m *ValidationManager) compareTwoTrees() []error { + return compareTwoTrees(m.templateTree.root, m.testTree.root, m.testTree.lines, m.skippedKeys) +} + +// yamlStringToBuffer reads the yaml value and returns +// the content (may be modified) in []byte,the original lines in []string and error. +func yamlStringToBuffer(value string, restoreComments, markLineno bool) ([]byte, []string, error) { + reader := bufio.NewReader(strings.NewReader(value)) + buffer := make([]byte, 0, 10) + // 1 for the first lineno + lines := make([]string, 1, 10) + + linenoReg := regexp.MustCompile(`:`) + for lineno := 1; ; lineno++ { + line, _, err := reader.ReadLine() + if err != nil { + if err == io.EOF { + break + } else { + return nil, nil, err + } + } + if !Contains(byte(':'), line) && !Contains(byte('-'), line) { + // if one line is pure comment, treat it as a blank line. + line = []byte("") + } + + lines = append(lines, strings.TrimSpace(string(line))) + if restoreComments { + line = trimComments(line) + } + if markLineno { + line = linenoReg.ReplaceAll(line, []byte(fmt.Sprintf("__lineno__%d:", lineno))) + } + buffer = append(buffer, append(line, '\n')...) + } + return buffer, lines, nil +} + +// bufferToMap unmarshals the byte buffer to map, +// keys are string and numbers are float64. +func bufferToMap(buffer []byte) (m map[string]interface{}, err error) { + err = yaml.Unmarshal(buffer, &m) + return +} + +// buildValidationTree builds the validation tree from yaml string. +func buildValidationTree(yamlString string, restoreComments, + markLineno bool) (*ValidationTree, error) { + yamlBuffer, lines, err := yamlStringToBuffer(yamlString, restoreComments, markLineno) + if err != nil { + return nil, err + } + yamlMap, err := bufferToMap(yamlBuffer) + if err != nil { + return nil, err + } + rawYamlMap, err := bufferToMap([]byte(yamlString)) + if err != nil { + return nil, err + } + root := mapToTreeNode(yamlMap, []string{""}) + return &ValidationTree{root, yamlMap, rawYamlMap, lines}, nil +} + +// getSkippedKeys returns the skippedKeys in a map. +// The type of skippedKeys array is []interface{}, +// so we need to transform every element to string. +func getSkippedKeys(m map[string]interface{}) (skippedKeys []string) { + value, ok := m["skippedKeys"] + if !ok { + return + } + slice, ok := value.([]interface{}) + if !ok { + return + } + for _, v := range slice { + skippedKeys = append(skippedKeys, v.(string)) + } + return skippedKeys +} + +// getModules extracts modules []string from map +func getModules(yamlMap map[string]interface{}) ([]string, error) { + modules, ok := yamlMap["modules"].([]interface{}) + if !ok { + return nil, ConfigError("the modules in your yaml is not valid") + } + var s []string + for _, module := range modules { + m := module.(string) + s = append(s, m) + } + return s, nil +} + +// getModules extracts backend components from map +func getBackend(yamlMap map[string]interface{}) (map[string]string, error) { + m := map[string]string{} + if computing, ok := yamlMap["computing"].(string); ok { + m["computing"] = computing + } else { + return nil, ConfigError("computing error, not found") + } + if federation, ok := yamlMap["federation"].(string); ok { + m["federation"] = federation + } else { + return nil, ConfigError("federation error, not found") + } + if storage, ok := yamlMap["storage"].(string); ok { + m["storage"] = storage + } else { + return nil, ConfigError("storage error, not found") + } + return m, nil +} + +// checkCommonModules checks common modules' presence +func checkCommonModules(modules []string) (errs []error) { + common := []string{"mysql", "python", "fateboard", "client"} + for _, c := range common { + if !Contains(c, modules) { + errs = append(errs, ConfigError(fmt.Sprintf("common module %s not enabled", c))) + } + } + return +} + +// checkModuleBackend checks each module has correct backend mapping +func checkModuleBackend(modules []string, backend map[string]string) (errs []error) { + for _, m := range modules { + switch m { + case "spark": + if c := backend["computing"]; c != "Spark" { + errs = append(errs, + ConfigError("module spark shall work with computing Spark but "+c)) + } + case "hdfs": + if s := backend["storage"]; s != "HDFS" { + errs = append(errs, + ConfigError("module hdfs shall work with computing HDFS but "+s)) + } + case "nginx": + if c := backend["computing"]; !Contains(c, []string{"Spark", "Spark_local"}) { + errs = append(errs, + ConfigError("module nginx shall work with computing Spark/Spark_local but "+c)) + } + case "pulsar": + if f := backend["federation"]; f != "Pulsar" { + errs = append(errs, + ConfigError("module pulsar shall work with federation Pulsar but "+f)) + } + case "rabbitmq": + if f := backend["federation"]; f != "RabbitMQ" { + errs = append(errs, + ConfigError("module rabbitmq shall work with federation RabbitMQ but "+f)) + } + } + } + return +} + +// moduleValidator validates yaml from modules' view +func moduleValidator(m *ValidationManager) (errs []error) { + yamlMap := m.testTree.rawYamlMap + if chartName, err := getChartName(yamlMap); err != nil || chartName != "fate" { + // if chart is not fate, just skip this validation + return []error{} + } + backend, err := getBackend(yamlMap) + if err != nil { + return []error{err} + } + modules, err := getModules(yamlMap) + if err != nil { + return []error{err} + } + errs = append(errs, checkCommonModules(modules)...) + errs = append(errs, checkModuleBackend(modules, backend)...) + return +} + +// checkComputing checks computing has correct modules enabled +func checkComputing(backend map[string]string, modules []string) (errs []error) { + key := "computing" + switch c := backend[key]; c { + case "Eggroll": + if !Contains("rollsite", modules) { + errs = append(errs, ConfigError(fmt.Sprintf("%s %s shall work with module %s", + key, c, "rollsite"))) + } + if !Contains("clustermanager", modules) { + errs = append(errs, ConfigError(fmt.Sprintf("%s %s shall work with module %s", + key, c, "clustermanager"))) + } + if !Contains("nodemanager", modules) { + errs = append(errs, ConfigError(fmt.Sprintf("%s %s shall work with module %s", + key, c, "nodemanager"))) + } + case "Spark": + if !Contains("spark", modules) { + errs = append(errs, ConfigError(fmt.Sprintf("%s %s shall work with module %s", + key, c, "spark"))) + } + if !Contains("nginx", modules) { + errs = append(errs, ConfigError(fmt.Sprintf("%s %s shall work with module %s", + key, c, "nginx"))) + } + case "Spark_local": + if !Contains("nginx", modules) { + errs = append(errs, ConfigError(fmt.Sprintf("%s %s shall work with module %s", + key, c, "nginx"))) + } + default: + errs = append(errs, ConfigError(fmt.Sprintf("%s module %s is not supported", key, c))) + } + return +} + +// checkFederation checks federation has correct modules enabled +func checkFederation(backend map[string]string, modules []string) (errs []error) { + key := "federation" + switch f := backend[key]; f { + case "Eggroll": + if !Contains("rollsite", modules) { + errs = append(errs, ConfigError(fmt.Sprintf("%s %s shall work with module %s", + key, f, "rollsite"))) + } + if !Contains("clustermanager", modules) { + errs = append(errs, ConfigError(fmt.Sprintf("%s %s shall work with module %s", + key, f, "clustermanager"))) + } + if !Contains("nodemanager", modules) { + errs = append(errs, ConfigError(fmt.Sprintf("%s %s shall work with module %s", + key, f, "nodemanager"))) + } + case "Pulsar": + if !Contains("pulsar", modules) { + errs = append(errs, ConfigError(fmt.Sprintf("%s %s shall work with module %s", + key, f, "pulsar"))) + } + case "RabbitMQ": + if !Contains("rabbitmq", modules) { + errs = append(errs, ConfigError(fmt.Sprintf("%s %s shall work with module %s", + key, f, "rabbitmq"))) + } + default: + errs = append(errs, ConfigError(fmt.Sprintf("%s module %s is not supported", key, f))) + } + return +} + +// checkStorage checks storage has correct modules enabled +func checkStorage(backend map[string]string, modules []string) (errs []error) { + key := "storage" + switch s := backend[key]; s { + case "Eggroll": + if !Contains("rollsite", modules) { + errs = append(errs, ConfigError(fmt.Sprintf("%s %s shall work with module %s", + key, s, "rollsite"))) + } + if !Contains("clustermanager", modules) { + errs = append(errs, ConfigError(fmt.Sprintf("%s %s shall work with module %s", + key, s, "clustermanager"))) + } + if !Contains("nodemanager", modules) { + errs = append(errs, ConfigError(fmt.Sprintf("%s %s shall work with module %s", + key, s, "nodemanager"))) + } + case "HDFS": + if !Contains("hdfs", modules) { + errs = append(errs, ConfigError(fmt.Sprintf("%s %s shall work with module %s", + key, s, "hdfs"))) + } + case "LocalFS": + default: + errs = append(errs, ConfigError(fmt.Sprintf("%s module %s is not supported", key, s))) + } + return +} + +func getChartName(m map[string]interface{}) (string, error) { + chartName, ok := m["chartName"] + err := errors.New("chart name not found") + if !ok { + return "", err + } + name, ok := chartName.(string) + if !ok || name != "fate" { + return "", err + } + return name, nil +} + +// backendValidator validates yaml from backend' view +func backendValidator(m *ValidationManager) (errs []error) { + yamlMap := m.testTree.rawYamlMap + if chartName, err := getChartName(yamlMap); err != nil || chartName != "fate" { + // if chart is not fate, just skip this validation + return []error{} + } + modules, err := getModules(yamlMap) + if err != nil { + return []error{err} + } + + backend, err := getBackend(yamlMap) + if err != nil { + return []error{err} + } + errs = append(errs, checkComputing(backend, modules)...) + errs = append(errs, checkFederation(backend, modules)...) + errs = append(errs, checkStorage(backend, modules)...) + return +} + +// ContainsSkipError returns if error list has SkipError +func ContainsSkipError(errs []error) bool { + var skip SkipError + for _, e := range errs { + if errors.As(e, &skip) { + return true + } + } + return false +} + +// RegisterPreprocessor registers a callback +func (m *ValidationManager) RegisterPreprocessor(p func(m *ValidationManager) []error) { + m.preprocessor = append(m.preprocessor, p) +} + +// preprocess runs callbacks +func (m *ValidationManager) preprocess() (errs []error) { + for _, p := range m.preprocessor { + errs = append(errs, p(m)...) + } + return +} + +// ValidateYaml validates the yaml file. +func ValidateYaml(templateValue, testValue string, skippedKeys []string, + preprocessors ...func(m *ValidationManager) []error) (errs []error) { + if templateValue == "" || testValue == "" { + return []error{SkipError("template or test yaml is empty")} + } + templateTree, err := buildValidationTree(templateValue, true, false) + if err != nil { + return []error{err} + } + testTree, err := buildValidationTree(testValue, false, true) + if err != nil { + return []error{err} + } + m := &ValidationManager{templateTree, testTree, skippedKeys, preprocessors} + if m.templateTree == nil || m.testTree == nil { + return []error{ConfigError("building validation tree failed")} + } + m.RegisterPreprocessor(moduleValidator) + m.RegisterPreprocessor(backendValidator) + errs = append(errs, m.preprocess()...) + errs = append(errs, m.compareTwoTrees()...) + return +} diff --git a/k8s-deploy/pkg/cli/validation_test.go b/k8s-deploy/pkg/cli/validation_test.go new file mode 100644 index 000000000..5415095f7 --- /dev/null +++ b/k8s-deploy/pkg/cli/validation_test.go @@ -0,0 +1,631 @@ +package cli + +import ( + "errors" + "fmt" + "reflect" + "testing" +) + +var ( + s0 = ` +chartName: fate +chartVersion: 1.9.0 +` + + s1 = ` +chartName: fate +chartVersion: 1.9.0 +skippedKeys: + - str +a: + b: 2 + c: [3, 4] +` + + s2 = ` +chartName: fate +chartVersion: 1.9.0 +skippedKeys: + - str +a: + b: 2 +` + + s3 = ` +chartName: fate +chartVersion: 1.9.0 +skippedKeys: + - str +a: + b: 2 + d: 3 +` + + s4 = ` +chartName: fate +chartVersion: 1.9.0 +skippedKeys: + - d +a: + b: 2 + d: 3 +` + + s5 = ` +backend: eggroll +modules: + - rollsite + - clustermanager +python: +` + + s6 = ` +backend: spark_rabbitmq +modules: + - pulsar + - rabbitmq +rollsite: +` + s7 = ` +modules: + - mysql + - python + - fateboard + - client + - pulsar + - hdfs + +computing: Spark +federation: Eggroll +storage: HDFS +` + + s8 = ` +modules: + - pulsar + - rabbitmq + +computing: Eggroll +federation: Pulsar +storage: HDFS +` + + s9 = ` +a: 1 +b: +- 2 +c: +- d: 1 +- d: 2 +` + + s10 = ` +a: +b: +c: +- e: +- d: +` + + sEggroll = ` +modules: + - rollsite + - clustermanager + - nodemanager + - mysql + - python + - fateboard + - client + +computing: Eggroll +federation: Eggroll +storage: Eggroll +` + + sSparkMQ = ` +modules: + - python + - mysql + - fateboard + - client + - spark + - hdfs + - nginx + - rabbitmq + +computing: Spark +federation: RabbitMQ +storage: HDFS +` + + sSparkPulsar = ` +modules: + - python + - mysql + - fateboard + - client + - spark + - hdfs + - nginx + - pulsar + +computing: Spark +federation: Pulsar +storage: HDFS +` + + sSparkLocalPulsar = ` +modules: + - python + - mysql + - fateboard + - client + - nginx + - pulsar + +computing: Spark_local +federation: Pulsar +storage: LocalFS +` +) + +var ( + m0 = map[string]interface{}{ + "chartName": "fate", + "chartVersion": "1.9.0", + } + m1 = map[string]interface{}{ + "chartName": "fate", + "chartVersion": "1.9.0", + "skippedKeys": []interface{}{"str"}, + "a": map[string]interface{}{ + "b": float64(2), + "c": []interface{}{float64(3), float64(4)}, + }, + } + m2 = map[string]interface{}{ + "chartName": "fate", + "chartVersion": "1.9.0", + "skippedKeys": []interface{}{"str"}, + "a": map[string]interface{}{ + "b": float64(2), + }, + } + m3 = map[string]interface{}{ + "chartName": "fate", + "chartVersion": "1.9.0", + "skippedKeys": []interface{}{"str"}, + "a": map[string]interface{}{ + "b": float64(2), + "d": float64(3), + }, + } + m4 = map[string]interface{}{ + "chartName": "fate", + "chartVersion": "1.9.0", + "skippedKeys": []interface{}{"d"}, + "a": map[string]interface{}{ + "b": float64(2), + "d": float64(3), + }, + } +) + +func Test_trimComments(t *testing.T) { + type args struct { + t []byte + } + tests := []struct { + name string + args args + want []byte + }{ + {"trimComment", args{[]byte("# comment\n")}, []byte("comment\n")}, + {"trimComments", args{[]byte(" # # # comment")}, []byte(" comment")}, + {"trimLeftComments", args{[]byte(" # # # comment # f")}, []byte(" comment # f")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := trimComments(tt.args.t); !reflect.DeepEqual(got, tt.want) { + t.Errorf("trimComments() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_deconstructKey(t *testing.T) { + type args struct { + k interface{} + } + tests := []struct { + name string + args args + want string + want1 int + }{ + {"deconstructKey", args{interface{}("key")}, "key", 0}, + {"deconstructKeyWithLineno", args{interface{}("key__lineno__21")}, "key", 21}, + {"deconstructNotValidKey", args{interface{}(21)}, "21", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := deconstructKey(tt.args.k) + if got != tt.want { + t.Errorf("deconstructKey() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("deconstructKey() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func Test_versionValid(t *testing.T) { + type args struct { + chartVersion string + startVersion []int + } + tests := []struct { + name string + args args + wantValid bool + }{ + {"versionEqual", args{"1.0.0", []int{1, 0, 0}}, true}, + {"versionLower", args{"1.0.0", []int{1, 1, 0}}, false}, + {"versionHigher", args{"1.2.0", []int{1, 1, 0}}, true}, + {"versionWithV", args{"v1.2.0", []int{1, 1, 0}}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if gotValid := versionValid(tt.args.chartVersion, tt.args.startVersion); gotValid != tt.wantValid { + t.Errorf("versionValid() = %v, want %v", gotValid, tt.wantValid) + } + }) + } +} + +func TestGetValueTemplateExample(t *testing.T) { + type args struct { + chartName string + chartVersion string + } + tests := []struct { + name string + args args + wantValue string + wantErr bool + }{ + {"getLowVersionFateValueTemplateExample", args{"fate", "1.8.0"}, "", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotValue, err := GetValueTemplateExample(tt.args.chartName, tt.args.chartVersion) + if (err != nil) != tt.wantErr { + t.Errorf("GetValueTemplateExample() error = %v, wantErr %v", err, tt.wantErr) + return + } + if gotValue != tt.wantValue { + t.Errorf("GetValueTemplateExample() = %v, want %v", gotValue, tt.wantValue) + } + }) + } +} + +func Test_bufferToMap(t *testing.T) { + type args struct { + buffer []byte + } + tests := []struct { + name string + args args + wantM map[string]interface{} + wantErr bool + }{ + {"bufferToMap0", args{[]byte(s0)}, m0, false}, + {"bufferToMap1", args{[]byte(s1)}, m1, false}, + {"bufferToMap2", args{[]byte(s2)}, m2, false}, + {"bufferToMap3", args{[]byte(s3)}, m3, false}, + {"bufferToMap4", args{[]byte(s4)}, m4, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotM, err := bufferToMap(tt.args.buffer) + if (err != nil) != tt.wantErr { + t.Errorf("bufferToMap() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotM, tt.wantM) { + t.Errorf("bufferToMap() = %v, want %v", gotM, tt.wantM) + } + }) + } +} + +func TestValidateYaml(t *testing.T) { + type args struct { + templateValue string + testValue string + skippedKeys []string + } + tests := []struct { + name string + args args + wantErrs []error + }{ + {"validateEmptyYaml", args{"", "", nil}, []error{SkipError("template or test yaml is empty")}}, + {"validateSameYaml", args{s1, s1, nil}, []error{ConfigError("computing error, not found"), ConfigError("the modules in your yaml is not valid")}}, + {"validateValidYaml", args{s1, s2, nil}, []error{ConfigError("computing error, not found"), ConfigError("the modules in your yaml is not valid")}}, + {"validateNotValidYaml", args{s1, s3, nil}, []error{ConfigError("computing error, not found"), ConfigError("the modules in your yaml is not valid"), ConfigError("your yaml at '/a/d', line 8 \n 'd: 3' may be redundant")}}, + {"validateYamlWithskippedKeys", args{s1, s4, []string{"d"}}, []error{ConfigError("computing error, not found"), ConfigError("the modules in your yaml is not valid")}}, + {"validateTestEmptyValue", args{s9, s10, nil}, []error{ConfigError("your yaml at '/c/@ArrayItem/e', line 5 \n '- e:' may be redundant")}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if gotErrs := ValidateYaml(tt.args.templateValue, tt.args.testValue, tt.args.skippedKeys); !reflect.DeepEqual(gotErrs, tt.wantErrs) { + if len(gotErrs) == 0 && len(tt.wantErrs) == 0 { + return + } + t.Errorf("ValidateYaml() = %v, want %v", gotErrs, tt.wantErrs) + } + }) + } +} + +func Test_getSkippedKeys(t *testing.T) { + type args struct { + m map[string]interface{} + } + m0, _ := bufferToMap([]byte(s0)) + m4, _ := bufferToMap([]byte(s4)) + tests := []struct { + name string + args args + wantSkippedKeys []string + }{ + {"getSkippedKeys", args{m0}, nil}, + {"getEmptySkippedKeys", args{m4}, []string{"d"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if gotSkippedKeys := getSkippedKeys(tt.args.m); !reflect.DeepEqual(gotSkippedKeys, tt.wantSkippedKeys) { + t.Errorf("getSkippedKeys() = %v, want %v", gotSkippedKeys, tt.wantSkippedKeys) + } + }) + } +} + +func Test_getModules(t *testing.T) { + type args struct { + yamlMap map[string]interface{} + } + m4, _ := bufferToMap([]byte(s4)) + m5, _ := bufferToMap([]byte(s5)) + tests := []struct { + name string + args args + want []string + wantErr bool + }{ + {"no modules", args{m4}, nil, true}, + {"with modules", args{m5}, []string{"rollsite", "clustermanager"}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getModules(tt.args.yamlMap) + if (err != nil) != tt.wantErr { + t.Errorf("getModules() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getModules() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getBackend(t *testing.T) { + type args struct { + yamlMap map[string]interface{} + } + + m6, _ := bufferToMap([]byte(s6)) + m7, _ := bufferToMap([]byte(s7)) + tests := []struct { + name string + args args + want map[string]string + wantErr bool + }{ + {"no backends", args{m6}, nil, true}, + {"With backends", args{m7}, map[string]string{ + "computing": "Spark", + "federation": "Eggroll", + "storage": "HDFS", + }, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getBackend(tt.args.yamlMap) + if (err != nil) != tt.wantErr { + t.Errorf("getBackend() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getBackend() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_checkCommonModules(t *testing.T) { + type args struct { + modules []string + } + common := []string{"mysql", "python", "fateboard", "client"} + errs := []error{} + for _, c := range common { + errs = append(errs, ConfigError(fmt.Sprintf("common module %s not enabled", c))) + } + tests := []struct { + name string + args args + wantErrs []error + }{ + {"no backends", args{nil}, errs}, + {"full backends", args{common}, nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if gotErrs := checkCommonModules(tt.args.modules); !reflect.DeepEqual(gotErrs, tt.wantErrs) { + t.Errorf("checkCommonModules() = %v, want %v", gotErrs, tt.wantErrs) + } + }) + } +} + +func sToModuleBackend(s string) (map[string]string, []string) { + m, _ := bufferToMap([]byte(s)) + backend, _ := getBackend(m) + module, _ := getModules(m) + return backend, module +} + +var ( + backend7, module7 = sToModuleBackend(s7) + backend8, module8 = sToModuleBackend(s8) + backendEggroll, moduleEggroll = sToModuleBackend(sEggroll) + backendSparkMQ, moduleSparkMQ = sToModuleBackend(sSparkMQ) + backendSparkPulsar, moduleSparkPulsar = sToModuleBackend(sSparkPulsar) + backendSparkLocalPulsar, moduleSparkLocalPulsar = sToModuleBackend(sSparkLocalPulsar) +) + +func Test_checkModuleBackend(t *testing.T) { + type args struct { + modules []string + backend map[string]string + } + tests := []struct { + name string + args args + wantErrs []error + }{ + {"7", args{module7, backend7}, []error{ConfigError("module pulsar shall work with federation Pulsar but Eggroll")}}, + {"8", args{module8, backend8}, []error{ConfigError("module rabbitmq shall work with federation RabbitMQ but Pulsar")}}, + {"Eggroll", args{moduleEggroll, backendEggroll}, nil}, + {"SparkMQ", args{moduleSparkMQ, backendSparkMQ}, nil}, + {"SparkPulsar", args{moduleSparkPulsar, backendSparkPulsar}, nil}, + {"SparkLocalPulsar", args{moduleSparkLocalPulsar, backendSparkLocalPulsar}, nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if gotErrs := checkModuleBackend(tt.args.modules, tt.args.backend); !reflect.DeepEqual(gotErrs, tt.wantErrs) { + t.Errorf("checkModuleBackend() = %v, want %v", gotErrs, tt.wantErrs) + } + }) + } +} + +func Test_checkComputing(t *testing.T) { + type args struct { + backend map[string]string + modules []string + } + tests := []struct { + name string + args args + wantErrs []error + }{ + {"7", args{backend7, module7}, []error{ConfigError("computing Spark shall work with module spark"), ConfigError("computing Spark shall work with module nginx")}}, + {"8", args{backend8, module8}, []error{ConfigError("computing Eggroll shall work with module rollsite"), ConfigError("computing Eggroll shall work with module clustermanager"), ConfigError("computing Eggroll shall work with module nodemanager")}}, + {"Eggroll", args{backendEggroll, moduleEggroll}, nil}, + {"SparkMQ", args{backendSparkMQ, moduleSparkMQ}, nil}, + {"SparkPulsar", args{backendSparkPulsar, moduleSparkPulsar}, nil}, + {"SparkLocalPulsar", args{backendSparkLocalPulsar, moduleSparkLocalPulsar}, nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if gotErrs := checkComputing(tt.args.backend, tt.args.modules); !reflect.DeepEqual(gotErrs, tt.wantErrs) { + t.Errorf("checkComputing() = %v, want %v", gotErrs, tt.wantErrs) + } + }) + } +} + +func Test_checkFederation(t *testing.T) { + type args struct { + backend map[string]string + modules []string + } + tests := []struct { + name string + args args + wantErrs []error + }{ + {"7", args{backend7, module7}, []error{ConfigError("federation Eggroll shall work with module rollsite"), ConfigError("federation Eggroll shall work with module clustermanager"), ConfigError("federation Eggroll shall work with module nodemanager")}}, + {"8", args{backend8, module8}, nil}, + {"Eggroll", args{backendEggroll, moduleEggroll}, nil}, + {"SparkMQ", args{backendSparkMQ, moduleSparkMQ}, nil}, + {"SparkPulsar", args{backendSparkPulsar, moduleSparkPulsar}, nil}, + {"SparkLocalPulsar", args{backendSparkLocalPulsar, moduleSparkLocalPulsar}, nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if gotErrs := checkFederation(tt.args.backend, tt.args.modules); !reflect.DeepEqual(gotErrs, tt.wantErrs) { + t.Errorf("checkFederation() = %v, want %v", gotErrs, tt.wantErrs) + } + }) + } +} + +func Test_checkStorage(t *testing.T) { + type args struct { + backend map[string]string + modules []string + } + tests := []struct { + name string + args args + wantErrs []error + }{ + {"7", args{backend7, module7}, nil}, + {"8", args{backend8, module8}, []error{ConfigError("storage HDFS shall work with module hdfs")}}, + {"Eggroll", args{backendEggroll, moduleEggroll}, nil}, + {"SparkMQ", args{backendSparkMQ, moduleSparkMQ}, nil}, + {"SparkPulsar", args{backendSparkPulsar, moduleSparkPulsar}, nil}, + {"SparkLocalPulsar", args{backendSparkLocalPulsar, moduleSparkLocalPulsar}, nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if gotErrs := checkStorage(tt.args.backend, tt.args.modules); !reflect.DeepEqual(gotErrs, tt.wantErrs) { + t.Errorf("checkStorage() = %v, want %v", gotErrs, tt.wantErrs) + } + }) + } +} + +func TestContainsSkipError(t *testing.T) { + type args struct { + errs []error + } + e1 := errors.New("") + e2 := ConfigError("") + e3 := SkipError("") + e4 := fmt.Errorf("error :%w ", e3) + tests := []struct { + name string + args args + want bool + }{ + {"without skipErrors", args{[]error{e1, e2}}, false}, + {"with skipErrors", args{[]error{e3}}, true}, + {"with wrappedSkipErrors", args{[]error{e4}}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ContainsSkipError(tt.args.errs); got != tt.want { + t.Errorf("ContainsSkipError() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/k8s-deploy/pkg/cli/version.go b/k8s-deploy/pkg/cli/version.go index 82a00e3b2..0f8330594 100644 --- a/k8s-deploy/pkg/cli/version.go +++ b/k8s-deploy/pkg/cli/version.go @@ -56,7 +56,12 @@ func GetServiceVersion() (string, error) { if serviceUrl == "" { serviceUrl = "localhost:8080/" } - Url := "http://" + serviceUrl + "/" + apiVersion + r.Path + safeconnect := viper.GetString("safeconnect") + scheme := "http://" + if safeconnect == "true" { + scheme = "https://" + } + Url := scheme + serviceUrl + "/" + apiVersion + r.Path body := bytes.NewReader(r.Body) log.Debug().Str("Type", r.Type).Str("url", Url).Msg("Request") request, err := http.NewRequest(r.Type, Url, body) diff --git a/k8s-deploy/pkg/job/cluster_install.go b/k8s-deploy/pkg/job/cluster_install.go index 5416ec1b9..50f4bd5a8 100644 --- a/k8s-deploy/pkg/job/cluster_install.go +++ b/k8s-deploy/pkg/job/cluster_install.go @@ -1,5 +1,5 @@ /* - * Copyright 2019-2021 VMware, Inc. + * Copyright 2019-2022 VMware, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/k8s-deploy/pkg/job/fate_upgrade_manager.go b/k8s-deploy/pkg/job/fate_upgrade_manager.go new file mode 100644 index 000000000..67e1d96e0 --- /dev/null +++ b/k8s-deploy/pkg/job/fate_upgrade_manager.go @@ -0,0 +1,130 @@ +/* + * Copyright 2019-2022 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package job + +import ( + "errors" + "fmt" + "github.com/FederatedAI/KubeFATE/k8s-deploy/pkg/modules" + "github.com/FederatedAI/KubeFATE/k8s-deploy/pkg/service" + "github.com/hashicorp/go-version" + "github.com/rs/zerolog/log" + "reflect" + "strings" + "time" +) + +const ( + fumChartName = "fate-upgrade-manager" + fumJobName = "fate-mysql-upgrade-job" +) + +type FateUpgradeManager struct { + namespace string + UpgradeManager +} + +func (fum *FateUpgradeManager) validate(specOld, specNew modules.MapStringInterface) error { + oldChartName := specOld["chartName"].(string) + newChartName := specNew["chartName"].(string) + oldVersion := specOld["chartVersion"].(string) + newVersion := specNew["chartVersion"].(string) + + // Chart must be the same fml framework + if oldChartName != newChartName { + return errors.New("doesn't support upgrade between different charts") + } + + // To upgrade, chart must have some difference + if reflect.DeepEqual(specOld, specNew) && oldVersion == newVersion { + return errors.New("the configuration file did not change") + } + + // Do not support downgrade + newVerFormatted, _ := version.NewVersion(newVersion) + oldVerFormatted, _ := version.NewVersion(oldVersion) + if newVerFormatted.LessThan(oldVerFormatted) { + return errors.New("using kubefate to downgrade a cluster is not supported yet") + } + + // KubeFATE cannot support rolling upgrade for FATE version <= 1.7.1 + if newVersion != oldVersion { + ver171, _ := version.NewVersion("1.7.1") + if oldVerFormatted.LessThanOrEqual(ver171) { + return errors.New("upgrade from FATE version <= 1.7.1 is not supported by KubeFATE") + } + } + log.Info().Msg("version validation for FATE cluster yaml passed") + return nil +} + +func (fum *FateUpgradeManager) getCluster(specOld, specNew modules.MapStringInterface) modules.Cluster { + fumCluster := modules.Cluster{ + Name: fumChartName, + NameSpace: fum.namespace, + ChartName: fumChartName, + ChartVersion: "v1.0.0", + Spec: constructFumSpec(specOld, specNew), + } + return fumCluster +} + +func (fum *FateUpgradeManager) waitFinish(interval, round int) bool { + var cycle int + for cycle = 0; cycle < round; cycle++ { + jobDone, err := service.CheckJobReadiness(fum.namespace, fumJobName) + if err != nil { + log.Error().Err(err).Msg("failed to check the upgrade job status") + } + if jobDone { + log.Info().Msgf("the upgrade manager finished its job in %d seconds", interval*cycle) + return true + } + log.Info().Msgf("the upgrade manager's job is not done yet, will recheck in %d seconds", interval) + time.Sleep(time.Second * time.Duration(interval)) + } + errMsg := fmt.Sprintf("the upgrade manager cannot finish the job in %d seconds", 30*cycle) + log.Error().Msg(errMsg) + return false +} + +func getMysqlCredFromSpec(clusterSpec modules.MapStringInterface) (username, password string) { + mysqlSpec := clusterSpec["mysql"].(map[string]interface{}) + if mysqlSpec["user"] == nil { + username = "fate" + } else { + username = mysqlSpec["user"].(string) + } + if mysqlSpec["password"] == nil { + password = "fate_dev" + } else { + password = mysqlSpec["password"].(string) + } + return +} + +func constructFumSpec(oldSpec, newSpec modules.MapStringInterface) (fumSpec modules.MapStringInterface) { + oldVersion := strings.ReplaceAll(oldSpec["chartVersion"].(string), "v", "") + newVersion := strings.ReplaceAll(newSpec["chartVersion"].(string), "v", "") + mysqlUsername, mysqlPassword := getMysqlCredFromSpec(newSpec) + res := modules.MapStringInterface{ + "username": mysqlUsername, + "password": mysqlPassword, + "start": oldVersion, + "target": newVersion, + } + return res +} diff --git a/k8s-deploy/pkg/job/fate_upgrade_manager_test.go b/k8s-deploy/pkg/job/fate_upgrade_manager_test.go new file mode 100644 index 000000000..ff9853c1e --- /dev/null +++ b/k8s-deploy/pkg/job/fate_upgrade_manager_test.go @@ -0,0 +1,95 @@ +/* + * Copyright 2019-2022 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package job + +import ( + "github.com/FederatedAI/KubeFATE/k8s-deploy/pkg/modules" + "github.com/stretchr/testify/assert" + "reflect" + "testing" +) + +func TestValidate(t *testing.T) { + specOld := modules.MapStringInterface{ + "chartName": "fate", + "chartVersion": "v1.7.2", + } + specNew := modules.MapStringInterface{ + "chartName": "fate", + "chartVersion": "v1.8.0", + } + fum := FateUpgradeManager{ + namespace: "blabla", + } + // Happy path + err := fum.validate(specOld, specNew) + assert.Nil(t, err) + + // framework different + specNew["chartName"] = "openfl" + err = fum.validate(specOld, specNew) + assert.NotNil(t, err) + specNew["chartName"] = "fate" + + // spec identical + specNew["chartVersion"] = "v1.7.2" + err = fum.validate(specOld, specNew) + assert.NotNil(t, err) + specNew["chartVersion"] = "v1.8.0" + + // do not support downgrade + specNew["chartVersion"] = "v1.6.0" + err = fum.validate(specOld, specNew) + assert.NotNil(t, err) + specNew["chartVersion"] = "v1.8.0" + + // fate version < 1.7.1 + specOld["chartVersion"] = "v1.7.0" + err = fum.validate(specOld, specNew) + assert.NotNil(t, err) +} + +func TestGetCluster(t *testing.T) { + specOld := modules.MapStringInterface{ + "chartName": "fate", + "chartVersion": "v1.7.2", + } + specNew := modules.MapStringInterface{ + "chartName": "fate", + "chartVersion": "v1.8.0", + "mysql": map[string]interface{}{ + "user": "fate", + "password": "fate_dev", + }, + } + fum := FateUpgradeManager{ + namespace: "blabla", + } + actual := fum.getCluster(specOld, specNew) + expect := modules.Cluster{ + Name: "fate-upgrade-manager", + NameSpace: "blabla", + ChartName: "fate-upgrade-manager", + ChartVersion: "v1.0.0", + Spec: modules.MapStringInterface{ + "password": "fate_dev", + "username": "fate", + "start": "1.7.2", + "target": "1.8.0", + }, + } + assert.True(t, reflect.DeepEqual(actual, expect)) +} diff --git a/k8s-deploy/pkg/job/job.go b/k8s-deploy/pkg/job/job.go index 8552185af..2c07596c8 100644 --- a/k8s-deploy/pkg/job/job.go +++ b/k8s-deploy/pkg/job/job.go @@ -1,5 +1,5 @@ /* - * Copyright 2019-2021 VMware, Inc. + * Copyright 2019-2022 VMware, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,7 +18,6 @@ package job import ( "fmt" - "reflect" "time" "github.com/FederatedAI/KubeFATE/k8s-deploy/pkg/modules" @@ -26,6 +25,10 @@ import ( "github.com/rs/zerolog/log" ) +const ( + fateChartName = "fate" +) + func stopJob(job *modules.Job, cluster *modules.Cluster) bool { if !cluster.IsExisted(cluster.Name, cluster.NameSpace) { return true @@ -145,10 +148,16 @@ func ClusterUpdate(clusterArgs *modules.ClusterArgs, creator string) (*modules.J var valuesOld = cluster.Values var valuesNew = clusterNew.Values - if reflect.DeepEqual(specOld, specNew) && - cluster.ChartName == clusterArgs.ChartName && - cluster.ChartVersion == clusterArgs.ChartVersion { - return nil, fmt.Errorf("the configuration file did not change") + var um UpgradeManager + switch cluster.ChartName { + case fateChartName: + um = &FateUpgradeManager{ + namespace: clusterArgs.Namespace, + } + } + err = um.validate(specOld, specNew) + if err != nil { + return nil, err } job := modules.NewJob(clusterArgs, "ClusterUpdate", creator, cluster.Uuid) @@ -174,6 +183,39 @@ func ClusterUpdate(clusterArgs *modules.ClusterArgs, creator string) (*modules.J log.Error().Err(dbErr).Msg("Cluster.SetStatus error") } + if specOld["chartVersion"].(string) != specNew["chartVersion"].(string) { + umCluster := um.getCluster(specOld, specNew) + // We will implicitly install a new cluster for the upgrade manager, and delete it after it finishes its job + err := umCluster.HelmInstall() + if err != nil { + log.Error().Err(err).Msgf("failed to install the upgrade manager's helm chart for cluster %s", cluster.ChartName) + dbErr := job.SetStatus(modules.JobStatusFailed) + if dbErr != nil { + log.Error().Err(dbErr).Msg("job.SetStatus error") + } + job.Status = modules.JobStatusFailed + log.Error().Msg("abort upgrade because failed to install upgrade manager") + return + } + finished := um.waitFinish(30, 20) + if !finished { + dbErr := job.SetStatus(modules.JobStatusFailed) + if dbErr != nil { + log.Error().Err(dbErr).Msg("job.SetStatus error") + } + } + if !clusterArgs.KeepUpgradeJob { + err = umCluster.HelmDelete() + if err != nil { + log.Error().Err(err).Msg("failed to delete the upgrade manager cluster, need a person to investigate why") + } + } + if job.Status == modules.JobStatusFailed { + log.Error().Msg("abort upgrade because upgrade manager cannot finish its job") + return + } + } + dbErr = cluster.SetValues(valuesNew) if dbErr != nil { log.Error().Err(dbErr).Msg("Cluster.SetSpec error") @@ -193,7 +235,7 @@ func ClusterUpdate(clusterArgs *modules.ClusterArgs, creator string) (*modules.J cluster.HelmRevision += 1 _, dbErr = cluster.UpdateByUuid(job.ClusterId) - if err != nil { + if dbErr != nil { log.Error().Err(dbErr).Interface("cluster", cluster).Msg("Update Cluster error") } @@ -221,7 +263,6 @@ func ClusterUpdate(clusterArgs *modules.ClusterArgs, creator string) (*modules.J } } - // for job.Status == modules.JobStatusRunning { if stopJob(job, &cluster) { continue @@ -282,7 +323,6 @@ func ClusterUpdate(clusterArgs *modules.ClusterArgs, creator string) (*modules.J // rollBACK if job.Status != modules.JobStatusSuccess && job.Status != modules.JobStatusCanceled { - //todo helm rollBack dbErr = cluster.SetValues(valuesOld) if dbErr != nil { log.Error().Err(dbErr).Msg("Cluster.SetSpec error") diff --git a/k8s-deploy/pkg/job/upgrade_manager.go b/k8s-deploy/pkg/job/upgrade_manager.go new file mode 100644 index 000000000..9611b39da --- /dev/null +++ b/k8s-deploy/pkg/job/upgrade_manager.go @@ -0,0 +1,24 @@ +/* + * Copyright 2019-2022 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package job + +import "github.com/FederatedAI/KubeFATE/k8s-deploy/pkg/modules" + +type UpgradeManager interface { + validate(specOld, specNew modules.MapStringInterface) error + getCluster(specOld, specNew modules.MapStringInterface) modules.Cluster + waitFinish(interval, round int) bool +} diff --git a/k8s-deploy/pkg/modules/helm_chart.go b/k8s-deploy/pkg/modules/helm_chart.go index 037f478d6..57c75b91c 100644 --- a/k8s-deploy/pkg/modules/helm_chart.go +++ b/k8s-deploy/pkg/modules/helm_chart.go @@ -26,14 +26,15 @@ import ( // HelmChart Helm Chart model type HelmChart struct { - Uuid string `json:"uuid" gorm:"type:varchar(36);index;unique"` - Name string `json:"name" gorm:"type:varchar(16);not null"` - Chart string `json:"chart" gorm:"type:text;not null"` - Values string `json:"values" gorm:"type:text;not null"` - ValuesTemplate string `json:"values_template" gorm:"type:text;not null"` - Templates Templates `json:"templates" gorm:"type:mediumblob" swaggerignore:"true"` - Version string `json:"version" gorm:"type:varchar(32);not null"` - AppVersion string `json:"app_version" gorm:"type:varchar(32);not null"` + Uuid string `json:"uuid" gorm:"type:varchar(36);index;unique"` + Name string `json:"name" gorm:"type:varchar(16);not null"` + Chart string `json:"chart" gorm:"type:text;not null"` + Values string `json:"values" gorm:"type:text;not null"` + ValuesTemplate string `json:"values_template" gorm:"type:text;not null"` + ValuesTemplateExample string `json:"values_template_example" gorm:"type:text;not null"` + Templates Templates `json:"templates" gorm:"type:mediumblob" swaggerignore:"true"` + Version string `json:"version" gorm:"type:varchar(32);not null"` + AppVersion string `json:"app_version" gorm:"type:varchar(32);not null"` gorm.Model } diff --git a/k8s-deploy/pkg/modules/helm_chart_special.go b/k8s-deploy/pkg/modules/helm_chart_special.go index 7af5166c2..4888bd451 100644 --- a/k8s-deploy/pkg/modules/helm_chart_special.go +++ b/k8s-deploy/pkg/modules/helm_chart_special.go @@ -39,6 +39,7 @@ func (e *HelmChart) ChartRequestedToHelmChart(chartRequested *chart.Chart) (*Hel var chartData string var valuesData string var ValuesTemplate string + var ValuesTemplateExample string for _, v := range chartRequested.Raw { if v.Name == "Chart.yaml" { chartData = string(v.Data) @@ -49,12 +50,16 @@ func (e *HelmChart) ChartRequestedToHelmChart(chartRequested *chart.Chart) (*Hel if v.Name == "values-template.yaml" { ValuesTemplate = string(v.Data) } + if v.Name == "values-template-example.yaml" { + ValuesTemplateExample = string(v.Data) + } } helmChart := NewHelmChart(chartRequested.Name(), chartData, valuesData, chartRequested.Templates, chartRequested.Metadata.Version, chartRequested.AppVersion()) helmChart.ValuesTemplate = ValuesTemplate + helmChart.ValuesTemplateExample = ValuesTemplateExample return helmChart, nil } @@ -103,6 +108,7 @@ func ChartRequestedToHelmChart(chartRequested *chart.Chart) (*HelmChart, error) var chartData string var valuesData string var ValuesTemplate string + var ValuesTemplateExample string for _, v := range chartRequested.Raw { if v.Name == "Chart.yaml" { chartData = string(v.Data) @@ -113,12 +119,16 @@ func ChartRequestedToHelmChart(chartRequested *chart.Chart) (*HelmChart, error) if v.Name == "values-template.yaml" { ValuesTemplate = string(v.Data) } + if v.Name == "values-template-example.yaml" { + ValuesTemplateExample = string(v.Data) + } } helmChart := NewHelmChart(chartRequested.Name(), chartData, valuesData, chartRequested.Templates, chartRequested.Metadata.Version, chartRequested.AppVersion()) helmChart.ValuesTemplate = ValuesTemplate + helmChart.ValuesTemplateExample = ValuesTemplateExample return helmChart, nil } diff --git a/k8s-deploy/pkg/modules/job.go b/k8s-deploy/pkg/modules/job.go index 91e0b5205..ce2e1bd17 100644 --- a/k8s-deploy/pkg/modules/job.go +++ b/k8s-deploy/pkg/modules/job.go @@ -1,5 +1,5 @@ /* - * Copyright 2019-2021 VMware, Inc. + * Copyright 2019-2022 VMware, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -55,12 +55,13 @@ func (s *States) Scan(v interface{}) error { } type ClusterArgs struct { - Name string `json:"name"` - Namespace string `json:"namespace"` - ChartName string `json:"chart_name"` - ChartVersion string `json:"chart_version"` - Cover bool `json:"cover"` - Data []byte `json:"data"` + Name string `json:"name"` + Namespace string `json:"namespace"` + ChartName string `json:"chart_name"` + ChartVersion string `json:"chart_version"` + Cover bool `json:"cover"` + Data []byte `json:"data"` + KeepUpgradeJob bool `json:"Keep_upgrade_job"` } type SubJobs map[string]SubJob diff --git a/k8s-deploy/pkg/modules/job_db.go b/k8s-deploy/pkg/modules/job_db.go index fa1688519..8520f9c5e 100644 --- a/k8s-deploy/pkg/modules/job_db.go +++ b/k8s-deploy/pkg/modules/job_db.go @@ -145,6 +145,7 @@ func (e *Job) Delete() (bool, error) { } func (e *Job) SetStatus(status JobStatus) error { + e.Status = status if err := DB.Model(e).Update("status", status).Error; err != nil { return err } diff --git a/k8s-deploy/pkg/service/info.go b/k8s-deploy/pkg/service/info.go index a8151140e..83d6f7b81 100644 --- a/k8s-deploy/pkg/service/info.go +++ b/k8s-deploy/pkg/service/info.go @@ -22,7 +22,7 @@ func GetClusterInfo(name, namespace string) (map[string]interface{}, error) { ip, err := GetNodeIP() if err != nil { log.Error().Str("func", "GetNodeIP()").Err(err).Msg("GetNodeIP error") - return nil, err + ip = []string{err.Error()} } port, err := GetProxySvcNodePorts(name, getDefaultNamespace(namespace)) if err != nil { diff --git a/k8s-deploy/pkg/service/kube.go b/k8s-deploy/pkg/service/kube.go index 506c152e2..3d66dac78 100644 --- a/k8s-deploy/pkg/service/kube.go +++ b/k8s-deploy/pkg/service/kube.go @@ -1,5 +1,5 @@ /* - * Copyright 2019-2021 VMware, Inc. + * Copyright 2019-2022 VMware, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,6 +12,7 @@ * limitations under the License. * */ + package service import ( @@ -33,6 +34,7 @@ type kubeClient interface { kube.Services kube.Log kube.Deployment + kube.Job } var KubeClient kubeClient = &kube.KUBE diff --git a/k8s-deploy/pkg/service/kube/job.go b/k8s-deploy/pkg/service/kube/job.go new file mode 100644 index 000000000..dad8676c3 --- /dev/null +++ b/k8s-deploy/pkg/service/kube/job.go @@ -0,0 +1,31 @@ +/* + * Copyright 2019-2022 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package kube + +import ( + "context" + v1 "k8s.io/api/batch/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type Job interface { + GetJobByName(namespace, jobName string) (*v1.Job, error) +} + +func (e *Kube) GetJobByName(namespace, jobName string) (*v1.Job, error) { + job, err := e.client.BatchV1().Jobs(namespace).Get(context.Background(), jobName, metav1.GetOptions{}) + return job, err +} diff --git a/k8s-deploy/pkg/service/kube_job.go b/k8s-deploy/pkg/service/kube_job.go new file mode 100644 index 000000000..aa8bfa0d9 --- /dev/null +++ b/k8s-deploy/pkg/service/kube_job.go @@ -0,0 +1,27 @@ +/* + * Copyright 2019-2022 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package service + +func CheckJobReadiness(namespace, jobName string) (bool, error) { + job, err := KubeClient.GetJobByName(namespace, jobName) + if err != nil { + return false, err + } + if job.Status.Succeeded > 0 { + return true, err + } + return false, err +} diff --git a/k8s-deploy/pkg/service/kube_namespace.go b/k8s-deploy/pkg/service/kube_namespace.go index 0d14914f6..ae2299209 100644 --- a/k8s-deploy/pkg/service/kube_namespace.go +++ b/k8s-deploy/pkg/service/kube_namespace.go @@ -16,6 +16,7 @@ package service import ( + "github.com/pkg/errors" v1 "k8s.io/api/core/v1" ) @@ -23,7 +24,7 @@ import ( func GetNamespaces() ([]v1.Namespace, error) { namespaceList, err := KubeClient.GetNamespaces() if err != nil { - return nil, err + return nil, errors.Wrap(err, "Get namespaces failed. Please check if user has the privilege to list resource \"namespaces\" at the cluster scope") } return namespaceList.Items, nil } diff --git a/k8s-deploy/pkg/service/kube_node.go b/k8s-deploy/pkg/service/kube_node.go index 69a27c4e2..6247d95ea 100644 --- a/k8s-deploy/pkg/service/kube_node.go +++ b/k8s-deploy/pkg/service/kube_node.go @@ -15,12 +15,14 @@ package service +import "github.com/pkg/errors" + // GetNodeIP get a node ip func GetNodeIP() ([]string, error) { svcs, err := KubeClient.GetNodes("") if err != nil { - return nil, err + return nil, errors.Wrap(err, "Get node ip failed. Please check if user has the privilege to list resource \"nodes\" at the cluster scope") } var nodeIP []string diff --git a/k8s-deploy/pkg/utils/k8sclient/client.go b/k8s-deploy/pkg/utils/k8sclient/client.go new file mode 100644 index 000000000..4344627d7 --- /dev/null +++ b/k8s-deploy/pkg/utils/k8sclient/client.go @@ -0,0 +1,521 @@ +package k8sclient + +import ( + "context" + "fmt" + "io" + "path/filepath" + "strings" + "time" + + "github.com/gosuri/uitable" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/homedir" +) + +type Client struct { + *kubernetes.Clientset +} + +type Namespace struct { + Name string + Ingresses []*Ingress + Services []*Service + Deployments []*Deployment + Pods []*Pod +} + +type Container struct { + Name string + Status string + Ready bool + Image string + ImageID string + ContainerID string + Log string +} + +type Pod struct { + Name string + Status string + Ready string + Containers []*Container + CreationTimestamp time.Time +} + +type Deployment struct { + Name string + Replicas int + UpdatedReplicas int + ReadyReplicas int + AvailableReplicas int + UnavailableReplicas int +} + +type Service struct { + Name string + Type string + ClusterIP string + ExternalIPs []string + Ports []string + CreationTimestamp time.Time +} + +type HostPath struct { + Path string + PathType string + Backend string +} + +type IngressRule struct { + Host string + Path []HostPath +} + +type Ingress struct { + Name string + Class string + Rules []IngressRule + Address string + Ports []string + CreationTimestamp time.Time +} + +// NewK8sClient returns a k8s restClient +func NewK8sClient(kubeconfig string) (c *Client, err error) { + config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + return + } + client, err := kubernetes.NewForConfig(config) + c = &Client{client} + return +} + +// NamespaceListToNames extracts and returns namespace list +func NamespaceListToNames(list *v1.NamespaceList) []string { + var names []string + for _, i := range list.Items { + names = append(names, i.Name) + } + return names +} + +// PodListToNames extracts and returns podname list +func PodListToNames(list *v1.PodList) []string { + var names []string + for _, i := range list.Items { + names = append(names, i.Name) + } + return names +} + +// GetNamespaceList returns namespaceList from k8s api +func (c *Client) GetNamespaceList() (*v1.NamespaceList, error) { + return c.CoreV1().Namespaces().List(context.TODO(), metav1.ListOptions{}) +} + +// GetNamespaceList returns specific namespace from k8s api +func (c *Client) GetNamespace(namespace string) (*v1.Namespace, error) { + return c.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}) +} + +// ToNamespace tansforms v1.Namespace to Namespace +func ToNamespace(n *v1.Namespace) *Namespace { + return &Namespace{ + Name: n.Name, + } +} + +// DescribeNamespace gets and returns all msg in specific namespace +func (c *Client) DescribeNamespace( + namespace string, tail int, podNames ...string) (n *Namespace, err error) { + n = &Namespace{Name: namespace} + d, err := c.DescribeDeployments(namespace) + if err != nil { + return + } + n.Deployments = d + + i, err := c.DescribeIngresses(namespace) + if err != nil { + return + } + n.Ingresses = i + + s, err := c.DescribeServices(namespace) + if err != nil { + return + } + n.Services = s + + var pods []*Pod + for _, pod := range podNames { + p, err := c.DescribePod(namespace, pod, tail) + if err != nil { + continue + } + pods = append(pods, p) + } + n.Pods = pods + return +} + +// GetDeploymentList returns DeploymentList from k8s api +func (c *Client) GetDeploymentList(namespace string) (*appsv1.DeploymentList, error) { + return c.AppsV1().Deployments(namespace).List(context.TODO(), metav1.ListOptions{}) +} + +// ToDeployment tansforms appsv1.Deployment to Deployment +func ToDeployment(d *appsv1.Deployment) *Deployment { + return &Deployment{ + Name: d.Name, + Replicas: int(d.Status.Replicas), + UpdatedReplicas: int(d.Status.UpdatedReplicas), + ReadyReplicas: int(d.Status.ReadyReplicas), + AvailableReplicas: int(d.Status.AvailableReplicas), + UnavailableReplicas: int(d.Status.UnavailableReplicas), + } +} + +// DescribeDeployments returns DeploymentList +func (c *Client) DescribeDeployments(namespace string) ([]*Deployment, error) { + d, err := c.GetDeploymentList(namespace) + if err != nil { + return nil, err + } + var deployments []*Deployment + for _, i := range d.Items { + deployments = append(deployments, ToDeployment(&i)) + } + return deployments, nil +} + +// SprintlnDeployments returns tabular output of DeploymentList +func SprintlnDeployments(items []*Deployment) string { + table := uitable.New() + table.AddRow("Deployments") + table.AddRow("Name", "Replicas", "UpdatedReplicas", + "ReadyReplicas", "AvailableReplicas", "UnavailableReplicas") + for _, r := range items { + table.AddRow(r.Name, r.Replicas, r.UpdatedReplicas, + r.ReadyReplicas, r.AvailableReplicas, r.UnavailableReplicas) + } + table.AddRow("") + return fmt.Sprintln(table) +} + +// GetServiceList returns v1.ServiceList from k8s api +func (c *Client) GetServiceList(namespace string) (*v1.ServiceList, error) { + return c.CoreV1().Services(namespace).List(context.TODO(), metav1.ListOptions{}) +} + +// GetService returns specific v1.Service from k8s api +func (c *Client) GetService(namespace, service string) (*v1.Service, error) { + return c.CoreV1().Services(namespace).Get(context.TODO(), service, metav1.GetOptions{}) +} + +// ToPorts tansforms v1.ServicePort to ports string +func ToPorts(p []v1.ServicePort) []string { + var ports []string + for _, i := range p { + ports = append(ports, fmt.Sprintf("%d/%s", i.Port, i.Protocol)) + } + if len(ports) == 0 { + ports = append(ports, "") + } + return ports +} + +// ToService tansforms v1.Service to Service +func ToService(s *v1.Service) *Service { + return &Service{ + Name: s.Name, + Type: string(s.Spec.Type), + ClusterIP: s.Spec.ClusterIP, + ExternalIPs: s.Spec.ExternalIPs, + Ports: ToPorts(s.Spec.Ports), + CreationTimestamp: s.CreationTimestamp.Time, + } +} + +// DescribeServices returns ServiceList +func (c *Client) DescribeServices(namespace string) ([]*Service, error) { + s, err := c.GetServiceList(namespace) + if err != nil { + return nil, err + } + var services []*Service + for _, i := range s.Items { + services = append(services, ToService(&i)) + } + return services, nil +} + +// SprintlnDeployments returns tabular output of DeploymentList +func SprintlnServices(items []*Service) string { + table := uitable.New() + table.AddRow("Services") + table.AddRow("Name", "Type", "ClusterIP", "ExternalIPs", "Ports", "CreationTime") + for _, r := range items { + table.AddRow(r.Name, r.Type, r.ClusterIP, + r.ExternalIPs, r.Ports, r.CreationTimestamp.Format(time.RFC3339)) + } + table.AddRow("") + return fmt.Sprintln(table) +} + +// GetIngressList returns IngressList from k8s api +func (c *Client) GetIngressList(namespace string) (*networkingv1.IngressList, error) { + return c.NetworkingV1().Ingresses(namespace).List(context.TODO(), metav1.ListOptions{}) +} + +// ToIngressAddress behaves mostly like a string interface and converts the given status to a string +func ToIngressAddress(s *networkingv1.IngressStatus) string { + ingress := s.LoadBalancer.Ingress + result := sets.NewString() + for i := range ingress { + if ingress[i].IP != "" { + result.Insert(ingress[i].IP) + } else if ingress[i].Hostname != "" { + result.Insert(ingress[i].Hostname) + } + } + return strings.Join(result.List(), ",") +} + +// backendStringer behaves just like a string interface and converts the given backend to a string +func backendStringer(backend *networkingv1.IngressBackend) string { + if backend == nil || backend.Service == nil { + return "" + } + return fmt.Sprintf("%v:%v", backend.Service.Name, backend.Service.Port.String()) +} + +// ToIngressRules returns a list of IngressRule +func ToIngressRules(rules []networkingv1.IngressRule) []IngressRule { + var result []IngressRule + for _, rule := range rules { + if rule.HTTP == nil { + continue + } + host := rule.Host + if len(host) == 0 { + host = "*" + } + paths := []HostPath{} + for _, p := range rule.HTTP.Paths { + paths = append(paths, HostPath{Path: p.Path, PathType: string(*p.PathType), Backend: backendStringer(&p.Backend)}) + } + result = append(result, IngressRule{Host: host, Path: paths}) + } + return result +} + +// ToIngressPorts returns a list of ports from the given []LoadBalancerIngress +func ToIngressPorts(ingresses []v1.LoadBalancerIngress) (ingressPorts []string) { + for _, i := range ingresses { + p := i.Ports + ports := make([]string, 0, len(p)) + for _, portStatus := range p { + ports = append(ports, fmt.Sprintf("%d/%s", portStatus.Port, portStatus.Protocol)) + } + ingressPorts = append(ingressPorts, strings.Join(ports, ",")) + } + return ingressPorts +} + +// ToIngress tansforms networkingv1.Ingress to Ingress +func ToIngress(i *networkingv1.Ingress) *Ingress { + return &Ingress{ + Name: i.Name, + Class: *i.Spec.IngressClassName, + Rules: ToIngressRules(i.Spec.Rules), + Address: ToIngressAddress(&i.Status), + Ports: ToIngressPorts(i.Status.LoadBalancer.Ingress), + CreationTimestamp: i.CreationTimestamp.Time, + } +} + +// DescribeIngresses returns IngressList +func (c *Client) DescribeIngresses(namespace string) ([]*Ingress, error) { + i, err := c.GetIngressList(namespace) + if err != nil { + return nil, err + } + var ingresses []*Ingress + for _, ingress := range i.Items { + ingresses = append(ingresses, ToIngress(&ingress)) + } + return ingresses, nil +} + +// SprintlnIngresses returns tabular output of IngressList +func SprintlnIngresses(items []*Ingress) string { + table := uitable.New() + table.Wrap = true + table.AddRow("Ingresses") + table.AddRow("Name", "Class", "Address", "Ports", "CreationTime", "Rules") + for _, r := range items { + // subtable of rules + subTable := uitable.New() + subTable.Wrap = true + subTable.AddRow("Host", "Path", "PathType", "Backend") + for _, rule := range r.Rules { + host := rule.Host + for _, path := range rule.Path { + subTable.AddRow(host, path.Path, path.PathType, path.Backend) + } + } + table.AddRow(r.Name, r.Class, r.Address, r.Ports, + r.CreationTimestamp.Format(time.RFC3339), subTable.String()) + } + table.AddRow("") + return fmt.Sprintln(table) +} + +// GetPodList returns PodList from k8s api +func (c *Client) GetPodList(namespace string) (*v1.PodList, error) { + return c.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{}) +} + +// GetPod returns specific Pod from k8s api +func (c *Client) GetPod(namespace, pod string) (*v1.Pod, error) { + return c.CoreV1().Pods(namespace).Get(context.TODO(), pod, metav1.GetOptions{}) +} + +// ToPod tansforms v1.Pod to Pod +func ToPod(p *v1.Pod) *Pod { + return &Pod{ + Name: p.Name, + Status: string(p.Status.Phase), + CreationTimestamp: p.CreationTimestamp.Time, + } +} + +// GetContainersFromPod fills a pod with it's containers +func (c *Client) GetContainersFromPod(p *v1.Pod, tail int) ([]*Container, error) { + var containers []*Container + for i := range p.Spec.Containers { + container := p.Spec.Containers[i] + status := p.Status.ContainerStatuses[i] + log, err := c.GetLog(p.Namespace, p.Name, container.Name, tail) + if err != nil { + log = "" + } + var state string + if status.State.Running != nil { + state = "Running" + } else if status.State.Waiting != nil { + state = "Waiting" + } else if status.State.Terminated != nil { + state = "Terminated" + } + _container := &Container{ + Name: container.Name, + Status: state, + Ready: status.Ready, + Image: container.Image, + ImageID: status.ImageID, + ContainerID: status.ContainerID, + Log: log, + } + containers = append(containers, _container) + } + return containers, nil +} + +// DescribePod returns specific Pod +func (c *Client) DescribePod(namespace, podName string, tail int) (*Pod, error) { + p, err := c.GetPod(namespace, podName) + if err != nil { + return nil, err + } + pod := ToPod(p) + pod.Containers, _ = c.GetContainersFromPod(p, tail) + var ready int + for _, c := range pod.Containers { + if c.Ready { + ready++ + } + } + pod.Ready = fmt.Sprintf("%d/%d", ready, len(pod.Containers)) + return pod, nil +} + +// DescribePods returns PodList +func (c *Client) DescribePods(namespace string, tail int) ([]*Pod, error) { + p, err := c.GetPodList(namespace) + if err != nil { + return nil, err + } + var pods []*Pod + for _, i := range p.Items { + if pod, err := c.DescribePod(namespace, i.Name, tail); err == nil { + pods = append(pods, pod) + } + } + return pods, nil +} + +// SprintlnPods returns tabular output of PodList +func SprintlnPods(items []*Pod) string { + table := uitable.New() + table.AddRow("Pods") + table.AddRow("Name", "Status", "Ready", "Containers", "CreationTime") + for _, r := range items { + var names []string + for _, c := range r.Containers { + names = append(names, c.Name) + } + table.AddRow(r.Name, r.Status, r.Ready, names, r.CreationTimestamp.Format(time.RFC3339)) + } + table.AddRow("") + return fmt.Sprintln(table) +} + +// SprintlnContainers returns tabular output of ContainerList +func SprintlnContainers(items []*Container) string { + table := uitable.New() + table.AddRow("Containers") + table.AddRow("Name", "Status", "Ready", "Image", "ImageID", "ContainerID") + for _, r := range items { + table.AddRow(r.Name, r.Status, r.Ready, r.Image, r.ImageID, r.ContainerID) + } + table.AddRow("") + return fmt.Sprintln(table) +} + +// GetLog returns log of one container +func (c *Client) GetLog(namespace, pod, container string, tail int) (string, error) { + tailLine := int64(tail) + logOpt := &v1.PodLogOptions{ + Container: container, + TailLines: &tailLine, + } + req := c.CoreV1().Pods(namespace).GetLogs(pod, logOpt) + r, err := req.Stream(context.TODO()) + if err != nil { + return "", err + } + defer r.Close() + buffer, err := io.ReadAll(r) + if err != nil { + return "", err + } + return string(buffer), nil +} + +// GetKubeconfig gets default kubeconfig +func GetKubeconfig() string { + if home := homedir.HomeDir(); home != "" { + return filepath.Join(home, ".kube", "config") + } + return "" +} diff --git a/k8s-deploy/pkg/utils/k8sclient/client_test.go b/k8s-deploy/pkg/utils/k8sclient/client_test.go new file mode 100644 index 000000000..83737b0cb --- /dev/null +++ b/k8s-deploy/pkg/utils/k8sclient/client_test.go @@ -0,0 +1,247 @@ +package k8sclient + +import ( + "reflect" + "testing" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + namespace1 = v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "1"}} + namespace2 = v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "2"}} + + deployment1 = appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "1"}} + deployment2 = appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "2"}} + + servicePorts1 = []v1.ServicePort{{Protocol: "tcp", Port: 8080}, {Protocol: "udp", Port: 8081}} + servicePorts2 = []v1.ServicePort{} + + service1 = v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "1"}, Spec: v1.ServiceSpec{Ports: servicePorts1}} + service2 = v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "2"}, Spec: v1.ServiceSpec{Ports: servicePorts2}} + + pathType1 = networkingv1.PathType("pathType1") + pathType2 = networkingv1.PathType("pathType2") + paths = []networkingv1.HTTPIngressPath{{Path: "/1", PathType: &pathType1}, {Path: "/2", PathType: &pathType2}} + ingressRules = []networkingv1.IngressRule{{Host: "example.com", IngressRuleValue: networkingv1.IngressRuleValue{HTTP: &networkingv1.HTTPIngressRuleValue{Paths: paths}}}} + ingressClassName = "cls1" + ingress1 = networkingv1.Ingress{ObjectMeta: metav1.ObjectMeta{Name: "1"}, Spec: networkingv1.IngressSpec{IngressClassName: &ingressClassName, Rules: ingressRules}} + + pod1 = v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "1"}} + pod2 = v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "2"}} +) + +func TestNewK8sClient(t *testing.T) { + type args struct { + kubeconfig string + } + tests := []struct { + name string + args args + wantC *Client + wantErr bool + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotC, err := NewK8sClient(tt.args.kubeconfig) + if (err != nil) != tt.wantErr { + t.Errorf("NewK8sClient() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotC, tt.wantC) { + t.Errorf("NewK8sClient() = %v, want %v", gotC, tt.wantC) + } + }) + } +} + +func TestNamespaceListToNames(t *testing.T) { + type args struct { + list *v1.NamespaceList + } + + tests := []struct { + name string + args args + want []string + }{ + {"test", args{&v1.NamespaceList{Items: []v1.Namespace{namespace1, namespace2}}}, []string{"1", "2"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := NamespaceListToNames(tt.args.list); !reflect.DeepEqual(got, tt.want) { + t.Errorf("NamespaceListToNames() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPodListToNames(t *testing.T) { + type args struct { + list *v1.PodList + } + tests := []struct { + name string + args args + want []string + }{ + {"test", args{&v1.PodList{Items: []v1.Pod{pod1, pod2}}}, []string{"1", "2"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := PodListToNames(tt.args.list); !reflect.DeepEqual(got, tt.want) { + t.Errorf("PodListToNames() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestToNamespace(t *testing.T) { + type args struct { + n *v1.Namespace + } + tests := []struct { + name string + args args + want *Namespace + }{ + {"1", args{&namespace1}, &Namespace{Name: "1"}}, + {"2", args{&namespace2}, &Namespace{Name: "2"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ToNamespace(tt.args.n); !reflect.DeepEqual(got, tt.want) { + t.Errorf("ToNamespace() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestToDeployment(t *testing.T) { + type args struct { + d *appsv1.Deployment + } + tests := []struct { + name string + args args + want *Deployment + }{ + {"1", args{&deployment1}, &Deployment{Name: "1"}}, + {"2", args{&deployment2}, &Deployment{Name: "2"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ToDeployment(tt.args.d); !reflect.DeepEqual(got, tt.want) { + t.Errorf("ToDeployment() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestToPorts(t *testing.T) { + type args struct { + p []v1.ServicePort + } + tests := []struct { + name string + args args + want []string + }{ + {"notEmpty", args{servicePorts1}, []string{"8080/tcp", "8081/udp"}}, + {"empty", args{servicePorts2}, []string{""}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ToPorts(tt.args.p); !reflect.DeepEqual(got, tt.want) { + t.Errorf("ToPorts() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestToService(t *testing.T) { + type args struct { + s *v1.Service + } + tests := []struct { + name string + args args + want *Service + }{ + {"1", args{&service1}, &Service{Name: "1", Ports: []string{"8080/tcp", "8081/udp"}}}, + {"1", args{&service2}, &Service{Name: "2", Ports: []string{""}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ToService(tt.args.s); !reflect.DeepEqual(got, tt.want) { + t.Errorf("ToService() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestToIngress(t *testing.T) { + type args struct { + i *networkingv1.Ingress + } + tests := []struct { + name string + args args + want *Ingress + }{ + {"1", args{&ingress1}, &Ingress{Name: "1", Class: "cls1", Rules: []IngressRule{{Host: "example.com", Path: []HostPath{{Path: "/1", PathType: string(pathType1), Backend: ""}, {Path: "/2", PathType: string(pathType2), Backend: ""}}}}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ToIngress(tt.args.i); !reflect.DeepEqual(got, tt.want) { + t.Errorf("ToIngress() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestToPod(t *testing.T) { + type args struct { + p *v1.Pod + } + tests := []struct { + name string + args args + want *Pod + }{ + {"1", args{&pod1}, &Pod{Name: "1"}}, + {"2", args{&pod2}, &Pod{Name: "2"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ToPod(tt.args.p); !reflect.DeepEqual(got, tt.want) { + t.Errorf("ToPod() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestToIngressRules(t *testing.T) { + type args struct { + rules []networkingv1.IngressRule + } + tests := []struct { + name string + args args + want []IngressRule + }{ + {"1", args{ingressRules}, []IngressRule{{Host: "example.com", Path: []HostPath{{Path: "/1", PathType: string(pathType1), Backend: ""}, {Path: "/2", PathType: string(pathType2), Backend: ""}}}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ToIngressRules(tt.args.rules); !reflect.DeepEqual(got, tt.want) { + t.Errorf("ToIngressRules() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/k8s-deploy/pkg/utils/supportbundle/supportbundle.go b/k8s-deploy/pkg/utils/supportbundle/supportbundle.go new file mode 100644 index 000000000..8ad39868c --- /dev/null +++ b/k8s-deploy/pkg/utils/supportbundle/supportbundle.go @@ -0,0 +1,193 @@ +package supportbundle + +import ( + "archive/zip" + "bytes" + "fmt" + "io/ioutil" + "os" + "path" + "strings" + + client "github.com/FederatedAI/KubeFATE/k8s-deploy/pkg/utils/k8sclient" +) + +type File struct { + Name string + Body []byte +} + +type Bundler struct { + KubeConfig string + CollectDir string + PackDir string + Client *client.Client +} + +// NewFile creates struct File with space-trimed and dash-joined name +func NewFile(name string, body []byte) *File { + group := strings.Split(name, " ") + newGroup := make([]string, 0, len(group)) + for _, s := range group { + if s != "" { + newGroup = append(newGroup, s) + } + } + newName := strings.Join(newGroup, "-") + return &File{newName, body} +} + +// NewBundler returns a bundle helper +func NewBundler(kubeconfig, collectDir, packDir string) (*Bundler, error) { + c, err := client.NewK8sClient(kubeconfig) + if err != nil { + return nil, err + } + return &Bundler{kubeconfig, collectDir, packDir, c}, nil +} + +// pathExists checks if path exists +func pathExists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, err + } + return false, err +} + +// CheckDir makes sure one directory is present +func CheckDir(dir string) error { + if exist, err := pathExists(dir); !exist || err != nil { + return os.MkdirAll(dir, os.ModePerm) + } + return nil +} + +// saveFile saves a file +func saveFile(file *File, dir string) error { + CheckDir(dir) + f, err := os.Create(path.Join(dir, file.Name)) + if err != nil { + return err + } + defer f.Close() + f.Write(file.Body) + return nil +} + +// dirToZip packs a directory into a zip file +func dirToZip(dir, dst string) error { + files, err := ioutil.ReadDir(dir) + if err != nil { + return err + } + fileDst, _ := os.Create(dst) + w := zip.NewWriter(fileDst) + defer w.Close() + for _, file := range files { + fw, err := w.Create(file.Name()) + if err != nil { + return err + } + fileContent, err := ioutil.ReadFile(path.Join(dir, file.Name())) + if err != nil { + return err + } + _, err = fw.Write(fileContent) + if err != nil { + return err + } + } + return nil +} + +// CollectIngresses collects ingresses in specific namespace +func (b *Bundler) CollectIngresses(namespace string) error { + i, err := b.Client.DescribeIngresses(namespace) + if err != nil { + return err + } + filename := fmt.Sprint("ingresses in ", namespace) + body := []byte(client.SprintlnIngresses(i)) + file := NewFile(filename, body) + return saveFile(file, b.CollectDir) +} + +// CollectPods collects pods in specific namespace +func (b *Bundler) CollectPods(namespace string, tail int) error { + p, err := b.Client.DescribePods(namespace, tail) + if err != nil { + return err + } + filename := fmt.Sprint("pods in ", namespace) + body := []byte(client.SprintlnPods(p)) + file := NewFile(filename, body) + return saveFile(file, b.CollectDir) +} + +// CollectContainers collects containers in specific namespace +func (b *Bundler) CollectContainers(namespace, podName string, tail int) error { + s, err := b.Client.DescribePod(namespace, podName, tail) + if err != nil { + return err + } + filename := fmt.Sprintf("containers in %s %s", namespace, podName) + body := []byte(client.SprintlnContainers(s.Containers)) + file := NewFile(filename, body) + return saveFile(file, b.CollectDir) +} + +// CollectServices collects services in specific namespace +func (b *Bundler) CollectServices(namespace string) error { + s, err := b.Client.DescribeServices(namespace) + if err != nil { + return err + } + filename := fmt.Sprint("services in ", namespace) + body := []byte(client.SprintlnServices(s)) + file := NewFile(filename, body) + return saveFile(file, b.CollectDir) +} + +// CollectNamespace collects ingresses,services,deployments,pods, +// containers as well as their logs in specific namespace and pods into one file +func (b *Bundler) CollectNamespace(namespace string, tail int, podNames ...string) error { + n, err := b.Client.DescribeNamespace(namespace, tail, podNames...) + if err != nil { + return err + } + + var buffer bytes.Buffer + buffer.WriteString(client.SprintlnDeployments(n.Deployments)) + buffer.WriteString(client.SprintlnServices(n.Services)) + buffer.WriteString(client.SprintlnIngresses(n.Ingresses)) + buffer.WriteString(client.SprintlnPods(n.Pods)) + + for _, pod := range n.Pods { + buffer.WriteString(client.SprintlnContainers(pod.Containers)) + for _, c := range pod.Containers { + filename := fmt.Sprintf("log of container %s in pod %s in namespace %s ", + c.Name, pod.Name, namespace) + file := NewFile(filename, []byte(c.Log)) + saveFile(file, b.CollectDir) + } + } + + namespaceFilename := fmt.Sprint("namespace ", namespace) + namespaceFile := NewFile(namespaceFilename, buffer.Bytes()) + err = saveFile(namespaceFile, b.CollectDir) + return err +} + +// Pack packs files into one zip file and removes the CollectDir +func (b *Bundler) Pack() error { + dst := path.Join(b.PackDir, "supportbundle.zip") + err := dirToZip(b.CollectDir, dst) + if err != nil { + return err + } + return os.RemoveAll(b.CollectDir) +} diff --git a/k8s-deploy/pkg/utils/supportbundle/supportbundle_test.go b/k8s-deploy/pkg/utils/supportbundle/supportbundle_test.go new file mode 100644 index 000000000..055b48970 --- /dev/null +++ b/k8s-deploy/pkg/utils/supportbundle/supportbundle_test.go @@ -0,0 +1,148 @@ +package supportbundle + +import ( + "os" + "reflect" + "testing" +) + +const ( + bundirDir = "test-bundle" +) + +var ( + file = File{"1-2-3", []byte("body")} +) + +func TestNewFile(t *testing.T) { + type args struct { + name string + body []byte + } + tests := []struct { + name string + args args + want *File + }{ + {"newFile", args{"1 2 3", []byte("body")}, &file}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := NewFile(tt.args.name, tt.args.body); !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewFile() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_pathExists(t *testing.T) { + type args struct { + path string + } + tests := []struct { + name string + args args + want bool + wantErr bool + }{ + {"pathExists", args{"."}, true, false}, + {"pathNotExists", args{bundirDir}, false, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := pathExists(tt.args.path) + if (err != nil) != tt.wantErr { + t.Errorf("pathExists() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("pathExists() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestCheckDir(t *testing.T) { + type args struct { + dir string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"checkExsitedDir", args{"."}, false}, + {"checkDirNotExist", args{bundirDir}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + exist, _ := pathExists(tt.args.dir) + if err := CheckDir(tt.args.dir); (err != nil) != tt.wantErr { + t.Errorf("CheckDir() error = %v, wantErr %v", err, tt.wantErr) + } + if !exist { + os.Remove(tt.args.dir) + } + }) + } +} + +func Test_saveFile(t *testing.T) { + type args struct { + file *File + dir string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"saveFile", args{&file, bundirDir}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := CheckDir(tt.args.dir); err != nil { + t.Errorf("Test_saveFile CheckDir() error = %v", err) + } + if err := saveFile(tt.args.file, tt.args.dir); (err != nil) != tt.wantErr { + t.Errorf("saveFile() error = %v, wantErr %v", err, tt.wantErr) + } + if err := os.RemoveAll(tt.args.dir); err != nil { + t.Errorf("Test_saveFile RemoveAll() error = %v", err) + } + }) + } +} + +func Test_dirToZip(t *testing.T) { + type args struct { + dir string + dst string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"dirToZip", args{bundirDir, "test.zip"}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := CheckDir(tt.args.dir); err != nil { + t.Errorf("Test_saveFile CheckDir() error = %v", err) + } + if err := dirToZip(tt.args.dir, tt.args.dst); (err != nil) != tt.wantErr { + t.Errorf("dirToZip() error = %v, wantErr %v", err, tt.wantErr) + } + if exist, err := pathExists(tt.args.dst); !exist || err != nil { + t.Errorf("dirToZip() pathExists error = %v", err) + } + if err := os.RemoveAll(tt.args.dir); err != nil { + t.Errorf("Test_dirToZip RemoveAll() dir error = %v", err) + } + if err := os.RemoveAll(tt.args.dst); err != nil { + t.Errorf("Test_dirToZip RemoveAll() dst error = %v", err) + } + }) + } +} diff --git a/k8s-deploy/rbac-config.yaml b/k8s-deploy/rbac-config.yaml index 24b70d218..3c293305e 100644 --- a/k8s-deploy/rbac-config.yaml +++ b/k8s-deploy/rbac-config.yaml @@ -111,8 +111,8 @@ rules: - apiGroups: - networking.istio.io resources: - - gateway - - virtualservice + - gateways + - virtualservices verbs: - get - create diff --git a/proposals/KubeFATE rolling upgrade proposal.md b/proposals/KubeFATE rolling upgrade proposal.md new file mode 100644 index 000000000..d2242695e --- /dev/null +++ b/proposals/KubeFATE rolling upgrade proposal.md @@ -0,0 +1,79 @@ +# Modification History + +| Date | Content | person | +|------------|---------------------------------------------------------|-----------| +| 2022/07/01 | initial version | Chen Jing | +| 2022/07/06 | Mainly change the upgrading matrix of using SC scenario | Chen Jing | + + +# The Goal +The goal of this feature is to make the upgrade smooth without any data loss. + +1. Reuse the command "kubefate cluster update -f ", make it able to detect the version change and do the rolling upgrade. +2. Make sure there is no data loss, every persistence volume of an older version FATE can be reclaimed by the newer FATE. +3. Make sure when rolling upgrade is failed, everything can be back to the origin status. + +# Functional spec +## Upgrade +### If the user doesn't enable persistence +Then definitely there will be data loss, even when a pod restarts. + +### If the user is using storage class to provision persistence volumes +We cannot support 100% automatic rolling upgrade for the older versions when we upgrade them to v1.9.0+ versions. Because in the version v1.9.0, we will change some FATE components to statefulSet, and it cannot re-attach the old pvc/pv because they used to be deployments. + + +| From/To | v1.7.1 | v1.7.2 | v1.8.0 | v1.9.0 | v1.10.0+ | +|---------|------------|------------|------------|---------|------------| +| v1.7.0 | no pv loss | no pv loss | no pv loss | pv loss | pv loss | +| v1.7.1 | | no pv loss | no pv loss | pv loss | pv loss | +| v1.7.2 | | | no pv loss | pv loss | pv loss | +| v1.8.0 | | | | pv loss | pv loss | +| v1.9.0 | | | | | no pv loss | + +#### Workaround +* Manually relocate the old files: If the storage facility can help to archive the old files, then the user can manually move the old files to the directories of the newer PV. +* Turn to use self-managed PVC: If the storage class has the "Retain" policy, then the old PVs will be preserved. The user can manage a PVC based on the old PV, and do not leverage storage class to automatically create PVC after upgrading. + +#### The influenced data of pv loss +* Mysql: +The metadata of the FATE jobs. + +* Jupyter Notebook client: +The new created files under the "persistence". + +* Node manager: +The uploaded dataset and the intermediate results. + +* HDFS: +The uploaded dataset and the intermediate results. + +### If the user is using existing PVC + +| From/To | v1.7.1 | v1.7.2 | v1.8.0 | v1.9.0 | v1.10.0+ | +|---------|------------|------------|------------|------------|------------| +| v1.7.0 | no pv loss | no pv loss | no pv loss | no pv loss | no pv loss | +| v1.7.1 | | no pv loss | no pv loss | no pv loss | no pv loss | +| v1.7.2 | | | no pv loss | no pv loss | no pv loss | +| v1.8.0 | | | | no pv loss | no pv loss | +| v1.9.0 | | | | | no pv loss | + +If the user is using self-managed PVC, then no data will be loss. + +## Rollback +If the upgrade failed, then we make sure to reproduce everything before the upgrade happens. + +# User story +Suppose the user's name is Tom. + +1. Tom has a cluster.yaml which declares a v1.7.0 FATE cluster, in which each component has been configured an existing PV, the FATE cluster has been up for a long time. +2. Tom changes the chart version and image version in the cluster.yaml file to v1.8.0. +3. Tom executes "kubefate cluster update -f cluster.yaml" in the terminal. +4. Tom sees that all the old pods turn into a "Terminating" status, and the new pods starting to pop out as "Running" +5. The old PVs and PVCs will be re-attached to the newer version pods automatically. +6. The sql update scripts will be executed against the mysql database to update the schema, based on the version diff. For example, we have 3 .sql scripts which are named as: 170-171.sql, 171-172.sql and 172-180.sql, then the three scripts will be executed against the mysql database in sequence when the upgrade is from v1.7.0 to v1.8.0. This step will only be conducted after the cluster turns into the "running" status. +7. If the upgrade fails, which means finally the cluster cannot turn into a "running" status, we need to fall back to the previous version. Kubefate service will help to fetch the previous cluster.yaml from the database and re-install it. As the pv/pvc doesn't change, and the mysql upgrade script hasn't been executed at this stage, nothing more need to be done. + +# Implementation plan +Plan to use the golang sdk of helm, and to use the helm upgrade functionality. It has already supported pv/pvc auto-reattach and rollback. We just need to make sure we run the mysql script against the mysql container after helm upgrade has been done. + +The mysql script is at: https://github.com/FederatedAI/FATE/tree/develop-1.9.0/deploy/upgrade/sql \ No newline at end of file diff --git a/registry/README.md b/registry/README.md index 39a39b8b1..3bad2e7cf 100644 --- a/registry/README.md +++ b/registry/README.md @@ -10,7 +10,7 @@ The requirements of the host to install Harbor are as follows: 3. Docker-Compose: 1.18+ -### Downloading the installer +## Downloading the installer The binary of the offline installer can be downloaded from the release page of [Harbor](https://github.com/goharbor/harbor/releases/tag/v1.9.3). Use *tar* command to extract the package. @@ -18,23 +18,23 @@ The binary of the offline installer can be downloaded from the release page of [ $ tar xvf harbor-offline-installer-v1.9.3.tgz ``` -### Configuring Harbor +## Configuring Harbor -Configuration parameters are located in the file `harbor.yml`. Change hostname the local IP adress of your Harbor host, such as 192.168.0.1: +Configuration parameters are located in the file `harbor.yml`. Change hostname the local IP address of your Harbor host, such as 192.168.10.1: ```bash hostname: 192.168.10.1 ``` -### Installing Harbor +## Installing Harbor Run the below command to install Harbor. After installation, Harbor registry service should be up and running: ```bash $ sudo ./install.sh ``` -### Configuring docker client: -The default installation of Harbor uses HTTP - as such, you need to add the option `--insecure-registry` to your client's Docker daemon and restart the Docker service. +## Configuring docker client: +The default installation of Harbor uses HTTP - as such, perhaps you need to add the option `--insecure-registry` to your client's Docker daemon and restart the Docker service. ```bash $ sudo vi /etc/docker/daemon.json @@ -51,21 +51,21 @@ Restart docker daemon: ``` -### Preparing FATE Images in Harbor +## Preparing FATE Images in Harbor After Harbor has been installed, FATE docker images must be pushed to Harbor registry so that other nodes can pull FATE images from Harbor. There are basically two ways to obtain the FATE images: - Build images from FATE source code and push to Harbor -- Replicate FATE imges from Docker Hub to Harbor +- Replicate FATE images from Docker Hub to Harbor -#### Build images from FATE source code +## Build images from FATE source code -Refer to [Build Document](https://github.com/FederatedAI/FATE/blob/master/docker-build/README.md) to create FATE iamges. Once images are built, push them to Harbor. +Refer to [Build Document](https://github.com/FederatedAI/FATE/blob/master/build/docker-build/README.md) to create FATE images. Once images are built, push them to Harbor. -Usually, a user does not need to take the lenghty time to build images from source. It is recommended to use te pre-built docker images of FATE directly. Refer to te below section to for more details. +Usually, a user does not need to take the long time to build images from source. It is recommended to use te pre-built docker images of FATE directly. Refer to te below section to for more details. -#### Replicating images from Docker Hub +## Replicating images from Docker Hub Log into Harbor's management portal using the URL `http:// - -
+For example, when MySQL version is `8`, we can configure as below.
- +
-MySQL version is `8` and Redis version is `5`. They both need to specify the **Destination namespace** as `federatedai`. +**IMPORTANT: We need to specify the Destination namespace as `federatedai`.** -**NOTE:** The replication rule continues to take effect until it is disabled or cancelled. If FATE images are avaiable for a new release, they will be replicated to Harbor automatically. +**NOTE:** The replication rule continues to take effect until it is disabled or cancelled. If FATE images are available for a new release, they will be replicated to Harbor automatically. diff --git a/registry/images/redis-replication.jpg b/registry/images/redis-replication.jpg deleted file mode 100644 index 371092525..000000000 Binary files a/registry/images/redis-replication.jpg and /dev/null differ