proto_ingest_legacy_to_bq.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1926a49b-0078-43d9-ae2a-1993b92bacb7",
   "metadata": {},
   "source": [
    "# First testing to read, test, ingest legacy catalog into bigquery\n",
    "\n",
    "Got this running fairly fast (~8 hours for the whole catalog)\n",
    "\n",
    "- [ ] Scale this out on a coiled cluster?\n",
    "- [ ] Apply same methodology to retesting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b0a52150-3822-4a09-9173-882e82e5b0fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "from leap_data_management_utils.cmip_testing import test_all\n",
    "from leap_data_management_utils import CMIPBQInterface, IIDEntry\n",
    "# # TODO refactor\n",
    "# from leap_data_management_utils.cmip_catalog import convert_cmip6_df_to_iid_df\n",
    "# from catalog_utils import convert_cmip6_df_to_iid_df\n",
    "import zarr\n",
    "import pandas as pd\n",
    "from tqdm.auto import tqdm\n",
    "import dask.bag as db\n",
    "from dask.diagnostics import ProgressBar\n",
    "from dask.distributed import LocalCluster, Client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "55475e8a-eb24-4cf3-a910-00f31e3258b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "    <div style=\"width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;\"> </div>\n",
       "    <div style=\"margin-left: 48px;\">\n",
       "        <h3 style=\"margin-bottom: 0px;\">Client</h3>\n",
       "        <p style=\"color: #9D9D9D; margin-bottom: 0px;\">Client-45125302-ba1c-11ee-a8d7-b2d4e2d4eb64</p>\n",
       "        <table style=\"width: 100%; text-align: left;\">\n",
       "\n",
       "        <tr>\n",
       "        \n",
       "            <td style=\"text-align: left;\"><strong>Connection method:</strong> Cluster object</td>\n",
       "            <td style=\"text-align: left;\"><strong>Cluster type:</strong> distributed.LocalCluster</td>\n",
       "        \n",
       "        </tr>\n",
       "\n",
       "        \n",
       "            <tr>\n",
       "                <td style=\"text-align: left;\">\n",
       "                    <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:8787/status\" target=\"_blank\">http://127.0.0.1:8787/status</a>\n",
       "                </td>\n",
       "                <td style=\"text-align: left;\"></td>\n",
       "            </tr>\n",
       "        \n",
       "\n",
       "        </table>\n",
       "\n",
       "        \n",
       "\n",
       "        \n",
       "            <details>\n",
       "            <summary style=\"margin-bottom: 20px;\"><h3 style=\"display: inline;\">Cluster Info</h3></summary>\n",
       "            <div class=\"jp-RenderedHTMLCommon jp-RenderedHTML jp-mod-trusted jp-OutputArea-output\">\n",
       "    <div style=\"width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;\">\n",
       "    </div>\n",
       "    <div style=\"margin-left: 48px;\">\n",
       "        <h3 style=\"margin-bottom: 0px; margin-top: 0px;\">LocalCluster</h3>\n",
       "        <p style=\"color: #9D9D9D; margin-bottom: 0px;\">a8760ff5</p>\n",
       "        <table style=\"width: 100%; text-align: left;\">\n",
       "            <tr>\n",
       "                <td style=\"text-align: left;\">\n",
       "                    <strong>Dashboard:</strong> <a href=\"http://127.0.0.1:8787/status\" target=\"_blank\">http://127.0.0.1:8787/status</a>\n",
       "                </td>\n",
       "                <td style=\"text-align: left;\">\n",
       "                    <strong>Workers:</strong> 4\n",
       "                </td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                <td style=\"text-align: left;\">\n",
       "                    <strong>Total threads:</strong> 12\n",
       "                </td>\n",
       "                <td style=\"text-align: left;\">\n",
       "                    <strong>Total memory:</strong> 64.00 GiB\n",
       "                </td>\n",
       "            </tr>\n",
       "            \n",
       "            <tr>\n",
       "    <td style=\"text-align: left;\"><strong>Status:</strong> running</td>\n",
       "    <td style=\"text-align: left;\"><strong>Using processes:</strong> True</td>\n",
       "</tr>\n",
       "\n",
       "            \n",
       "        </table>\n",
       "\n",
       "        <details>\n",
       "            <summary style=\"margin-bottom: 20px;\">\n",
       "                <h3 style=\"display: inline;\">Scheduler Info</h3>\n",
       "            </summary>\n",
       "\n",
       "            <div style=\"\">\n",
       "    <div>\n",
       "        <div style=\"width: 24px; height: 24px; background-color: #FFF7E5; border: 3px solid #FF6132; border-radius: 5px; position: absolute;\"> </div>\n",
       "        <div style=\"margin-left: 48px;\">\n",
       "            <h3 style=\"margin-bottom: 0px;\">Scheduler</h3>\n",
       "            <p style=\"color: #9D9D9D; margin-bottom: 0px;\">Scheduler-dfb056bb-3ee5-4b3c-9c7a-197d3d60e67e</p>\n",
       "            <table style=\"width: 100%; text-align: left;\">\n",
       "                <tr>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Comm:</strong> tcp://127.0.0.1:54628\n",
       "                    </td>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Workers:</strong> 4\n",
       "                    </td>\n",
       "                </tr>\n",
       "                <tr>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Dashboard:</strong> <a href=\"http://127.0.0.1:8787/status\" target=\"_blank\">http://127.0.0.1:8787/status</a>\n",
       "                    </td>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Total threads:</strong> 12\n",
       "                    </td>\n",
       "                </tr>\n",
       "                <tr>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Started:</strong> Just now\n",
       "                    </td>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Total memory:</strong> 64.00 GiB\n",
       "                    </td>\n",
       "                </tr>\n",
       "            </table>\n",
       "        </div>\n",
       "    </div>\n",
       "\n",
       "    <details style=\"margin-left: 48px;\">\n",
       "        <summary style=\"margin-bottom: 20px;\">\n",
       "            <h3 style=\"display: inline;\">Workers</h3>\n",
       "        </summary>\n",
       "\n",
       "        \n",
       "        <div style=\"margin-bottom: 20px;\">\n",
       "            <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
       "            <div style=\"margin-left: 48px;\">\n",
       "            <details>\n",
       "                <summary>\n",
       "                    <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 0</h4>\n",
       "                </summary>\n",
       "                <table style=\"width: 100%; text-align: left;\">\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Comm: </strong> tcp://127.0.0.1:54640\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Total threads: </strong> 3\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:54643/status\" target=\"_blank\">http://127.0.0.1:54643/status</a>\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Memory: </strong> 16.00 GiB\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Nanny: </strong> tcp://127.0.0.1:54631\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\"></td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td colspan=\"2\" style=\"text-align: left;\">\n",
       "                            <strong>Local directory: </strong> /var/folders/m2/vwrjf3zx1bvdbhfkw7bxpz7r0000gn/T/dask-scratch-space/worker-fbcavg5b\n",
       "                        </td>\n",
       "                    </tr>\n",
       "\n",
       "                    \n",
       "\n",
       "                    \n",
       "\n",
       "                </table>\n",
       "            </details>\n",
       "            </div>\n",
       "        </div>\n",
       "        \n",
       "        <div style=\"margin-bottom: 20px;\">\n",
       "            <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
       "            <div style=\"margin-left: 48px;\">\n",
       "            <details>\n",
       "                <summary>\n",
       "                    <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 1</h4>\n",
       "                </summary>\n",
       "                <table style=\"width: 100%; text-align: left;\">\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Comm: </strong> tcp://127.0.0.1:54639\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Total threads: </strong> 3\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:54644/status\" target=\"_blank\">http://127.0.0.1:54644/status</a>\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Memory: </strong> 16.00 GiB\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Nanny: </strong> tcp://127.0.0.1:54632\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\"></td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td colspan=\"2\" style=\"text-align: left;\">\n",
       "                            <strong>Local directory: </strong> /var/folders/m2/vwrjf3zx1bvdbhfkw7bxpz7r0000gn/T/dask-scratch-space/worker-s8ullavb\n",
       "                        </td>\n",
       "                    </tr>\n",
       "\n",
       "                    \n",
       "\n",
       "                    \n",
       "\n",
       "                </table>\n",
       "            </details>\n",
       "            </div>\n",
       "        </div>\n",
       "        \n",
       "        <div style=\"margin-bottom: 20px;\">\n",
       "            <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
       "            <div style=\"margin-left: 48px;\">\n",
       "            <details>\n",
       "                <summary>\n",
       "                    <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 2</h4>\n",
       "                </summary>\n",
       "                <table style=\"width: 100%; text-align: left;\">\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Comm: </strong> tcp://127.0.0.1:54642\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Total threads: </strong> 3\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:54647/status\" target=\"_blank\">http://127.0.0.1:54647/status</a>\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Memory: </strong> 16.00 GiB\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Nanny: </strong> tcp://127.0.0.1:54633\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\"></td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td colspan=\"2\" style=\"text-align: left;\">\n",
       "                            <strong>Local directory: </strong> /var/folders/m2/vwrjf3zx1bvdbhfkw7bxpz7r0000gn/T/dask-scratch-space/worker-8b1gkbsl\n",
       "                        </td>\n",
       "                    </tr>\n",
       "\n",
       "                    \n",
       "\n",
       "                    \n",
       "\n",
       "                </table>\n",
       "            </details>\n",
       "            </div>\n",
       "        </div>\n",
       "        \n",
       "        <div style=\"margin-bottom: 20px;\">\n",
       "            <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
       "            <div style=\"margin-left: 48px;\">\n",
       "            <details>\n",
       "                <summary>\n",
       "                    <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 3</h4>\n",
       "                </summary>\n",
       "                <table style=\"width: 100%; text-align: left;\">\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Comm: </strong> tcp://127.0.0.1:54641\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Total threads: </strong> 3\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:54648/status\" target=\"_blank\">http://127.0.0.1:54648/status</a>\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Memory: </strong> 16.00 GiB\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Nanny: </strong> tcp://127.0.0.1:54634\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\"></td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td colspan=\"2\" style=\"text-align: left;\">\n",
       "                            <strong>Local directory: </strong> /var/folders/m2/vwrjf3zx1bvdbhfkw7bxpz7r0000gn/T/dask-scratch-space/worker-t2zzqs69\n",
       "                        </td>\n",
       "                    </tr>\n",
       "\n",
       "                    \n",
       "\n",
       "                    \n",
       "\n",
       "                </table>\n",
       "            </details>\n",
       "            </div>\n",
       "        </div>\n",
       "        \n",
       "\n",
       "    </details>\n",
       "</div>\n",
       "\n",
       "        </details>\n",
       "    </div>\n",
       "</div>\n",
       "            </details>\n",
       "        \n",
       "\n",
       "    </div>\n",
       "</div>"
      ],
      "text/plain": [
       "<Client: 'tcp://127.0.0.1:54628' processes=4 threads=12, memory=64.00 GiB>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client = Client()\n",
    "client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "074a1625-c011-4251-84f6-810f88289882",
   "metadata": {},
   "outputs": [],
   "source": [
    "table_id = 'leap-pangeo.testcmip6.cmip6_consolidated_manual_testing'\n",
    "bq = CMIPBQInterface(table_id=table_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "411374e9-ef4e-4f0e-ac5e-977bcd2a3025",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fix_legacy_df(df: pd.DataFrame) -> pd.DataFrame:\n",
    "    \"\"\"There were several errors in the original dataframe. This should fix it and bring it into official CMIP conventions\n",
    "    Source: https://docs.google.com/document/d/1h0r8RZr_f3-8egBMMh7aqLwy3snpD6_MrDz1q8n5XUk/edit\n",
    "    1. `member_id` and `variant_label` are mixed up. \n",
    "    2. `dcpp_init` year is not mentioned anywhere in the CMIP docs. I suspect this is `sub_experiment_id`\n",
    "    3. Some values are saved as floats causing issues with generating proper iids\n",
    "    \"\"\"\n",
    "    df = df.copy()\n",
    "    # df = df.drop(columns='variant_label')\n",
    "    df = df.rename(columns={'dcpp_init_year':'sub_experiment_id', 'member_id':'variant_label'})\n",
    "    # convert sub_experiment column to str and prepend an `s` (see: https://github.com/WCRP-CMIP/CMIP6_CVs/blob/master/CMIP6_sub_experiment_id.json)\n",
    "    df['sub_experiment_id'] = df['sub_experiment_id'].fillna(1e6).astype(int)\n",
    "    #\n",
    "    df['sub_experiment_id'] = 's'+df['sub_experiment_id'].astype('str')\n",
    "    # remove nan values\n",
    "    df.loc[df['sub_experiment_id'].isin(['s1000000']), \"sub_experiment_id\"] = 'none'\n",
    "    return df\n",
    "\n",
    "\n",
    "def _maybe_join(iterable):\n",
    "    assert len(iterable) == 2 \n",
    "    sub_experiment_id = iterable.iloc[0]\n",
    "    variant_label = iterable.iloc[1]\n",
    "    if sub_experiment_id != 'none':\n",
    "        return f\"{sub_experiment_id}-{variant_label}\"\n",
    "    else:\n",
    "        return variant_label\n",
    "\n",
    "def convert_cmip6_df_to_iid_df(df: pd.DataFrame) -> pd.DataFrame:\n",
    "    # now remove the ones already in the pangeo catalog\n",
    "    df = df.copy()\n",
    "    df['member_id'] = df[[\"sub_experiment_id\", \"variant_label\"]].agg(_maybe_join, axis=1)\n",
    "    df['version'] = 'v'+df['version'].astype(str)\n",
    "    df['instance_id'] = df[['activity_id', 'institution_id', 'source_id', 'experiment_id',\n",
    "        'member_id', 'table_id', 'variable_id', 'grid_label', 'version']].astype(str).agg('.'.join, axis=1).tolist()\n",
    "    df['instance_id'] = 'CMIP6.'+df['instance_id']\n",
    "    df['store'] = df['zstore']\n",
    "    # add current time as bigquery timestamp\n",
    "    df['timestamp'] = pd.Timestamp.now(tz='UTC')\n",
    "    df = df[['instance_id','store', 'timestamp']]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0dac6aa9-7a8e-4212-8086-5fb284132a5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_legacy_raw = pd.read_csv('https://storage.googleapis.com/cmip6/pangeo-cmip6.csv')\n",
    "df_legacy_fixed_raw = fix_legacy_df(df_legacy_raw)\n",
    "df_legacy = convert_cmip6_df_to_iid_df(df_legacy_fixed_raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a9cf54bb-3f2c-4d8b-8432-7cf7ccc144ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(store_url,iid):\n",
    "    store = zarr.storage.FSStore(store_url)\n",
    "    try:\n",
    "        test_all(store, iid, verbose=False)\n",
    "        test_passed = True\n",
    "    except Exception as e: \n",
    "        test_passed = False\n",
    "    return test_passed "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7aa64a71-d6d7-45d4-938b-52ffe28755e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e2dbeb3a05a448dfaaffab3d99579108",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/52 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# filter for iids that do not exist\n",
    "# need to do this in batches\n",
    "iids = df_legacy['instance_id'].tolist()\n",
    "iids_in_bq = []\n",
    "batchsize = 10000 #10k seems to be the limit \n",
    "iid_batches = [iids[i:i+batchsize] for i in range(0,len(iids), batchsize)]\n",
    "for iids_batch in tqdm(iid_batches):\n",
    "    iids_in_bq_batch = bq.iid_list_exists(iids_batch)\n",
    "    iids_in_bq.extend(iids_in_bq_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "72fd8886-8732-4319-af7e-6de563ef5916",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_legacy_filtered = df_legacy[~df_legacy.instance_id.isin(iids_in_bq)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8693001b-3f28-4cf4-bcf6-e517b893f19d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_input = df_legacy_filtered.iloc[:300000, :] # for now only a representative example?"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2d82f9c-8db4-4366-a7d7-c20931855539",
   "metadata": {},
   "source": [
    "## Parallel loading and testing xarray datasets and adding to bq in batches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1810c3f2-4780-4e05-9ded-7062aa5623d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch_test_add(batch_df: pd.DataFrame, bq_client: CMIPBQInterface):\n",
    "    seq = [(a[1].store,a[1].instance_id) for a in batch_df.iterrows()]\n",
    "    bag = db.from_sequence(seq)\n",
    "    with ProgressBar():\n",
    "        results_dask = bag.map(lambda a: (*a, test(*a))).compute()\n",
    "    results_dask_iid_entry = [IIDEntry(iid=a[1], store=a[0], retracted=False, tests_passed=a[2]) for a in results_dask]\n",
    "    bq_client.insert_multiple_iids(results_dask_iid_entry)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae738269-f028-498e-83da-bab85ea68623",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "12769b0dda5f4b3484be0e15372bde10",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/30 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-01-22 21:24:08,678 - distributed.utils_perf - WARNING - full garbage collections took 13% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:24:30,697 - distributed.utils_perf - WARNING - full garbage collections took 13% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:24:46,293 - distributed.utils_perf - WARNING - full garbage collections took 13% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:24:48,848 - distributed.utils_perf - WARNING - full garbage collections took 13% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:24:58,780 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:01,678 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:04,151 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:05,973 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:07,377 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:08,506 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:09,957 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:11,197 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:13,048 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:14,895 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:18,296 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:22,371 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:25,225 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:27,173 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:28,802 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:30,706 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:32,009 - distributed.utils_perf - WARNING - full garbage collections took 15% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:33,980 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:36,456 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:36,554 - distributed.utils_perf - WARNING - full garbage collections took 15% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:38,515 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:40,502 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:41,139 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:42,975 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:44,805 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:44,823 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:47,311 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:50,276 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:25:54,963 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:05,945 - distributed.utils_perf - WARNING - full garbage collections took 13% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:10,486 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:11,251 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:12,153 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:12,806 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:13,922 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:15,200 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:15,941 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:16,373 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:17,400 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:17,608 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:18,387 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:18,833 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:19,461 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:19,972 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:20,160 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:21,260 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:21,390 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:21,636 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:22,470 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:22,737 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:23,932 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:24,952 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:25,058 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:25,607 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:26,422 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:26,508 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:27,669 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:27,950 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:28,547 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:28,872 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:29,019 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:29,560 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:31,281 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:31,494 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:31,920 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:33,290 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:34,023 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:34,229 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:35,450 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:35,627 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:36,606 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:36,837 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:37,298 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:39,483 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:39,676 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:40,004 - distributed.utils_perf - WARNING - full garbage collections took 15% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:40,988 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:41,118 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:41,982 - distributed.utils_perf - WARNING - full garbage collections took 15% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:43,069 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:43,143 - distributed.utils_perf - WARNING - full garbage collections took 15% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:43,186 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:44,152 - distributed.utils_perf - WARNING - full garbage collections took 15% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:45,349 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:46,542 - distributed.utils_perf - WARNING - full garbage collections took 15% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:46,901 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:48,128 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:49,121 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:51,654 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:51,699 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:51,907 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:55,010 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:55,732 - distributed.utils_perf - WARNING - full garbage collections took 15% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:56,750 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:57,186 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:58,743 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:26:59,257 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:00,708 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:09,348 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:14,441 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:16,135 - distributed.utils_perf - WARNING - full garbage collections took 15% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:19,379 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:31,709 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:31,925 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:33,396 - distributed.utils_perf - WARNING - full garbage collections took 15% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:34,923 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:35,985 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:36,170 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:36,999 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:37,476 - distributed.utils_perf - WARNING - full garbage collections took 15% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:37,990 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:38,629 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:39,858 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:40,466 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:40,588 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:41,150 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:42,920 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:42,988 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:43,216 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:43,772 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:45,314 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:45,493 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:46,157 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:48,843 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:49,331 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:49,478 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:50,324 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:51,180 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:51,348 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:53,424 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:53,471 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:54,121 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:55,127 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:55,952 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:57,385 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:58,330 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:58,824 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:59,429 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:27:59,763 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:00,908 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:01,922 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:04,282 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:06,789 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:09,786 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:12,051 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:14,424 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:19,894 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:22,938 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:24,090 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:25,482 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:26,568 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:27,593 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:28,966 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:30,749 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:33,378 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:37,002 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:41,374 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:41,533 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:42,179 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:42,716 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:44,843 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:45,316 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:48,080 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:49,183 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:50,820 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:51,669 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:53,362 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:54,166 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:55,661 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:56,986 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:59,297 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:28:59,789 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:00,868 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:07,614 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:11,060 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:12,691 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:14,276 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:14,328 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:15,535 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:20,129 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:28,473 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:32,186 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:33,915 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:34,932 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:37,534 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:38,363 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:43,318 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:44,988 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:46,655 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:47,715 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:51,470 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:51,694 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:53,592 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:55,363 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:55,487 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:55,922 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:29:57,140 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:30:00,072 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:30:01,636 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:30:02,915 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:30:02,963 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:30:06,534 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:30:11,025 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:30:11,291 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:30:11,336 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:30:13,758 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:30:16,990 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:30:17,550 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:30:18,098 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:30:31,297 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:30:31,327 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:30:37,583 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:30:49,636 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:30:53,919 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:30:56,984 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:30:57,030 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:31:00,313 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:31:00,738 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:31:00,795 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:31:03,209 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:31:04,400 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:31:09,077 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:31:12,904 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:31:14,866 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:31:19,107 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:31:23,163 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:31:32,120 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:31:35,077 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:31:41,973 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:31:44,076 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:31:45,159 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:31:47,097 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:31:59,064 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:32:01,543 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:32:03,291 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:32:11,235 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:32:16,042 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:32:25,991 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:32:35,114 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:32:37,259 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:32:48,387 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:32:57,576 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:00,160 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:02,974 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:04,827 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:04,838 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:07,076 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:08,587 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:09,396 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:13,935 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:19,306 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:19,451 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:23,059 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:24,291 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:32,686 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:36,362 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:37,101 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:39,019 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:40,189 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:40,796 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:43,886 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:46,205 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:53,671 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:54,000 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:55,457 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:33:57,328 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:34:00,117 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:34:01,167 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:34:01,590 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:34:03,552 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:34:13,432 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:34:16,067 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:34:16,133 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:34:16,248 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:34:19,910 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:34:24,700 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:34:34,960 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:34:39,545 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:34:41,385 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:34:41,589 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:34:46,348 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:34:48,025 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:34:48,842 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:34:52,981 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:34:53,187 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:34:54,520 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:34:58,487 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:34:58,739 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:01,789 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:01,898 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:02,820 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:04,659 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:09,706 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:10,494 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:11,779 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:12,756 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:14,875 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:16,267 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:18,058 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:19,826 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:20,044 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:20,834 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:23,590 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:26,250 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:29,868 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:32,103 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:33,060 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:37,410 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:37,924 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:39,208 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:42,622 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:45,094 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:45,195 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:50,406 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:52,314 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:53,073 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:35:58,369 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:36:08,518 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:36:14,625 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:36:17,649 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:36:17,801 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:36:18,637 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:36:26,692 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:36:28,466 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:36:29,605 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:36:30,987 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:36:32,767 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:36:37,203 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:36:44,524 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:36:44,618 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:36:47,546 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:36:47,862 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:36:50,034 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:36:51,289 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:36:52,851 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:36:53,719 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:36:58,522 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:36:59,521 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:00,526 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:01,733 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:03,795 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:07,530 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:08,488 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:09,569 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:10,857 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:11,391 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:14,400 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:15,950 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:17,562 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:19,465 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:23,304 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:26,039 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:27,539 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:28,793 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:29,503 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:30,334 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:32,222 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:35,398 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:35,470 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:35,530 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:35,852 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:40,437 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:41,016 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:41,559 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:44,648 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:45,882 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:47,399 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:48,006 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:50,391 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:51,806 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:53,195 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:55,992 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:57,851 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:37:58,451 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:38:00,688 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:38:05,667 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:38:05,778 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:38:14,701 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:38:17,880 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:38:26,667 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:38:37,365 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:38:42,994 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:19,295 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:20,122 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:23,095 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:26,108 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:28,627 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:30,032 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:33,231 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:33,315 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:34,230 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:37,012 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:37,325 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:39,713 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:40,981 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:41,272 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:42,325 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:47,262 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:50,018 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:50,697 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:51,833 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:52,603 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:54,405 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:56,808 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:39:58,422 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:40:02,506 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:40:02,663 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:40:10,028 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:40:12,650 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:40:14,343 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:40:17,907 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:40:20,161 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:40:20,669 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:40:22,342 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:40:24,437 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:40:26,979 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:40:28,364 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:40:30,239 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:40:30,490 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:40:34,538 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:40:55,113 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:41:04,948 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:41:14,459 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:41:30,755 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:41:37,725 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:41:40,540 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:41:45,817 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:41:46,956 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:41:48,132 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:41:49,856 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:41:49,962 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:41:51,551 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:41:52,310 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:41:53,981 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:41:54,675 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:41:55,918 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:41:58,180 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:41:59,977 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:42:00,371 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:42:03,097 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:42:05,175 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:42:06,892 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:42:14,591 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:42:15,234 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:42:33,466 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:42:36,577 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:42:36,599 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:42:40,746 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:42:42,245 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:42:58,271 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:42:58,979 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:00,984 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:03,255 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:07,495 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:09,226 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:11,835 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:11,964 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:12,608 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:16,337 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:16,953 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:18,145 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:20,537 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:22,260 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:23,647 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:25,212 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:26,898 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:27,337 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:28,987 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:30,451 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:31,210 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:32,234 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:32,487 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:34,844 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:35,281 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:36,345 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:40,160 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:46,676 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:51,255 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:51,594 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:54,860 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:57,352 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:43:59,053 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:44:04,404 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:44:04,421 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:44:12,700 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:44:14,404 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:44:15,475 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:44:26,784 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:44:32,833 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:44:37,939 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:44:38,957 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:44:41,089 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:44:42,583 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:44:46,835 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:44:55,906 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:44:56,198 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:44:58,057 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:45:00,500 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:45:07,203 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:45:08,173 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:45:15,803 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:45:21,589 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:45:31,166 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:45:42,932 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:45:44,128 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:45:45,885 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:45:50,997 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:45:52,525 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:45:58,791 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:02,559 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:04,943 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:05,140 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:05,150 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:09,605 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:11,284 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:11,518 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:12,228 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:15,613 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:17,977 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:18,654 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:20,793 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:21,091 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:23,109 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:24,286 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:26,569 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:26,796 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:27,219 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:28,854 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:32,293 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:33,399 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:33,954 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:39,469 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:40,752 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:43,284 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:43,324 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:48,095 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:52,244 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:46:59,966 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:00,977 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:02,896 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:03,874 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:05,897 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:06,416 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:08,557 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:08,754 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:09,459 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:11,825 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:11,847 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:12,870 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:13,130 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:14,764 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:16,150 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:16,823 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:17,205 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:17,414 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:18,727 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:21,367 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:21,894 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:24,416 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:26,399 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:31,253 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:34,471 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:34,530 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:34,911 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:42,421 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:44,488 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:45,148 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:45,829 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:48,783 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:50,256 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:50,896 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:53,320 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:55,931 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:56,624 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:47:57,439 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:01,129 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:01,445 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:01,762 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:04,982 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:07,592 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:10,023 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:12,499 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:14,355 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:16,523 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:19,732 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:21,235 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:22,470 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:26,822 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:26,879 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:27,373 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:27,709 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:32,369 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:33,224 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:34,611 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:34,699 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:36,835 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:38,567 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:39,916 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:40,675 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:43,223 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:45,197 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:48:48,085 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:49:05,175 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:49:08,139 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:49:14,525 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:49:16,023 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:49:20,026 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:49:24,829 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:49:26,918 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:49:28,134 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:49:31,207 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:49:33,438 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:49:33,889 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:49:41,847 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:49:47,891 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:49:48,220 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:49:48,501 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:49:56,505 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:49:57,177 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:50:18,010 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:50:20,483 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:50:21,401 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:50:21,951 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:50:28,667 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:50:30,716 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:50:33,079 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:50:33,173 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:50:42,141 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:50:42,232 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:50:47,941 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:50:49,829 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:50:50,397 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:50:52,265 - distributed.utils_perf - WARNING - full garbage collections took 27% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:50:55,713 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:50:56,384 - distributed.utils_perf - WARNING - full garbage collections took 27% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:50:59,014 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:51:04,022 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:51:05,727 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:51:06,731 - distributed.utils_perf - WARNING - full garbage collections took 27% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:51:08,751 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:51:15,809 - distributed.utils_perf - WARNING - full garbage collections took 27% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:51:19,148 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:51:28,085 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:51:28,878 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:51:29,364 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:51:33,341 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:51:40,038 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:51:42,743 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:51:49,162 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:51:51,190 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:51:51,558 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:51:51,562 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:51:56,332 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:51:56,457 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:51:58,684 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:52:00,635 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:52:01,180 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:52:11,601 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:52:12,677 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:52:17,583 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:52:18,471 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:52:20,121 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:52:25,516 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:52:27,202 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:52:33,472 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:52:33,527 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:52:35,256 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:52:37,849 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:53:12,612 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:53:14,095 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:53:14,856 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:53:16,121 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:53:21,672 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:53:22,256 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:53:23,816 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:53:41,363 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:54:11,755 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:54:15,541 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:54:16,131 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:54:40,537 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:55:11,329 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:55:15,319 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:55:18,179 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:55:37,481 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:55:57,956 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:00,582 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:01,562 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:04,607 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:06,375 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:06,650 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:11,306 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:12,458 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:15,646 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:17,975 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:20,451 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:21,527 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:21,695 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:21,785 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:23,013 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:23,253 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:23,387 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:25,158 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:25,266 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:27,282 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:28,363 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:32,404 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:35,519 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:40,655 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:40,801 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:42,497 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:47,409 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:56:53,796 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:04,470 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:07,695 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:09,079 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:09,498 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:10,757 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:14,110 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:14,172 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:15,585 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:15,899 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:17,840 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:18,214 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:20,833 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:22,095 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:22,896 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:27,006 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:28,598 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:34,517 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:35,277 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:35,908 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:38,383 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:39,195 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:40,132 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:43,568 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:43,740 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:44,865 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:45,067 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:47,068 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:47,169 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:51,609 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:53,768 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:55,509 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:56,536 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:58,570 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:57:58,868 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:58:00,438 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:58:01,281 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:58:08,861 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:58:16,275 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:58:16,530 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:58:21,219 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:58:24,633 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:58:27,027 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:58:28,385 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:58:31,855 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:58:35,456 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:58:38,186 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:58:40,629 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:58:50,728 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:58:51,225 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:58:54,414 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:58:55,956 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:58:59,452 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:00,429 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:03,951 - distributed.utils_perf - WARNING - full garbage collections took 27% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:04,122 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:06,318 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:07,449 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:07,460 - distributed.utils_perf - WARNING - full garbage collections took 27% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:10,803 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:14,286 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:15,002 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:17,129 - distributed.utils_perf - WARNING - full garbage collections took 27% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:21,091 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:21,780 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:25,104 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:26,108 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:27,192 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:28,535 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:30,098 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:30,940 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:33,610 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:33,886 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:34,288 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:34,413 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:42,457 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:48,474 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:48,537 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:51,545 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:52,198 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:53,670 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:54,664 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:57,241 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:57,348 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:58,040 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 21:59:58,100 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:01,856 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:02,050 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:03,534 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:04,002 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:08,228 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:08,302 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:11,607 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:12,130 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:12,249 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:16,276 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:16,333 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:16,410 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:23,152 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:29,626 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:30,630 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:31,497 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:32,490 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:34,814 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:36,095 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:36,410 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:38,273 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:38,414 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:39,961 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:41,642 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:42,211 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:44,569 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:46,063 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:00:59,100 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:01:05,979 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:01:15,207 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:01:15,573 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:01:19,158 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:01:24,509 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:01:30,533 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:01:34,750 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:01:37,462 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:01:38,119 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:01:43,367 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:01:45,552 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:01:50,341 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:01:55,627 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:01:56,112 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:02:06,597 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:02:07,083 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:02:11,219 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:02:22,705 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:02:28,630 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:02:30,072 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:02:33,145 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:02:45,873 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:02:57,991 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:01,731 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:02,287 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:03,145 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:06,504 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:08,428 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:09,942 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:14,128 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:15,655 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:16,889 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:19,886 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:22,264 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:24,152 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:24,900 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:25,049 - distributed.utils_perf - WARNING - full garbage collections took 27% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:30,582 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:30,628 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:31,072 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:31,318 - distributed.utils_perf - WARNING - full garbage collections took 27% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:34,215 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:35,039 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:37,816 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:39,098 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:41,237 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:43,055 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:43,198 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:43,280 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:47,792 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:47,896 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:49,528 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:51,198 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:52,990 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:54,792 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:03:59,724 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:02,000 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:02,871 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:06,937 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:09,769 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:09,892 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:12,703 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:14,495 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:14,779 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:23,476 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:25,114 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:26,430 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:28,782 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:31,312 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:32,246 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:33,004 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:33,616 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:38,304 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:40,229 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:41,273 - distributed.utils_perf - WARNING - full garbage collections took 27% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:47,461 - distributed.utils_perf - WARNING - full garbage collections took 27% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:48,552 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:52,371 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:53,244 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:57,419 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:04:57,487 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:05:00,045 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:05:02,119 - distributed.utils_perf - WARNING - full garbage collections took 27% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:05:02,404 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:05:07,749 - distributed.utils_perf - WARNING - full garbage collections took 26% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:05:24,455 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:05:56,712 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:05:57,907 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:06:31,668 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:06:55,812 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:07:01,681 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:07:07,362 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:07:20,683 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:07:23,231 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:07:42,856 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:08:43,493 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:08:49,797 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:09:04,476 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:09:13,646 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:09:48,277 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:10:09,123 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:10:14,254 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:10:32,317 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:11:06,791 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:11:42,770 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:11:55,969 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:12:00,724 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:12:15,134 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:12:34,278 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:13:09,673 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:13:09,739 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:13:13,984 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:13:20,009 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:13:20,111 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:13:22,994 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:13:29,089 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:13:33,715 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:13:34,948 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:13:38,940 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:13:40,096 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:13:40,500 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:13:41,164 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:13:47,438 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:13:49,851 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:13:50,939 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:13:51,469 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:13:54,434 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:13:56,151 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:13:56,565 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:13:59,202 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:14:07,631 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:14:09,665 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:14:13,567 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:14:14,697 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:14:24,088 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:14:26,280 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:14:42,913 - distributed.utils_perf - WARNING - full garbage collections took 15% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:14:50,690 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:15:29,123 - distributed.utils_perf - WARNING - full garbage collections took 25% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:16:24,912 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:17:18,642 - distributed.utils_perf - WARNING - full garbage collections took 15% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:17:21,708 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:18:08,501 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:18:51,770 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:19:37,885 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:19:50,805 - distributed.utils_perf - WARNING - full garbage collections took 15% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:20:35,442 - distributed.utils_perf - WARNING - full garbage collections took 15% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:21:24,338 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:22:11,981 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:22:24,424 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:23:11,662 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:24:02,292 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:24:52,591 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:25:03,516 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:25:49,142 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:26:35,746 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:27:27,747 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:27:38,429 - distributed.utils_perf - WARNING - full garbage collections took 10% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:28:21,205 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:29:05,725 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:29:59,462 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:30:13,194 - distributed.utils_perf - WARNING - full garbage collections took 10% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:30:50,135 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:31:38,159 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:32:30,286 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:33:23,093 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:35:01,520 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:37:33,261 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:40:04,353 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:42:26,266 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:42:45,682 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:43:04,796 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:43:13,797 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:43:37,320 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:44:39,971 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 22:45:31,072 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "2024-01-22 23:29:41,710 - distributed.utils_perf - WARNING - full garbage collections took 10% CPU time recently (threshold: 10%)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "2024-01-22 23:30:06,204 - distributed.utils_perf - WARNING - full garbage collections took 10% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:30:10,603 - distributed.utils_perf - WARNING - full garbage collections took 10% CPU time recently (threshold: 10%)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "2024-01-22 23:30:17,052 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:30:17,659 - distributed.utils_perf - WARNING - full garbage collections took 34% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:30:21,107 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:30:23,814 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:30:24,485 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:30:28,157 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:30:31,234 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:30:32,065 - distributed.utils_perf - WARNING - full garbage collections took 34% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:30:35,206 - distributed.utils_perf - WARNING - full garbage collections took 34% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:30:38,711 - distributed.utils_perf - WARNING - full garbage collections took 34% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:30:42,797 - distributed.utils_perf - WARNING - full garbage collections took 34% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:30:45,333 - distributed.utils_perf - WARNING - full garbage collections took 34% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:30:48,718 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:30:54,731 - distributed.utils_perf - WARNING - full garbage collections took 34% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:30:57,010 - distributed.utils_perf - WARNING - full garbage collections took 34% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:30:59,155 - distributed.utils_perf - WARNING - full garbage collections took 34% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:30:59,711 - distributed.utils_perf - WARNING - full garbage collections took 34% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:31:02,064 - distributed.utils_perf - WARNING - full garbage collections took 35% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:31:05,697 - distributed.utils_perf - WARNING - full garbage collections took 35% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:31:09,625 - distributed.utils_perf - WARNING - full garbage collections took 34% CPU time recently (threshold: 10%)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "2024-01-22 23:31:16,659 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "2024-01-22 23:31:21,671 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:31:23,890 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "2024-01-22 23:31:26,041 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:31:28,694 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "2024-01-22 23:31:32,244 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:31:40,822 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:31:42,375 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "2024-01-22 23:31:47,592 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:31:56,641 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:31:58,578 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:31:59,386 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:32:20,177 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:32:21,099 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:32:23,615 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:32:30,507 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:32:32,611 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:32:33,372 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:32:36,529 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:32:37,164 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:32:40,282 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:32:41,204 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:32:44,625 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:32:47,547 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:32:54,014 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:32:58,866 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:33:02,674 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:33:10,017 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:33:17,313 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:33:25,619 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:33:26,577 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:33:37,980 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:33:40,993 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:33:43,322 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:34:21,437 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:34:32,733 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:35:01,634 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:35:07,184 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:35:07,363 - distributed.utils_perf - WARNING - full garbage collections took 10% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:35:16,577 - distributed.utils_perf - WARNING - full garbage collections took 10% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:35:21,345 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:35:24,563 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:35:34,288 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:35:36,194 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:35:42,642 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:35:55,029 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "2024-01-22 23:36:00,734 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:36:04,067 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "2024-01-22 23:36:10,182 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:36:22,162 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "2024-01-22 23:36:33,744 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:36:41,303 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:36:51,749 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:37:03,650 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:37:35,527 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "/Users/juliusbusecke/miniforge3/envs/cmip6-leap-feedstock-maintenance/lib/python3.10/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "2024-01-22 23:37:40,926 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:37:42,401 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:37:47,241 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:37:50,858 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:37:56,582 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:38:07,592 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:38:15,298 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:38:20,151 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:38:22,747 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:38:32,097 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:38:35,310 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:38:35,483 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:38:39,565 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:38:40,319 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:38:43,718 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:38:43,779 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:38:44,004 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:38:47,365 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:38:47,666 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:38:50,214 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:38:51,654 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:38:53,547 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:38:53,674 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:38:53,800 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:38:56,320 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:38:58,439 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:38:58,547 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:39:00,699 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:39:01,182 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:39:02,315 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:39:02,769 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:39:09,486 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:39:19,681 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:39:21,569 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:40:03,379 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:40:04,310 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:40:10,102 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:40:11,309 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:40:15,715 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:40:19,432 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:40:19,728 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:40:21,563 - distributed.utils_perf - WARNING - full garbage collections took 33% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:40:22,914 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:40:28,115 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:40:28,207 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:40:36,065 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:40:40,866 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:40:42,244 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:40:44,338 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:40:47,895 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:40:48,287 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:40:52,980 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:40:55,759 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:40:57,565 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:40:59,938 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:41:00,320 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:41:02,249 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:41:04,421 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:41:04,768 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:41:06,191 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:41:08,248 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:41:12,266 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:41:12,330 - distributed.utils_perf - WARNING - full garbage collections took 32% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:41:29,202 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:41:36,307 - distributed.utils_perf - WARNING - full garbage collections took 27% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:41:37,074 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:02,977 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:09,629 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:14,287 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:19,543 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:24,567 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:28,212 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:29,153 - distributed.utils_perf - WARNING - full garbage collections took 10% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:30,388 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:32,690 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:34,069 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:35,055 - distributed.utils_perf - WARNING - full garbage collections took 31% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:35,118 - distributed.utils_perf - WARNING - full garbage collections took 10% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:37,786 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:39,057 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:40,565 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:42,892 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:43,345 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:45,085 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:46,697 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:47,462 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:47,634 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:49,072 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:53,386 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:42:53,430 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:43:01,587 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:43:08,573 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:43:11,852 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:43:12,944 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:43:16,076 - distributed.utils_perf - WARNING - full garbage collections took 19% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:43:18,596 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:43:19,093 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:43:21,416 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:43:22,351 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:43:55,497 - distributed.utils_perf - WARNING - full garbage collections took 27% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:44:36,243 - distributed.utils_perf - WARNING - full garbage collections took 30% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:45:21,681 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:45:26,438 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:45:31,406 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:45:38,749 - distributed.utils_perf - WARNING - full garbage collections took 27% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:46:05,590 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:46:10,346 - distributed.utils_perf - WARNING - full garbage collections took 29% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:46:12,812 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:46:18,490 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:46:30,948 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:46:35,920 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:46:41,275 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:46:59,504 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:47:07,692 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:47:21,457 - distributed.utils_perf - WARNING - full garbage collections took 27% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:47:48,070 - distributed.utils_perf - WARNING - full garbage collections took 28% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:48:16,600 - distributed.utils_perf - WARNING - full garbage collections took 27% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:48:18,064 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:48:20,511 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:49:21,747 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:50:04,271 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:50:07,722 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:50:14,882 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:51:30,002 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:51:36,065 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:51:39,697 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:51:40,788 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:51:43,002 - distributed.utils_perf - WARNING - full garbage collections took 18% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:51:43,222 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:51:44,615 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:51:47,659 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:51:49,565 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:51:49,627 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:51:50,783 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:51:51,440 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:51:54,103 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:51:55,810 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:51:55,945 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:52:02,749 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:52:05,375 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:52:05,797 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:52:07,845 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:52:10,851 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:52:12,971 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:52:13,165 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:52:14,381 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:52:16,641 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:52:19,648 - distributed.utils_perf - WARNING - full garbage collections took 21% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:52:58,835 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:53:16,594 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:53:36,157 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:54:01,887 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:55:47,271 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:56:01,171 - distributed.utils_perf - WARNING - full garbage collections took 20% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:56:30,566 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:58:15,948 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:58:34,729 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-22 23:59:06,645 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:00:26,721 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:00:47,353 - distributed.utils_perf - WARNING - full garbage collections took 15% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:01:06,050 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:01:30,446 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:01:34,376 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:01:34,968 - distributed.utils_perf - WARNING - full garbage collections took 15% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:01:36,551 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:01:39,615 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:01:40,072 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:01:42,076 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:01:45,620 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:02:29,049 - distributed.utils_perf - WARNING - full garbage collections took 15% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:02:57,265 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:03:03,757 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:03:26,544 - distributed.utils_perf - WARNING - full garbage collections took 17% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:04:00,920 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:04:05,706 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:04:05,764 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:04:17,336 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:04:43,527 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:04:49,904 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:04:52,904 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:04:53,626 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:04:59,023 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:05:12,391 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:05:15,179 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:05:31,109 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:05:38,724 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:05:41,746 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:05:52,803 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:05:58,642 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:00,660 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:04,847 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:09,089 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:09,935 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:10,808 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:11,966 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:14,353 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:18,177 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:18,316 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:19,243 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:21,573 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:23,355 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:24,182 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:28,531 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:30,229 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:31,093 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:37,033 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:37,669 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:37,829 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:38,363 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:39,231 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:39,487 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:41,798 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:44,379 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:45,551 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:49,736 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:51,649 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:53,849 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:54,735 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:54,897 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:57,576 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:06:57,666 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:00,106 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:00,990 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:01,156 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:07,626 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:15,853 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:20,596 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:21,920 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:33,982 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:34,745 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:35,256 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:36,303 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:37,227 - distributed.utils_perf - WARNING - full garbage collections took 22% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:39,833 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:45,659 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:46,045 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:47,032 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:47,756 - distributed.utils_perf - WARNING - full garbage collections took 23% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:48,212 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:48,779 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:50,168 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:50,838 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:51,176 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:51,703 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:51,923 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:52,550 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:53,119 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:53,896 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:54,543 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:54,875 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:55,135 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:55,598 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:55,645 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:56,311 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:56,416 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:56,524 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:58,464 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:58,825 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:07:59,376 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:08:00,012 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:08:00,725 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:08:01,803 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:08:03,867 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:08:04,296 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:08:04,512 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:08:04,968 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:08:05,214 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:08:05,844 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:08:07,439 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:08:07,861 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:08:08,090 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:08:10,372 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:08:11,226 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:08:11,645 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:08:22,469 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:08:42,130 - distributed.utils_perf - WARNING - full garbage collections took 14% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:08:45,036 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:08:47,416 - distributed.utils_perf - WARNING - full garbage collections took 16% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:08:55,867 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:09:20,156 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:09:49,034 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:10:05,482 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:10:48,321 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:11:02,457 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:11:03,501 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:11:06,077 - distributed.utils_perf - WARNING - full garbage collections took 24% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:11:07,192 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:11:11,715 - distributed.utils_perf - WARNING - full garbage collections took 12% CPU time recently (threshold: 10%)\n",
      "2024-01-23 00:11:28,330 - distributed.utils_perf - WARNING - full garbage collections took 11% CPU time recently (threshold: 10%)\n"
     ]
    }
   ],
   "source": [
    "n=10000\n",
    "df_input_batches = [df_input[i:i+n] for i in range(0,len(df_input),n)]\n",
    "for df_batch in tqdm(df_input_batches):\n",
    "    batch_test_add(df_batch, bq)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "512d6ffc-df9e-4974-a3d6-ba7ee7029c0c",
   "metadata": {},
   "source": [
    "## show all legacy stores that had failing tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de9eca6d-4e5d-4bbf-969e-77f0cb30ac1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_read = bq.get_latest()\n",
    "failed_legacy = df_read[df_read['tests_passed'].isin([False])]\n",
    "failed_legacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e0a4d5ff-5a0d-44e6-ac8d-69a8fc05b48b",
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (332580067.py, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  Cell \u001b[0;32mIn[13], line 1\u001b[0;36m\u001b[0m\n\u001b[0;31m    break this shit\u001b[0m\n\u001b[0m          ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "break this shit"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba555eb7-59f3-45a3-89dd-df05983377fb",
   "metadata": {},
   "source": [
    "## Running in serial and profiling\n",
    "- Opening the zarr in xarray takes by far the longest\n",
    "    - I was able to shave quite a bit of time by only opening the dataset once and then passing that object around\n",
    "- The API calls to google are not negligible (~25% of the wall time)\n",
    "- I guess to speed this up we would need to parallelize the loading of datasets and testing and then add batches of results to the bq! Doable but not exactly what I want to focus on right now.\n",
    "- Snakeviz adds significant overhead! Run without if we want to speed this up. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2528319-3161-4f94-83c0-2abdfe13e546",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_and_add(iid, store_url, bq_client):\n",
    "    test_passed = test(iid, store_url)\n",
    "    iid_entry = IIDEntry(\n",
    "            iid,\n",
    "            store_url, # should this be the url (str) or the store ( zarr.storage.FSStore)?\n",
    "            retracted = False, \n",
    "            tests_passed=False\n",
    "        ) \n",
    "    bq_client.insert_iid(iid_entry)\n",
    "    return iid, store_url, test_passed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32d1075a-81c8-46eb-b471-967bf2a5e5b6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%load_ext snakeviz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2da37a7-8f91-4594-ae43-8c1b0da77f60",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%snakeviz\n",
    "results = []\n",
    "for idx,row in tqdm(df_input.iterrows(),total=len(df_input)):\n",
    "    iid = row.instance_id\n",
    "    store_url = row.store\n",
    "    results.append(test_and_add(iid, store_url, bq))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "267d7344-f502-4ed0-8e17-8c9f5e18fbba",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "results = []\n",
    "for idx,row in tqdm(df_input.iterrows(),total=len(df_input)):\n",
    "    iid = row.instance_id\n",
    "    store_url = row.store\n",
    "    results.append(test_and_add(iid, store_url, bq))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}