examples/JAX_+_Triton_Flash_Attention.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "JAX + Triton Flash Attention",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# JAX + Triton Flash Attention\n"
      ],
      "metadata": {
        "id": "Gf_QOFlcBskR"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f8ZH9ERvVzDF"
      },
      "source": [
        "Copyright 2022 Google LLC\n",
        "\n",
        "Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "you may not use this file except in compliance with the License.\n",
        "You may obtain a copy of the License at\n",
        "\n",
        "     https://www.apache.org/licenses/LICENSE-2.0\n",
        "\n",
        "Unless required by applicable law or agreed to in writing, software\n",
        "distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "See the License for the specific language governing permissions and\n",
        "limitations under the License."
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Installs"
      ],
      "metadata": {
        "id": "pw5qfsHYD13n"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -U --pre triton\n",
        "!pip install chex\n",
        "!pip install cmake"
      ],
      "metadata": {
        "id": "IYq4sTNWBak3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ca0fcaa8-9638-46d9-fd50-ec0905bc8a18"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting triton\n",
            "  Downloading triton-2.0.0.dev20220721-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (18.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 18.7 MB 1.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from triton) (1.12.0+cu113)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from triton) (3.7.1)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.7/dist-packages (from triton) (3.22.5)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch->triton) (4.1.1)\n",
            "Installing collected packages: triton\n",
            "Successfully installed triton-2.0.0.dev20220721\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting chex\n",
            "  Downloading chex-0.1.3-py3-none-any.whl (72 kB)\n",
            "\u001b[K     |████████████████████████████████| 72 kB 618 kB/s \n",
            "\u001b[?25hRequirement already satisfied: jax>=0.1.55 in /usr/local/lib/python3.7/dist-packages (from chex) (0.3.14)\n",
            "Requirement already satisfied: jaxlib>=0.1.37 in /usr/local/lib/python3.7/dist-packages (from chex) (0.3.14+cuda11.cudnn805)\n",
            "Requirement already satisfied: toolz>=0.9.0 in /usr/local/lib/python3.7/dist-packages (from chex) (0.12.0)\n",
            "Requirement already satisfied: dm-tree>=0.1.5 in /usr/local/lib/python3.7/dist-packages (from chex) (0.1.7)\n",
            "Requirement already satisfied: absl-py>=0.9.0 in /usr/local/lib/python3.7/dist-packages (from chex) (1.2.0)\n",
            "Requirement already satisfied: numpy>=1.18.0 in /usr/local/lib/python3.7/dist-packages (from chex) (1.21.6)\n",
            "Requirement already satisfied: etils[epath] in /usr/local/lib/python3.7/dist-packages (from jax>=0.1.55->chex) (0.6.0)\n",
            "Requirement already satisfied: opt-einsum in /usr/local/lib/python3.7/dist-packages (from jax>=0.1.55->chex) (3.3.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from jax>=0.1.55->chex) (4.1.1)\n",
            "Requirement already satisfied: scipy>=1.5 in /usr/local/lib/python3.7/dist-packages (from jax>=0.1.55->chex) (1.7.3)\n",
            "Requirement already satisfied: flatbuffers<3.0,>=1.12 in /usr/local/lib/python3.7/dist-packages (from jaxlib>=0.1.37->chex) (2.0)\n",
            "Requirement already satisfied: importlib_resources in /usr/local/lib/python3.7/dist-packages (from etils[epath]->jax>=0.1.55->chex) (5.8.0)\n",
            "Requirement already satisfied: zipp in /usr/local/lib/python3.7/dist-packages (from etils[epath]->jax>=0.1.55->chex) (3.8.1)\n",
            "Installing collected packages: chex\n",
            "Successfully installed chex-0.1.3\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.7/dist-packages (3.22.5)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Install JAX + Triton { vertical-output: true }\n",
        "%cd /root\n",
        "!pip uninstall jax -y\n",
        "!git clone https://github.com/sharadmv/jax.git\n",
        "%cd jax\n",
        "!git checkout triton\n",
        "!pip install -I -U \".[cuda11_cudnn82]\" -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html\n",
        "!pip install pybind11\n",
        "%cd triton\n",
        "!make\n",
        "!pip install .\n",
        "%cd /root"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "cellView": "form",
        "id": "TgpBQ2XdH6Kn",
        "outputId": "3a89904e-fe46-4c80-f39e-c830fee55788"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/root\n",
            "Found existing installation: jax 0.3.14\n",
            "Uninstalling jax-0.3.14:\n",
            "  Successfully uninstalled jax-0.3.14\n",
            "Cloning into 'jax'...\n",
            "remote: Enumerating objects: 72139, done.\u001b[K\n",
            "remote: Counting objects: 100% (269/269), done.\u001b[K\n",
            "remote: Compressing objects: 100% (134/134), done.\u001b[K\n",
            "remote: Total 72139 (delta 159), reused 215 (delta 135), pack-reused 71870\u001b[K\n",
            "Receiving objects: 100% (72139/72139), 52.81 MiB | 17.69 MiB/s, done.\n",
            "Resolving deltas: 100% (56480/56480), done.\n",
            "/root/jax\n",
            "Branch 'triton' set up to track remote branch 'triton' from 'origin'.\n",
            "Switched to a new branch 'triton'\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Looking in links: https://storage.googleapis.com/jax-releases/jax_cuda_releases.html\n",
            "Processing /root/jax\n",
            "\u001b[33m  DEPRECATION: A future pip version will change local packages to be built in-place without first copying to a temporary directory. We recommend you use --use-feature=in-tree-build to test your packages with this new behavior before it becomes the default.\n",
            "   pip 21.3 will remove support for this functionality. You can find discussion regarding this at https://github.com/pypa/pip/issues/7555.\u001b[0m\n",
            "Collecting absl-py\n",
            "  Downloading absl_py-1.2.0-py3-none-any.whl (123 kB)\n",
            "\u001b[K     |████████████████████████████████| 123 kB 8.0 MB/s \n",
            "\u001b[?25hCollecting numpy>=1.19\n",
            "  Downloading numpy-1.21.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (15.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 15.7 MB 87 kB/s \n",
            "\u001b[?25hCollecting opt_einsum\n",
            "  Downloading opt_einsum-3.3.0-py3-none-any.whl (65 kB)\n",
            "\u001b[K     |████████████████████████████████| 65 kB 4.9 MB/s \n",
            "\u001b[?25hCollecting scipy>=1.5\n",
            "  Downloading scipy-1.7.3-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (38.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 38.1 MB 1.2 MB/s \n",
            "\u001b[?25hCollecting typing_extensions\n",
            "  Downloading typing_extensions-4.3.0-py3-none-any.whl (25 kB)\n",
            "Collecting etils[epath]\n",
            "  Downloading etils-0.6.0-py3-none-any.whl (98 kB)\n",
            "\u001b[K     |████████████████████████████████| 98 kB 8.9 MB/s \n",
            "\u001b[?25hCollecting jaxlib==0.3.14+cuda11.cudnn82\n",
            "  Downloading https://storage.googleapis.com/jax-releases/cuda11/jaxlib-0.3.14%2Bcuda11.cudnn82-cp37-none-manylinux2014_x86_64.whl (161.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 161.9 MB 75 kB/s \n",
            "\u001b[?25hCollecting flatbuffers<3.0,>=1.12\n",
            "  Downloading flatbuffers-2.0-py2.py3-none-any.whl (26 kB)\n",
            "Collecting zipp\n",
            "  Downloading zipp-3.8.1-py3-none-any.whl (5.6 kB)\n",
            "Collecting importlib_resources\n",
            "  Downloading importlib_resources-5.9.0-py3-none-any.whl (33 kB)\n",
            "Building wheels for collected packages: jax\n",
            "  Building wheel for jax (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for jax: filename=jax-0.3.15-py3-none-any.whl size=1181768 sha256=d4fac2b0e93b9c99832fe4b9273e63a796754bf717a5814b72d8bbc0802892ad\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-hboiw5g3/wheels/86/d7/53/899202b8f11885ecc5a020cea4adf3759eb700ccb98870eecb\n",
            "Successfully built jax\n",
            "Installing collected packages: zipp, typing-extensions, etils, numpy, importlib-resources, scipy, opt-einsum, flatbuffers, absl-py, jaxlib, jax\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "thinc 8.1.0 requires typing-extensions<4.2.0,>=3.7.4.1; python_version < \"3.8\", but you have typing-extensions 4.3.0 which is incompatible.\n",
            "spacy 3.4.0 requires typing-extensions<4.2.0,>=3.7.4; python_version < \"3.8\", but you have typing-extensions 4.3.0 which is incompatible.\n",
            "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\n",
            "albumentations 0.1.12 requires imgaug<0.2.7,>=0.2.5, but you have imgaug 0.2.9 which is incompatible.\u001b[0m\n",
            "Successfully installed absl-py-1.2.0 etils-0.6.0 flatbuffers-2.0 importlib-resources-5.9.0 jax-0.3.15 jaxlib-0.3.14+cuda11.cudnn82 numpy-1.21.6 opt-einsum-3.3.0 scipy-1.7.3 typing-extensions-4.3.0 zipp-3.8.1\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "numpy"
                ]
              }
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting pybind11\n",
            "  Downloading pybind11-2.10.0-py3-none-any.whl (213 kB)\n",
            "\u001b[K     |████████████████████████████████| 213 kB 7.8 MB/s \n",
            "\u001b[?25hInstalling collected packages: pybind11\n",
            "Successfully installed pybind11-2.10.0\n",
            "/root/jax/triton\n",
            "g++ --std=c++17 custom_call.cc -o jax_triton/custom_call.so -L/usr/local/cuda/lib64 -L/usr/local/cuda/lib64/stubs -shared -fPIC -I/usr/include/python3.7m -I/usr/local/lib/python3.7/dist-packages/pybind11/include -I/usr/local/cuda/include -lcuda -fpermissive\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Processing /root/jax/triton\n",
            "\u001b[33m  DEPRECATION: A future pip version will change local packages to be built in-place without first copying to a temporary directory. We recommend you use --use-feature=in-tree-build to test your packages with this new behavior before it becomes the default.\n",
            "   pip 21.3 will remove support for this functionality. You can find discussion regarding this at https://github.com/pypa/pip/issues/7555.\u001b[0m\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Building wheels for collected packages: jax-triton\n",
            "  Building wheel for jax-triton (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for jax-triton: filename=jax_triton-0.1.0-py3-none-any.whl size=185263 sha256=6a2e9ade40d0bae62abb616868c0e4d1fbd20bb43a0e6bd1d7fa335bf3e340b7\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-3yb0dir1/wheels/df/eb/41/0d937c00487fd1f775c9e088cd16a406ba36162860f54ec746\n",
            "Successfully built jax-triton\n",
            "Installing collected packages: jax-triton\n",
            "Successfully installed jax-triton-0.1.0\n",
            "/root\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Imports"
      ],
      "metadata": {
        "id": "ofyANXztDkUm"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MDqK1FIyBM7X",
        "outputId": "246d125b-1d64-4bac-c64d-751974f63847"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Tue Jul 26 01:09:19 2022       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 460.32.03    Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   37C    P8     9W /  70W |      3MiB / 15109MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ],
      "source": [
        "import jax\n",
        "import jax.numpy as jnp\n",
        "import chex\n",
        "import numpy as np\n",
        "import time\n",
        "\n",
        "import pytest\n",
        "import torch\n",
        "\n",
        "import triton\n",
        "import triton.language as tl\n",
        "\n",
        "# Verify GPU type\n",
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Attention"
      ],
      "metadata": {
        "id": "SLMwL-o0GpUB"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Triton Flash Attention"
      ],
      "metadata": {
        "id": "dskUHUMiEPX9"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "FlashAttention was added recently to OpenAI Triton (https://github.com/openai/triton/issues/540#issuecomment-1170219125 and implementation https://github.com/openai/triton/blob/master/python/tutorials/06-fused-attention.py). This is just the copy-pasted implementation:"
      ],
      "metadata": {
        "id": "XUkihzdgD63T"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "@triton.jit\n",
        "def _fwd_kernel(\n",
        "    Q, K, V,\n",
        "    TMP, L, M,  # NOTE: TMP is a scratchpad buffer to workaround a compiler bug\n",
        "    Out,\n",
        "    stride_qz, stride_qh, stride_qm, stride_qk,\n",
        "    stride_kz, stride_kh, stride_kk, stride_kn,\n",
        "    stride_vz, stride_vh, stride_vk, stride_vn,\n",
        "    stride_oz, stride_oh, stride_om, stride_on,\n",
        "    Z, H, N_CTX,\n",
        "    BLOCK_M: tl.constexpr, BLOCK_DMODEL: tl.constexpr,\n",
        "    BLOCK_N: tl.constexpr,\n",
        "):\n",
        "    start_qm = tl.program_id(0)\n",
        "    off_hz = tl.program_id(1)\n",
        "    # initialize offsets\n",
        "    offs_m = start_qm * BLOCK_M + tl.arange(0, BLOCK_M)\n",
        "    offs_n = tl.arange(0, BLOCK_N)\n",
        "    offs_d = tl.arange(0, BLOCK_DMODEL)\n",
        "    off_q = off_hz * stride_qh + offs_m[:, None] * stride_qm + offs_d[None, :] * stride_qk\n",
        "    off_k = off_hz * stride_qh + offs_n[None, :] * stride_kn + offs_d[:, None] * stride_kk\n",
        "    off_v = off_hz * stride_qh + offs_n[:, None] * stride_qm + offs_d[None, :] * stride_qk\n",
        "    # Initialize pointers to Q, K, V\n",
        "    q_ptrs = Q + off_q\n",
        "    k_ptrs = K + off_k\n",
        "    v_ptrs = V + off_v\n",
        "    # initialize pointer to m and l\n",
        "    t_ptrs = TMP + off_hz * N_CTX + offs_m\n",
        "\n",
        "    acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32)\n",
        "    m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float(\"inf\")\n",
        "    l_i = tl.zeros([BLOCK_M], dtype=tl.float32)\n",
        "\n",
        "    q = tl.load(q_ptrs)\n",
        "    for start_n in range(0, start_qm + 1):\n",
        "        # -- compute qk ----\n",
        "        k = tl.load(k_ptrs)\n",
        "        qk = tl.dot(q, k)\n",
        "        qk += tl.where(offs_m[:, None] >= (start_n * BLOCK_N + offs_n[None, :]), 0, float(\"-inf\"))\n",
        "        # -- compute m_ij, p, l_ij\n",
        "        m_ij = tl.max(qk, 1)\n",
        "        p = tl.exp(qk - m_ij[:, None])\n",
        "        l_ij = tl.sum(p, 1)\n",
        "        # -- update m_i and l_i\n",
        "        m_i_new = tl.maximum(m_i, m_ij)\n",
        "        alpha = tl.exp(m_i - m_i_new)\n",
        "        beta = tl.exp(m_ij - m_i_new)\n",
        "        l_i_new = alpha * l_i + beta * l_ij\n",
        "        # -- update output accumulator --\n",
        "        # scale p\n",
        "        p_scale = beta / l_i_new\n",
        "        p = p * p_scale[:, None]\n",
        "        p = p.to(tl.float16)\n",
        "        # scale acc\n",
        "        acc_scale = l_i / l_i_new * alpha\n",
        "        tl.store(t_ptrs, acc_scale)\n",
        "        acc_scale = tl.load(t_ptrs)  # BUG: have to store and immediately load\n",
        "        acc = acc * acc_scale[:, None]\n",
        "        # update acc\n",
        "        v = tl.load(v_ptrs)\n",
        "        acc += tl.dot(p, v)\n",
        "        k_ptrs += BLOCK_N * stride_kn\n",
        "        v_ptrs += BLOCK_N * stride_vk\n",
        "        # r_ptrs += BLOCK_N\n",
        "        l_i = l_i_new\n",
        "        m_i = m_i_new\n",
        "\n",
        "    start_qm = tl.program_id(0)\n",
        "    offs_m = start_qm * BLOCK_M + tl.arange(0, BLOCK_M)\n",
        "    # write back l and m\n",
        "    l_ptrs = L + off_hz * N_CTX + offs_m\n",
        "    m_ptrs = M + off_hz * N_CTX + offs_m\n",
        "    tl.store(l_ptrs, l_i)\n",
        "    tl.store(m_ptrs, m_i)\n",
        "    # initialize pointers to output\n",
        "    offs_n = tl.arange(0, BLOCK_DMODEL)\n",
        "    off_out = off_hz * stride_oh + offs_m[:, None] * stride_om + offs_n[None, :] * stride_on\n",
        "    out_ptrs = Out + off_out\n",
        "    tl.store(out_ptrs, acc)\n",
        "\n",
        "\n",
        "class _attention(torch.autograd.Function):\n",
        "\n",
        "    @staticmethod\n",
        "    def forward(ctx, q, k, v):\n",
        "        BLOCK = 128\n",
        "        # shape constraints\n",
        "        Lq, Lk = q.shape[-1], k.shape[-2]\n",
        "        assert Lq == Lk\n",
        "        o = torch.empty_like(q)\n",
        "        grid = (triton.cdiv(q.shape[2], BLOCK), q.shape[0] * q.shape[1])\n",
        "        tmp = torch.empty((q.shape[0] * q.shape[1], q.shape[2]), device=q.device, dtype=torch.float32)\n",
        "        L = torch.empty((q.shape[0] * q.shape[1], q.shape[2]), device=q.device, dtype=torch.float32)\n",
        "        m = torch.empty((q.shape[0] * q.shape[1], q.shape[2]), device=q.device, dtype=torch.float32)\n",
        "        _fwd_kernel[grid](\n",
        "            q, k, v,\n",
        "            tmp, L, m,\n",
        "            o,\n",
        "            q.stride(0), q.stride(1), q.stride(2), q.stride(3),\n",
        "            k.stride(0), k.stride(1), k.stride(2), k.stride(3),\n",
        "            v.stride(0), v.stride(1), v.stride(2), v.stride(3),\n",
        "            o.stride(0), o.stride(1), o.stride(2), o.stride(3),\n",
        "            q.shape[0], q.shape[1], q.shape[2],\n",
        "            BLOCK_M=BLOCK, BLOCK_N=BLOCK,\n",
        "            BLOCK_DMODEL=64, num_warps=4,\n",
        "            num_stages=1,\n",
        "        )\n",
        "        ctx.save_for_backward(q, k, v, o, L, m)\n",
        "        ctx.BLOCK = BLOCK\n",
        "        ctx.grid = grid\n",
        "        return o\n",
        "\n",
        "\n",
        "attention = _attention.apply\n",
        "\n",
        "\n",
        "@pytest.mark.parametrize('Z, H, N_CTX, D_MODEL', [(2, 3, 1024, 64)])\n",
        "def test_op(Z, H, N_CTX, D_MODEL, dtype=torch.float16):\n",
        "    torch.manual_seed(20)\n",
        "    q = .5 * torch.randn((Z, H, N_CTX, D_MODEL), dtype=dtype, device=\"cuda\", requires_grad=True)\n",
        "    k = .5 * torch.randn((Z, H, D_MODEL, N_CTX), dtype=dtype, device=\"cuda\", requires_grad=True)\n",
        "    v = .5 * torch.randn((Z, H, N_CTX, D_MODEL), dtype=dtype, device=\"cuda\", requires_grad=True)\n",
        "    # triton implementation\n",
        "    tri_out = attention(q, k, v)\n",
        "    # reference implementation\n",
        "    M = torch.tril(torch.ones((N_CTX, N_CTX), device=\"cuda\"))\n",
        "    ref_qk = torch.matmul(q, k)\n",
        "    for z in range(Z):\n",
        "        for h in range(H):\n",
        "            ref_qk[:, :, M == 0] = float(\"-inf\")\n",
        "    ref_qk = torch.softmax(ref_qk, dim=-1)\n",
        "    ref_out = torch.matmul(ref_qk, v)\n",
        "    # compare\n",
        "    triton.testing.assert_almost_equal(ref_out, tri_out)\n",
        "\n",
        "\n",
        "try:\n",
        "    from flash_attn.flash_attn_interface import flash_attn_func\n",
        "    HAS_FLASH = True\n",
        "except BaseException:\n",
        "    HAS_FLASH = False\n",
        "\n",
        "BATCH, N_HEADS, N_CTX, D_HEAD = 4, 64, 2048, 64\n",
        "# vary batch size for fixed heads / seq\n",
        "batch_bench = triton.testing.Benchmark(\n",
        "    x_names=['BATCH'],\n",
        "    x_vals=[2**i for i in range(0, 8)],\n",
        "    line_arg='provider',\n",
        "    line_vals=['triton'] + (['flash'] if HAS_FLASH else []),\n",
        "    line_names=['Triton'] + (['Flash'] if HAS_FLASH else []),\n",
        "    styles=[('red', '-'), ('blue', '-')],\n",
        "    ylabel='ms',\n",
        "    plot_name=f'fused-attention-seq{N_CTX}-head{N_HEADS}-d{D_HEAD}',\n",
        "    args={'H': N_HEADS, 'N_CTX': N_CTX, 'D_MODEL': D_HEAD, 'dtype': torch.float16}\n",
        ")\n",
        "# vary seq length for fixed head and batch=4\n",
        "seq_bench = triton.testing.Benchmark(\n",
        "    x_names=['N_CTX'],\n",
        "    x_vals=[2**i for i in range(10, 16)],\n",
        "    line_arg='provider',\n",
        "    line_vals=['triton'] + (['flash'] if HAS_FLASH else []),\n",
        "    line_names=['Triton'] + (['Flash'] if HAS_FLASH else []),\n",
        "    styles=[('red', '-'), ('blue', '-')],\n",
        "    ylabel='ms',\n",
        "    plot_name=f'fused-attention-batch{BATCH}-head{N_HEADS}-d{D_HEAD}',\n",
        "    args={'H': D_HEAD, 'BATCH': BATCH, 'D_MODEL': D_HEAD, 'dtype': torch.float16}\n",
        ")\n",
        "\n",
        "\n",
        "@triton.testing.perf_report([batch_bench, seq_bench])\n",
        "def bench_flash_attention(BATCH, H, N_CTX, D_MODEL, provider, dtype=torch.float16, device=\"cuda\"):\n",
        "    warmup = 25\n",
        "    rep = 500\n",
        "    if provider == \"triton\":\n",
        "        q = torch.randn((BATCH, H, N_CTX, D_MODEL), dtype=dtype, device=\"cuda\", requires_grad=True)\n",
        "        k = torch.randn((BATCH, H, D_MODEL, N_CTX), dtype=dtype, device=\"cuda\", requires_grad=True)\n",
        "        v = torch.randn((BATCH, H, N_CTX, D_MODEL), dtype=dtype, device=\"cuda\", requires_grad=True)\n",
        "        fn = lambda: attention(q, k, v)\n",
        "        ms = triton.testing.do_bench(fn, percentiles=None, warmup=warmup, rep=rep)\n",
        "        return ms\n",
        "    if provider == \"flash\":\n",
        "        lengths = torch.full((BATCH,), fill_value=N_CTX, device=device)\n",
        "        cu_seqlens = torch.zeros((BATCH + 1,), device=device, dtype=torch.int32)\n",
        "        cu_seqlens[1:] = lengths.cumsum(0)\n",
        "        qkv = torch.randn((BATCH * N_CTX, 3, H, D_MODEL), dtype=dtype, device=device, requires_grad=True)\n",
        "        fn = lambda: flash_attn_func(qkv, cu_seqlens, 0., N_CTX, causal=True)\n",
        "        ms = triton.testing.do_bench(fn, percentiles=None, warmup=warmup, rep=rep)\n",
        "        return ms\n",
        "\n",
        "\n",
        "#bench_flash_attention.run(save_path='.', print_data=True)"
      ],
      "metadata": {
        "id": "vmuhnKjYGuFM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## JAX"
      ],
      "metadata": {
        "id": "SyNCzJuoWdsj"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Verify equivalence to JAX:"
      ],
      "metadata": {
        "id": "YkRE5zA7e8zc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from types import SimpleNamespace\n",
        "import jax_triton as jt\n",
        "\n",
        "import jax\n",
        "import jax.numpy as jnp\n",
        "from jax import random\n",
        "import numpy as np\n",
        "\n",
        "def _strides(shape):\n",
        "  all = np.prod(shape)\n",
        "  for s in shape:\n",
        "    all = all // s\n",
        "    yield int(all)\n",
        "\n",
        "@triton.jit\n",
        "def fused_attention_kernel(\n",
        "    Q, K, V,\n",
        "    TMP, L, M,  # NOTE: TMP is a scratchpad buffer to workaround a compiler bug\n",
        "    Out,\n",
        "    stride_qz: tl.constexpr, stride_qh: tl.constexpr, stride_qm: tl.constexpr, stride_qk: tl.constexpr,\n",
        "    stride_kz: tl.constexpr, stride_kh: tl.constexpr, stride_kk: tl.constexpr, stride_kn: tl.constexpr,\n",
        "    stride_vz: tl.constexpr, stride_vh: tl.constexpr, stride_vk: tl.constexpr, stride_vn: tl.constexpr,\n",
        "    stride_oz: tl.constexpr, stride_oh: tl.constexpr, stride_om: tl.constexpr, stride_on: tl.constexpr,\n",
        "    Z: tl.constexpr, H: tl.constexpr, N_CTX: tl.constexpr,\n",
        "    BLOCK_M: tl.constexpr, BLOCK_DMODEL: tl.constexpr,\n",
        "    BLOCK_N: tl.constexpr,\n",
        "):\n",
        "    start_qm = tl.program_id(0)\n",
        "    off_hz = tl.program_id(1)\n",
        "    # initialize offsets\n",
        "    offs_m = start_qm * BLOCK_M + tl.arange(0, BLOCK_M)\n",
        "    offs_n = tl.arange(0, BLOCK_N)\n",
        "    offs_d = tl.arange(0, BLOCK_DMODEL)\n",
        "    off_q = off_hz * stride_qh + offs_m[:, None] * stride_qm + offs_d[None, :] * stride_qk\n",
        "    off_k = off_hz * stride_qh + offs_n[None, :] * stride_kn + offs_d[:, None] * stride_kk\n",
        "    off_v = off_hz * stride_qh + offs_n[:, None] * stride_qm + offs_d[None, :] * stride_qk\n",
        "    # Initialize pointers to Q, K, V\n",
        "    q_ptrs = Q + off_q\n",
        "    k_ptrs = K + off_k\n",
        "    v_ptrs = V + off_v\n",
        "    # initialize pointer to m and l\n",
        "    t_ptrs = TMP + off_hz * N_CTX + offs_m\n",
        "\n",
        "    acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32)\n",
        "    m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float(\"inf\")\n",
        "    l_i = tl.zeros([BLOCK_M], dtype=tl.float32)\n",
        "\n",
        "    q = tl.load(q_ptrs)\n",
        "    for start_n in range(0, start_qm + 1):\n",
        "        # -- compute qk ----\n",
        "        k = tl.load(k_ptrs)\n",
        "        qk = tl.dot(q, k)\n",
        "        qk += tl.where(offs_m[:, None] >= (start_n * BLOCK_N + offs_n[None, :]), 0, float(\"-inf\"))\n",
        "        # -- compute m_ij, p, l_ij\n",
        "        m_ij = tl.max(qk, 1)\n",
        "        p = tl.exp(qk - m_ij[:, None])\n",
        "        l_ij = tl.sum(p, 1)\n",
        "        # -- update m_i and l_i\n",
        "        m_i_new = tl.maximum(m_i, m_ij)\n",
        "        alpha = tl.exp(m_i - m_i_new)\n",
        "        beta = tl.exp(m_ij - m_i_new)\n",
        "        l_i_new = alpha * l_i + beta * l_ij\n",
        "        # -- update output accumulator --\n",
        "        # scale p\n",
        "        p_scale = beta / l_i_new\n",
        "        p = p * p_scale[:, None]\n",
        "        p = p.to(tl.float16)\n",
        "        # scale acc\n",
        "        acc_scale = l_i / l_i_new * alpha\n",
        "        tl.store(t_ptrs, acc_scale)\n",
        "        acc_scale = tl.load(t_ptrs)  # BUG: have to store and immediately load\n",
        "        acc = acc * acc_scale[:, None]\n",
        "        # update acc\n",
        "        v = tl.load(v_ptrs)\n",
        "        acc += tl.dot(p, v)\n",
        "        k_ptrs += BLOCK_N * stride_kn\n",
        "        v_ptrs += BLOCK_N * stride_vk\n",
        "        # r_ptrs += BLOCK_N\n",
        "        l_i = l_i_new\n",
        "        m_i = m_i_new\n",
        "\n",
        "    start_qm = tl.program_id(0)\n",
        "    offs_m = start_qm * BLOCK_M + tl.arange(0, BLOCK_M)\n",
        "    # write back l and m\n",
        "    l_ptrs = L + off_hz * N_CTX + offs_m\n",
        "    m_ptrs = M + off_hz * N_CTX + offs_m\n",
        "    tl.store(l_ptrs, l_i)\n",
        "    tl.store(m_ptrs, m_i)\n",
        "    # initialize pointers to output\n",
        "    offs_n = tl.arange(0, BLOCK_DMODEL)\n",
        "    off_out = off_hz * stride_oh + offs_m[:, None] * stride_om + offs_n[None, :] * stride_on\n",
        "    out_ptrs = Out + off_out\n",
        "    tl.store(out_ptrs, acc)\n",
        "    \n",
        "@jax.jit\n",
        "def jax_attention_triton(q: jnp.ndarray, k: jnp.ndarray, v: jnp.ndarray) -> jnp.ndarray:\n",
        "  BLOCK = 128\n",
        "  Lq, Lk = q.shape[-1], k.shape[-2]\n",
        "  assert Lq == Lk\n",
        "  grid = lambda _: (triton.cdiv(q.shape[2], BLOCK), q.shape[0] * q.shape[1])\n",
        "  out_shape = [\n",
        "      SimpleNamespace(shape=(q.shape[0] * q.shape[1], q.shape[2]), dtype=q.dtype),\n",
        "      SimpleNamespace(shape=(q.shape[0] * q.shape[1], q.shape[2]), dtype=q.dtype),\n",
        "      SimpleNamespace(shape=(q.shape[0] * q.shape[1], q.shape[2]), dtype=q.dtype),\n",
        "      SimpleNamespace(shape=q.shape, dtype=q.dtype)]\n",
        "  stride_qz, stride_qh, stride_qm, stride_qk = _strides(q.shape)\n",
        "  stride_kz, stride_kh, stride_kk, stride_kn = _strides(k.shape)\n",
        "  stride_vz, stride_vh, stride_vk, stride_vn = _strides(v.shape)\n",
        "  stride_oz, stride_oh, stride_om, stride_on = _strides(out_shape[-1].shape)\n",
        "  \n",
        "  metaparams = dict(\n",
        "    BLOCK_M=BLOCK,\n",
        "    BLOCK_N=BLOCK,\n",
        "    BLOCK_DMODEL=64,\n",
        "    stride_qz=stride_qz, stride_qh=stride_qh, stride_qm=stride_qm, stride_qk=stride_qk,\n",
        "    stride_kz=stride_kz, stride_kh=stride_kh, stride_kk=stride_kk, stride_kn=stride_kn,\n",
        "    stride_vz=stride_vz, stride_vh=stride_vh, stride_vk=stride_vk, stride_vn=stride_vn,\n",
        "    stride_oz=stride_oz, stride_oh=stride_oh, stride_om=stride_om, stride_on=stride_on,\n",
        "    Z=q.shape[0], H=q.shape[1], N_CTX=q.shape[2],\n",
        "    num_warps=4, num_stages=1\n",
        "  )\n",
        "  _, _, _, output = jt.triton_call(q, k, v, kernel=fused_attention_kernel,\n",
        "      out_shape=out_shape, grid=grid, **metaparams)\n",
        "  return output\n",
        "\n",
        "@jax.jit\n",
        "def jax_attention(q, k, v):\n",
        "  n_seq = q.shape[-2]\n",
        "  logits = jnp.matmul(q, k)\n",
        "  mask = jnp.tril(jnp.ones((1, 1, n_seq, n_seq), dtype=q.dtype))\n",
        "  mask = jnp.broadcast_to(mask, logits.shape)\n",
        "  logits = jnp.where(mask, logits, float('-inf'))\n",
        "  ref_qk = jax.nn.softmax(logits)\n",
        "  return jnp.matmul(ref_qk, v)\n",
        "\n",
        "def test_triton_jax(batch, heads, seq_len, d_model, dtype=torch.float16):\n",
        "  torch.manual_seed(20)\n",
        "  q = .5 * torch.randn((batch, heads, seq_len, d_model), dtype=dtype, device=\"cuda\", requires_grad=True)\n",
        "  k = .5 * torch.randn((batch, heads, d_model, seq_len), dtype=dtype, device=\"cuda\", requires_grad=True)\n",
        "  v = .5 * torch.randn((batch, heads, seq_len, d_model), dtype=dtype, device=\"cuda\", requires_grad=True)\n",
        "  # triton implementation\n",
        "  tri_out = attention(q, k, v).cpu().detach().numpy()\n",
        "\n",
        "  q_jax = jnp.array(q.cpu().detach().numpy(), dtype=jnp.float16)\n",
        "  del q\n",
        "  k_jax = jnp.array(k.cpu().detach().numpy(), dtype=jnp.float16)\n",
        "  del k\n",
        "  v_jax = jnp.array(v.cpu().detach().numpy(), dtype=jnp.float16)\n",
        "  del v\n",
        "\n",
        "  jax_out = jax_attention(q_jax, k_jax, v_jax)\n",
        "  chex.assert_trees_all_close(tri_out, jax_out, atol=0.003)\n",
        "  jax_out = jax_attention_triton(q_jax, k_jax, v_jax)\n",
        "  chex.assert_trees_all_close(tri_out, jax_out, atol=0.003)\n",
        "\n",
        "test_triton_jax(2, 32, 2048, 64)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cArp-W5vTWfz",
        "outputId": "b4f9f520-3eb8-400b-ceb7-d9ef613fa99b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:absl:chex.assert_trees_all_close has been renamed to chex.assert_trees_all_close, please update your code.\n",
            "WARNING:absl:chex.assert_trees_all_close has been renamed to chex.assert_trees_all_close, please update your code.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The setup `BATCH, N_HEADS, N_CTX, D_HEAD = 8, 4, 2048, 64` took 2.44ms on Triton based on timings above, as speedup of 8x:"
      ],
      "metadata": {
        "id": "9oZSrxbrwDi5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "BATCH, N_HEADS, N_CTX, D_HEAD = 8, 64, 2048, 64\n",
        "\n",
        "\n",
        "# warmup\n",
        "def bench_jax_triton(batch, heads, seq_len, d_model):\n",
        "  q_jax = jnp.ones((batch, heads, seq_len, d_model), dtype=jnp.float16)\n",
        "  k_jax = jnp.ones((batch, heads, d_model, seq_len), dtype=jnp.float16)\n",
        "  v_jax = jnp.ones((batch, heads, seq_len, d_model), dtype=jnp.float16)\n",
        "  # warmup\n",
        "  jax_attention_triton(q_jax, k_jax, q_jax).block_until_ready()\n",
        "  jax_attention_triton(q_jax, k_jax, q_jax).block_until_ready()\n",
        "\n",
        "  t1 = time.time()\n",
        "  num_runs = 100\n",
        "  for _ in range(num_runs):\n",
        "    jax_attention_triton(q_jax, k_jax, q_jax).block_until_ready()\n",
        "  estimate_ms = 1000 * (time.time() - t1) / num_runs\n",
        "  return estimate_ms\n",
        "\n",
        "print(bench_jax_triton(batch=BATCH, heads=N_HEADS, seq_len=N_CTX, d_model=D_HEAD))\n",
        "\n",
        "# warmup\n",
        "def bench_jax(batch, heads, seq_len, d_model):\n",
        "  q_jax = jnp.ones((batch, heads, seq_len, d_model), dtype=jnp.float16)\n",
        "  k_jax = jnp.ones((batch, heads, d_model, seq_len), dtype=jnp.float16)\n",
        "  v_jax = jnp.ones((batch, heads, seq_len, d_model), dtype=jnp.float16)\n",
        "  # warmup\n",
        "  jax_attention(q_jax, k_jax, q_jax).block_until_ready()\n",
        "  jax_attention(q_jax, k_jax, q_jax).block_until_ready()\n",
        "\n",
        "  t1 = time.time()\n",
        "  num_runs = 100\n",
        "  for _ in range(num_runs):\n",
        "    jax_attention(q_jax, k_jax, q_jax).block_until_ready()\n",
        "  estimate_ms = 1000 * (time.time() - t1) / num_runs\n",
        "  return estimate_ms\n",
        "\n",
        "print(bench_jax(batch=BATCH, heads=N_HEADS, seq_len=N_CTX, d_model=D_HEAD))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hV0u8v3uaVrH",
        "outputId": "c529370a-b3af-4be6-d81a-ce7735adf921"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "35.6231689453125\n",
            "110.03916501998901\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Rather than using the official Triton benchmarking utils, do a sanity check:"
      ],
      "metadata": {
        "id": "whN5Alokwckn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def bench_triton(batch, heads, seq_len, d_model):\n",
        "  dtype = torch.float16\n",
        "  warmup = 25\n",
        "  rep = 500\n",
        "  q = torch.randn((batch, heads, seq_len, d_model), dtype=dtype, device=\"cuda\", requires_grad=True)\n",
        "  k = torch.randn((batch, heads, d_model, seq_len), dtype=dtype, device=\"cuda\", requires_grad=True)\n",
        "  v = torch.randn((batch, heads, seq_len, d_model), dtype=dtype, device=\"cuda\", requires_grad=True)\n",
        "  fn = lambda: attention(q, k, v)\n",
        "\n",
        "  # Warmup\n",
        "  fn()\n",
        "  fn()\n",
        "  torch.cuda.synchronize()\n",
        "  t1 = time.time()\n",
        "  num_runs = 100\n",
        "  for _ in range(num_runs):\n",
        "      fn()\n",
        "  torch.cuda.synchronize()\n",
        "  estimate_ms = 1000 * (time.time() - t1) / num_runs\n",
        "  return estimate_ms\n",
        "\n",
        "bench_triton(batch=BATCH, heads=N_HEADS, seq_len=N_CTX, d_model=D_HEAD)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nZ5BqZhDgdcU",
        "outputId": "93e04cff-43d0-4b98-eaf1-fb2754d0e2a8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "33.43979358673096"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Plot:"
      ],
      "metadata": {
        "id": "v_ggO79Qyohi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "seq_lens = [128, 256, 512, 1024]\n",
        "y_jax, y_jax_triton, y_trit = [], [], []\n",
        "\n",
        "for s in seq_lens:\n",
        "  y_jax.append(bench_jax(batch=BATCH, heads=N_HEADS, seq_len=s, d_model=D_HEAD))\n",
        "  y_jax_triton.append(bench_jax_triton(batch=BATCH, heads=N_HEADS, seq_len=s, d_model=D_HEAD))\n",
        "  y_trit.append(bench_triton(batch=BATCH, heads=N_HEADS, seq_len=s, d_model=D_HEAD))\n",
        "\n",
        "plt.plot(seq_lens, y_jax, label='jax')\n",
        "plt.plot(seq_lens, y_jax_triton, label='jax+triton')\n",
        "plt.plot(seq_lens, y_trit, label='triton')\n",
        "plt.title('Time with sequence length')\n",
        "plt.ylabel('time (ms)')\n",
        "plt.xlabel('Sequence Length')\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "obiF1ZrNzevb",
        "outputId": "5faf1c5f-b5c2-4135-936c-2edb12a3b723"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEWCAYAAABhffzLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd1xV9/348debJcpQAUVQAbfiQgVBrdFokmaaZjTNjppoVtOkTUeapmnadKS/NvPbpG2WGLOXSZu2SVOzVRBQ494CIiBL2fPez++PcxgaUVQu98J9Px+P++Des+77nnt538/9nM8QYwxKKaW8h4+7A1BKKdW1NPErpZSX0cSvlFJeRhO/Ukp5GU38SinlZTTxK6WUl9HEr9olIltFZK6742gmIn8TkV+eYP1DIvJyV8bU3YjIXBHJc9Nz6/vjIfzcHYByHxGpavOwD1APOOzHtxpjxnd9VO0zxtzWfN/+QnrZGDPEfRGp9uj749k08XsxY0xw830RyQZuMcb8z30RKaW6glb1qHaJSLaInGPff0hE3hKRl0WkUkQ2i8hoEfm5iBSJyAEROa/Nvn1F5AURKRCRgyLyWxHxPc5zBIpIrYhE2I9/ISJNIhJqP35YRJ6w76faxwkC/gNEi0iVfYu2DxkgIi/ZMW4VkcR2XpuIyON27BX265lgr+slIn8WkVwROWRXMfVus+9P7NeVLyKLRcSIyEh73WcickubbReKyFdtHo8VkY9FpExEdorIVW3WpYrI0yLyLzv+dBEZ0Wb9+Db7HhKR++3lPiJyn4jsFZFSEXlTRMI6+B5Hi8g7IlIsIvtF5Adt1j1kH+u451NEporIBnvdWyLyRme9P8q1NPGrU3EJsALoD2wAPsL6DA0GfgP8vc22qUATMBKYApwH3MIxjDF1QAYwx140B8gBZrV5/Pkx+1QDFwD5xphg+5Zvr14AvA70A/4B/KWd13IecBYwGugLXAWU2usesZcn2PEPBh4EEJHzgR8D5wKjgHPaOf432AnxY+BVYCBwNfCMiMS32exq4NdY53gP8Dt73xDgf8CHQLQd1yp7n7uA72Cdq2jgMPB0B+LxAf4JfG2/xvnAPSLy7TabHfd8ikgAsBLrfQ4DXgMug057f5QLaeJXp+JLY8xHxpgm4C1gAPCIMaYR6585TkT6iUgkcCFwjzGm2hhTBDyOldSO53Ngjoj4AZOAp+zHgUAS8MUpxPiVMebfxhgH1pfU5Ha2awRCgLGAGGO2G2MKRESApcAPjTFlxphK4PdtYr8KWGaM2WInuIdOIbaLgWxjzDJjTJMxZgPwDvDdNtusNMass8/xK1hfPs37FhpjHjXG1BljKo0x6fa624BfGGPyjDH1dkxX2ufzRJKAAcaY3xhjGowx+4DnOPp9au98pmBVFT9ljGk0xrwLrOvAOejo+6NcSOv41ak41OZ+LVBi/wM3PwYIxip1+gMFVh4FrELGgXaO+znwGDAV2IxVKn4BK7nsMcaUtrPf8RS2uV8DBIqIn51IWxhjPhGRv2CVjGNF5F2sknwg1oXurDaxC9BcTRUNZLU5VM4pxBYLJIvIkTbL/LASYHvxN1+HGQrsPcFxV4qIs80yBxAJHDxJPNHHxOMLfHmCeALtL5Ro4KA5epTH9t7ftjr0/ijX0sSvXOEAVguhiA7+Q68BxmBVFXxujNkmIjFYvxo+b2efMx5W1hjzFPCUiAwE3gR+AvwK60tsvDHmeEmzACsJN4s5Zn011hdHs0Ft7h/Aen3nnka4B2j/F9MBYLExZvVpHHO/MWbUacRTAAwWEWmT/Nt+Oemwvx5Mq3pUpzPGFAD/BR4VkVD74uMIEZnTzvY1WKXoO2lN9GuwqjDaS/yHgHAR6Xs6MYpIkogki4g/VrKuA5zGGCdWdcfj9hcCIjK4Tb33m8BCEYkXkT5YXxRtbQQuF5E+9gXfm9us+wAYLSI3iIi/fUsSkXEdCPkDIEpE7rEvPoeISLK97m/A70Qk1o53gIhc2oFjrgMqReRnItJbRHxFZIKIJHVg37VYvyq+LyJ+9vNNb7P+jN4f5Vqa+JWr3AgEANuwLja+DUSdYPvPsaqH1rV5HEI79fvGmB1YFxT3iciRNq1GOioUK8EfxqquKQX+ZK/7GdaF1TQRqcC6qDrGft7/AE8An9jbfHLMcR8HGrAS33KsevrmmCuxLipfDeRjVXv8Eeh1smDtfc/FusBeCOwGzrZXP4l1ofS/IlIJpAHJxzvOMcd0YF07SAD2AyXA81gXu0+2bwNwOdYX2xHgeqwvp3p7/Zm+P8qFRCdiUerMiIgBRhlj9rg7FncSkXTgb8aYZe6ORZ2YlviVUqdFROaIyCC7qucmrBZZH7o7LnVyenFXKXW6xmBd8wgC9gFX2td3lIfTqh6llPIyWtWjlFJepltU9URERJi4uDh3h6GUUt1KVlZWiTFmwLHLu0Xij4uLIzMz091hKKVUtyIix+1ZrlU9SinlZTTxK6WUl9HEr5RSXqZb1PEfT2NjI3l5edTV1bk7lB4jMDCQIUOG4O/v7+5QlFIu1G0Tf15eHiEhIcTFxdFm+Fx1mowxlJaWkpeXx7Bhw9wdjlLKhbptVU9dXR3h4eGa9DuJiBAeHq6/oJTyAt028QOa9DuZnk+lvEO3TvxKKdVTVdU38dA/tlJR19jpx9bEf4Zmzpzp7hCUUj1MQXkt3/3bWlak5ZCxv6zTj99tL+56ijVr1rg7BKVUD7I1v5zFqRlU1zt4cWESc0Z/Y8SFM6Yl/jMUHBxMVVUV8+fPZ+rUqUycOJH3338fgIyMDCZNmkRdXR3V1dWMHz+eLVu2uDlipZSn+nRHEd/921p8RXj79hkuSfrQQ0r8v/7nVrblV3TqMeOjQ/nVJeM7tG1gYCArV64kNDSUkpISUlJSWLBgAUlJSSxYsIAHHniA2tparr/+eiZMmNCpcSqleoYVa7P51T+2Eh8dygs3JREZGuiy5+oRid/djDHcf//9fPHFF/j4+HDw4EEOHTrEoEGDePDBB0lKSiIwMJCnnnrK3aEqpTyMw2n4w7+38/xX+zln3ECeumYKfQJcm5p7ROLvaMncVV555RWKi4vJysrC39+fuLi4lvbwpaWlVFVV0djYSF1dHUFBQW6NVSnlOWobHNzzxgY+2nqIhTPj+OXF8fj6uL5Ztdbxd4Ly8nIGDhyIv78/n376KTk5rSOh3nrrrTz88MNcd911/OxnP3NjlEopT1JUWcfVz67l422H+NUl8Ty0YHyXJH3oISV+dxIRrrvuOi655BImTpxIYmIiY8eOBeCll17C39+fa6+9FofDwcyZM/nkk0+YN2+em6NWSrnTrkOVLFqWQVl1A8/ekMg58ZFd+vya+M9AaWkpYWFhREREsHbt2m+sj4uL48YbbwTA19eX9PT0rg5RKeVhVu8p4baXswj09+XNW2cwcUjfLo9Bq3pOU35+PjNmzODHP/6xu0NRSnUTb2Yc4KYX1zG4X2/eu3OWW5I+aIn/tEVHR7Nr1y53h6GU6gacTsOjH+/k6U/3MntUBM9cN5WQQPcNf66JXymlXKiu0cFP3t7EP7/O55rpQ/nNpRPw93VvZYsmfqWUcpGy6gaWvpRJZs5h7rtgLLeeNdwjRsHVxK+UUi6wr7iKRakZFJTX8fS1U7loUpS7Q2qhiV8ppTrZuv1lLF2Ria8Iry1JYVpsf3eHdBRt1XOGXDUs8xNPPEFNTU2762+55Ra2bdsGwO9//3uXxKCUOnXvbTjI9c+nExYUwMo7Znlc0gdN/GfsTIdlTk1N5aGHHvrG8hMlfofDwfPPP098fDygiV8pT2CM4alVu7nnjY1MienHu7fPJCa8j7vDOi5N/GfIFcMyP/XUU+Tn53P22Wdz9tlntzzPvffey+TJk1m7di1z584lMzOT++67j9raWhISErjuuusAeOyxx5gwYQITJkzgiSeeACA7O5tx48axZMkSxo8fz3nnnUdtba2LzopS3qWhycmP39rEYx/v4vIpg1lxczL9+gS4O6x29Yw6/v/cB4WbO/eYgybCBY90aNPOHpb5Bz/4AY899hiffvopERERAFRXV5OcnMyjjz561LaPPPIIf/nLX9i4cSMAWVlZLFu2jPT0dIwxJCcnM2fOHPr378/u3bt57bXXeO6557jqqqt45513uP7660/xxCil2iqvaeS2l7NYu6+Ue84Zxd3zR3lEy50T6RmJ381OdVjm0tJS5s+fD0BZWRkNDQ289957AKxYsYKJEyd+4zl8fX254oorThrLV199xWWXXdYyCujll1/Ol19+yYIFCxg2bBgJCQkATJs2jezs7M54+Up5rQNlNSxcto7cshoe/95kLpsyxN0hdUjPSPwdLJm7yqkOyxweHt5SQk9NTSU7O/u49fxtBQYG4uvre0Zx9urVq+W+r6+vVvUodQY25B7mluWZNDkNK25OJmV4uLtD6jCt4+8ErhiWOSQkhMrKyg5t6+/vT2NjIwCzZ8/mvffeo6amhurqalauXMns2bNP7QUppU7oP5sLuPrZNIJ6+fHuHTO7VdIHF5b4RWQo8BIQCRjgWWPMkyLyELAEKLY3vd8Y829XxeFqrhqWeenSpZx//vlER0fz6aefnnTbSZMmMXXqVF555RUWLlzI9OnTAavZ55QpU7RaR6lOYIzhuS/38Yf/7GDK0H48d2Mi4cG9Tr6jhxFjjGsOLBIFRBlj1otICJAFfAe4Cqgyxvy5o8dKTEw0mZmZRy3bvn0748aN68yQT1lpaSlTp049qoTf3XnCeVXKEzU5nDz4j628mp7LRZOiePS7kwn0P7PqV1cTkSxjTOKxy11W4jfGFAAF9v1KEdkODHbV83W1/Px85s6dq8MyK+UFKusa+f6rG/h8VzF3zB3Bj88bg08XzZblCl1ycVdE4oApQDowC/i+iNwIZAL3GmMOH2efpcBSgJiYmK4I85TosMxKeYf8I7UsTs1gd1EVj1w+kaune14+OlUuv7grIsHAO8A9xpgK4K/ACCAB6xfBo8fbzxjzrDEm0RiTOGDAAFeHqZRS37DlYDmXPbOag4drSV2U1COSPri4xC8i/lhJ/xVjzLsAxphDbdY/B3zgyhiUUup0rNp+iLte20D/PgG8fXsyYwaFuDukTuOyEr9YXddeALYbYx5rs7zt2KSXAScfw0AppbrQ8jXZLHkpkxEDgll5x8welfTBtSX+WcANwGYR2Wgvux+4RkQSsJp4ZgO3ujAGpZTqMIfT8Lt/befF1fs5Nz6SJ69OoE9Az+jn2pbLSvzGmK+MMWKMmWSMSbBv/zbG3GCMmWgvX2C3/ul2jhw5wjPPPNPu+ubhmrOzs3n11Ve7Kiyl1GmqaWjitpezeHH1fhbPGsbfrp/WI5M+aM/d09Ze4m9qagJah2vWxK+U5yuqrOPqZ9NYtf0Qv14wngcvice3GzfXPBlN/KfpvvvuY+/evSQkJJCUlMTs2bNZsGBByxj5wcHBLdt9+eWXJCQk8Pjjj1NXV8eiRYuYOHEiU6ZMaemVm5qayuWXX87555/PqFGj+OlPf+q216aUN9lZWMllT69hT1EVz92YyE0z49wdksv1iN8xf1z3R3aU7ejUY44NG8vPprc/ts4jjzzCli1b2LhxI5999hkXXXQRW7ZsYdiwYd/Y7s9//jMffGA1Xnr00UcRETZv3syOHTs477zzWvoDbNy4kQ0bNtCrVy/GjBnDXXfdxdChQzv1dSmlWn2xq5g7X1lP7wBf3rx1BhMG93V3SF1CS/ydZPr06d9I+sfz1VdftYyBP3bsWGJjY1sS//z58+nbty+BgYHEx8f3qKEglPI0r6/LZVFqBoP79+a9O2d5TdKHHlLiP1HJvKs0j39/Jo4dNrn5eoFSqvM4nYY//Xcnf/1sL3NGD+Av104hJNDf3WF1KS3xn6aODpt87HazZ8/mlVdeAWDXrl3k5uYyZswYl8WplGpV1+jgrtc38NfP9nJtcgwv3JTodUkfekiJ3x3Cw8OZNWsWEyZMoHfv3kRGRh53u0mTJuHr68vkyZNZuHAhd9xxB7fffjsTJ07Ez8+P1NTUo0r6SinXKK2qZ8lLmazPPcL9F45lyezhHj9Foqu4bFjmzuSpwzL3RHpeVU+0t7iKRcsyOFRRxxPfS+CCiVEn36kH6PJhmZVSyhOk7Svl1hVZ+PkIry1NYWpMf3eH5Haa+JVSPdbKDXn89O1NxIT1IXXRdIaG9XF3SB6hWyd+Y4zX1tG5Qneo9lOqI4wxPLlqN0/8bzczhofzt+un0beP913EbU+3TfyBgYGUlpYSHh6uyb8TGGMoLS0lMDDQ3aEodUYampzc9+4m3l1/kCumDuEPl08kwE8bMLbVbRP/kCFDyMvLo7i4+OQbqw4JDAxkyJAh7g5DqdNWXtPIrS9nkravjHvPHc33543UguFxdNvE7+/v36Geskop75BbWsPC1HXkldXy5NUJXJrQY6b47nTdNvErpVSz9bmHWbI8E4cxvHxLMtOHhbk7JI+miV8p1a39a1MBP3pzI1F9A3lxYRLDBwS7OySPp4lfKdUtGWP4+xf7eOQ/O0iM7c+zNyYSFhTg7rC6BU38Sqlup9Hh5MH3t/LaulwumRzNn66cRKC/r7vD6jY08SulupXKukbueGU9X+4u4c6zR3DvuWPw6cGzZbmCJn6lVLeRf6SWxakZ7Cmq4v9dMYmrknSiotOhiV8p1S1szivn5uUZ1DY4SF00nW+NinB3SN2WJn6llMf7eNshfvDaBsKCAnj5jmRGR4a4O6RuTRO/UsqjLVu9n998sI1Jg/vy3E2JDAzRYUXOlCZ+pZRHcjgND3+wjdQ12ZwXH8mTV0+hd4C23OkMmviVUh6nur6Ju1/fwP+2F3HLt4bx8wvH4astdzqNJn6llEc5VFHHzcsz2JZfwcOXjueGGXHuDqnH0cSvlPIYOworWLwsgyO1jTx/UyLzxh5/Lmt1Zlw2SLWIDBWRT0Vkm4hsFZG77eVhIvKxiOy2/+o8aEopPt9VzJV/XYvDGN66bYYmfRdy5ewETcC9xph4IAW4U0TigfuAVcaYUcAq+7FSyou9mp7L4tQMhob14b07ZzE+uq+7Q+rRXFbVY4wpAArs+5Uish0YDFwKzLU3Ww58BvzMVXEopTyX02n440c7+Pvn+5g7ZgB/uXYqwb20BtrVuuQMi0gcMAVIByLtLwWAQuC4v+dEZCmwFCAmJsb1QSqlulRdo4MfvbmRf28u5PqUGB66ZDx+vjpFYldweeIXkWDgHeAeY0xF22nQjDFGRI47w7cx5lngWYDExESdBVypHqSkqp4lL2Wy8cARHrhoHDd/a5hOkdiFXJr4RcQfK+m/Yox51158SESijDEFIhIFFLkyBqWUZ9lTVMWi1HUUV9bz1+umcf6EQe4Oyeu4slWPAC8A240xj7VZ9Q/gJvv+TcD7ropBKeVZ1u4t5fJnVlPb4OD1pTM06buJK0v8s4AbgM0istFedj/wCPCmiNwM5ABXuTAGpZSHeCcrj/ve3URceBAvLkxiaFgfd4fktVzZqucroL1Ku/muel6llGcxxvDE/3bz5KrdzBoZzjPXTaNvb393h+XVtN2UUspl6psc3PfOZlZuOMh3pw3hd5dNJMBPW+64myZ+pZRLHKlpYOmKLNbtL+Mn3x7DHXNHaMsdD6GJXynV6XJKq1m0LIO8w7U8eXUClyYMdndIqg1N/EqpTpWVU8aSl7IwxvDKkmSS4sLcHZI6hiZ+pVSn+efX+dz71tcM7tebFxcmMSwiyN0hqePQxK+UOmPGGJ75bC9/+mgnSXH9efaGRPoHBbg7LNUOTfxKqTPS6HDywMotvJF5gAWTo/l/V04i0F+nSPRkmviVUqetoq6RO15ez1d7Srhr3kh+dO5obbnTDWjiV0qdlrzDNSxOzWBfcTX/78pJXJU41N0hqQ7SxK+UOmWb8o5w8/JM6hodvLR4OjNHRrg7JHUKOpT4RWQg1tg70UAtsAXINMY4XRibUsoD/XdrIT94fQMRwb149ZZkRkWGuDskdYpOmPhF5GysqRHDgA1YQygHAt8BRojI28CjxpgKVweqlHIvYwwvrs7mt//axqQh/Xj+xkQGhPRyd1jqNJysxH8hsMQYk3vsChHxAy4GzsUac18p1UM1OZw8/ME2lq/N4fzxg3j8ewn0DtCWO93VCRO/MeYnJ1jXBLzX6REppTxKdX0Td722gU92FLH0rOHcd/5YfHy05U531qFh8kTkbhEJFcsLIrJeRM5zdXBKKfc6VFHHVX9fy2c7i3j4OxO4/8JxmvR7gI6Oj7rYrsc/D+iPNcHKIy6LSinldtvyK/jO06vJLqnmhYVJ3JAS6+6QVCfpaHPO5q/4C4EVxpitor00lOqxPttZxJ2vrCck0J+3bptJfHSou0NSnaijiT9LRP4LDAN+LiIhgDblVKoHeiU9hwff38rYQSG8cFMSg/oGujsk1ck6mvhvBhKAfcaYGhEJBxa5LiylVFdzOg2PfLiDZ7/Yx7yxA/m/a6YQ1Ev7ePZEHXpXjTFOEWkCzrKbcTbb5JqwlFJdqa7RwQ/f2Mh/thRy44xYHrw4Hj9fnSKxp+poz90XgUnAVlqreAzwroviUkp1keLKepa8lMnXeUf45cXxLJ4VpwOt9XAd/R2XYoyJd2kkSqkut6eokoXLMiipqudv10/j2+MHuTsk1QU6mvjXiki8MWabS6NRSnWZNXtLuG1FFgF+vrx56wwmDenn7pBUF+lo4n8JK/kXAvVYzTuNMWaSyyJTSrnM21l53PfOJoYPCOLFhUkM6d/H3SGpLtTRxP8CVqetzWgzTqW6LWMMj3+8i6c+2cO3RkbwzPVTCQ30d3dYqot1NPEXG2P+4dJIlFIuVd/k4Kdvb+L9jfl8L3Eov71sAv7acscrdTTxbxCRV4F/YlX1AGCMabdVj90S6GKgyBgzwV72ELAEKLY3u98Y8+/TiFspdQoOVzewdEUmGdmH+cm3x3DH3BHacseLdTTx98ZK+G0HZjtZc85U4C9Y1wfaetwY8+eOBqiUOjPZJdUsSs3g4JFa/u+aKVwyOdrdISk362gHrlPupWuM+UJE4k51P6VU58nILmPpS5kAvHpLMolxYW6OSHmCE1bwicgDItLuJ0VE5onIxaf4nN8XkU0i8qKI9D/BsZeKSKaIZBYXF7e3mVKqHf/4Op/rnkunf58AVt4xS5O+anGyEv9m4J8iUgesx6qbDwRGYY3d8z/g96fwfH8FHsaqJnoYeBRYfLwNjTHPAs8CJCYmmlN4DqW8mjGGZz7by58+2sn0uDD+fsM0+gcFuDss5UFONgPX+8D7IjIKa7L1KKACeBlYaoypPZUnM8Ycar4vIs8BH5xyxEqpdjU6nPxi5WbezMzjOwnR/PHKSfTy0ykS1dE6Wse/G9h9pk8mIlHGmAL74WXAljM9plLKUl7byB2vZLF6Tyk/mD+KH54zSlvuqONy2ZirIvIaMBeIEJE84FfAXBFJwKrqyQZuddXzK+VNDpTVsDg1g+zSav783clcOW2Iu0NSHsxlid8Yc81xFr/gqudTylt9feAINy/PpKHJwfLF05k5IsLdISkPp7MsKNWNfbilkHve2MCAkF68vjSZkQND3B2S6gY61F9bREaLyCoR2WI/niQiD7g2NKVUe4wxPP/lPm5/JYtxUaGsvGOWJn3VYR0dqOM54OdAI4AxZhNwtauCUkq1r8nh5MH3t/Lbf23nggmDeG1JChHBvdwdlupGOlrV08cYs+6YFgJNLohHKXUCVfVN3PXqej7dWcytc4bzs2+PxcdHW+6oU9PRxF8iIiOwWuMgIlcCBSfeRSnVmQrL61icmsHOQ5X8/rKJXJsc4+6QVDfV0cR/J1Yv2rEichDYD1zvsqiUUkfZll/B4tQMquqbeHFhEnNGD3B3SKob62gHrn3AOSISBPgYYypdG5ZSqtmnO4r4/qvrCe3tz1u3zWBcVKi7Q1LdXIcSv4j0A24E4gC/5rp+Y8wPXBaZUooVaTn86v0tjIsK5cWFSUSGBro7JNUDdLSq599AGjr1olJdwuE0/OHf23n+q/2cM24gT149haBe2u1GdY6OfpICjTE/cmkkSikAahsc3PPGBj7aeoiFM+P45cXx+GrLHdWJOpr4V4jIEqzRNNtOvVjmkqiU8lJFlXUsWZ7JpoPl/OqSeBbNGubukFQP1NHE3wD8CfgFdpNO++9wVwSllDfadaiSRcsyKKtu4NkbEjk3PtLdIakeqqOJ/15gpDGmxJXBKOWtVu8p4baXswj09+XNW2cwcUhfd4ekerCOJv49QI0rA1HKW72ZcYD7V25mxIBgXlyUxOB+vd0dkurhOpr4q4GNIvIpR9fxa3NOpU6T02l49OOdPP3pXmaPiuDp66YSGujv7rCUF+ho4n/PvimlOkFdo4OfvL2Jf36dzzXTh/KbSyfg79vRMROVOjMd7bm73NWBKOUtyqobWPpSJpk5h/nZ+WO5bc5wnSJRdakTJn4RedMYc5WIbKa1NU8LY8wkl0WmVA+0v6SaRcvWkV9ex9PXTuWiSVHuDkl5oZOV+O+2/17s6kCU6unW7S9j6YpMfER4bUkK02L7uzsk5aVOWKlojGkeevkOY0xO2xtwh+vDU6pneH/jQa5/Pp2woABW3jFTk75yq45eTTr3OMsu6MxAlOqJjDH836rd3P36RqbE9OPd22cSGx7k7rCUlztZHf/tWCX74SKyqc2qEGC1KwNTqrtraHJy/8rNvJ2Vx+VTBvOHKybSy8/X3WEpddI6/leB/wB/AO5rs7xSx+lRqn3ltY3c/nIWa/aWcs85o7h7/ihtuaM8xgkTvzGmHCgHrumacJTq/g6U1bAoNYOc0moe/95kLpsyxN0hKXUUHeBbqU60IfcwS17KpNFhWHFzMinDw90dklLfoIlfqU7y4ZYC7n59I5GhgSxblMSIAcHuDkmp49LEr9QZMsbw/Jf7+f1/tjNlaD+euzGR8OBe7g5LqXa5bHAQEXlRRIpEZEubZWEi8rGI7Lb/amNm1a01OZw88N4Wfvfv7Vw4IYpXl6Ro0lcez5WjQqUC5x+z7D5glTFmFLCKo1sKKSCTEw8AACAASURBVNWtVNY1cvPyTF5Jz+X2uSP4v2umEOivzTWV53NZVY8x5gsRiTtm8aXAXPv+cuAz4GeuikEpV8k/Usvi1Ax2F1Xxh8sncs30GHeHpFSHdXUdf2SbYSAKgXbnlhORpcBSgJgY/adSnmPLwXJuXp5BTb2D1EVJzB41wN0hKXVK3DYAuDHGcJwRP9usf9YYk2iMSRwwQP+xlGdYtf0QV/19Lb4ivH37TE36yrWcDnA6O/2wXZ34D4lIFID9t6iLn1+p07Z8TTZLXspkxIBg3rtzFmMGhbg7JNUTlR+E9SvgrUXwpxGQv77Tn6Krq3r+AdwEPGL/fb+Ln1+pU+ZwGn73r+28uHo/54yL5KlrEugToC2hVSdpqIGcNbD3E9i7Cop3WMuDB8HoC8C/8+dgdtmnV0Rew7qQGyEiecCvsBL+myJyM5ADXOWq51eqM9Q0NHH36xv5eNshFs2K44GL4vH10TF31BkwBg5tbU30OWvBUQ++vSB2Jky5HkbMg4Hx4KLxnVzZqqe98X3mu+o5lepMRZV13LI8ky0Hy3nokngWzhrm7pBUd1VVDPs+sxL93k+g6pC1fMA4mL4ERpwNsbOOW7o3xnT6AH/6e1Wp49hZWMni1AzKqht49oZEzolvtwGaUt/U1AAH0qwkv2cVFNqj2vcOs5L8iHnWLTT6G7uW1ZWxrmAdaQVppBWk8eicRxkfMb5Tw9PEr9QxvtxdzB0vr6d3gC9v3TaDCYP7ujsk5emMgdI9rYk++ytorAYfPxiaDPMegBHzIWoy+Bzdya+msYb1RetJy08jvTCdHWVWHX+IfwhJg5LABbU9mviVauONjFx+sXILIwcG8+LCJKL7df6FNdVD1B6G/V9YiX7vp1Ceay0PGw4J11ol+mGzodfRrb+anE1sKdlCWkEa6QXpbCzeSJOzCX8ff6YMnMIPpvyAlKgUxoWPw8/HNSlaE79SgNNp+PN/d/LMZ3s5a/QAnr52CiGB/u4OS3kSR5PVtHKPXU9/MBOME3qFwrCz4Fv3WMk+7OhrQcYY9pXva6m6ySzMpKqxCkEYGzaWG+JvICUqhSkDp9Dbr2sKGpr4ldera3Tw47e+5oNNBVwzPYbfXDoef1+39W1UnuRIbmui3/851JUDAoOnwewfw8j51n3fowsJhdWFpBekk16QTlpBGsW1xQAMDRnKBcMuICUqhemDptMvsJ8bXpQmfuXlSqvqWboii6ycw/z8grEsPWu4TpHozeqrrPr55qaWpXus5aGDYdwCK9EPmwN9wo7araKhgozCjJZEv798PwBhgWEkD0omJTqF5KhkBgcP7upXdFya+JXX2ltcxeLUDArL63jmuqlcODHK3SGpruZ0Wi1u9n5i3XLTwNkIfr0h7luQdItVfRMx+qg29Q2OBjYWbWypp99SugWncdLbrzfTIqdxxagrSIlKYVT/UfiI5/161MSvvFL6vlKWrsjCz0d4bWkKU2N0agivUVnYmuj3fgo1JdbyyIkw4w4r0cfMAL/WeRWcxsmO0u0tiX79ofXUOerwFV8mRkxk6aSlJA9KZvKAyfj7ev61IU38yuu8t+EgP317E0PDepO6aDpDw/q4OyTlSo11kGsPibDnEyjaai0PGmBV3YyYB8PPhpDWvhrGGPIqDrC2YC3pBemsK1zHkfojAIzsN5IrR19JSlQK0yKnERzQ/abY1MSvvIYxhqdW7eHx/+1ixvBw/nb9NPr28fzSmTpFxljj3TS3qc9ZDU114BsAMSlwzq+tZB85AXxaq2GO7Th1sOogAJF9IpkzZI5VTz8omQF9uv+IrJr4lVdoaHLy83c38876PK6YOoQ/XD6RAD/Pq3tVp6mmDPZ9apXo934ClfnW8ojRMG2RlejjZkFAUOsujTVk5We1XJDdeXgnYHWcmh41nYXjF5IclUxcaFyPu+CviV/1eBsPHOH3/97Ouv1l/Ojc0dw1b2SP+0f2Oo5GyMtobWqZvwEwENgPhs9tHRKh39CWXRqdjWy1L8imFaTxdfHXLR2npg6cyt1T7yZ5UDLx4fH4+vTsKTQ18aseqaHJyX+2FLBsdTYbDxwhuJcfT3wvge9M8YzmdOo0lO1r7SW7/wtoqATxhSFJMPfnVn199JSWIRGMMew7stdK9PlpZBzKoLqxGkEYFz6OG+NvJDkquUs7TnkKTfyqRymurOe1dbm8nJZDUWU9wyKC+PWC8VwxbQjBvfTj3q3UlcP+L1vb1B/Otpb3i4GJV1qJPm429G7tBNXccaq59U1zx6mYkBguGnYRyVHJbu045Sn0P0H1CJvzylm2Zj8ffF1Ag8PJnNED+OOVccwZNQAfHT+/e3A6IH9j69DFB9aBcUBAsJXgZ3zfHhJheEub+oqGCjJyV5GWb1XfZFdkA3bHqahkUqI8q+OUp9DEr7qtRoeTD7cUkromm6ycwwQF+HLN9KHcODOOEQO6XxM7r1Se19qmft9n1sBnCEQn2GPfzLeqcvwCAKh31PN1YUZLPf3W0q0tHacSIxNbmll6ascpT6GJX3U7pVX1vJ5xgBVrcyisqCM2vA8PXhzPlYlDCNWB1TxbQ43VvLK5qWWJ1ZKGkCgYc6Hdpn4uBEUAdsepsh0t9fQbija0dJyaNGASSyctJSUqhUkRk7pFxylPoYlfdRtb88tJXZ3N+1/n09DkZPaoCH532QTmjhmo0yF6KmPg0JbWRJ+7FhwN4BdoTTM49UZ7msFxIGJ1nKrMY+3ON0krSGNd4TrK68uBntFxylNo4lcercnh5L/bDpG6Opt12WX09vflqsQh3DQjjlGRISc/gOp6VcV2m/pV1t/maQYHxsP0pVaij53ZMs1gaW0p67I/bCnV51dbbfAj+0Qyd8jcHtVxylNo4lce6XB1A69l5PLy2hzyy+sYGtabBy4ax3cTh9K3t/6k9yhN9XAgvbVNffM0g33CraEQWqYZtAbBq2msIetQZks9/a7DuwAICQhh+qDpLJqwiJSoFGJDY7W/hYto4lceZXtBBcvXZLNyw0Hqm5zMGhnOry+dwLyxWp3jMYyBkt2tF2Wzv4TGGnuawRSY90urqeWgyeDjY3WcKtnK2v3vkZafxqaSTTQ5mwjwCWDKwCncPfVua8apsHE9vuOUp9DEr9zO4TR8vO0QqWv2k7avjEB/Hy6fOoSFM+MYM0irczxC7WHY97nd1PJTKD9gLQ8bAQnX2W3qvwW9QjDGsPfIXtJ2vGrNOHUo8xsdp5pnnAr0C3Tv6/JSmviV2xypaeCNjAO8tDaHg0dqGdyvNz+/YCzfSxpKvz4B7g7Puzma4GBWa5v6g1n2NIN9YfhZMPtHVvVN/zjA6jiVdmBVS8epklprqOPmjlMp0daMU3176cT1nkATv+pyuw5Vkromm3fX51HX6CRleBi/vDiec8YNxE+nPHSfwzmtiX7fF1BfDuJjTS141k+sNvWDp4GvH+X15WQWZrJ258ukF6R/o+PUjKgZJEclEx0c7d7XpI5LE7/qEg6n4ZMdRaSu2c/qPaX08vPhOwmDuWlmHPHRoe4OzzvVV1n1881NLcv2WstDh8D4S61EP+ws6BNGvaPemnHq66dJL0j/Rsep747+LinRKYzqN0ovyHYDmviVS5XXNvJW5gGWr83mQFktUX0D+en5Y7g6KYawIK3O6VJOJxR+3TohyYF0a5pB/z5W/XxzU8uIUTiMkx2Hd5C2d6U141TReuod9S0dp26ddCspUSlMjJioHae6IU38yiX2FFWyfE0O76zPo6bBwfS4MH5+wTjOi4/U6pyuVFFwdJv6mlJr+aCJMONOe5rBFIxvAAcqD1hNLLf+7Rsdp747+rvMiJ7BtMhpBPkHneAJVXfglsQvItlAJeAAmowxie6IQ3Uup9Pw2a4ilq3O5svdJQT4+rAgIZqFM+OYMFgv6nWJxlrIWdM6n2zLNIMDYeS5dpv6syF4ICW1JawrWEf6ut8f1XFqUNAgzh56dssAZxG9I9z4gpQruLPEf7YxpsSNz686SWVdI29l5vHS2myyS2uIDO3Fj88bzdXTY4gI7nXS/dUZMAaKtrcOXZyzps00gzOsaQZHzoeB46lx1JF5KJP07cu/0XEqeVAyiycsJiU6hZiQGK2n7+G0qkedtn3FVby0Noe3Mg9Q3eBgWmx/7j1vDOdPGIS/Vue4TnWpVW3T3IGqssBaHjEGEhfbQyLMotHPny0lW0jL/4q0DX9mU/EmmozdcSpSO055M3clfgP8V0QM8HdjzLPHbiAiS4GlADExMV0cnmqP02n4YncxqWuy+WxnMf6+wiWTorlpZhyTh3r35BYu09RgTTPY3NQyfyMt0wyOaB0SwYQOZs+RPVZb+i9/SkZhBjVNNQhCfHg8N42/qWXGKe045d3EGNP1Tyoy2BhzUEQGAh8Ddxljvmhv+8TERJOZmdl1AapvqKpv4p2sPJavyWZfSTUDQnpxfXIs1yQPZWCIJpFOZYw1zWBziX7/F9BQZU0zOHS6nejnQ3QChbXFLWPetO04FRsaS0pUCilRKSQNStKOU15KRLKOdw3VLSV+Y8xB+2+RiKwEpgPtJn7lPtkl1Sxfm81bmXlU1TcxeWg/nrw6gQsmRBHgp9U5naau3ErwzW3qj+RYy/vFwqSr7Db1sykXyCjMIC3vP6RnPHRUx6nmRK8dp9TJdHniF5EgwMcYU2nfPw/4TVfHodpnjOGrPSUsW53NpzuL8BXhoklRLJwZx5SY/u4Or2dwOiB/Q2uiz8tonWZw2Fkw8y4YMY/6fkPYULTBmkf2f6+wrWxbS8eppEFJ2nFKnRZ3lPgjgZX2h9QPeNUY86Eb4lDHqK5v4t0NB1m+Jps9RVVEBAdw17xRXJccQ2SoVuecsfK81qGL930GdUdonWbwhzBiHo7B09hRsdeaQzbrETYUbaDeUY+f+DFpwCRum3QbyVHJ2nFKnZEuT/zGmH3A5K5+XtW+3NIaXlqbzRuZB6isa2Li4L48dtVkLpoURS8/be1x2hqqIXt1a1PLEqv5JCFRMPZiGHE2Zthccp3VVok++x3S1/6UioYKAEb1H8VVY65qmXFKO06pzqLNOb2UMYa1e0t5cXU2q3YcwkeECyYMYtGsOKbG9Ndqg9NhDBRubk30uWltphmcBdMWwoh5lAQPYF3hOuui7IfPUlBtNcccFDSI+THzSY5K1o5TyqU08XuZ2gYHKzccJHXNfnYdqiIsKIA7547kupQYovr2dnd43U9VkdVDtnmc+uoia/nA8ZB8K4yYR01UApllW6xEn/YLdh/eDUBoQCjJUcncPOFm7TilupQmfi+Rd7iGFWtzeD3jAOW1jcRHhfKnKydxyeRoAv21OqfDmuqtknxzm/rCzdbyPuEt7ekb42azpaHEqqfftZxNq+9t6Tg1NXIqF069kBlRMxgbNlY7Tim30MTfgxljSN9fxrLV+/l42yFEhG+Pj2ThzGEkxWl1Toe0TDNoJ/rsr+xpBv0hJgXmP4gZPo89gX1IO7SOtILVZG59sqXj1Pjw8dw0/iZSolNIGJCgHaeUR9DE3wPVNTp4f+NBlq3OZkdhJf36+HPrnBFcnxLL4H5anXNSNWWw//PW4Ysr8qzl4SNhyvUwYj4FA0aQVmpV36R/9UNK66xRL+NC47hkxCXacUp5NE38PUj+kVpWpOXw2rpcjtQ0MnZQCH+8YiKXJgzW6pwTcTTBwczWppb564+eZvCsH1M+dDoZdQVWPf22v5BTYXWwCg8MJzkquaXzVFRwlJtfjFInp4m/mzPGkJlzmGWr9/PR1kMYYzg33qrOSRkeptU57Tmc3Zro938B9RX2NIOJcNZPqY+bzQZ/Ie1QBmkFH7Ft6+MYDH38+pA4KJHvjfkeKVEpjOw3Us+x6nY08XdTdY0O/vl1PqlrstmaX0FooB+3fGsY16fEMjSsj7vD8zz1lbD/y9amlmX7rOV9h8L4y3CMOJsd/QaxtmwraQVpbFz9zlEdp26ffDsp0SlMiJiAv492nFLdmyb+bqawvI6X03J4dV0uZdUNjI4M5veXTeQ7U6LpE+Dlb6cxUFkIpbutzlIle+z7u+FILmDsaQZnY6bfSm7UONJqC0grSGfdpj9rxynlNbw8U3QPxhjW5x5m2epsPtxSiMMY5o+NZNGsOGaOCPe+qobGWijdayX30j1WYi/dbSX6hsrW7fyDIGIkDEmChOsoiRpPuo+DtKJM0vPepWCn1XEqKiiK+THzSYlKYXrUdO04pXo8TfwerL7JwQdfF5C6JpvNB8sJCfRj4cw4bpwRR0x4D6/OMcaaYKRkl53Y97SW4ssPYE3pYOsbYyX4hGsx4SMp7TuIHP9e5DqqyanMJbcyl71H1rAv52WgtePULRNvISUqhaEhQ73vy1N5NU38Hqiooo6X03N5NT2HkqoGRgwI4uHvTODyKYMJ6tXD3rKGGiupN5fYS3ZZ90v3WmPQNwsItppTxiRjwq/nSL9ocgL7kIOTnJoCcitzya3IJnfvF1Q3Vrfs5ufjx5DgIcSFxrFgxAJSolMY2187Tinv1sOySPfV5HCy8cARVqTl8K9NBTQ5DfPGDmThzDi+NTICH59uXCI1BioOHlNyt++XH2izoUC/oRA+CmJmUt5/CLm9Q8jx8yW3odwqvVfkklPwPpU5rVU6vuJLdHA0MaExTBk4hdjQWGJDY4kJjSEqKAo/H/2YK9WW/ke4SV2jg68PHCEju4x12YdZn3OYqvomgnv5ccOMWG6cEcewiG52UbGhurXOvaXe3S69tymFExBiVc3EzqS6fxw5QX3J9Q8gh3pyqwvIrsgmt3w1R4qOtOwiiJXcQ2K4cNiFrck9JIbBIYO1pY1Sp0ATfxepqGskK/sw67LLyNhfxqa8chocTgBGRwZzaUI004eFMX9cJMGeXJ3jdFql9+ak3vbCanMPV8AqvcdAxGhqYlM4EBxOTq9Acn0MOXVl5FbmklOxi9LctUcdPrJPJLGhsZwTew6xIVapPTY0liEhQ+jl26trX6tSPZQHZ5juraiiriXJr8s+zI7CCowBPx9hwuC+LJwVR1JcGImx/ekfFODucL+pvuqYFjO7W6tnmmpbt+sVCuEjqY+dyYG+A8kJDCLXz4ecpmpyqw+SU5FDUdFOKGrdJaJ3BDEhMZw15CxiQmOIC40jJjSGoSFD6e2nQ0oo5Wqa+DuBMYac0po2ib6MnNIaAHr7+zI1th93zx/F9LgwEmL6eU57e6fTKqUf1ebdvl+Z37qd+EC/GBrDR5E3dCq5QaFk+/mRa+rJqS0mtzKXwqp0TFVrS5v+vfq3TPjdXN/eXILXdvFKuZeHZKDuxeE07CisIGN/GRl29U1xZT0A/fr4kxgbxnXJMSTFhTFhcF/8fd08KXl9ZZsLq7tb27+X7j2m9N6XpoiR5MdNJyeoP7m9AskRh31h9QD51btxlu4EazwyQgNCiQ2NZWrk1KOqZWJCYwgNCHXPa1VKnZQm/g6ob3KwKa+cdfvLyMguIyvnMJV1TQBE9w1k5ohwkuLCmD4sjJEDgt3TAsfpsFrIHFVyt5N9ZUHrduKDo38shWFx5AyOJzewDzk+kOuoJremkLzKgzRVFYPdkjLIP4iYkBgmREzgwuEXtlTLxIbE0i+wX9e/TqXUGdPEfxyVdY1k5RwmI7uMjP2H2Zh3hIYm60LsyIHBXDwpmunD+pMUF8aQ/l3ckaqu4ptt3kv2QNleaKpr2cwZ2JeiiBHkxkwhu89Z5Pr5kuOsI7eulANVeTQ69sLhvQD09utNTEgMo/qP5pzYc4kJaS25hwd6Yc9gpXq4np34a8qsJoY+vlY9tdh/fXyOelxc3URWbjnpOUfIyDnCtvwKnAZ8fYTx0aHcmBJL0jDrQmx4cBe0LHE6rLFljm3zXrIbqgpbNjPiS0lYLDn9B5MbNZycgF7k0khOQzkHqvOpc5RATQnUQIBPgHUhte8w5gyd21otExLDwD4DNbkr5UXEGHPyrdwsMTHRZGZmnvqO/7oXMp4/5d2c+ICPLyI+SNsvjWO+MFq/UHzaPG67To55fKJ19nWAI7lW3bvDumZggMN9+pMbHktOSAQ5vXqT6+Mkp6mS3JoiappqWuJu7qXaXFpvWy0TGRSJj7j5WoNSqkuJSJYxJvHY5T26xL9/xBwKg/tyuLqeQ+U1HCqvpbiijrqGRgRDoB9EhgQwMDiAAUEB9O/jh68YxOkEnNZkHMbYf5vvO8AYxNjbOK31gtNqJWMM4AAD4nQArfuLcYJpBNMADmudtBzbiQNDfmg/ciPPIsfXh1xnDbl1JVQ2VgNlUFuGb50vg4MHExMaw7TBs46qltFeqkqpjujRWeK+LV+yrfrD1gW97dvxVNs3VxP71l7h21GNVLXppRqdor1UlVKdqkcn/rOjriQ0L4mxUaGMGxTCgNAAhKPrsg0nr+o6tjrs2H2Od4yT7WMvPJrAoD6DGBIyhABfD+zUpZTqEXp04r9tVjKQ7O4wlFLKo7jlap+InC8iO0Vkj4jc544YlFLKW3V54hcRX+Bp4AIgHrhGROK7Og6llPJW7ijxTwf2GGP2GWMagNeBS90Qh1JKeSV3JP7BQNvZN/LsZUcRkaUikikimcXFxV0WnFJK9XQe26PHGPOsMSbRGJM4YMAAd4ejlFI9hjsS/0FgaJvHQ+xlSimluoA7En8GMEpEholIAHA18A83xKGUUl6py9vxG2OaROT7wEeAL/CiMWZrV8ehlFLeqlsM0iYixUCOu+M4gQigxN1BeBA9H0fT89FKz8XRXH0+Yo0x37hI2i0Sv6cTkczjjYDnrfR8HE3PRys9F0dz1/nw2FY9SimlXEMTv1JKeRlN/J3jWXcH4GH0fBxNz0crPRdHc8v50Dp+pZTyMlriV0opL6OJXymlvIwm/pMQkaEi8qmIbBORrSJyt708TEQ+FpHd9t/+9nIRkafsuQY2ichU974C1xARXxHZICIf2I+HiUi6/brfsHtlIyK97Md77PVx7ozbFUSkn4i8LSI7RGS7iMzw5s+HiPzQ/l/ZIiKviUigN30+RORFESkSkS1tlp3y50FEbrK33y0iN3VmjJr4T64JuNcYEw+kAHfa8wfcB6wyxowCVtmPwZpnYJR9Wwr8tetD7hJ3A9vbPP4j8LgxZiRwGLjZXn4zcNhe/ri9XU/zJPChMWYsMBnrvHjl50NEBgM/ABKNMROweudfjXd9PlKB849ZdkqfBxEJA36FNYXgdOBXzV8WncIYo7dTuAHvA+cCO4Eoe1kUsNO+/3fgmjbbt2zXU25YA+utAuYBH2BNH18C+NnrZwAf2fc/AmbY9/3s7cTdr6ETz0VfYP+xr8lbPx+0DrseZr/fHwDf9rbPBxAHbDndzwNwDfD3NsuP2u5Mb1riPwX2z9ApQDoQaYwpsFcVApH2/Q7NN9DNPQH8FHDaj8OBI8aYJvtx29fccj7s9eX29j3FMKAYWGZXfT0vIkF46efDGHMQ+DOQCxRgvd9ZeO/no9mpfh5c+jnRxN9BIhIMvAPcY4ypaLvOWF/JXtEuVkQuBoqMMVnujsVD+AFTgb8aY6YA1bT+jAe87vPRH2tGvWFANBDEN6s9vJonfB408XeAiPhjJf1XjDHv2osPiUiUvT4KKLKX9/T5BmYBC0QkG2vazHlYddz9RKR5tNe2r7nlfNjr+wKlXRmwi+UBecaYdPvx21hfBN76+TgH2G+MKTbGNALvYn1mvPXz0exUPw8u/Zxo4j8JERHgBWC7MeaxNqv+ATRfab8Jq+6/efmN9tX6FKC8zU+8bs8Y83NjzBBjTBzWRbtPjDHXAZ8CV9qbHXs+ms/Tlfb2Pab0a4wpBA6IyBh70XxgG176+cCq4kkRkT72/07z+fDKz0cbp/p5+Ag4T0T627+izrOXdQ53XwTx9BvwLayfZZuAjfbtQqx6yFXAbuB/QJi9vQBPA3uBzVitG9z+Olx0buYCH9j3hwPrgD3AW0Ave3mg/XiPvX64u+N2wXlIADLtz8h7QH9v/nwAvwZ2AFuAFUAvb/p8AK9hXd9oxPpFePPpfB6AxfZ52QMs6swYdcgGpZTyMlrVo5RSXkYTv1JKeRlN/Eop5WU08SullJfRxK+UUl5GE7/ySCLyC3uEx00islFEkt0d05kQkVQRufLkW5728eeKyMyuej7VvfmdfBOlupaIzAAuBqYaY+pFJAIIcHNYnm4uUAWscXMcqhvQEr/yRFFAiTGmHsAYU2KMyQcQkWki8rmIZInIR226wU8Tka/t25+ax0IXkYUi8pfmA4vIByIy175/noisFZH1IvKWPR4TIpItIr+2l28WkbH28mARWWYv2yQiV5zoOCcj1pwGfxKRDPt4t9rL54rIZ9I6xv8rdi9YRORCe1mWPY77B/bggbcBP7R/Hc22n+IsEVkjIvu09K/a0sSvPNF/gaEisktEnhGROdAyZtL/AVcaY6YBLwK/s/dZBtxljJnckSewf0U8AJxjjJmK1fP2R202KbGX/xX4sb3sl1hd6icaYyYBn3TgOCdys328JCAJWCIiw+x1U4B7gHisXq+zRCQQa3jeC+zXPwDAGJMN/A1rvPsEY8yX9jGisHqeXww80sGYlBfQqh7lcYwxVSIyDZgNnA28ISL3YSXVCcDHdgHYFygQkX5AP2PMF/YhVmBNcHEiKVhJdbV9rABgbZv1zYPxZQGX2/fPwRqfqDnOw/ZopSc6zomcB0xqUxrvizUhRwOwzhiTByAiG7HGd68C9hlj9tvbv4Y1eUd73jPGOIFtIhJ5gu2Ul9HErzySMcYBfAZ8JiKbsQa2ygK2GmNmtN3WTvztaeLoX7aBzbsBHxtjrmlnv3r7r4MT/5+c7DgnIli/Uo4afMuuiqpvs+hkMbSn7THkNPZXPZRW9SiPIyJjRGRUm0UJQA7W7EQD7Iu/iIi/iIw3xhwBjojIt+ztr2uzbzaQICI+IjIUp71NkgAAASJJREFUaxo7gDSs6pOR9rGCRGT0SUL7GLizTZz9T/M4zT4CbrersBCR0WJN4tKencBwaZ2X9ntt1lUCIR18XuXlNPErTxQMLBdrgvtNWFUpDxljGrCG7v2jiHyNNVJqcxPGRcDTdrVI29LtaqypEbcBTwHrAYwxxcBC4DX7OdYCY08S12+B/mJNIv41cPYpHufvIpJn39YCz9txrbcvRv+dE5TsjTG1wB3AhyKShZXsy+3V/4T/394d2wAIAlEAhd7lHMJ5rN3DVawcBguwMfYE7r0Jrvr55JIjrZ/lLvxynZPptEZ8lvrZ91RyzkvbgbznfO9Syt57Lsai8cNYtvaquVJdBh+d52FAGj9AMBo/QDCCHyAYwQ8QjOAHCEbwAwTzALy6jjLpgGjlAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    }
  ]
}