From 07d68e17ba177d05b0d6e3fbd7bbd9b6f58a73a2 Mon Sep 17 00:00:00 2001 From: JernKunpittaya <61564542+JernKunpittaya@users.noreply.github.com> Date: Thu, 29 Feb 2024 18:20:05 +0700 Subject: [PATCH] delete copy where --- .../copy where+mean/copy2where+mean.ipynb | 407 ------------------ examples/copy where+mean/copywhere+mean.ipynb | 342 --------------- examples/copy where+mean/data.json | 6 - 3 files changed, 755 deletions(-) delete mode 100644 examples/copy where+mean/copy2where+mean.ipynb delete mode 100644 examples/copy where+mean/copywhere+mean.ipynb delete mode 100644 examples/copy where+mean/data.json diff --git a/examples/copy where+mean/copy2where+mean.ipynb b/examples/copy where+mean/copy2where+mean.ipynb deleted file mode 100644 index af8eded..0000000 --- a/examples/copy where+mean/copy2where+mean.ipynb +++ /dev/null @@ -1,407 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: ezkl==7.0.0 in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from -r ../../requirements.txt (line 1)) (7.0.0)\n", - "Requirement already satisfied: torch in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from -r ../../requirements.txt (line 2)) (2.2.0)\n", - "Requirement already satisfied: requests in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from -r ../../requirements.txt (line 3)) (2.31.0)\n", - "Requirement already satisfied: scipy in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from -r ../../requirements.txt (line 4)) (1.12.0)\n", - "Requirement already satisfied: numpy in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from -r ../../requirements.txt (line 5)) (1.26.3)\n", - "Requirement already satisfied: matplotlib in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from -r ../../requirements.txt (line 6)) (3.8.2)\n", - "Requirement already satisfied: statistics in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from -r ../../requirements.txt (line 7)) (1.0.3.5)\n", - "Requirement already satisfied: onnx in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from -r ../../requirements.txt (line 8)) (1.15.0)\n", - "Requirement already satisfied: filelock in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from torch->-r ../../requirements.txt (line 2)) (3.13.1)\n", - "Requirement already satisfied: typing-extensions>=4.8.0 in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from torch->-r ../../requirements.txt (line 2)) (4.9.0)\n", - "Requirement already satisfied: sympy in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from torch->-r ../../requirements.txt (line 2)) (1.12)\n", - "Requirement already satisfied: networkx in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from torch->-r ../../requirements.txt (line 2)) (3.2.1)\n", - "Requirement already satisfied: jinja2 in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from torch->-r ../../requirements.txt (line 2)) (3.1.3)\n", - "Requirement already satisfied: fsspec in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from torch->-r ../../requirements.txt (line 2)) (2023.12.2)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from requests->-r ../../requirements.txt (line 3)) (3.3.2)\n", - "Requirement already satisfied: idna<4,>=2.5 in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from requests->-r ../../requirements.txt (line 3)) (3.6)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from requests->-r ../../requirements.txt (line 3)) (2.2.0)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from requests->-r ../../requirements.txt (line 3)) (2024.2.2)\n", - "Requirement already satisfied: contourpy>=1.0.1 in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from matplotlib->-r ../../requirements.txt (line 6)) (1.2.0)\n", - "Requirement already satisfied: cycler>=0.10 in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from matplotlib->-r ../../requirements.txt (line 6)) (0.12.1)\n", - "Requirement already satisfied: fonttools>=4.22.0 in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from matplotlib->-r ../../requirements.txt (line 6)) (4.47.2)\n", - "Requirement already satisfied: kiwisolver>=1.3.1 in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from matplotlib->-r ../../requirements.txt (line 6)) (1.4.5)\n", - "Requirement already satisfied: packaging>=20.0 in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from matplotlib->-r ../../requirements.txt (line 6)) (23.2)\n", - "Requirement already satisfied: pillow>=8 in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from matplotlib->-r ../../requirements.txt (line 6)) (10.2.0)\n", - "Requirement already satisfied: pyparsing>=2.3.1 in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from matplotlib->-r ../../requirements.txt (line 6)) (3.1.1)\n", - "Requirement already satisfied: python-dateutil>=2.7 in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from matplotlib->-r ../../requirements.txt (line 6)) (2.8.2)\n", - "Requirement already satisfied: docutils>=0.3 in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from statistics->-r ../../requirements.txt (line 7)) (0.20.1)\n", - "Requirement already satisfied: protobuf>=3.20.2 in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from onnx->-r ../../requirements.txt (line 8)) (4.25.2)\n", - "Requirement already satisfied: six>=1.5 in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from python-dateutil>=2.7->matplotlib->-r ../../requirements.txt (line 6)) (1.16.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from jinja2->torch->-r ../../requirements.txt (line 2)) (2.1.4)\n", - "Requirement already satisfied: mpmath>=0.19 in /Users/jernkun/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages (from sympy->torch->-r ../../requirements.txt (line 2)) (1.3.0)\n", - "\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.0\u001b[0m\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", - "Note: you may need to restart the kernel to use updated packages.\n" - ] - } - ], - "source": [ - "pip install -r ../../requirements.txt" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "import ezkl\n", - "import torch\n", - "from torch import nn\n", - "import json\n", - "import os\n", - "import time\n", - "import scipy\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "import statistics\n", - "import math" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "%run -i ../../zkstats/core.py" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "# init path\n", - "os.makedirs(os.path.dirname('shared/'), exist_ok=True)\n", - "os.makedirs(os.path.dirname('prover/'), exist_ok=True)\n", - "verifier_model_path = os.path.join('shared/verifier.onnx')\n", - "prover_model_path = os.path.join('prover/prover.onnx')\n", - "verifier_compiled_model_path = os.path.join('shared/verifier.compiled')\n", - "prover_compiled_model_path = os.path.join('prover/prover.compiled')\n", - "pk_path = os.path.join('shared/test.pk')\n", - "vk_path = os.path.join('shared/test.vk')\n", - "proof_path = os.path.join('shared/test.pf')\n", - "settings_path = os.path.join('shared/settings.json')\n", - "srs_path = os.path.join('shared/kzg.srs')\n", - "witness_path = os.path.join('prover/witness.json')\n", - "# this is private to prover since it contains actual data\n", - "sel_data_path = os.path.join('prover/sel_data.json')\n", - "# this is just dummy random value\n", - "sel_dummy_data_path = os.path.join('shared/sel_dummy_data.json')" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "======================= ZK-STATS FLOW =======================" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "data_path = os.path.join('data.json')\n", - "dummy_data_path = os.path.join('shared/dummy_data.json')\n", - "\n", - "data = np.array(json.loads(open(data_path, \"r\").read())['col_name'])\n", - "data_tensor = torch.reshape(torch.tensor(data),(1,-1, 1))\n", - "# remember that our create_dummy creates random numbers in range 1 - 30, so need to make sure this example the filter doesnt\n", - "# get filter out all data\n", - "create_dummy(data_path, dummy_data_path)\n", - "dummy_data = np.array(json.loads(open(dummy_data_path, \"r\").read())['col_name'])\n", - "dummy_data_tensor = torch.reshape(torch.tensor(dummy_data), (1,-1,1))\n", - "\n", - "# # where(element > 20)\n", - "gt20_dummy_data_tensor = dummy_data_tensor[dummy_data_tensor > 20].reshape(1,-1,1)\n", - "dummy_theory_output = torch.mean(gt20_dummy_data_tensor)\n", - "\n", - "gt20_data_tensor = data_tensor[data_tensor > 20].reshape(1,-1,1)\n", - "theory_output = torch.mean(gt20_data_tensor)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.Size([1, 16, 1])\n", - "torch.Size([1, 20, 1])\n", - "tensor(True)\n" - ] - } - ], - "source": [ - "print(gt20_data_tensor.size())\n", - "print(dummy_data_tensor.size())\n", - "print(data_tensor[0][1][0]<20)" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [], - "source": [ - "scales = [5]\n", - "selected_columns = ['col_name']\n", - "commitment_maps = get_data_commitment_maps(data_path, scales)" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/89/y9dw12v976ngdmqz4l7wbsnr0000gn/T/ipykernel_8994/1800029062.py:12: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.\n", - " result = torch.tensor([0]*self.filtered.size()[1])\n" - ] - }, - { - "ename": "RuntimeError", - "evalue": "graph_ == value->owningGraph() INTERNAL ASSERT FAILED at \"/Users/runner/work/pytorch/pytorch/pytorch/torch/csrc/jit/ir/ir.cpp\":1426, please report a bug to PyTorch. ", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[35], line 27\u001b[0m\n\u001b[1;32m 23\u001b[0m fil_X \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mwhere(\u001b[39mfilter\u001b[39m, X, \u001b[39m0.0\u001b[39m)\n\u001b[1;32m 24\u001b[0m \u001b[39mreturn\u001b[39;00m (torch\u001b[39m.\u001b[39mabs(torch\u001b[39m.\u001b[39msum(fil_X)\u001b[39m-\u001b[39mnum_fil_X\u001b[39m*\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mw)\u001b[39m<\u001b[39m\u001b[39m=\u001b[39mtorch\u001b[39m.\u001b[39mabs(\u001b[39m0.01\u001b[39m\u001b[39m*\u001b[39mnum_fil_X\u001b[39m*\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mw), \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mw)\n\u001b[0;32m---> 27\u001b[0m verifier_define_calculation(dummy_data_path, selected_columns,sel_dummy_data_path,verifier_model, verifier_model_path)\n", - "File \u001b[0;32m~/Desktop/zk-stats-lib/zkstats/core.py:26\u001b[0m, in \u001b[0;36mverifier_define_calculation\u001b[0;34m(dummy_data_path, col_array, dummy_sel_data_path, verifier_model, verifier_model_path)\u001b[0m\n\u001b[1;32m 24\u001b[0m dummy_data_tensor_array \u001b[39m=\u001b[39m _process_data(dummy_data_path, col_array, dummy_sel_data_path)\n\u001b[1;32m 25\u001b[0m \u001b[39m# export onnx file\u001b[39;00m\n\u001b[0;32m---> 26\u001b[0m _export_onnx(verifier_model, dummy_data_tensor_array, verifier_model_path)\n", - "File \u001b[0;32m~/Desktop/zk-stats-lib/zkstats/core.py:257\u001b[0m, in \u001b[0;36m_export_onnx\u001b[0;34m(model, data_tensor_array, model_loc)\u001b[0m\n\u001b[1;32m 254\u001b[0m dynamic_axes[\u001b[39m\"\u001b[39m\u001b[39moutput\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m {\u001b[39m0\u001b[39m : \u001b[39m'\u001b[39m\u001b[39mbatch_size\u001b[39m\u001b[39m'\u001b[39m}\n\u001b[1;32m 256\u001b[0m \u001b[39m# Export the model\u001b[39;00m\n\u001b[0;32m--> 257\u001b[0m torch\u001b[39m.\u001b[39;49monnx\u001b[39m.\u001b[39;49mexport(circuit, \u001b[39m# model being run\u001b[39;49;00m\n\u001b[1;32m 258\u001b[0m data_tensor_tuple, \u001b[39m# model input (or a tuple for multiple inputs)\u001b[39;49;00m\n\u001b[1;32m 259\u001b[0m model_loc, \u001b[39m# where to save the model (can be a file or file-like object)\u001b[39;49;00m\n\u001b[1;32m 260\u001b[0m export_params\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m, \u001b[39m# store the trained parameter weights inside the model file\u001b[39;49;00m\n\u001b[1;32m 261\u001b[0m opset_version\u001b[39m=\u001b[39;49m\u001b[39m11\u001b[39;49m, \u001b[39m# the ONNX version to export the model to\u001b[39;49;00m\n\u001b[1;32m 262\u001b[0m do_constant_folding\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m, \u001b[39m# whether to execute constant folding for optimization\u001b[39;49;00m\n\u001b[1;32m 263\u001b[0m input_names \u001b[39m=\u001b[39;49m input_names, \u001b[39m# the model's input names\u001b[39;49;00m\n\u001b[1;32m 264\u001b[0m output_names \u001b[39m=\u001b[39;49m [\u001b[39m'\u001b[39;49m\u001b[39moutput\u001b[39;49m\u001b[39m'\u001b[39;49m], \u001b[39m# the model's output names\u001b[39;49;00m\n\u001b[1;32m 265\u001b[0m dynamic_axes\u001b[39m=\u001b[39;49mdynamic_axes)\n", - "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages/torch/onnx/utils.py:516\u001b[0m, in \u001b[0;36mexport\u001b[0;34m(model, args, f, export_params, verbose, training, input_names, output_names, operator_export_type, opset_version, do_constant_folding, dynamic_axes, keep_initializers_as_inputs, custom_opsets, export_modules_as_functions, autograd_inlining)\u001b[0m\n\u001b[1;32m 189\u001b[0m \u001b[39m@_beartype\u001b[39m\u001b[39m.\u001b[39mbeartype\n\u001b[1;32m 190\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mexport\u001b[39m(\n\u001b[1;32m 191\u001b[0m model: Union[torch\u001b[39m.\u001b[39mnn\u001b[39m.\u001b[39mModule, torch\u001b[39m.\u001b[39mjit\u001b[39m.\u001b[39mScriptModule, torch\u001b[39m.\u001b[39mjit\u001b[39m.\u001b[39mScriptFunction],\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 208\u001b[0m autograd_inlining: Optional[\u001b[39mbool\u001b[39m] \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m,\n\u001b[1;32m 209\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 210\u001b[0m \u001b[39m \u001b[39m\u001b[39mr\u001b[39m\u001b[39m\"\"\"Exports a model into ONNX format.\u001b[39;00m\n\u001b[1;32m 211\u001b[0m \n\u001b[1;32m 212\u001b[0m \u001b[39m If ``model`` is not a :class:`torch.jit.ScriptModule` nor a\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 513\u001b[0m \u001b[39m All errors are subclasses of :class:`errors.OnnxExporterError`.\u001b[39;00m\n\u001b[1;32m 514\u001b[0m \u001b[39m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 516\u001b[0m _export(\n\u001b[1;32m 517\u001b[0m model,\n\u001b[1;32m 518\u001b[0m args,\n\u001b[1;32m 519\u001b[0m f,\n\u001b[1;32m 520\u001b[0m export_params,\n\u001b[1;32m 521\u001b[0m verbose,\n\u001b[1;32m 522\u001b[0m training,\n\u001b[1;32m 523\u001b[0m input_names,\n\u001b[1;32m 524\u001b[0m output_names,\n\u001b[1;32m 525\u001b[0m operator_export_type\u001b[39m=\u001b[39;49moperator_export_type,\n\u001b[1;32m 526\u001b[0m opset_version\u001b[39m=\u001b[39;49mopset_version,\n\u001b[1;32m 527\u001b[0m do_constant_folding\u001b[39m=\u001b[39;49mdo_constant_folding,\n\u001b[1;32m 528\u001b[0m dynamic_axes\u001b[39m=\u001b[39;49mdynamic_axes,\n\u001b[1;32m 529\u001b[0m keep_initializers_as_inputs\u001b[39m=\u001b[39;49mkeep_initializers_as_inputs,\n\u001b[1;32m 530\u001b[0m custom_opsets\u001b[39m=\u001b[39;49mcustom_opsets,\n\u001b[1;32m 531\u001b[0m export_modules_as_functions\u001b[39m=\u001b[39;49mexport_modules_as_functions,\n\u001b[1;32m 532\u001b[0m autograd_inlining\u001b[39m=\u001b[39;49mautograd_inlining,\n\u001b[1;32m 533\u001b[0m )\n", - "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages/torch/onnx/utils.py:1613\u001b[0m, in \u001b[0;36m_export\u001b[0;34m(model, args, f, export_params, verbose, training, input_names, output_names, operator_export_type, export_type, opset_version, do_constant_folding, dynamic_axes, keep_initializers_as_inputs, fixed_batch_size, custom_opsets, add_node_names, onnx_shape_inference, export_modules_as_functions, autograd_inlining)\u001b[0m\n\u001b[1;32m 1610\u001b[0m dynamic_axes \u001b[39m=\u001b[39m {}\n\u001b[1;32m 1611\u001b[0m _validate_dynamic_axes(dynamic_axes, model, input_names, output_names)\n\u001b[0;32m-> 1613\u001b[0m graph, params_dict, torch_out \u001b[39m=\u001b[39m _model_to_graph(\n\u001b[1;32m 1614\u001b[0m model,\n\u001b[1;32m 1615\u001b[0m args,\n\u001b[1;32m 1616\u001b[0m verbose,\n\u001b[1;32m 1617\u001b[0m input_names,\n\u001b[1;32m 1618\u001b[0m output_names,\n\u001b[1;32m 1619\u001b[0m operator_export_type,\n\u001b[1;32m 1620\u001b[0m val_do_constant_folding,\n\u001b[1;32m 1621\u001b[0m fixed_batch_size\u001b[39m=\u001b[39;49mfixed_batch_size,\n\u001b[1;32m 1622\u001b[0m training\u001b[39m=\u001b[39;49mtraining,\n\u001b[1;32m 1623\u001b[0m dynamic_axes\u001b[39m=\u001b[39;49mdynamic_axes,\n\u001b[1;32m 1624\u001b[0m )\n\u001b[1;32m 1626\u001b[0m \u001b[39m# TODO: Don't allocate a in-memory string for the protobuf\u001b[39;00m\n\u001b[1;32m 1627\u001b[0m defer_weight_export \u001b[39m=\u001b[39m (\n\u001b[1;32m 1628\u001b[0m export_type \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m _exporter_states\u001b[39m.\u001b[39mExportTypes\u001b[39m.\u001b[39mPROTOBUF_FILE\n\u001b[1;32m 1629\u001b[0m )\n", - "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages/torch/onnx/utils.py:1135\u001b[0m, in \u001b[0;36m_model_to_graph\u001b[0;34m(model, args, verbose, input_names, output_names, operator_export_type, do_constant_folding, _disable_torch_constant_prop, fixed_batch_size, training, dynamic_axes)\u001b[0m\n\u001b[1;32m 1132\u001b[0m args \u001b[39m=\u001b[39m (args,)\n\u001b[1;32m 1134\u001b[0m model \u001b[39m=\u001b[39m _pre_trace_quant_model(model, args)\n\u001b[0;32m-> 1135\u001b[0m graph, params, torch_out, module \u001b[39m=\u001b[39m _create_jit_graph(model, args)\n\u001b[1;32m 1136\u001b[0m params_dict \u001b[39m=\u001b[39m _get_named_param_dict(graph, params)\n\u001b[1;32m 1138\u001b[0m \u001b[39mtry\u001b[39;00m:\n", - "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages/torch/onnx/utils.py:1011\u001b[0m, in \u001b[0;36m_create_jit_graph\u001b[0;34m(model, args)\u001b[0m\n\u001b[1;32m 1006\u001b[0m graph \u001b[39m=\u001b[39m _C\u001b[39m.\u001b[39m_propagate_and_assign_input_shapes(\n\u001b[1;32m 1007\u001b[0m graph, flattened_args, param_count_list, \u001b[39mFalse\u001b[39;00m, \u001b[39mFalse\u001b[39;00m\n\u001b[1;32m 1008\u001b[0m )\n\u001b[1;32m 1009\u001b[0m \u001b[39mreturn\u001b[39;00m graph, params, torch_out, \u001b[39mNone\u001b[39;00m\n\u001b[0;32m-> 1011\u001b[0m graph, torch_out \u001b[39m=\u001b[39m _trace_and_get_graph_from_model(model, args)\n\u001b[1;32m 1012\u001b[0m _C\u001b[39m.\u001b[39m_jit_pass_onnx_lint(graph)\n\u001b[1;32m 1013\u001b[0m state_dict \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mjit\u001b[39m.\u001b[39m_unique_state_dict(model)\n", - "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages/torch/onnx/utils.py:915\u001b[0m, in \u001b[0;36m_trace_and_get_graph_from_model\u001b[0;34m(model, args)\u001b[0m\n\u001b[1;32m 913\u001b[0m prev_autocast_cache_enabled \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mis_autocast_cache_enabled()\n\u001b[1;32m 914\u001b[0m torch\u001b[39m.\u001b[39mset_autocast_cache_enabled(\u001b[39mFalse\u001b[39;00m)\n\u001b[0;32m--> 915\u001b[0m trace_graph, torch_out, inputs_states \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39;49mjit\u001b[39m.\u001b[39;49m_get_trace_graph(\n\u001b[1;32m 916\u001b[0m model,\n\u001b[1;32m 917\u001b[0m args,\n\u001b[1;32m 918\u001b[0m strict\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m 919\u001b[0m _force_outplace\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m 920\u001b[0m _return_inputs_states\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m,\n\u001b[1;32m 921\u001b[0m )\n\u001b[1;32m 922\u001b[0m torch\u001b[39m.\u001b[39mset_autocast_cache_enabled(prev_autocast_cache_enabled)\n\u001b[1;32m 924\u001b[0m warn_on_static_input_change(inputs_states)\n", - "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages/torch/jit/_trace.py:1296\u001b[0m, in \u001b[0;36m_get_trace_graph\u001b[0;34m(f, args, kwargs, strict, _force_outplace, return_inputs, _return_inputs_states)\u001b[0m\n\u001b[1;32m 1294\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39misinstance\u001b[39m(args, \u001b[39mtuple\u001b[39m):\n\u001b[1;32m 1295\u001b[0m args \u001b[39m=\u001b[39m (args,)\n\u001b[0;32m-> 1296\u001b[0m outs \u001b[39m=\u001b[39m ONNXTracedModule(\n\u001b[1;32m 1297\u001b[0m f, strict, _force_outplace, return_inputs, _return_inputs_states\n\u001b[1;32m 1298\u001b[0m )(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 1299\u001b[0m \u001b[39mreturn\u001b[39;00m outs\n", - "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1509\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_compiled_call_impl(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs) \u001b[39m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1510\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_call_impl(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n", - "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1515\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1516\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1518\u001b[0m \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1519\u001b[0m \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 1522\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 1523\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n", - "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages/torch/jit/_trace.py:138\u001b[0m, in \u001b[0;36mONNXTracedModule.forward\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m 135\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 136\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mtuple\u001b[39m(out_vars)\n\u001b[0;32m--> 138\u001b[0m graph, out \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39;49m_C\u001b[39m.\u001b[39;49m_create_graph_by_tracing(\n\u001b[1;32m 139\u001b[0m wrapper,\n\u001b[1;32m 140\u001b[0m in_vars \u001b[39m+\u001b[39;49m module_state,\n\u001b[1;32m 141\u001b[0m _create_interpreter_name_lookup_fn(),\n\u001b[1;32m 142\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mstrict,\n\u001b[1;32m 143\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_force_outplace,\n\u001b[1;32m 144\u001b[0m )\n\u001b[1;32m 146\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_return_inputs:\n\u001b[1;32m 147\u001b[0m \u001b[39mreturn\u001b[39;00m graph, outs[\u001b[39m0\u001b[39m], ret_inputs[\u001b[39m0\u001b[39m]\n", - "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages/torch/jit/_trace.py:129\u001b[0m, in \u001b[0;36mONNXTracedModule.forward..wrapper\u001b[0;34m(*args)\u001b[0m\n\u001b[1;32m 127\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_return_inputs_states:\n\u001b[1;32m 128\u001b[0m inputs_states\u001b[39m.\u001b[39mappend(_unflatten(in_args, in_desc))\n\u001b[0;32m--> 129\u001b[0m outs\u001b[39m.\u001b[39mappend(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49minner(\u001b[39m*\u001b[39;49mtrace_inputs))\n\u001b[1;32m 130\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_return_inputs_states:\n\u001b[1;32m 131\u001b[0m inputs_states[\u001b[39m0\u001b[39m] \u001b[39m=\u001b[39m (inputs_states[\u001b[39m0\u001b[39m], trace_inputs)\n", - "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1509\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_compiled_call_impl(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs) \u001b[39m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1510\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_call_impl(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n", - "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1515\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1516\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1518\u001b[0m \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1519\u001b[0m \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 1522\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 1523\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n", - "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/zkstats-OJpceffF-py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._slow_forward\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1499\u001b[0m recording_scopes \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m\n\u001b[1;32m 1500\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m-> 1501\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mforward(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 1502\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 1503\u001b[0m \u001b[39mif\u001b[39;00m recording_scopes:\n", - "Cell \u001b[0;32mIn[35], line 13\u001b[0m, in \u001b[0;36mverifier_model.forward\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m 11\u001b[0m filtered_index \u001b[39m=\u001b[39m \u001b[39m0\u001b[39m\n\u001b[1;32m 12\u001b[0m result \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mtensor([\u001b[39m0\u001b[39m]\u001b[39m*\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfiltered\u001b[39m.\u001b[39msize()[\u001b[39m1\u001b[39m])\n\u001b[0;32m---> 13\u001b[0m \u001b[39mfor\u001b[39;00m x_val \u001b[39min\u001b[39;00m X[\u001b[39m0\u001b[39;49m]:\n\u001b[1;32m 14\u001b[0m result[filtered_index] \u001b[39m=\u001b[39m x_val[\u001b[39m0\u001b[39m]\u001b[39m==\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfiltered[\u001b[39m0\u001b[39m][filtered_index][\u001b[39m0\u001b[39m]\n\u001b[1;32m 15\u001b[0m filtered_index \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mwhere(x_val[\u001b[39m0\u001b[39m]\u001b[39m>\u001b[39m\u001b[39m20\u001b[39m, filtered_index\u001b[39m+\u001b[39m\u001b[39m1\u001b[39m, filtered_index)\n", - "\u001b[0;31mRuntimeError\u001b[0m: graph_ == value->owningGraph() INTERNAL ASSERT FAILED at \"/Users/runner/work/pytorch/pytorch/pytorch/torch/csrc/jit/ir/ir.cpp\":1426, please report a bug to PyTorch. " - ] - } - ], - "source": [ - "# Verifier/ data consumer side: send desired calculation\n", - "class verifier_model(nn.Module):\n", - " def __init__(self):\n", - " super(verifier_model, self).__init__()\n", - " # w represents mean in this case\n", - " self.w = nn.Parameter(data = dummy_theory_output, requires_grad = False)\n", - " self.filtered = nn.Parameter(data = gt20_dummy_data_tensor, requires_grad = False)\n", - "\n", - " def forward(self,X):\n", - " arb = X.size()[1]>0\n", - " filtered_index = 0\n", - " result = torch.tensor([0]*self.filtered.size()[1])\n", - " filtered_index = filtered_index+1\n", - " # for x_val in X[0]:\n", - " # result[filtered_index] = x_val[0]==self.filtered[0][filtered_index][0]\n", - " # filtered_index = torch.where(x_val[0]>20, filtered_index+1, filtered_index)\n", - " \n", - " print('final', filtered_index)\n", - " bool_output = torch.abs(torch.sum(self.filtered)-self.filtered.size()[1]*(self.w))<=torch.abs(0.01*self.filtered.size()[1]*self.w)\n", - " return (torch.logical_and(arb, bool_output), self.w)\n", - " # where part\n", - " filter = (X>20)\n", - " num_fil_X = torch.sum(filter.float())\n", - " fil_X = torch.where(filter, X, 0.0)\n", - " return (torch.abs(torch.sum(fil_X)-num_fil_X*self.w)<=torch.abs(0.01*num_fil_X*self.w), self.w)\n", - "\n", - "\n", - "verifier_define_calculation(dummy_data_path, selected_columns,sel_dummy_data_path,verifier_model, verifier_model_path)" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Theory_output: tensor(50.3750, dtype=torch.float64)\n", - "==== Generate & Calibrate Setting ====\n", - "scale: [5]\n", - "setting: {\"run_args\":{\"tolerance\":{\"val\":0.0,\"scale\":1.0},\"input_scale\":5,\"param_scale\":5,\"scale_rebase_multiplier\":10,\"lookup_range\":[0,0],\"logrows\":11,\"num_inner_cols\":2,\"variables\":[[\"batch_size\",1]],\"input_visibility\":{\"Hashed\":{\"hash_is_public\":true,\"outlets\":[]}},\"output_visibility\":\"Public\",\"param_visibility\":\"Private\"},\"num_rows\":1312,\"total_assignments\":20,\"total_const_size\":0,\"model_instance_shapes\":[[1],[1]],\"model_output_scales\":[0,5],\"model_input_scales\":[5],\"module_sizes\":{\"kzg\":[],\"poseidon\":[1312,[1]],\"elgamal\":[0,[0]]},\"required_lookups\":[],\"check_mode\":\"UNSAFE\",\"version\":\"7.0.0\",\"num_blinding_factors\":null}\n" - ] - } - ], - "source": [ - "# prover calculates settings, send to verifier\n", - "gt30_data_tensor = data_tensor[data_tensor > 20].reshape(1,-1,1)\n", - "theory_output = torch.mean(gt30_data_tensor)\n", - "print(\"Theory_output: \", theory_output)\n", - "class prover_model(nn.Module):\n", - " def __init__(self):\n", - " super(prover_model, self).__init__()\n", - " # w represents mean in this case\n", - " self.w = nn.Parameter(data = theory_output, requires_grad = False)\n", - " self.filtered = nn.Parameter(data = gt20_data_tensor, requires_grad = False)\n", - "\n", - " def forward(self,X):\n", - " # where part\n", - " arb = X.size()[1]>0\n", - " filtered_index = 0\n", - " result = torch.tensor([0]*self.filtered.size()[1])\n", - " for x_val in X[0]:\n", - " result[filtered_index] = x_val[0]==self.filtered[0][filtered_index][0]\n", - " filtered_index = torch.where(x_val[0]>20, filtered_index+1, filtered_index)\n", - " print(filtered_index)\n", - " bool_output = torch.abs(torch.sum(self.filtered)-self.filtered.size()[1]*(self.w))<=torch.abs(0.01*self.filtered.size()[1]*self.w)\n", - " return (torch.logical_and(arb, bool_output), self.w)\n", - " filter = (X>20)\n", - " num_fil_X = torch.sum(filter.float())\n", - " fil_X = torch.where(filter, X, 0.0)\n", - "\n", - " return (torch.abs(torch.sum(fil_X)-num_fil_X*self.w)<=torch.abs(0.01*num_fil_X*self.w), self.w)\n", - "\n", - "\n", - "prover_gen_settings(data_path, selected_columns, sel_data_path, prover_model,prover_model_path, scales, \"resources\", settings_path)" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "spawning module 0\n", - "spawning module 2\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "==== setting up ezkl ====\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "spawning module 0\n", - "spawning module 2\n", - "spawning module 0\n", - "spawning module 2\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Time setup: 0.25963401794433594 seconds\n", - "=======================================\n", - "Theory output: tensor(50.3750, dtype=torch.float64)\n", - "==== Generating Witness ====\n", - "witness boolean: 1.0\n", - "witness result 1 : 50.375\n", - "==== Generating Proof ====\n", - "proof: {'instances': [[[16266678685338052443, 7438397994847379675, 5674719498686815871, 2240658077530776119], [12436184717236109307, 3962172157175319849, 7381016538464732718, 1011752739694698287], [17049824896927915697, 16648112313320311147, 14284844057324796177, 2517225790866338397]]], 'proof': '152f97b2b107c76372d8f3fbed4dba39ae37ef6b1773a88485d93ed36861c30c232f111689976debfdd1ea5a3bca37b96b76b994b1753c33de5ffc8ef3140f68271ec13c48cb354a50d9ee0b8906009a156e8a23f9f0aaff2be1ebd5b4655f940163fc3f2f755d8298c6cfd8fd01a9f13d12423108846780d1237dc9745c63f91d1ed8eb3224fe059f197eb6686829186673a576ff34c45bf9501f1f66cc54bd08b8651f7ae2753362901bb6309f9a72c7445ffcd3b0ed63b00091746e703cbf0c9c106e72e36fce2d39765a494d19b79f0b2342d98f2ade25f3e4aa28e967431ee6cfa836c8550bc6f7cf2bb95576fda960ed1bf83a30144bc3518485fd9b131fcd84eeacb10719792785c65ba093343ca25c8ec5a1fbd391475d43bfbfbab82ba04bb948f6cc5a82c2aa7a9f25fc3f943caee704fa8d064f332af507030fe1275e41a13a27fd1a6080c6a0e9ed3531af4872148cb136f67ae5a2f1b3a0c1b6206d617453617554246b93092aec66ca7d66c2afc00e1291b8a6cba9dad537131c697d22394bce2764bdd24ea24b9f645b1b4345c2108687bcd25dc475b2964a22cf704e9732dec165e81e6673dd7acb39491f46bc1a6d15ac3c86a040df864b14f4aebfdc89187138f7b6226ded8152f073133a288b20aeda4410a31091f0930dfa8fb9945027e3312c811d0337de24d4d88587173d0122a0df7f595dbf463d239b520d82e47469f4fc9fc6a72cdcca6f159f104180dd7820525490d068ff1e0a2b85db44b0b08a14d1fa7e73cc95ffe412bb6737fb84bd8cfb03f88a773a090bcd80881550425763b64da40bdc3c37e1d2dbb73012df0d650fee8d27f69e971ed7210be2af028fd5a677e70aa057a76278a809b1f796c2dbb92be5be9f9cbb0c324420393acd808c95bea3d84da24cce5da7c6f25b5dc5664bc4fa2151ab5d1f48303cbf17c89405ab623e31471515f21066582f3d9e427d0e9c7f559f2f8a0e02763808780c900161304dddced50a1a7354e5d2ef83b5ba6b370d27c2a96c303318cddfc0a33922631d969bb9f2fb574a697e7cf64d7be180a742c81d05e22daceb6e8523ac08097242f50a7bb553436f80c539c6f0b97ab7c669479b37a2269c02eaa113df19885f05479f3d1cada5596b9310a714dfa32f0553ec2bb3b4074a33ce7ff9939476685a70577b316aee442454220bb086617257ef65fc03611280b0d96151eabedd7ed63f699ba7a3c6ee47f304e5cef3248ac44b019e94ee1b0f7bad2f5dbef8c8d5c25ffc42031039b312c4ca82ed9037c1e9b4240814b11ca0c5196702bd7e18da80d8d60df0522cd44ad64a59cb5951aa39c5ff606d5c093c199445f9b148400677ec7a7295a38b2646c0636ca89d5ddf89886bb600e32149f8299e5f8d15b962b9bf3800c855b24965ac32a9be53a89c1c01b4b5b51b0b157fb987877148cf88df49c3291ff31e687d62c2385c526cadd07f33010c561b2c8199dae1ca3f432fcaa22e4e3c788b6f4460ea6669f8efedc680d4a974101770cda6ecbfbf7ffdd49bc0216b406ee2d4474b598cb9e6877e12718f1f2f9e1027a6f8523b8d459f484e7bfcda3a4d7b6fc57115b046bf1e88ee91c9853bda286152016a7be9137a4edf5ba886c00837f36fcce65b3eb0ccf831e41ad5a5a52b096220b2ecd364a36adf3ea536d62d684c0fe581125f75e6833a34b2d3a55d15b198bbef5be73f08e2fd947cbd1d879ab2905688b91aaf9832b7018ab80e46054682194bb2550f34220960f88f971fd944b813ad6e4ab5bef13a2adabac19b2006412e6dd801626bbf3be973fd2b455f7566b046f637a433267be5395b4ec422258c0447f9454ef1c018a17165888c7cc5eeb9ececc409b6c8658732e88d491230fe89151c363732fa359cd77b7be69a87309cdda537ebb4c0b5f3c3996235232ad0fc90f919e0d74abb1a96023867dc2e6c3e6eda87ddfb47f432fb3dd20f2368146a2fd788cf61aaf9459fd84d3aa9deadecabc3d01b5f9507d14fa34c621d035e2b6b895310ada71ec815455f253816d5059e3e9c3cdb3a0bb09870548a0b821e1cf61e8ae1ea34ffe6ef9d2a60bd6280b6194119a0776f7d1cf05397b61a107e8d492ed672de6cb6b469529c55b66718ffa7afafe21bf7f4327a2276192cd059180adcad314011edeed33c35a9eeaaa830882415fb5334ca5ea3f1f71911cc66a936e2cc25f9e36d4dcdd3a9fb42f4683d45e7ea73ed6678a8e745c5501b16ad04e2bdfd2379343d2122da32ee6b0549fb00397e1033e8dc35f9f574981e73a09350daffc7dd07ae0065fdacec01a20f618151d4ee97d4747c53af90000b565ca2c1e11673614d862bd711aaec6884ee69bee60450f540dabc76b5ef2e27f0cd19f6529b4d0151f491f4f1bd3111c990a9b66c029344a8209f1f4c11e407cf9bf382f2fc332380ac9f831ea3c2ddd04b94f5ae717a1865d9af7d0b307503a24e84a71a917ed56ec50db10cfa9d49a6d79190ab05cf9ba397dc24e39f2a035f7cd727e347aa5b490911a4b89818bcd414e8510c0868d81223aaa831e2d619bd8b68f9a3dd0dd5f2832ab60c6c868ab429e1c9f975f288497887e7e9fb7a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000121e71177322cb9ff7939844d474c138eb89bf8dbcd69c4931372674c9ac9a302878609f6eea53b4020d91bf39a5fa2f838ee23d420a0c76cc031047f6123af1370bca6ed4b188c3947adcb22cd71f0494002bd39df9066ab3a6ac75cdacb8c28b35e5f30a024ec7ca39f61454a2555abce212e42203f5f18b3346d0dec9b91262aabe3a70432e26827c1dc18240f4d49145e5775cb42d648ebed790217ced51ed414c52d81e68c472bacb05943c91c5e4896903680070d8fe6be32eec27ee827eadea01c92976a7850d3c9b0698f3e295dac6b45e4caf5697e9257ce02a2241ed14f5321f194389698dcfd635132bfcdc24a57b76565304be8a214f84c994528c6c250591780446c5a0975dc777b47351cfb75618921427f31331a1007e7e627210a1e3c3c11d87d7bdcbd17415222c20f0e6bb7d0f6d6ee561857b7abe39624e3f0d43c4de785f40db95a6e070716c95e0af550c46a9689729d906d41fff712288340193e9fdbe7753011308d2f4a32b613ae0370ce4d4fbc7dc47750492406e6dbaefa68db057b40f23ac1aac88149989c8ad72d331c0102d65a3191c70c1937c461747f7a411ec07da8f220cf9781ac77c8cbfb94405c26fb8d13ac633e18a51e75127a90f26d0f1dfd429e2e540744a3787312c7a889c0c466429f815a13c51d6c4511351fd37288e599c8b02786bfbb50b5c1bd9a131de6d8bf707e6122eed404dfc3df9563e2c815e82bd2e72c6f7cc82912aeb40cdb0b77cb9e9fec22157f7af4e6372492c436b9c0a8025710398deff9ba72fd0a539ad9c211f3fc0d3973c24f5f7984a494f64bdce785d45d0a497bc3724f3e73f7af1d50078ea519effe2fa9bd0bff3eb7e131605002312be83e0bf00af8ab8071aec6ce39352e24c07c23c217f75fb714f812c003f060db49b9d38c0bf7e4eb46b6056578bb802d0ea5d99ece8fb57fadb807b15e835628bd9e9a91a5da8c1e6656e12731bcd00946de23798294724900c2abe47f856c89b9c34c1a58ba4d0efa981d7050ceaa0c15bba197a16a43b565b94ae722c37fd8a8298609ccdaceb20b4502f11e512516a004aed29241e65c833a82c858f81c55bd4dbc051d51144849c85a214f47a4', 'transcript_type': 'EVM'}\n", - "Time gen prf: 0.26849365234375 seconds\n" - ] - } - ], - "source": [ - "# Here verifier & prover can concurrently call setup since all params are public to get pk.\n", - "# Here write as verifier function to emphasize that verifier must calculate its own vk to be sure\n", - "verifier_setup(verifier_model_path, verifier_compiled_model_path, settings_path,vk_path, pk_path )\n", - "\n", - "print(\"=======================================\")\n", - "# Prover generates proof\n", - "print(\"Theory output: \", theory_output)\n", - "prover_gen_proof(prover_model_path, sel_data_path, witness_path, prover_compiled_model_path, settings_path, proof_path, pk_path)" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "50.375" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Verifier verifies\n", - "verifier_verify(proof_path, settings_path, vk_path, selected_columns, commitment_maps)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/copy where+mean/copywhere+mean.ipynb b/examples/copy where+mean/copywhere+mean.ipynb deleted file mode 100644 index d2f097c..0000000 --- a/examples/copy where+mean/copywhere+mean.ipynb +++ /dev/null @@ -1,342 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: ezkl==7.0.0 in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from -r ../../requirements.txt (line 1)) (7.0.0)\n", - "Requirement already satisfied: torch in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from -r ../../requirements.txt (line 2)) (2.1.1)\n", - "Requirement already satisfied: requests in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from -r ../../requirements.txt (line 3)) (2.31.0)\n", - "Requirement already satisfied: scipy in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from -r ../../requirements.txt (line 4)) (1.11.4)\n", - "Requirement already satisfied: numpy in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from -r ../../requirements.txt (line 5)) (1.26.2)\n", - "Requirement already satisfied: matplotlib in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from -r ../../requirements.txt (line 6)) (3.8.2)\n", - "Requirement already satisfied: statistics in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from -r ../../requirements.txt (line 7)) (1.0.3.5)\n", - "Requirement already satisfied: onnx in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from -r ../../requirements.txt (line 8)) (1.15.0)\n", - "Requirement already satisfied: fsspec in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from torch->-r ../../requirements.txt (line 2)) (2023.10.0)\n", - "Requirement already satisfied: networkx in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from torch->-r ../../requirements.txt (line 2)) (3.2.1)\n", - "Requirement already satisfied: filelock in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from torch->-r ../../requirements.txt (line 2)) (3.13.1)\n", - "Requirement already satisfied: sympy in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from torch->-r ../../requirements.txt (line 2)) (1.12)\n", - "Requirement already satisfied: typing-extensions in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from torch->-r ../../requirements.txt (line 2)) (4.8.0)\n", - "Requirement already satisfied: jinja2 in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from torch->-r ../../requirements.txt (line 2)) (3.1.2)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from requests->-r ../../requirements.txt (line 3)) (3.3.2)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from requests->-r ../../requirements.txt (line 3)) (2023.11.17)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from requests->-r ../../requirements.txt (line 3)) (2.1.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from requests->-r ../../requirements.txt (line 3)) (3.6)\n", - "Requirement already satisfied: python-dateutil>=2.7 in /Users/jernkun/Library/Python/3.10/lib/python/site-packages (from matplotlib->-r ../../requirements.txt (line 6)) (2.8.2)\n", - "Requirement already satisfied: pillow>=8 in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from matplotlib->-r ../../requirements.txt (line 6)) (10.1.0)\n", - "Requirement already satisfied: fonttools>=4.22.0 in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from matplotlib->-r ../../requirements.txt (line 6)) (4.45.1)\n", - "Requirement already satisfied: cycler>=0.10 in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from matplotlib->-r ../../requirements.txt (line 6)) (0.12.1)\n", - "Requirement already satisfied: kiwisolver>=1.3.1 in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from matplotlib->-r ../../requirements.txt (line 6)) (1.4.5)\n", - "Requirement already satisfied: contourpy>=1.0.1 in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from matplotlib->-r ../../requirements.txt (line 6)) (1.2.0)\n", - "Requirement already satisfied: packaging>=20.0 in /Users/jernkun/Library/Python/3.10/lib/python/site-packages (from matplotlib->-r ../../requirements.txt (line 6)) (23.2)\n", - "Requirement already satisfied: pyparsing>=2.3.1 in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from matplotlib->-r ../../requirements.txt (line 6)) (3.1.1)\n", - "Requirement already satisfied: docutils>=0.3 in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from statistics->-r ../../requirements.txt (line 7)) (0.20.1)\n", - "Requirement already satisfied: protobuf>=3.20.2 in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from onnx->-r ../../requirements.txt (line 8)) (4.25.1)\n", - "Requirement already satisfied: six>=1.5 in /Users/jernkun/Library/Python/3.10/lib/python/site-packages (from python-dateutil>=2.7->matplotlib->-r ../../requirements.txt (line 6)) (1.16.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from jinja2->torch->-r ../../requirements.txt (line 2)) (2.1.3)\n", - "Requirement already satisfied: mpmath>=0.19 in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from sympy->torch->-r ../../requirements.txt (line 2)) (1.3.0)\n", - "\u001b[33mWARNING: You are using pip version 21.2.3; however, version 23.3.2 is available.\n", - "You should consider upgrading via the '/usr/local/bin/python3 -m pip install --upgrade pip' command.\u001b[0m\n", - "Note: you may need to restart the kernel to use updated packages.\n" - ] - } - ], - "source": [ - "pip install -r ../../requirements.txt" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [], - "source": [ - "import ezkl\n", - "import torch\n", - "from torch import nn\n", - "import json\n", - "import os\n", - "import time\n", - "import scipy\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "import statistics\n", - "import math" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [], - "source": [ - "%run -i ../../zkstats/core.py" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [], - "source": [ - "# init path\n", - "os.makedirs(os.path.dirname('shared/'), exist_ok=True)\n", - "os.makedirs(os.path.dirname('prover/'), exist_ok=True)\n", - "verifier_model_path = os.path.join('shared/verifier.onnx')\n", - "prover_model_path = os.path.join('prover/prover.onnx')\n", - "verifier_compiled_model_path = os.path.join('shared/verifier.compiled')\n", - "prover_compiled_model_path = os.path.join('prover/prover.compiled')\n", - "pk_path = os.path.join('shared/test.pk')\n", - "vk_path = os.path.join('shared/test.vk')\n", - "proof_path = os.path.join('shared/test.pf')\n", - "settings_path = os.path.join('shared/settings.json')\n", - "srs_path = os.path.join('shared/kzg.srs')\n", - "witness_path = os.path.join('prover/witness.json')\n", - "# this is private to prover since it contains actual data\n", - "sel_data_path = os.path.join('prover/sel_data.json')\n", - "# this is just dummy random value\n", - "sel_dummy_data_path = os.path.join('shared/sel_dummy_data.json')" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "======================= ZK-STATS FLOW =======================" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [], - "source": [ - "data_path = os.path.join('data.json')\n", - "dummy_data_path = os.path.join('shared/dummy_data.json')\n", - "\n", - "data = np.array(json.loads(open(data_path, \"r\").read())['col_name'])\n", - "data_tensor = torch.reshape(torch.tensor(data),(1,-1, 1))\n", - "# remember that our create_dummy creates random numbers in range 1 - 30, so need to make sure this example the filter doesnt\n", - "# get filter out all data\n", - "create_dummy(data_path, dummy_data_path)\n", - "dummy_data = np.array(json.loads(open(dummy_data_path, \"r\").read())['col_name'])\n", - "dummy_data_tensor = torch.reshape(torch.tensor(dummy_data), (1,-1,1))\n", - "\n", - "# # where(element > 20)\n", - "gt20_dummy_data_tensor = dummy_data_tensor[dummy_data_tensor > 20].reshape(1,-1,1)\n", - "dummy_theory_output = torch.mean(gt20_dummy_data_tensor)\n", - "\n", - "gt20_data_tensor = data_tensor[data_tensor > 20].reshape(1,-1,1)\n", - "theory_output = torch.mean(gt20_data_tensor)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "scales = [2]\n", - "selected_columns = ['col_name']\n", - "commitment_maps = get_data_commitment_maps(data_path, scales)" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [], - "source": [ - "# Verifier/ data consumer side: send desired calculation\n", - "class verifier_model(nn.Module):\n", - " def __init__(self):\n", - " super(verifier_model, self).__init__()\n", - " # w represents mean in this case\n", - " self.w = nn.Parameter(data = dummy_theory_output, requires_grad = False)\n", - "\n", - " def forward(self,X):\n", - " # where part\n", - " filter = (X>20)\n", - " num_fil_X = torch.sum(filter.double())\n", - " fil_X = torch.where(filter, X, 0.0)\n", - " return (torch.abs(torch.sum(fil_X)-num_fil_X*self.w)<=torch.abs(0.01*num_fil_X*self.w), self.w)\n", - "\n", - "\n", - "verifier_define_calculation(dummy_data_path, selected_columns,sel_dummy_data_path,verifier_model, verifier_model_path)" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Theory_output: tensor(50.2196, dtype=torch.float64)\n", - "==== Generate & Calibrate Setting ====\n", - "scale: [2]\n", - "setting: {\"run_args\":{\"tolerance\":{\"val\":0.0,\"scale\":1.0},\"input_scale\":2,\"param_scale\":2,\"scale_rebase_multiplier\":10,\"lookup_range\":[-154296,1206],\"logrows\":18,\"num_inner_cols\":2,\"variables\":[[\"batch_size\",1]],\"input_visibility\":{\"Hashed\":{\"hash_is_public\":true,\"outlets\":[]}},\"output_visibility\":\"Public\",\"param_visibility\":\"Private\"},\"num_rows\":14432,\"total_assignments\":3318,\"total_const_size\":306,\"model_instance_shapes\":[[1],[1]],\"model_output_scales\":[0,2],\"model_input_scales\":[2],\"module_sizes\":{\"kzg\":[],\"poseidon\":[14432,[1]],\"elgamal\":[0,[0]]},\"required_lookups\":[\"Abs\",{\"Div\":{\"denom\":100.0}},{\"GreaterThan\":{\"a\":0.0}}],\"check_mode\":\"UNSAFE\",\"version\":\"7.0.0\",\"num_blinding_factors\":null}\n" - ] - } - ], - "source": [ - "# prover calculates settings, send to verifier\n", - "gt30_data_tensor = data_tensor[data_tensor > 20].reshape(1,-1,1)\n", - "theory_output = torch.mean(gt30_data_tensor)\n", - "print(\"Theory_output: \", theory_output)\n", - "class prover_model(nn.Module):\n", - " def __init__(self):\n", - " super(prover_model, self).__init__()\n", - " # w represents mean in this case\n", - " self.w = nn.Parameter(data = theory_output, requires_grad = False)\n", - "\n", - " def forward(self,X):\n", - " # where part\n", - " filter = (X>20)\n", - " num_fil_X = torch.sum(filter.double())\n", - " fil_X = torch.where(filter, X, 0.0)\n", - "\n", - " return (torch.abs(torch.sum(fil_X)-num_fil_X*self.w)<=torch.abs(0.01*num_fil_X*self.w), self.w)\n", - "\n", - "\n", - "prover_gen_settings(data_path, selected_columns, sel_data_path, prover_model,prover_model_path, scales, \"resources\", settings_path)" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "spawning module 0\n", - "spawning module 2\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "==== setting up ezkl ====\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "spawning module 0\n", - "spawning module 2\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Time setup: 26.11175513267517 seconds\n", - "=======================================\n", - "Theory output: tensor(50.2196, dtype=torch.float64)\n", - "!@# compiled_model exists? False\n", - "!@# compiled_model exists? True\n", - "==== Generating Witness ====\n", - "witness boolean: 1.0\n", - "witness result 1 : 50.25\n", - "==== Generating Proof ====\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "spawning module 0\n", - "spawning module 2\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "proof: {'instances': [[[12572659313263335624, 14443766455855958404, 432930639589567449, 1881177029071802301], [12436184717236109307, 3962172157175319849, 7381016538464732718, 1011752739694698287], [2359119529189047257, 1186270854317446351, 12298916256516019502, 1116401204062057155]]], 'proof': '2fdcccf3147d0fd78981aacedc04a84fd5f85175fe3f82d29da419f241a79e622777e8afa6fb96db586ace343bfd30b3c714ef283afe03649b62428dcd461a9c0a51ac5443fbf47f630dfbf0097b0846d3ea8272662d047d14363e1becf379782e1fe34bbb01dbdda6fe850372f2f8760a8c8a9c0ef659f6ac9885a5b6e6a28005307dcdc7153f81dd80287f0cad008d5e5ccf9798f87f959e797a6f822e113d1175179b47d9f81149e2852ff55b940f0c337d357b51042953e8652827a626b71151076631997d34c44a03948cedea275fe232d1302f92ff54203999c9e2ca4624e5a9f7b8985e85f4a43d504d556372dbcbb4495661ab1f1cc2f4dd49434c0f251de3691943b7d0afbd76ff1f9f9ce1db9d93a254ab559788a1027bdf7f71d809062d6cf12a15c722006b47c0e9c6f7d4f264336e39a072468ca5166710891d098c93611eda31ca03739aa5c90bd1822aea2d1e251627341cb99473eb0439e126a3a831fe95fc7ca410ddd0516487042c93e6ace4f12f5fba4fc8411706b0b1233c3706eb61c31ee8d8ac38f97e4572fecee1311c98e9ff41ec9d6cca9cbf16050fcd1291f4aac758dfabbd4c8d802484e83e6ebdbb347848f87184a3fd22ad21dc2a9c181731bda7b076a9d9bfa7c6c0554e348b050f6cc1092a8359123dda1008ce50679999eb180141575be504d8a9f232387401d1476b149f622f3adccc2cd793580df8028fbd9a02997aa381591228b231449a0b81b212c3ab2f1eaaaa2f3686dee9801ae8ffe100d697d07cc50eb737c29bf4f564e12948ae520e74a92e4bab5b9005277f8775de4dacd3a228ba311b93503a4bfedeba1f18719bc8e60ce6f0a42116e709c417a29e66b4bf5386419d6bdc7b2cc1ad63df2d8894ba7821b7af4eb20134b11b07c441a0061a984ee2ae9b4ce489aa8414d4f9e81b51e623233f9bb5b7ebfdc5b4a8caff774ed29fac406aed5f03b033e913a0bdb4dc2d0cbe6555dd663344a4cc8e9b21dc734a78f6b239dcb1390b9aa9f246023b23110b1772e571d06ef1fb46273007ff93f1ca51e2332cbb9838a1e5f1040fe9ee911cc28134f62ae8698540f8d07d28a9354ba493055d55d7301d4091939fde37530e1cd3f0fdf9ffa4dc5db1ce50104795a503100bab90766811aad1d84707509702da6a7f27f5fb834ad1c62739946dd0e25139dd724cb32a11b18ec2e704d461003ba79ce2fb819fc6e35ca29eeaf2e00219c9478855b828a2f73a84067bdc332ab7014768e92e38e2a971e1dfef6a73c0dd32fd20b8976496b47deea031e3f425a9e0be1c043f94cd919319a29d4494b3f32459e58d188ed650675540a3d8662cc95de50b3c9cd70bcbdbe4c8a08e6099299444703fef8fccab7c782f8eebd100376fe8153d63807041f27ff20f3c7546c159f123c07cd4b21eebe38cad5a3d0dccafa4e3aa16c2dc9bed7464efd43bc3741a2b9274a7bf3afe35ed1c0b910e1edd1dc4612f2b53265b894b896f4706bc918d4966b2a7d728c490df4d76aa401216b1cc71cc9f574b6008e02653710b78204fdd82c09c1156693193c3c7154f25e92fbe099391335da50c7a1db87bc104a53773053ad5a572b584c5cddfb64d02f81511ca83117fd013470ab332aea731c62fbf6bd0e644099db570e7f030ce13d57d0dd22f770dc365429ad6e19719ebc8409252c97e508c66fde78f5b054b00737a62bf81874afb5fd59eb8f8eb4c513c6a0b3787cfff9fa564d1a00c2cf92f69fe858f0e2c1bf0adcde93cbc00b1310346d8a96804cbeedc91584f24f86416fcb091e3e03ca2927120be744666a9b31b98f9a37d21a38b11bc4954119d791166a6d133d8d3de27861c02e62c2293f3d7226df4b7f2698ff8a76cbab4bb9705300976836ba21ef9c1d5f92334a547468193d0a4c21ba2e8c9042cc8410c662e378b4f701ad77d453b033f869d3f4a496dd4eac399a89e4b7b034b38342aba2eb972947665f8e5d8ed5ce62e73e65663b706c5340b247669617f7ef5afdf4d0aceff062c52229d3cafc947b704729f7a9921aca5e37539615ab757c27a067a0e29c39555ea3d322d713033c589c39796bc218953c745f446adae02774ff17828f3ab94e7246ed36ee7104bd4eae72400ee42ee253bf0bf24d6241aa06780ae266382b27a8d57fda333a1f7edfb3c09a19f7a0e3b6e3b5c8e856203f5ea1ffc236fa71ea163d2ee8f4baefd8821068c10e9711b2cca31284dacdd44ac3003ff2b20aa6fa42dba2d8bbf7731152a74cb348ff286a10508fa10b2c872dafd6a6f1e733b02229dfc8a76c76c091276fc30e65662eaddc02f18e2f818c5d6dc407e21efe1f2f8810805f0603730e0c54ac03486d029b0eb3c8593f6a1b5b2277e7d07adf8fc71fd572471b6305d85928218fd6a85cf591ca797d214567935ff0c442faaa30ba3ed1842de9fb514d34ad58a63bf909a3c3a70ce0af1bd4aafaa683006f270aca66c4799fa0009101f327e62c46248fdb902871ba201931e821fcf7703071baa6c96c736c30464e108b18d975eb963b36641df28613116c067a4d5f713af6c7a31f40bb092c0cdb43ff1519b47871710257816316f89cdb0807d4c992eec4c0031fb230064fa1d5551a0485625642466575712e7d317a3591e82db0f24157eb338d4fa82570232413b8db86f7fe00815306979a5da2478db2e044e771f0a829d62a38ef91a5290ec96fe7a4a21b154f22acbc3d29308d4c5a4a68072138fed754f45d3fc42b445faafed45c6d7ac37c62b4de7077bccdb5c3d17ac081587ba7ad735f95da682e95ccab51468487f7bfbeaf6558f5a6897aaaa65415a05f04687000a0bbfc3ae88089e955e404595c57ceceda74d94d1c3da21f082862f77dc9c0fae3ecaeefc70f378fc6e632e6530bee3050c1bb8ac967281d1592727c56773649731c37971fdb1d0a6520c2fd19ad6367591646b6c3bf9130a2f0c150148b864f6a86b71150995422e4d74539d70e06cf4482cce0c806acad07db10d2a15a50528453f0dc60c2dc5934c6fd3bced765f8dd16ac77c3c3ca6f15b3408f5bd78f7deab02fbd90fae0f78211934919edf53214aa47f1382cb10030bb42c3a4a6f009c0fefb328ae52d8e72ca09121f7fa90d348b64b542d0c117b40a01de8c410b2f0721732275e9165e618c885c0f9435a281dd9525b66abad6e16f10b19a4159558f055ca074f5ff1661584f13fd4e8a81da625072f52ddd6653cac20e76d117448870de7e22d8423fe6569c039aba51764dc629320f7ed882cbbd32da2bb6d7ca31e5ebf7cd544abd5f326fec648ebe9363495d5f4da961b6f481a01e8ebb007a27cc407499f083e8382f25fa2f12dd57b98be4473ce8643b0dd5425fbebc5c8edc1275548d0b21c8967f1f39e05c4347ed3d3b1bed69c72f81e0503bbab9a55ebd90d138cbc67761ddfa4d00679b7368df0c66e55f96add45cc5c27000617680b4c04c1628142545025734fb2d5a29d20e20e97116b4156ebedd710d257fecf4ee7df152c455f2e67a52fcf30ee91d2d5e11b8aec208f9415e22e1ca8cc59732100840f3b82b1a6033b091547e4436ff7c70d6fe10d80fe0b37cf0777ba633f8c3bc0c8384b8c2f02d59f991460a484691189e0f01a9680c9318301dc058889d9170c17575f2504a98ea24010cd39cf0bf1c991f77d03a6bb3bfb06e27e6a9a2e07a917e7111eaa64e946f50cedf9b03001f8f352cde1aa25c9e9191ffda954f9a9bc3eb7cecbd320f43ede62cfb9d3dd67bf6fe2e3d4dfac48fa099de2955a0bac775e26a8c121c1636efc19862469225d452794e0cacca1b2020000000000000000000000000000000000000000000000000000000000000000191ffda954f9a9bc3eb7cecbd320f43ede62cfb9d3dd67bf6fe2e3d4dfac48fa00a907c0033870e5d0edc36890d2b6a6d7c5666da5096aa947d8d811ad8e894d118a1423749b0438cfbf42bee4deced29ac0c466c5a981be0459bed4760fb6e7037b5c9e019f31cd2ee9208f9b5e1df814ef428881bb196f9004ffe86080137e1c6d5c626951028db2c97da9c31a23bda5b1a4e05824e868003fe1356fdb5775020d9effe5fc6ee86a74d5e2ca6d7692b20bf556e739de580af7826067280d1711e866c6a5074a7e0d69db4764e52ea82e4828b4bf4749142ae83750ed2f6d9625b9a1af03ddc6a1169d6368c585560a16265a472a3fead5c40f4f31ebb075290800e750da132857e842db7034fac387aa6cf6de154dc7f32ff8c26c4c6430c924b3db48223f57a2f6ec31c14c96133187f859852f9b4132e9588ccb6bb7efac1ee1bd6d83e9734460aab83eb2a0f622ae801ad8d70d6bf699cbaee7add5aa93001ec040a90c411188a1671598bdba42e5cd697e8a39d103f061a0efa59457f8073c90b2b9516f4f67a0eb747119cc8e56a814a66f6a16f8540aa78ea754cc840b96d6b109f347709e32213bef57f045c10457c3522db50868cbf22d87574f9a04776204ccc4d4fda17134387657dea4f98133bd3d5352f8498ae7255b985d2627d281118df0e8dcf4bde40621bb6f32b0528fbb43d53be5864a14a9f61fdb2623322c9ff40527f6c8b96b2b60640adbb2ee0c4c7c8717af0c7df33211cf492f21cea87b01e93214d27623b5f3d011a19a0a66e3f29a74278df2760b47f0cb222e1bbc3372cb99d53ae3c207fc40fb860746077a44ebc6d3393be89b191c73212c128b5c9e723a0dcc41b51a54beb191399eddd1525f0c89720e926f16135c2f1f53aa73e9cead06f82e0d7d2823f28cb4d29a047efcd8c4b48386444c21e43507b97cf098389764fe5a0a41e274045bcbec58fddca7d184761adc6ab831a2b72de6518fdbec821cfa9ad477eb16d45515a99a6f4845962cbd0e059926ee95911c4547fdb840a9956776948b12878821da6c973c7a2a1e2cc05bc115c0e7e40920d7e7e6207e9e513dc81b783c793e1d502a4b246ad8a61503231948b73e8dae06836d6321f1e20ea45b5cb7a78b2d3b250b1d46f0dd2ef5e5f3ffee014296b31dabe8d700b662d46f165cc7d745e0f74e4f0d6a33ffbb2903d9ea9b3a32f9732e7021fbde764128e74954f720e9cd1c0151f435b84a8eb57c49de180ea821421ede9e17e535a9d51a4908b8312344c8d95139098fd4e331122ad5ad235416502bd9d0211eb1580b5aece365d17ceddd888f845d52fdf95bd2955d83f95313e82866ff0e2501597a793c742a90c52776ead93fc596e76ba3ba862c0a03b736840472d17a64e0777ffaa69b43bc2e82ddbd39002aeeb6b019603fe43453302f0118e483588acbf45935e8dedb3861b9792964b12795f6b191f82bedc9376822ee184a562921200243778bf9fd7d2398f95f6a361b2454acfae4b094d508aba0ec0ea95b429b27ea4df4f688b8f0d663cf9531921ff9f6d25c83ac77db898ecbfd06da3608ce418fd3becb4f4e91dfaa72b6e7cef756585fae45f81ecca49756f001c7026c6fec7bb16d10c5ea4a5da34339fabc5df66ae4ccd90bd684061419ab054e8145f8ac37eb8f5e3fcfc3b7b317a243df87e519221a99b4ef0fee65fe2a011d6d33a6ca994b12dca055bb6eabae8c733e9b29e07e68dd369053e68d46ad2a04940fe79428858f0586e81ff0718d4e37f37cf4e7ab45581be2e38b91581821d655b2030c3b92ceb8417f4af1009db1a375c371f3694251f98921e0fef6f40352ebb756cb2e11b6698e390c331cb3b688715819e34a865c6968f50f01a2c625cfc66819c45a77d4fcf29f900842a277fc5d0653bfc18395d58d316ac22bac27ff3a17098ac5deb83392b4655255a17389a8a5c1b84e2e84a9f7da52fa03e61a0c9524024a81596865f42e2a199ea51f6afed30f5653f504307d7d4bfb418101eaa6405d677a4a600ff733bfdea08fb822e12a026ae95fc36c816ac777a2d30d36e5abf57e8852698c0481a03467ccef5857100ae5071a172827cfe38e51c32b36a7426aeb812317e949aa8cdfaba4cf137d93ccaf520ae3d488402d8fb0681ec77aa87c06a064aa81698b023f3ee19f7dfc7e46df45605dce26fe0cc81dd91911799bbd9987ae058b4e48342e89043bac062148a525de3b3d6418bd0266cd0fd8bc0a4d2abe4d9b7f64fef8b191cd07329c4df8fa0d934387764d3d81e978141da6cff3d13942c9df5194709c544cdd028fbcf481ba2ff24b89a4e57300920963ddbf9de96b13e6bbcdbaf55672c532a29264f4eb1a502cb3c523e570658f28d0b99968357d74f26fbe336614bf12807d4308eff13e2a96c4c9f716adf8a1', 'transcript_type': 'EVM'}\n", - "Time gen prf: 35.637612104415894 seconds\n" - ] - } - ], - "source": [ - "# Here verifier & prover can concurrently call setup since all params are public to get pk.\n", - "# Here write as verifier function to emphasize that verifier must calculate its own vk to be sure\n", - "verifier_setup(verifier_model_path, verifier_compiled_model_path, settings_path,vk_path, pk_path )\n", - "\n", - "print(\"=======================================\")\n", - "# Prover generates proof\n", - "print(\"Theory output: \", theory_output)\n", - "prover_gen_proof(prover_model_path, sel_data_path, witness_path, prover_compiled_model_path, settings_path, proof_path, pk_path)" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "num_inputs: 1\n", - "prf instances: [[[12572659313263335624, 14443766455855958404, 432930639589567449, 1881177029071802301], [12436184717236109307, 3962172157175319849, 7381016538464732718, 1011752739694698287], [2359119529189047257, 1186270854317446351, 12298916256516019502, 1116401204062057155]]]\n", - "proof boolean: 1.0\n", - "proof result 1 : 50.25\n", - "verified\n" - ] - } - ], - "source": [ - "# Verifier verifies\n", - "verifier_verify(proof_path, settings_path, vk_path, selected_columns, commitment_maps)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.0" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/copy where+mean/data.json b/examples/copy where+mean/data.json deleted file mode 100644 index fa97ade..0000000 --- a/examples/copy where+mean/data.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "col_name": [ - 33.0, 15.0, 38.0, 38.0, 70.0, 14.0, 34.0, 67.0, 54.0, 78.0, 80.0, 21.0, - 41.0, 47.0, 57.0, 5.0, 6.0, 43.0, 51.0, 14.0 - ] -}