Virtual region managers and dynamic lookup support

.github/workflows/ci.yml

@@ -11,7 +11,7 @@ env:
 
 jobs:
   build:
-    runs-on: ubuntu-latest-m
+    runs-on: ubuntu-latest-64core-256ram
 
     steps:
       - uses: actions/checkout@v3
@@ -24,7 +24,7 @@ jobs:
       - name: Run halo2-ecc tests (mock prover)
         working-directory: "halo2-ecc"
         run: |
-          cargo test --lib -- --skip bench --test-threads=2
+          cargo test --lib -- --skip bench
       - name: Run halo2-ecc tests (real prover)
         working-directory: "halo2-ecc"
         run: |
@@ -39,7 +39,7 @@ jobs:
       - name: Run zkevm tests
         working-directory: "hashes/zkevm"
         run: |
-          cargo test
+          cargo test packed_multi_keccak_prover::k_14
 
   lint:
     name: Lint

halo2-base/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "halo2-base"
-version = "0.3.2"
+version = "0.4.0"
 edition = "2021"
 
 [dependencies]
@@ -18,9 +18,9 @@ getset = "0.1.2"
 ark-std = { version = "0.3.0", features = ["print-trace"], optional = true }
 
 # Use Axiom's custom halo2 monorepo for faster proving when feature = "halo2-axiom" is on
-halo2_proofs_axiom = { git = "https://github.com/axiom-crypto/halo2.git", package = "halo2_proofs", optional = true }
+halo2_proofs_axiom = { git = "https://github.com/axiom-crypto/halo2.git", package = "halo2_proofs", optional = true, branch = "revert_cell_noref" }
 # Use PSE halo2 and halo2curves for compatibility when feature = "halo2-pse" is on
-halo2_proofs = { git = "https://github.com/privacy-scaling-explorations/halo2.git", rev = "f348757", optional = true }
+halo2_proofs = { git = "https://github.com/privacy-scaling-explorations/halo2.git", rev = "0f00047", optional = true }
 
 # This is Scroll's audited poseidon circuit. We only use it for the Native Poseidon spec. We do not use the halo2 circuit at all (and it wouldn't even work because the halo2_proofs tag is not compatbile).
 # We forked it to upgrade to ff v0.13 and removed the circuit module
@@ -39,6 +39,8 @@ pprof = { version = "0.11", features = ["criterion", "flamegraph"] }
 criterion = "0.4"
 criterion-macro = "0.4"
 test-case = "3.1.0"
+test-log = "0.2.12"
+env_logger = "0.10.0"
 proptest = "1.1.0"
 # native poseidon for testing
 pse-poseidon = { git = "https://github.com/axiom-crypto/pse-poseidon.git" }
@@ -50,7 +52,7 @@ jemallocator = { version = "0.5", optional = true }
 mimalloc = { version = "0.1", default-features = false, optional = true }
 
 [features]
-default = ["halo2-axiom", "display"]
+default = ["halo2-axiom", "display", "test-utils"]
 asm = ["halo2_proofs_axiom?/asm"]
 dev-graph = [
     "halo2_proofs?/dev-graph",

halo2-base/benches/inner_product.rs

@@ -1,4 +1,4 @@
-use halo2_base::gates::builder::{GateThreadBuilder, RangeCircuitBuilder};
+use halo2_base::gates::circuit::{builder::RangeCircuitBuilder, CircuitBuilderStage};
 use halo2_base::gates::flex_gate::{GateChip, GateInstructions};
 use halo2_base::halo2_proofs::{
     arithmetic::Field,
@@ -36,20 +36,20 @@ fn inner_prod_bench<F: ScalarField>(ctx: &mut Context<F>, a: Vec<F>, b: Vec<F>)
 fn bench(c: &mut Criterion) {
     let k = 19u32;
     // create circuit for keygen
-    let mut builder = GateThreadBuilder::new(false);
+    let mut builder =
+        RangeCircuitBuilder::from_stage(CircuitBuilderStage::Keygen).use_k(k as usize);
     inner_prod_bench(builder.main(0), vec![Fr::zero(); 5], vec![Fr::zero(); 5]);
-    let config_params = builder.config(k as usize, Some(20));
-    let circuit = RangeCircuitBuilder::mock(builder, config_params.clone());
+    let config_params = builder.calculate_params(Some(20));
 
     // check the circuit is correct just in case
-    MockProver::run(k, &circuit, vec![]).unwrap().assert_satisfied();
+    MockProver::run(k, &builder, vec![]).unwrap().assert_satisfied();
 
     let params = ParamsKZG::<Bn256>::setup(k, OsRng);
-    let vk = keygen_vk(&params, &circuit).expect("vk should not fail");
-    let pk = keygen_pk(&params, vk, &circuit).expect("pk should not fail");
+    let vk = keygen_vk(&params, &builder).expect("vk should not fail");
+    let pk = keygen_pk(&params, vk, &builder).expect("pk should not fail");
 
-    let break_points = circuit.0.break_points.take();
-    drop(circuit);
+    let break_points = builder.break_points();
+    drop(builder);
 
     let mut group = c.benchmark_group("plonk-prover");
     group.sample_size(10);
@@ -58,17 +58,12 @@ fn bench(c: &mut Criterion) {
         &(&params, &pk),
         |bencher, &(params, pk)| {
             bencher.iter(|| {
-                let mut builder = GateThreadBuilder::new(true);
+                let mut builder =
+                    RangeCircuitBuilder::prover(config_params.clone(), break_points.clone());
                 let a = (0..5).map(|_| Fr::random(OsRng)).collect_vec();
                 let b = (0..5).map(|_| Fr::random(OsRng)).collect_vec();
                 inner_prod_bench(builder.main(0), a, b);
-                let circuit = RangeCircuitBuilder::prover(
-                    builder,
-                    config_params.clone(),
-                    break_points.clone(),
-                );
-
-                gen_proof(params, pk, circuit);
+                gen_proof(params, pk, builder);
             })
         },
     );

halo2-base/benches/mul.rs

@@ -1,4 +1,4 @@
-use halo2_base::gates::builder::{GateThreadBuilder, RangeCircuitBuilder};
+use halo2_base::gates::circuit::{builder::RangeCircuitBuilder, CircuitBuilderStage};
 use halo2_base::gates::flex_gate::{GateChip, GateInstructions};
 use halo2_base::halo2_proofs::{
     halo2curves::bn256::{Bn256, Fr},
@@ -31,16 +31,16 @@ fn mul_bench<F: ScalarField>(ctx: &mut Context<F>, inputs: [F; 2]) {
 
 fn bench(c: &mut Criterion) {
     // create circuit for keygen
-    let mut builder = GateThreadBuilder::new(false);
+    let mut builder =
+        RangeCircuitBuilder::from_stage(CircuitBuilderStage::Keygen).use_k(K as usize);
     mul_bench(builder.main(0), [Fr::zero(); 2]);
-    let config_params = builder.config(K as usize, Some(9));
-    let circuit = RangeCircuitBuilder::keygen(builder, config_params.clone());
+    let config_params = builder.calculate_params(Some(9));
 
     let params = ParamsKZG::<Bn256>::setup(K, OsRng);
-    let vk = keygen_vk(&params, &circuit).expect("vk should not fail");
-    let pk = keygen_pk(&params, vk, &circuit).expect("pk should not fail");
+    let vk = keygen_vk(&params, &builder).expect("vk should not fail");
+    let pk = keygen_pk(&params, vk, &builder).expect("pk should not fail");
 
-    let break_points = circuit.0.break_points.take();
+    let break_points = builder.break_points();
 
     let a = Fr::random(OsRng);
     let b = Fr::random(OsRng);
@@ -50,16 +50,12 @@ fn bench(c: &mut Criterion) {
         &(&params, &pk, [a, b]),
         |bencher, &(params, pk, inputs)| {
             bencher.iter(|| {
-                let mut builder = GateThreadBuilder::new(true);
+                let mut builder =
+                    RangeCircuitBuilder::prover(config_params.clone(), break_points.clone());
                 // do the computation
                 mul_bench(builder.main(0), inputs);
-                let circuit = RangeCircuitBuilder::prover(
-                    builder,
-                    config_params.clone(),
-                    break_points.clone(),
-                );
 
-                gen_proof(params, pk, circuit);
+                gen_proof(params, pk, builder);
             })
         },
     );

halo2-base/examples/inner_product.rs

@@ -1,7 +1,7 @@
 #![cfg(feature = "test-utils")]
 use halo2_base::gates::flex_gate::{GateChip, GateInstructions};
+use halo2_base::gates::RangeInstructions;
 use halo2_base::halo2_proofs::{arithmetic::Field, halo2curves::bn256::Fr};
-use halo2_base::safe_types::RangeInstructions;
 use halo2_base::utils::testing::base_test;
 use halo2_base::utils::ScalarField;
 use halo2_base::{Context, QuantumCell::Existing};
@@ -32,8 +32,8 @@ fn main() {
             (0..5).map(|_| Fr::random(OsRng)).collect_vec(),
             (0..5).map(|_| Fr::random(OsRng)).collect_vec(),
         ),
-        |builder, range, (a, b)| {
-            inner_prod_bench(builder.main(0), range.gate(), a, b);
+        |pool, range, (a, b)| {
+            inner_prod_bench(pool.main(), range.gate(), a, b);
         },
     );
 }