Remove dependency to common ConstraintSystem in the backend (#290)

* refactor: generalize ExpressionMid

* refactor: move ConstraintSystem::from(ConstraintSystemV2Backend) to backend

* refactor: generalize Expression type

* feat: define ExpressionMid as an alias

* refactor: rename ConstraintSystemV2Back to ConstraintSystemMid, GateV2Back to GateMid

* refactor: use ConstraintSystemBack in halo2_backend

* fix: warnings

* refactor: simplify Query type

* feat(backend): rewrite pk/vk serialization

- Rewrite VerifyingKey and ProvingKey methods in terms of the
  ConstraintSystemBack so that the backend becomes independent of the
  Circuit trait (which belongs to the frontend)
- Add `vk_read` and `pk_read` legacy functions in halo2_proofs for
  compatiblity.
- Split the implementation of converting selectors to fixed columns into
  two parts:
  - One part just converts the ConstraintSystem, transforming the
    selectors into fixed columns (compressed and direct versions)
  - The other part transforms the assignments of selector columns into
    assignments of fixed columns based on the mappings calculated in
    part one.

* feat: remove feature circuit-params from halo2_backend

* wip: clean up common+backend

* wip: clean up frontend

* wip: clean common plonk folder

* refactor: move Error to frontend

* refactor: move Error to backend

* refactor: clean up error types

* fix: use errors instead of temporary panics

* feat: annotate columns in test

* fix: remove unnecessary pub, set correct SelectorsToFixed.compressed value, add safety check

In selectors_to_fixed_compressed and selectors_to_fixed_direct add
safety check via the ConstraintSystem.selectors_to_fixed status field
such that the methods can only be called once.  Calling them multiple
times would lead to an invalid ConstraintSystem with unused duplicated
fixed columns.

* fix: ponk_api unit tests

* fix: clippy warnings, common dependencies

* fix: remove old TODO

* chore: bump VK version

* chore: remove unnecessary code

* chore: remove virtual variable rules in middleware Expression

* chore: remove unused evaluate_lazy, deprecate directly_convert_selectors_to_fixed

Cargo.toml

@@ -6,4 +6,5 @@ members = [
     "halo2_middleware",
     "halo2_backend",
     "halo2_common",
-]
+]
+resolver = "2"

halo2_backend/Cargo.toml

@@ -57,7 +57,6 @@ bits = ["halo2curves/bits"]
 gadget-traces = ["backtrace"]
 sanity-checks = []
 batch = ["rand_core/getrandom"]
-circuit-params = []
 cost-estimator = ["serde", "serde_derive"]
 derive_serde = ["halo2curves/derive_serde"]
 

halo2_backend/src/helpers.rs

@@ -3,7 +3,7 @@ pub(crate) use halo2_common::helpers::{SerdeFormat, SerdePrimeField};
 use halo2_middleware::ff::PrimeField;
 use std::io;
 
-pub(crate) use halo2_common::helpers::{pack, unpack, CurveRead, SerdeCurveAffine};
+pub(crate) use halo2_common::helpers::{CurveRead, SerdeCurveAffine};
 
 /// Reads a vector of polynomials from buffer
 pub(crate) fn read_polynomial_vec<R: io::Read, F: SerdePrimeField, B>(

halo2_backend/src/lib.rs

@@ -5,6 +5,5 @@ pub mod poly;
 pub mod transcript;
 
 // Internal re-exports
-pub use halo2_common::circuit;
 pub use halo2_common::multicore;
 pub use halo2_common::SerdeFormat;

halo2_backend/src/plonk.rs

@@ -3,22 +3,22 @@ use group::ff::{Field, FromUniformBytes, PrimeField};
 
 use crate::arithmetic::CurveAffine;
 use crate::helpers::{
-    self, polynomial_slice_byte_length, read_polynomial_vec, write_polynomial_slice,
-    SerdeCurveAffine, SerdePrimeField,
+    polynomial_slice_byte_length, read_polynomial_vec, write_polynomial_slice, SerdeCurveAffine,
+    SerdePrimeField,
 };
+use crate::plonk::circuit::{ConstraintSystemBack, PinnedConstraintSystem};
 use crate::poly::{
     Coeff, EvaluationDomain, ExtendedLagrangeCoeff, LagrangeCoeff, PinnedEvaluationDomain,
     Polynomial,
 };
 use crate::transcript::{ChallengeScalar, EncodedChallenge, Transcript};
 pub(crate) use evaluation::Evaluator;
-use halo2_common::plonk::{Circuit, ConstraintSystem, PinnedConstraintSystem};
 use halo2_common::SerdeFormat;
 
 use std::io;
 
-pub(crate) use halo2_common::plonk::Error;
-
+mod circuit;
+mod error;
 mod evaluation;
 pub mod keygen;
 mod lookup;
@@ -28,6 +28,8 @@ mod shuffle;
 mod vanishing;
 pub mod verifier;
 
+pub use error::*;
+
 /// This is a verifying key which allows for the verification of proofs for a
 /// particular circuit.
 #[derive(Clone, Debug)]
@@ -39,20 +41,18 @@ pub struct VerifyingKey<C: CurveAffine> {
     /// Permutation verifying key
     permutation: permutation::VerifyingKey<C>,
     /// Constraint system
-    cs: ConstraintSystem<C::Scalar>,
+    cs: ConstraintSystemBack<C::Scalar>,
     /// Cached maximum degree of `cs` (which doesn't change after construction).
     cs_degree: usize,
     /// The representative of this `VerifyingKey` in transcripts.
     transcript_repr: C::Scalar,
-    /// Selectors
-    selectors: Vec<Vec<bool>>,
-    // TODO: Use setter/getter https://github.com/privacy-scaling-explorations/halo2/issues/259
-    /// Whether selector compression is turned on or not.
-    pub compress_selectors: bool,
+    /// Legacy field that indicates wether the circuit was compiled with compressed selectors or
+    /// not using the legacy API.
+    pub compress_selectors: Option<bool>,
 }
 
 // Current version of the VK
-const VERSION: u8 = 0x03;
+const VERSION: u8 = 0x04;
 
 impl<C: SerdeCurveAffine> VerifyingKey<C>
 where
@@ -74,23 +74,12 @@ where
         assert!(*k <= C::Scalar::S);
         // k value fits in 1 byte
         writer.write_all(&[*k as u8])?;
-        writer.write_all(&[self.compress_selectors as u8])?;
         writer.write_all(&(self.fixed_commitments.len() as u32).to_le_bytes())?;
         for commitment in &self.fixed_commitments {
             commitment.write(writer, format)?;
         }
         self.permutation.write(writer, format)?;
 
-        if !self.compress_selectors {
-            assert!(self.selectors.is_empty());
-        }
-        // write self.selectors
-        for selector in &self.selectors {
-            // since `selector` is filled with `bool`, we pack them 8 at a time into bytes and then write
-            for bits in selector.chunks(8) {
-                writer.write_all(&[helpers::pack(bits)])?;
-            }
-        }
         Ok(())
     }
 
@@ -104,10 +93,10 @@ where
     /// Checks that field elements are less than modulus, and then checks that the point is on the curve.
     /// - `RawBytesUnchecked`: Reads an uncompressed curve element with coordinates in Montgomery form;
     /// does not perform any checks
-    pub fn read<R: io::Read, ConcreteCircuit: Circuit<C::Scalar>>(
+    pub fn read<R: io::Read>(
         reader: &mut R,
         format: SerdeFormat,
-        #[cfg(feature = "circuit-params")] params: ConcreteCircuit::Params,
+        cs: ConstraintSystemBack<C::Scalar>,
     ) -> io::Result<Self> {
         let mut version_byte = [0u8; 1];
         reader.read_exact(&mut version_byte)?;
@@ -131,20 +120,7 @@ where
                 ),
             ));
         }
-        let mut compress_selectors = [0u8; 1];
-        reader.read_exact(&mut compress_selectors)?;
-        if compress_selectors[0] != 0 && compress_selectors[0] != 1 {
-            return Err(io::Error::new(
-                io::ErrorKind::InvalidData,
-                "unexpected compress_selectors not boolean",
-            ));
-        }
-        let compress_selectors = compress_selectors[0] == 1;
-        let (domain, cs, _) = keygen::create_domain::<C, ConcreteCircuit>(
-            k as u32,
-            #[cfg(feature = "circuit-params")]
-            params,
-        );
+        let domain = keygen::create_domain::<C>(&cs, k as u32);
         let mut num_fixed_columns = [0u8; 4];
         reader.read_exact(&mut num_fixed_columns)?;
         let num_fixed_columns = u32::from_le_bytes(num_fixed_columns);
@@ -155,36 +131,7 @@ where
 
         let permutation = permutation::VerifyingKey::read(reader, &cs.permutation, format)?;
 
-        let (cs, selectors) = if compress_selectors {
-            // read selectors
-            let selectors: Vec<Vec<bool>> = vec![vec![false; 1 << k]; cs.num_selectors]
-                .into_iter()
-                .map(|mut selector| {
-                    let mut selector_bytes = vec![0u8; (selector.len() + 7) / 8];
-                    reader.read_exact(&mut selector_bytes)?;
-                    for (bits, byte) in selector.chunks_mut(8).zip(selector_bytes) {
-                        helpers::unpack(byte, bits);
-                    }
-                    Ok(selector)
-                })
-                .collect::<io::Result<_>>()?;
-            let (cs, _) = cs.compress_selectors(selectors.clone());
-            (cs, selectors)
-        } else {
-            // we still need to replace selectors with fixed Expressions in `cs`
-            let fake_selectors = vec![vec![]; cs.num_selectors];
-            let (cs, _) = cs.directly_convert_selectors_to_fixed(fake_selectors);
-            (cs, vec![])
-        };
-
-        Ok(Self::from_parts(
-            domain,
-            fixed_commitments,
-            permutation,
-            cs,
-            selectors,
-            compress_selectors,
-        ))
+        Ok(Self::from_parts(domain, fixed_commitments, permutation, cs))
     }
 
     /// Writes a verifying key to a vector of bytes using [`Self::write`].
@@ -195,17 +142,12 @@ where
     }
 
     /// Reads a verification key from a slice of bytes using [`Self::read`].
-    pub fn from_bytes<ConcreteCircuit: Circuit<C::Scalar>>(
+    pub fn from_bytes(
         mut bytes: &[u8],
         format: SerdeFormat,
-        #[cfg(feature = "circuit-params")] params: ConcreteCircuit::Params,
+        cs: ConstraintSystemBack<C::Scalar>,
     ) -> io::Result<Self> {
-        Self::read::<_, ConcreteCircuit>(
-            &mut bytes,
-            format,
-            #[cfg(feature = "circuit-params")]
-            params,
-        )
+        Self::read(&mut bytes, format, cs)
     }
 }
 
@@ -216,21 +158,13 @@ impl<C: CurveAffine> VerifyingKey<C> {
     {
         10 + (self.fixed_commitments.len() * C::byte_length(format))
             + self.permutation.bytes_length(format)
-            + self.selectors.len()
-                * (self
-                    .selectors
-                    .get(0)
-                    .map(|selector| (selector.len() + 7) / 8)
-                    .unwrap_or(0))
     }
 
     fn from_parts(
         domain: EvaluationDomain<C::Scalar>,
         fixed_commitments: Vec<C>,
         permutation: permutation::VerifyingKey<C>,
-        cs: ConstraintSystem<C::Scalar>,
-        selectors: Vec<Vec<bool>>,
-        compress_selectors: bool,
+        cs: ConstraintSystemBack<C::Scalar>,
     ) -> Self
     where
         C::ScalarExt: FromUniformBytes<64>,
@@ -246,8 +180,7 @@ impl<C: CurveAffine> VerifyingKey<C> {
             cs_degree,
             // Temporary, this is not pinned.
             transcript_repr: C::Scalar::ZERO,
-            selectors,
-            compress_selectors,
+            compress_selectors: None,
         };
 
         let mut hasher = Blake2bParams::new()
@@ -300,7 +233,7 @@ impl<C: CurveAffine> VerifyingKey<C> {
     }
 
     /// Returns `ConstraintSystem`
-    pub fn cs(&self) -> &ConstraintSystem<C::Scalar> {
+    pub fn cs(&self) -> &ConstraintSystemBack<C::Scalar> {
         &self.cs
     }
 
@@ -400,17 +333,12 @@ where
     /// Checks that field elements are less than modulus, and then checks that the point is on the curve.
     /// - `RawBytesUnchecked`: Reads an uncompressed curve element with coordinates in Montgomery form;
     /// does not perform any checks
-    pub fn read<R: io::Read, ConcreteCircuit: Circuit<C::Scalar>>(
+    pub fn read<R: io::Read>(
         reader: &mut R,
         format: SerdeFormat,
-        #[cfg(feature = "circuit-params")] params: ConcreteCircuit::Params,
+        cs: ConstraintSystemBack<C::Scalar>,
     ) -> io::Result<Self> {
-        let vk = VerifyingKey::<C>::read::<R, ConcreteCircuit>(
-            reader,
-            format,
-            #[cfg(feature = "circuit-params")]
-            params,
-        )?;
+        let vk = VerifyingKey::<C>::read::<R>(reader, format, cs)?;
         let l0 = Polynomial::read(reader, format)?;
         let l_last = Polynomial::read(reader, format)?;
         let l_active_row = Polynomial::read(reader, format)?;
@@ -440,17 +368,12 @@ where
     }
 
     /// Reads a proving key from a slice of bytes using [`Self::read`].
-    pub fn from_bytes<ConcreteCircuit: Circuit<C::Scalar>>(
+    pub fn from_bytes(
         mut bytes: &[u8],
         format: SerdeFormat,
-        #[cfg(feature = "circuit-params")] params: ConcreteCircuit::Params,
+        cs: ConstraintSystemBack<C::Scalar>,
     ) -> io::Result<Self> {
-        Self::read::<_, ConcreteCircuit>(
-            &mut bytes,
-            format,
-            #[cfg(feature = "circuit-params")]
-            params,
-        )
+        Self::read(&mut bytes, format, cs)
     }
 }