mirror of
https://github.com/luxfi/fhe-ml.git
synced 2026-10-30 19:52:59 +00:00
3030 lines
111 KiB
Python
3030 lines
111 KiB
Python
"""Quantized versions of the ONNX operators for post training quantization."""
|
|
|
|
# pylint: disable=too-many-lines
|
|
|
|
# This file is too long and should be split
|
|
# FIXME: https://github.com/luxfi/luxfhe-ml-internal/issues/1018
|
|
|
|
from typing import Any, Dict, Optional, Sequence, Set, Union
|
|
|
|
import numpy
|
|
from concrete.fhe import conv as fhe_conv
|
|
from concrete.fhe import maxpool as fhe_maxpool
|
|
from concrete.fhe import tag, univariate, zeros
|
|
from typing_extensions import SupportsIndex
|
|
|
|
from ..common.debugging import assert_false, assert_true
|
|
from ..onnx.onnx_impl_utils import (
|
|
compute_onnx_pool_padding,
|
|
numpy_onnx_pad,
|
|
onnx_avgpool_compute_norm_const,
|
|
)
|
|
from ..onnx.ops_impl import RawOpOutput
|
|
from .base_quantized_op import (
|
|
ONNXOpInputOutputType,
|
|
QuantizedMixingOp,
|
|
QuantizedOp,
|
|
QuantizedOpUnivariateOfEncrypted,
|
|
)
|
|
from .quantizers import (
|
|
QuantizationOptions,
|
|
QuantizedArray,
|
|
UniformQuantizationParameters,
|
|
UniformQuantizer,
|
|
)
|
|
|
|
|
|
def _check_op_input_zero_point(zero_point: Any, op_name: Optional[str]):
|
|
"""Check that an operation's quantized input zero-point is a single value.
|
|
|
|
Args:
|
|
zero_point (Any): The input zero-point
|
|
op_name (str): The name of the operation that is checking its input
|
|
|
|
"""
|
|
|
|
# Checks with assert to also to ensure type safety
|
|
assert zero_point is not None and (
|
|
isinstance(zero_point, (int, float))
|
|
or numpy.isscalar(zero_point)
|
|
or (isinstance(zero_point, numpy.ndarray) and zero_point.size == 1)
|
|
), (
|
|
f"Operation {op_name} is trying to use an input with a zero-point that is "
|
|
"not a single value. Only model output quantizers can have zero-points that are arrays. "
|
|
)
|
|
|
|
|
|
class QuantizedSigmoid(QuantizedOp):
|
|
"""Quantized sigmoid op."""
|
|
|
|
_impl_for_op_named: str = "Sigmoid"
|
|
|
|
|
|
class QuantizedHardSigmoid(QuantizedOp):
|
|
"""Quantized HardSigmoid op."""
|
|
|
|
_impl_for_op_named: str = "HardSigmoid"
|
|
|
|
|
|
class QuantizedRelu(QuantizedOp):
|
|
"""Quantized Relu op."""
|
|
|
|
_impl_for_op_named: str = "Relu"
|
|
|
|
|
|
class QuantizedPRelu(QuantizedOp):
|
|
"""Quantized PRelu op."""
|
|
|
|
_impl_for_op_named: str = "PRelu"
|
|
|
|
|
|
class QuantizedLeakyRelu(QuantizedOp):
|
|
"""Quantized LeakyRelu op."""
|
|
|
|
_impl_for_op_named: str = "LeakyRelu"
|
|
|
|
|
|
class QuantizedHardSwish(QuantizedOp):
|
|
"""Quantized Hardswish op."""
|
|
|
|
_impl_for_op_named: str = "HardSwish"
|
|
|
|
|
|
class QuantizedElu(QuantizedOp):
|
|
"""Quantized Elu op."""
|
|
|
|
_impl_for_op_named: str = "Elu"
|
|
|
|
|
|
class QuantizedSelu(QuantizedOp):
|
|
"""Quantized Selu op."""
|
|
|
|
_impl_for_op_named: str = "Selu"
|
|
|
|
|
|
class QuantizedCelu(QuantizedOp):
|
|
"""Quantized Celu op."""
|
|
|
|
_impl_for_op_named: str = "Celu"
|
|
|
|
|
|
class QuantizedClip(QuantizedOp):
|
|
"""Quantized clip op."""
|
|
|
|
_impl_for_op_named: str = "Clip"
|
|
|
|
|
|
class QuantizedRound(QuantizedOp):
|
|
"""Quantized round op."""
|
|
|
|
_impl_for_op_named: str = "Round"
|
|
|
|
|
|
class QuantizedPow(QuantizedOpUnivariateOfEncrypted, QuantizedOp):
|
|
"""Quantized pow op.
|
|
|
|
Only works for a float constant power. This operation will be fused to a (potentially
|
|
larger) TLU.
|
|
"""
|
|
|
|
_impl_for_op_named: str = "Pow"
|
|
|
|
|
|
class QuantizedGemm(QuantizedMixingOp):
|
|
"""Quantized Gemm op."""
|
|
|
|
_impl_for_op_named: str = "Gemm"
|
|
|
|
def __init__(
|
|
self,
|
|
n_bits_output: int,
|
|
op_instance_name: str,
|
|
int_input_names: Optional[Set[str]] = None,
|
|
constant_inputs: Optional[Union[Dict[str, Any], Dict[int, Any]]] = None,
|
|
input_quant_opts: Optional[QuantizationOptions] = None,
|
|
**attrs,
|
|
) -> None:
|
|
super().__init__(
|
|
n_bits_output,
|
|
op_instance_name,
|
|
int_input_names,
|
|
constant_inputs,
|
|
input_quant_opts,
|
|
**attrs,
|
|
)
|
|
|
|
alpha = self.attrs.get("alpha", 1)
|
|
beta = self.attrs.get("beta", 1)
|
|
|
|
assert_true(
|
|
alpha == 1 and beta in [0, 1],
|
|
f"{self.__class__.__name__} currently only supports alpha == 1 and beta in [0, 1].\n"
|
|
f"Got alpha == {alpha} and beta == {beta}.",
|
|
)
|
|
|
|
@classmethod
|
|
def supported_by_linear_backend(cls) -> bool:
|
|
return True
|
|
|
|
# pylint: disable-next=too-many-statements,too-many-locals
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
calibrate_rounding: bool = False,
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
|
|
alpha = self.attrs.get("alpha", 1)
|
|
beta = self.attrs.get("beta", 1)
|
|
|
|
# If self.constant_inputs is empty this is an encrypted gemm
|
|
# There might be caveats here
|
|
# (for example when one of the input is passed in clear with encrypted statuses.)
|
|
# FIXME: https://github.com/luxfi/luxfhe-ml-internal/issues/4132
|
|
is_encrypted_gemm = isinstance(self.constant_inputs, dict) and not self.constant_inputs
|
|
|
|
# If alpha != 1 or beta not in [0, 1], this function must be modified
|
|
assert_true(alpha == 1)
|
|
assert_true(beta in {0, 1})
|
|
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=True
|
|
)
|
|
|
|
q_input1 = prepared_inputs[0]
|
|
assert isinstance(q_input1, QuantizedArray)
|
|
q_input2 = prepared_inputs[1]
|
|
assert isinstance(q_input2, QuantizedArray)
|
|
|
|
# In the operation Y = alpha * A' * B' + beta * C, q_bias is used for
|
|
# generalised matrix multiplication. q_bias is set to None for standard
|
|
# matrix multiplication (beta == 0 or only two inputs)
|
|
q_bias = None if len(prepared_inputs) == 2 or beta == 0 else prepared_inputs[2]
|
|
assert isinstance(q_bias, (type(None), QuantizedArray))
|
|
|
|
# Using snake case here to please the Python format, the original attrs don't have the '_'
|
|
# Use default false so we also support MatMul impl, MatMul does not have these flags
|
|
transpose_inputs1 = attrs.get("transA", False)
|
|
with tag(self.op_instance_name + ".input"):
|
|
input1_q_values = (
|
|
numpy.transpose(q_input1.qvalues) if transpose_inputs1 else q_input1.qvalues
|
|
)
|
|
|
|
transpose_inputs2 = attrs.get("transB", False)
|
|
input2_q_values = (
|
|
numpy.transpose(q_input2.qvalues) if transpose_inputs2 else q_input2.qvalues
|
|
)
|
|
|
|
assert_true(
|
|
input2_q_values.ndim in {2, 3},
|
|
f"Unsupported dimension for the weight input of the gemm: {input2_q_values.ndim}",
|
|
)
|
|
|
|
# For mypy
|
|
assert self.output_quant_params is not None
|
|
assert self.output_quant_params.scale is not None
|
|
assert self.output_quant_params.zero_point is not None
|
|
|
|
assert q_input2.quantizer.scale is not None
|
|
assert q_input2.quantizer.zero_point is not None
|
|
|
|
assert q_input1.quantizer.scale is not None
|
|
assert q_input1.quantizer.zero_point is not None
|
|
|
|
# The following MatMul is done with integers, and thus, does not use of any PBS.
|
|
# Rescaling the output of the integer MatMul to handle scale changes is done
|
|
# in float and will thus be fused with any float processing that follows this layer.
|
|
|
|
# Here we follow Eq.7 in https://arxiv.org/abs/1712.05877 to split the core computation
|
|
# from the zero points and scales.
|
|
|
|
p = input2_q_values.shape[-2]
|
|
|
|
# Remove the manual matrix multiplication when we can handle input precision with rounding
|
|
# FIXME: https://github.com/luxfi/concrete-internal/issues/512
|
|
def enc_mul(x, y):
|
|
r"""Encrypted multiplication of two input arrays.
|
|
|
|
This function computes the encrypted multiplication of two numpy arrays.
|
|
It uses the following equality:
|
|
|
|
\[
|
|
(x + y)^2 - (x - y)^2 = 4xy
|
|
\]
|
|
|
|
This equation simplifies to the standard multiplication operation.
|
|
|
|
In TFHE, this allows to do encrypted multiplication with 2 PBS.
|
|
|
|
Args:
|
|
x (numpy.ndarray): The first input numpy array.
|
|
y (numpy.ndarray): The second input numpy array.
|
|
|
|
Returns:
|
|
numpy.ndarray: The result of the encrypted multiplication.
|
|
"""
|
|
with tag("pbs_multiplication"):
|
|
# Compute sum and difference of x and y
|
|
add = x + y
|
|
sub = x - y
|
|
|
|
# Apply Concrete rounding to the addition and substraction
|
|
with tag(self.op_instance_name + ".pbs_matmul_rounding_add"):
|
|
add = self.cnp_round(add, calibrate_rounding, rounding_operation_id="add")
|
|
with tag(self.op_instance_name + ".pbs_matmul_rounding_sub"):
|
|
sub = self.cnp_round(sub, calibrate_rounding, rounding_operation_id="sub")
|
|
|
|
# Square the rounded sums and differences, and divide by 4
|
|
add_pow = (add.astype(numpy.float64)) ** 2
|
|
sub_pow = (sub.astype(numpy.float64)) ** 2
|
|
add_pow_divide = (add_pow / 4.0).astype(numpy.int64)
|
|
sub_pow_divide = (sub_pow / 4.0).astype(numpy.int64)
|
|
|
|
# Return the result of the multiplication
|
|
return add_pow_divide - sub_pow_divide
|
|
|
|
# Remove the manual matrix multiplication when we can handle input precision with rounding
|
|
# FIXME: https://github.com/luxfi/concrete-internal/issues/512
|
|
def matmul(a, b):
|
|
"""Matrix multiplication of two input arrays, supporting 2D or 3D.
|
|
|
|
This function performs matrix multiplication on either 2D or 3D numpy arrays.
|
|
It supports batch processing, where either or both inputs can be a batch
|
|
(3D array), and handles the reshaping and summation operations required
|
|
for matrix multiplication.
|
|
|
|
Args:
|
|
a (numpy.ndarray): The first input array, can be 2D or 3D.
|
|
b (numpy.ndarray): The second input array, can be 2D or 3D.
|
|
|
|
Returns:
|
|
numpy.ndarray: The result of the matrix multiplication.
|
|
"""
|
|
with tag("encrypted_matmul"):
|
|
# Determine the dimensions of inputs and handle 3D (batch) inputs
|
|
a_3d = a.ndim == 3
|
|
b_3d = b.ndim == 3
|
|
|
|
# Extract shapes and batch sizes
|
|
if a_3d:
|
|
batch_a, m, n = a.shape
|
|
else:
|
|
m, n = a.shape
|
|
batch_a = 1
|
|
|
|
if b_3d:
|
|
batch_b, n_b, p = b.shape
|
|
else:
|
|
n_b, p = b.shape
|
|
batch_b = 1
|
|
|
|
# Check for dimension compatibility
|
|
assert_true(n == n_b, "Inner dimensions do not match for matrix multiplication")
|
|
assert (
|
|
batch_a == batch_b or batch_a == 1 or batch_b == 1
|
|
), "Batch sizes must be equal or one must be 1"
|
|
|
|
# Determine the batch size for the operation
|
|
batch_size = batch_a
|
|
c = zeros(shape=(batch_size, m, p))
|
|
|
|
# Perform batched matrix multiplication
|
|
for i in range(batch_size):
|
|
# Slice the batch or use the whole array if not batched
|
|
a_slice = a[i] if a_3d else a
|
|
b_slice = b[i] if b_3d else b
|
|
|
|
# Reshape for element-wise multiplication
|
|
a_reshaped = a_slice.reshape((m, n, 1))
|
|
b_reshaped = b_slice.reshape((1, n, p))
|
|
|
|
# Perform encrypted multiplication and sum along the axis
|
|
enc_mul_result = enc_mul(a_reshaped, b_reshaped)
|
|
c[i] = numpy.sum(enc_mul_result, axis=1)
|
|
|
|
# Squeeze the first dimension if both inputs were 2D
|
|
if not a_3d and not b_3d:
|
|
c = numpy.squeeze(c, axis=0)
|
|
|
|
# Return the result of matrix multiplication
|
|
return c
|
|
|
|
# Remove the manual matrix multiplication when we can handle input precision with rounding
|
|
# FIXME: https://github.com/luxfi/luxfhe-ml-internal/issues/4127
|
|
@univariate
|
|
def copy_function(x):
|
|
return x
|
|
|
|
# Copy trick explanation:
|
|
# The copy_function is used to preserve the original precision of input values across
|
|
# various operations. Operations like addition, subtraction, sum, and matmul can
|
|
# unintentionally increase precision ('precision raising').
|
|
#
|
|
# Precision raising in one of these operations can inadvertently affect the precision of
|
|
# the same value in other branches of the code. By creating copies of the input values,
|
|
# any precision changes are limited to these copies, not the original values.
|
|
#
|
|
# This strategy is particularly important to make sure PBS in all branches are done on the
|
|
# pre-defined precision. The use of the copy is conditional, applied only when needed
|
|
# to optimize performance.
|
|
|
|
input1_q_values_copy = (
|
|
copy_function(input1_q_values) if is_encrypted_gemm else input1_q_values
|
|
)
|
|
input2_q_values_copy = (
|
|
copy_function(input2_q_values) if is_encrypted_gemm else input2_q_values
|
|
)
|
|
|
|
# Core matmul operation in full integers with a shape change (INTEGERS)
|
|
with tag(self.op_instance_name + ".matmul"):
|
|
# We implement our own encrypted matmul to be able to round before PBS
|
|
if is_encrypted_gemm:
|
|
matmul = matmul(input1_q_values_copy, input2_q_values_copy)
|
|
# Otherwise we let concrete do it
|
|
else:
|
|
matmul = input1_q_values_copy @ input2_q_values_copy
|
|
|
|
input1_q_values_copy = (
|
|
copy_function(input1_q_values) if is_encrypted_gemm else input1_q_values
|
|
)
|
|
|
|
# If the weights have symmetric quantization, their zero point will be 0
|
|
# The following check avoids the computation of the sum of the inputs, which may have
|
|
# large bit-width, in the case where it would be multiplied by zero
|
|
if q_input2.quantizer.zero_point != 0:
|
|
# Sum operation in full integers resulting in large integers (INTEGERS)
|
|
with tag(self.op_instance_name + ".matmul_inputsum"):
|
|
sum_input = -q_input2.quantizer.zero_point * numpy.sum(
|
|
input1_q_values_copy, axis=-1, keepdims=True
|
|
)
|
|
|
|
with tag(self.op_instance_name + ".matmul_add_inputsum"):
|
|
# Last part that has to be done in integer
|
|
numpy_q_out = matmul + sum_input
|
|
else:
|
|
numpy_q_out = matmul
|
|
|
|
if self.debug_value_tracker is not None:
|
|
# pylint: disable-next=unsubscriptable-object
|
|
self.debug_value_tracker[self.op_instance_name]["output"] = numpy_q_out # type: ignore
|
|
|
|
input2_q_values_copy = (
|
|
copy_function(input2_q_values) if is_encrypted_gemm else input2_q_values
|
|
)
|
|
|
|
with tag(self.op_instance_name + ".sum_weights_times_zero_point"):
|
|
|
|
# sum_weights is a constant
|
|
sum_weights = q_input1.quantizer.zero_point * numpy.sum(
|
|
input2_q_values_copy, axis=-2, keepdims=True
|
|
)
|
|
|
|
final_term = p * q_input1.quantizer.zero_point * q_input2.quantizer.zero_point
|
|
|
|
# Note that here we do not rescale to the output_scale and we do not add a zero-point
|
|
# Any following Gemm/MatMul/Conv layers will do the rescaling (during re-quantization)
|
|
# by calling _prepare_inputs_with_constants(...quantize_real_values=True)
|
|
m_matmul = q_input1.quantizer.scale * q_input2.quantizer.scale
|
|
|
|
# If this operation's result are network outputs, return
|
|
# directly the integer values and a appropriate quantization parameters that
|
|
# allow direct in-the-clear de-quantization, including the bias
|
|
if self.produces_graph_output and not is_encrypted_gemm:
|
|
out_zp: Union[int, numpy.ndarray] = sum_weights - final_term
|
|
if q_bias is not None:
|
|
# Make mypy happy
|
|
assert q_bias is not None
|
|
# Reshape the biases to broadcast them to each neuron
|
|
bias_out = q_bias.values if isinstance(q_bias, QuantizedArray) else q_bias
|
|
out_zp = out_zp + bias_out / (-m_matmul)
|
|
|
|
# We identify terms in the above equation to determine what
|
|
# the scale/zero-point of the in-the-clear quantizer should be
|
|
# to properly de-quantize numpy_q_out
|
|
return self.make_output_quant_parameters(numpy_q_out, m_matmul, out_zp)
|
|
|
|
# Integer biases are only supported for Brevitas QAT which sets is_precomputed_qat to true
|
|
# These biases are produced by QuantizedBrevitasQuant ops
|
|
if q_bias is not None and q_bias.quantizer.is_precomputed_qat:
|
|
# Make sure the scale was correctly matching during training
|
|
# The bias scale should be the same scale as the one of the weights * inputs
|
|
assert q_bias.quantizer.scale is not None
|
|
assert numpy.isclose(q_bias.quantizer.scale, m_matmul)
|
|
numpy_q_out += q_bias.qvalues
|
|
|
|
# If weights are not encrypted then we can round as the next
|
|
# line is going to be done in a PBS
|
|
if not is_encrypted_gemm:
|
|
with tag(self.op_instance_name + ".matmul_rounding"):
|
|
# Apply Concrete rounding (if relevant)
|
|
numpy_q_out = self.cnp_round(
|
|
numpy_q_out, calibrate_rounding, rounding_operation_id="matmul"
|
|
)
|
|
|
|
# Force a PBS with astype float64
|
|
numpy_q_out = numpy_q_out.astype(numpy.float64)
|
|
|
|
# Quantization scales and zero points
|
|
# This is done in a PBS if is_encrypted_gemm == False
|
|
# (along with the following activation function)
|
|
# Otherwise it is done in FHE
|
|
numpy_q_out = numpy_q_out + final_term - sum_weights
|
|
|
|
if is_encrypted_gemm:
|
|
with tag(self.op_instance_name + ".matmul_rounding"):
|
|
# Apply Concrete rounding (if relevant)
|
|
numpy_q_out = self.cnp_round(
|
|
numpy_q_out, calibrate_rounding, rounding_operation_id="matmul"
|
|
)
|
|
|
|
numpy_q_out = m_matmul * numpy_q_out
|
|
|
|
if q_bias is not None and not q_bias.quantizer.is_precomputed_qat:
|
|
# The bias is handled as a float and will be fused
|
|
numpy_q_out = numpy_q_out + q_bias.values
|
|
|
|
# Return the float values, so that Concrete can fuse any following float operations
|
|
# We also keep track of the scaling factor and zero-point, since these will be
|
|
# applied by the following layers.
|
|
return QuantizedArray(
|
|
self.n_bits,
|
|
numpy_q_out,
|
|
value_is_float=True,
|
|
options=self._get_output_quant_opts(),
|
|
stats=self.output_quant_stats,
|
|
params=self.output_quant_params,
|
|
)
|
|
|
|
def calibrate_analytical_output(self, *inputs: QuantizedArray) -> UniformQuantizer:
|
|
"""Calibrate output quantization based on analytical formulas.
|
|
|
|
Args:
|
|
*inputs (QuantizedArray): quantized operation inputs. Quantized weights
|
|
are stored in the op instance
|
|
|
|
Returns:
|
|
res (UniformQuantizer): the quantizer of the operation's output values
|
|
that can be used to de-quantize these values.
|
|
"""
|
|
|
|
q_input1 = inputs[0]
|
|
assert isinstance(q_input1, QuantizedArray)
|
|
q_input2 = inputs[1]
|
|
assert isinstance(q_input2, QuantizedArray)
|
|
|
|
# In the operation Y = alpha * A' * B' + beta * C, q_bias is used for
|
|
# generalised matrix multiplication. q_bias is set to None for standard
|
|
# matrix multiplication (beta == 0 or only two inputs)
|
|
q_bias = None if len(inputs) == 2 or self.attrs["beta"] == 0 else inputs[2]
|
|
assert isinstance(q_bias, (type(None), QuantizedArray))
|
|
|
|
# Using snake case here to please the Python format, the original attrs don't have the '_'
|
|
# Use default false so we also support MatMul impl, MatMul does not have these flags
|
|
transpose_inputs2 = self.attrs.get("transB", False)
|
|
|
|
p = q_input2.qvalues.shape[-2]
|
|
|
|
assert q_input1.quantizer.scale is not None
|
|
assert q_input2.quantizer.scale is not None
|
|
m_matmul = q_input1.quantizer.scale * q_input2.quantizer.scale
|
|
|
|
input2_q_values = (
|
|
numpy.transpose(q_input2.qvalues) if transpose_inputs2 else q_input2.qvalues
|
|
)
|
|
|
|
# Compute the third term, the sum of the weights which is a constant
|
|
sum_weights = q_input1.quantizer.zero_point * numpy.sum(
|
|
input2_q_values, axis=-2, keepdims=True
|
|
)
|
|
|
|
assert q_input1.quantizer.zero_point is not None
|
|
assert q_input2.quantizer.zero_point is not None
|
|
final_term = p * q_input1.quantizer.zero_point * q_input2.quantizer.zero_point
|
|
|
|
out_zp: Union[int, numpy.ndarray] = sum_weights - final_term
|
|
if q_bias is not None:
|
|
# Make mypy happy
|
|
assert q_bias is not None
|
|
# Reshape the biases to broadcast them to each neuron
|
|
bias_out = q_bias.values if isinstance(q_bias, QuantizedArray) else q_bias
|
|
out_zp = out_zp + bias_out / (-m_matmul)
|
|
|
|
out_params = UniformQuantizationParameters(
|
|
scale=m_matmul,
|
|
zero_point=out_zp,
|
|
offset=0,
|
|
)
|
|
|
|
return UniformQuantizer(self._get_output_quant_opts(), self.output_quant_stats, out_params)
|
|
|
|
|
|
class QuantizedMatMul(QuantizedGemm):
|
|
"""Quantized MatMul op."""
|
|
|
|
_impl_for_op_named: str = "MatMul"
|
|
|
|
|
|
class QuantizedAdd(QuantizedMixingOp):
|
|
"""Quantized Addition operator.
|
|
|
|
Can add either two variables (both encrypted) or a variable and a constant
|
|
"""
|
|
|
|
_impl_for_op_named: str = "Add"
|
|
b_sign: int = 1
|
|
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
# If operating over all raw inputs, just perform the op in the clear
|
|
if all(isinstance(q_input, RawOpOutput) for q_input in q_inputs):
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=False
|
|
)
|
|
return self.call_impl(*prepared_inputs, **attrs).view(RawOpOutput)
|
|
|
|
# For mypy
|
|
assert self.output_quant_params is not None
|
|
assert self.output_quant_params.scale is not None
|
|
assert self.output_quant_params.zero_point is not None
|
|
|
|
# Optimize computation when adding constants, or tensors obtained from a unique integer
|
|
# tensor. Optimization allows univariate float subgraph fusion to a TLU
|
|
execute_in_float = len(self.constant_inputs) > 0 or self.can_fuse()
|
|
assert_true(
|
|
len(self.constant_inputs) < 2,
|
|
"Constant folding should have eliminated a two constant-input add node",
|
|
)
|
|
|
|
if execute_in_float:
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=False
|
|
)
|
|
|
|
return QuantizedArray(
|
|
self.n_bits,
|
|
prepared_inputs[0] + self.b_sign * prepared_inputs[1],
|
|
value_is_float=True,
|
|
options=self._get_output_quant_opts(),
|
|
stats=self.output_quant_stats,
|
|
params=self.output_quant_params,
|
|
)
|
|
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=True
|
|
)
|
|
|
|
q_input_0: QuantizedArray = prepared_inputs[0]
|
|
q_input_1: QuantizedArray = prepared_inputs[1]
|
|
|
|
assert q_input_0.quantizer.scale is not None
|
|
assert q_input_0.quantizer.zero_point is not None
|
|
|
|
assert q_input_1.quantizer.scale is not None
|
|
assert q_input_1.quantizer.zero_point is not None
|
|
|
|
# Dequantize
|
|
input_0 = q_input_0.dequant()
|
|
input_1 = q_input_1.dequant()
|
|
|
|
# If this operator is the last one in the graph,
|
|
# we rescale using the smallest scale to keep all information
|
|
if self.produces_graph_output:
|
|
common_scale = min(q_input_0.quantizer.scale, q_input_1.quantizer.scale)
|
|
# Otherwise we use the output op quantization scale
|
|
else:
|
|
common_scale = self.output_quant_params.scale
|
|
|
|
common_zero_point = 0
|
|
offset = 0
|
|
|
|
output_quant_params = UniformQuantizationParameters(
|
|
scale=common_scale,
|
|
zero_point=common_zero_point,
|
|
offset=offset,
|
|
)
|
|
|
|
quantizer = UniformQuantizer(params=output_quant_params, no_clipping=True)
|
|
|
|
# Re-quantize using the common quantization paramaters
|
|
q_input_0_rescaled = quantizer.quant(input_0)
|
|
q_input_1_rescaled = quantizer.quant(input_1)
|
|
|
|
# The sum of quantized encrypted integer values
|
|
# This sum has << max(in_bits0, in_bits1) + 1 >> bits
|
|
# Moreover, the zero-point will be sum of input zero-points
|
|
assert self.b_sign in [-1, 1]
|
|
|
|
sum_q = q_input_0_rescaled + self.b_sign * q_input_1_rescaled
|
|
|
|
if self.produces_graph_output:
|
|
return self.make_output_quant_parameters(sum_q, common_scale, common_zero_point)
|
|
|
|
# But we would like the output to have n_bits, so we de-quantize
|
|
dequant_sum = quantizer.dequant(sum_q)
|
|
|
|
# Return the raw float values without re-quantizing them to the new scale, as any
|
|
# following Gemm/Add/Conv will quantize them with _prepare_inputs_with_constants(...)
|
|
return QuantizedArray(
|
|
self.n_bits,
|
|
dequant_sum,
|
|
value_is_float=True,
|
|
options=self._get_output_quant_opts(),
|
|
stats=self.output_quant_stats,
|
|
params=self.output_quant_params,
|
|
)
|
|
|
|
def can_fuse(self) -> bool:
|
|
"""Determine if this op can be fused.
|
|
|
|
Add operation can be computed in float and fused if it operates over inputs produced
|
|
by a single integer tensor. For example the expression x + x * 1.75, where x is
|
|
an encrypted tensor, can be computed with a single TLU.
|
|
|
|
Returns:
|
|
bool: Whether the number of integer input tensors allows computing this op as a TLU
|
|
"""
|
|
|
|
return len(self._int_input_names) == 1
|
|
|
|
|
|
class QuantizedTanh(QuantizedOp):
|
|
"""Quantized Tanh op."""
|
|
|
|
_impl_for_op_named: str = "Tanh"
|
|
|
|
|
|
class QuantizedSoftplus(QuantizedOp):
|
|
"""Quantized Softplus op."""
|
|
|
|
_impl_for_op_named: str = "Softplus"
|
|
|
|
|
|
class QuantizedExp(QuantizedOp):
|
|
"""Quantized Exp op."""
|
|
|
|
_impl_for_op_named: str = "Exp"
|
|
|
|
|
|
class QuantizedLog(QuantizedOp):
|
|
"""Quantized Log op."""
|
|
|
|
_impl_for_op_named: str = "Log"
|
|
|
|
|
|
class QuantizedAbs(QuantizedOp):
|
|
"""Quantized Abs op."""
|
|
|
|
_impl_for_op_named: str = "Abs"
|
|
|
|
|
|
class QuantizedIdentity(QuantizedOp):
|
|
"""Quantized Identity op."""
|
|
|
|
_impl_for_op_named: str = "Identity"
|
|
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
assert_true(len(q_inputs) == 1, "Identity does not work with multiple QuantizedArray")
|
|
|
|
# This op takes only encrypted inputs in the form of QuantizedArray
|
|
assert isinstance(q_inputs[0], QuantizedArray)
|
|
|
|
self.output_quant_params = q_inputs[0].quantizer.quant_params
|
|
return super().q_impl(*q_inputs, **attrs)
|
|
|
|
|
|
class QuantizedReshape(QuantizedOp):
|
|
"""Quantized Reshape op."""
|
|
|
|
_impl_for_op_named: str = "Reshape"
|
|
quantize_inputs_with_model_outputs_precision = True
|
|
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
"""Reshape the input integer encrypted tensor.
|
|
|
|
Args:
|
|
q_inputs: an encrypted integer tensor at index 0 and one constant shape at index 1
|
|
attrs: additional optional reshape options
|
|
|
|
Returns:
|
|
result (QuantizedArray): reshaped encrypted integer tensor
|
|
"""
|
|
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=True
|
|
)
|
|
|
|
# This op takes only encrypted inputs in the form of QuantizedArray
|
|
assert isinstance(q_inputs[0], QuantizedArray)
|
|
|
|
newshape = prepared_inputs[1]
|
|
assert_true(numpy.issubdtype(newshape.dtype, numpy.integer))
|
|
|
|
# Return a new quantized array with the same quantization parameters
|
|
return QuantizedArray(
|
|
q_inputs[0].quantizer.n_bits,
|
|
self.call_impl(prepared_inputs[0].qvalues, newshape, **attrs),
|
|
value_is_float=False,
|
|
options=prepared_inputs[0].quantizer.quant_options,
|
|
stats=prepared_inputs[0].quantizer.quant_stats,
|
|
params=prepared_inputs[0].quantizer.quant_params,
|
|
)
|
|
|
|
def can_fuse(self) -> bool:
|
|
"""Determine if this op can be fused.
|
|
|
|
Reshape operation can not be fused since it must be performed over integer tensors.
|
|
|
|
Returns:
|
|
bool: False, this operation can not be fused.
|
|
"""
|
|
return False
|
|
|
|
|
|
class QuantizedConv(QuantizedMixingOp):
|
|
"""Quantized Conv op."""
|
|
|
|
_impl_for_op_named: str = "Conv"
|
|
|
|
def __init__(
|
|
self,
|
|
n_bits_output: int,
|
|
op_instance_name: str,
|
|
int_input_names: Set[str] = None,
|
|
constant_inputs: Optional[Union[Dict[str, Any], Dict[int, Any]]] = None,
|
|
input_quant_opts: QuantizationOptions = None,
|
|
**attrs,
|
|
) -> None:
|
|
"""Construct the quantized convolution operator and retrieve parameters.
|
|
|
|
Args:
|
|
n_bits_output: number of bits for the quantization of the outputs of this operator
|
|
op_instance_name (str): The name that should be assigned to this operation, used
|
|
to retrieve it later or get debugging information about this op (bit-width, value
|
|
range, integer intermediary values, op-specific error messages). Usually this name
|
|
is the same as the ONNX operation name for which this operation is constructed.
|
|
int_input_names: names of integer tensors that are taken as input for this operation
|
|
constant_inputs: the weights and activations
|
|
input_quant_opts: options for the input quantizer
|
|
attrs: convolution options
|
|
dilations (Tuple[int]): dilation of the kernel. Default to 1 on all dimensions.
|
|
group (int): number of convolution groups. Default to 1.
|
|
kernel_shape (Tuple[int]): shape of the kernel. Should have 2 elements for 2d conv
|
|
pads (Tuple[int]): padding in ONNX format (begin, end) on each axis
|
|
strides (Tuple[int]): stride of the convolution on each axis
|
|
"""
|
|
|
|
super().__init__(
|
|
n_bits_output,
|
|
op_instance_name,
|
|
int_input_names,
|
|
constant_inputs,
|
|
input_quant_opts,
|
|
**attrs,
|
|
)
|
|
|
|
# Get the ONNX parameters
|
|
self.group = attrs.get("group", 1)
|
|
self.kernel_shape = attrs.get("kernel_shape", None)
|
|
self.pads = attrs.get("pads", tuple([0] * 2 * (len(self.kernel_shape) - 2)))
|
|
self.dilations = attrs.get("dilations", tuple([1] * len(self.kernel_shape)))
|
|
self.strides = attrs.get("strides", tuple([1] * len(self.kernel_shape)))
|
|
|
|
# Validate the parameters
|
|
assert_true(
|
|
len(self.kernel_shape) in (1, 2),
|
|
"The convolution operator currently only supports 1d or 2d. "
|
|
f"Got {len(self.kernel_shape)}-d",
|
|
)
|
|
assert_true(
|
|
len(self.kernel_shape) == len(self.strides),
|
|
"The convolution operator requires the number of strides to "
|
|
"be the same as the number of kernel dimensions",
|
|
)
|
|
assert_true(
|
|
bool(numpy.all(numpy.asarray(self.dilations) == 1)),
|
|
"The convolution operator in Concrete does not support dilation",
|
|
)
|
|
assert_true(
|
|
len(self.pads) == 2 * len(self.kernel_shape),
|
|
"The convolution operator in Concrete ML requires padding to be specified as "
|
|
" (pad_left_dim1, pad_right_dim1, pad_left_dim2, pad_right_dim2, ...), following ONNX"
|
|
" standard",
|
|
)
|
|
|
|
# pylint: disable-next=too-many-statements, too-many-locals
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
calibrate_rounding: bool = False,
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
"""Compute the quantized convolution between two quantized tensors.
|
|
|
|
Allows an optional quantized bias.
|
|
|
|
Args:
|
|
q_inputs: input tuple, contains
|
|
x (numpy.ndarray): input data. Shape is N x C x H x W for 2d
|
|
w (numpy.ndarray): weights tensor. Shape is (O x I x Kh x Kw) for 2d
|
|
b (numpy.ndarray, Optional): bias tensor, Shape is (O,)
|
|
calibrate_rounding (bool): Whether to calibrate rounding
|
|
attrs: convolution options handled in constructor
|
|
|
|
Returns:
|
|
res (QuantizedArray): result of the quantized integer convolution
|
|
"""
|
|
|
|
# For mypy
|
|
assert self.output_quant_params is not None
|
|
assert self.output_quant_params.scale is not None
|
|
assert self.output_quant_params.zero_point is not None
|
|
|
|
# Retrieve the quantized inputs
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=True
|
|
)
|
|
q_input: QuantizedArray = prepared_inputs[0]
|
|
q_weights: QuantizedArray = prepared_inputs[1]
|
|
q_bias: Optional[QuantizedArray] = None if len(prepared_inputs) == 2 else prepared_inputs[2]
|
|
|
|
in_channels = q_input.values.shape[1]
|
|
weight_channels = q_weights.values.shape[1]
|
|
assert_true(
|
|
weight_channels == in_channels / self.group,
|
|
f"Expected number of channels in weight to be {in_channels / self.group} "
|
|
f"(C / group). Got {weight_channels}.",
|
|
)
|
|
|
|
out_channels = q_weights.values.shape[0]
|
|
assert_true(
|
|
out_channels % self.group == 0,
|
|
f"Expected number of output channels O ({out_channels}) to be a multiple of "
|
|
f"group ({self.group}).",
|
|
)
|
|
|
|
assert q_weights.quantizer.scale is not None
|
|
assert q_weights.quantizer.zero_point is not None
|
|
|
|
assert q_input.quantizer.scale is not None
|
|
assert q_input.quantizer.zero_point is not None
|
|
|
|
# Can only pad with scalar zero-points, but zero-points can be float in special cases
|
|
# for output layers
|
|
_check_op_input_zero_point(q_input.quantizer.zero_point, self.op_instance_name)
|
|
pad_value = int(q_input.quantizer.zero_point)
|
|
q_input_pad = numpy_onnx_pad(q_input.qvalues, self.pads, pad_value, True)
|
|
|
|
is_conv1d = len(self.kernel_shape) == 1
|
|
|
|
q_weights_values = q_weights.qvalues
|
|
kernel_shape = self.kernel_shape
|
|
strides = self.strides
|
|
dilations = self.dilations
|
|
|
|
# Workaround for handling torch's Conv1d operator until it is supported by Concrete Python
|
|
# FIXME: https://github.com/luxfi/luxfhe-ml-internal/issues/4117
|
|
if is_conv1d:
|
|
q_input_pad = numpy.expand_dims(q_input_pad, axis=-2)
|
|
q_weights_values = numpy.expand_dims(q_weights_values, axis=-2)
|
|
kernel_shape = (1, kernel_shape[0])
|
|
strides = (1, strides[0])
|
|
dilations = (1, dilations[0])
|
|
|
|
# Prepare a constant tensor to compute the sum of the inputs
|
|
q_weights_1 = numpy.ones_like(q_weights_values)
|
|
|
|
# We follow the Quantized Gemm implementation
|
|
# which in turn follows Eq.7 in https://arxiv.org/abs/1712.05877
|
|
# to split the core computation from the zero points and scales.
|
|
|
|
# Compute the first encrypted term that convolves weights and inputs
|
|
# Force padding to 0 as padding needs to use a custom padding initializer
|
|
# and is thus manually performed in the code above
|
|
fake_pads = [0, 0] * len(kernel_shape)
|
|
|
|
with tag(self.op_instance_name + ".conv"):
|
|
conv_wx = fhe_conv(
|
|
q_input_pad,
|
|
q_weights_values,
|
|
bias=None,
|
|
pads=fake_pads,
|
|
kernel_shape=kernel_shape,
|
|
strides=strides,
|
|
dilations=dilations,
|
|
group=self.group,
|
|
)
|
|
|
|
# The total number of elements that are convolved by the application of a single kernel
|
|
n_weights = numpy.prod(q_weights_values.shape[1:])
|
|
|
|
# If the weights have symmetric quantization, their zero point will be 0
|
|
# The following check avoids the computation of the sum of the inputs, which may have
|
|
# large bit-width, in the case where it would be multiplied by zero
|
|
if q_weights.quantizer.zero_point != 0:
|
|
# Compute the sum of the inputs (second encrypted term)
|
|
assert_true(
|
|
isinstance(q_weights.quantizer.zero_point, (int, numpy.int_)),
|
|
f"Zero point of weights tensor in {self.op_type} "
|
|
f"op {self.op_instance_name} must be integer",
|
|
)
|
|
with tag(self.op_instance_name + ".conv_inputsum"):
|
|
zw_conv_1x = -q_weights.quantizer.zero_point * fhe_conv(
|
|
q_input_pad,
|
|
q_weights_1,
|
|
bias=None,
|
|
pads=fake_pads,
|
|
kernel_shape=kernel_shape,
|
|
strides=strides,
|
|
dilations=dilations,
|
|
group=self.group,
|
|
)
|
|
|
|
with tag(self.op_instance_name + ".conv_add_inputsum"):
|
|
numpy_q_out = conv_wx + zw_conv_1x
|
|
else:
|
|
numpy_q_out = conv_wx
|
|
|
|
# Workaround for handling torch's Conv1d operator until it is supported by Concrete Python
|
|
# FIXME: https://github.com/luxfi/luxfhe-ml-internal/issues/4117
|
|
if is_conv1d:
|
|
numpy_q_out = numpy.squeeze(numpy_q_out, axis=-2)
|
|
|
|
if self.debug_value_tracker is not None:
|
|
# pylint: disable-next=unsubscriptable-object
|
|
self.debug_value_tracker[self.op_instance_name]["output"] = numpy_q_out
|
|
|
|
weight_sum_axes = (1, 2) if is_conv1d else (1, 2, 3)
|
|
weight_transpose_axes = (1, 0, 2) if is_conv1d else (1, 0, 2, 3)
|
|
|
|
# Compute the third term, the sum of the weights which is a constant
|
|
sum_weights = q_input.quantizer.zero_point * numpy.sum(
|
|
q_weights.qvalues, axis=weight_sum_axes, keepdims=True
|
|
).transpose(*weight_transpose_axes)
|
|
|
|
# Compute the forth term which is a constant
|
|
final_term = n_weights * q_input.quantizer.zero_point * q_weights.quantizer.zero_point
|
|
|
|
# Compute the rescaling factor that de-quantizes the input
|
|
# This is going to be compiled with a PBS (along with the following activation function)
|
|
# Note that we don't re-quantize the output of the conv, this will be done by
|
|
# any Gemm/Add/Conv layers that follow
|
|
m_matmul = q_input.quantizer.scale * q_weights.quantizer.scale
|
|
|
|
bias_shape = (1, -1, 1) if is_conv1d else (1, -1, 1, 1)
|
|
|
|
# If this operation's result are network outputs, return
|
|
# directly the integer values and an appropriate quantization parameters that
|
|
# allow direct in-the-clear de-quantization, including the bias
|
|
if self.produces_graph_output:
|
|
# Note that to use the bias, we need to rescale it to the output scale
|
|
# For Eq. 7 in https://arxiv.org/abs/1712.05877, we can write:
|
|
# S_out(q_out - zp_out) = S_x * S_w (multisum + bias / (S_x * S_w))
|
|
# where multisum is the dot product of quantized inputs and quantized weights
|
|
# Then we identify terms:
|
|
# S_out = S_x * S_w
|
|
# q_out = multisum terms involving inputs
|
|
# zp_out = -(multisum terms involving weights + bias / (S_x * S_w))
|
|
out_zp: Union[int, numpy.ndarray] = sum_weights - final_term
|
|
if q_bias is not None:
|
|
# Reshape the biases to broadcast them to each channel
|
|
out_zp = out_zp - q_bias.values.reshape(bias_shape) / m_matmul
|
|
|
|
# We identify terms in the above equation to determine what
|
|
# the scale/zero-point of the in-the-clear quantizer should be
|
|
# to properly de-quantize numpy_q_out
|
|
return self.make_output_quant_parameters(numpy_q_out, m_matmul, out_zp)
|
|
|
|
if q_bias is not None and q_bias.quantizer.is_precomputed_qat:
|
|
# Make sure the scale was correctly matching during training
|
|
# The bias scale should be the same scale as the one of the weights * inputs
|
|
assert q_bias.quantizer.scale is not None
|
|
assert numpy.isclose(q_bias.quantizer.scale, m_matmul)
|
|
|
|
numpy_q_out += q_bias.qvalues.reshape(bias_shape)
|
|
|
|
with tag(self.op_instance_name + ".conv_rounding"):
|
|
# Apply Concrete rounding (if relevant)
|
|
numpy_q_out = self.cnp_round(
|
|
numpy_q_out, calibrate_rounding, rounding_operation_id="matmul"
|
|
)
|
|
|
|
# Now compute the whole sum (sum of the four terms)
|
|
numpy_q_out = numpy_q_out.astype(numpy.float64) + final_term - sum_weights
|
|
|
|
# Rescale from scale=scale_inputs x scale_outputs to output scale
|
|
numpy_q_out = m_matmul * numpy_q_out
|
|
|
|
if q_bias is not None and not q_bias.quantizer.is_precomputed_qat:
|
|
# The bias addition is handled in float and will be fused into a TLU
|
|
# Reshape the biases to broadcast them to each channel
|
|
numpy_q_out = numpy_q_out + q_bias.values.reshape(bias_shape) # bias_part
|
|
|
|
# And return as a QuantizedArray initialized from the float data, keeping
|
|
# track of the quantization parameters
|
|
return QuantizedArray(
|
|
self.n_bits,
|
|
numpy_q_out,
|
|
value_is_float=True,
|
|
options=self._get_output_quant_opts(),
|
|
stats=self.output_quant_stats,
|
|
params=self.output_quant_params,
|
|
)
|
|
|
|
|
|
class QuantizedAvgPool(QuantizedMixingOp):
|
|
"""Quantized Average Pooling op."""
|
|
|
|
_impl_for_op_named: str = "AveragePool"
|
|
|
|
# Since this op takes a single input, we can set int_input_names to a single default id
|
|
def __init__(
|
|
self,
|
|
n_bits_output: int,
|
|
op_instance_name: str,
|
|
int_input_names: Set[str] = None,
|
|
constant_inputs: Optional[Union[Dict[str, Any], Dict[int, Any]]] = None,
|
|
input_quant_opts: QuantizationOptions = None,
|
|
**attrs,
|
|
) -> None:
|
|
|
|
super().__init__(
|
|
n_bits_output,
|
|
op_instance_name,
|
|
int_input_names,
|
|
constant_inputs,
|
|
input_quant_opts,
|
|
**attrs,
|
|
)
|
|
|
|
# Get the ONNX parameters
|
|
self.ceil_mode = attrs.get("ceil_mode", 0)
|
|
self.auto_pad = attrs.get("auto_pad", "NOTSET")
|
|
self.kernel_shape = attrs.get("kernel_shape", None)
|
|
|
|
assert_true(self.kernel_shape is not None, "Setting parameter 'kernel_shape' is required.")
|
|
|
|
self.count_include_pad = attrs.get("count_include_pad", 1)
|
|
self.pads = attrs.get("pads", tuple([0] * 2 * (len(self.kernel_shape) - 2)))
|
|
self.dilations = attrs.get("dilations", tuple([1] * len(self.kernel_shape)))
|
|
self.strides = attrs.get("strides", tuple([1] * len(self.kernel_shape)))
|
|
|
|
# Validate the parameters
|
|
assert_true(
|
|
self.auto_pad == "NOTSET",
|
|
"The 'auto_pad' parameter is not supported. Please keep the the default 'NOTSET' value "
|
|
"and provide explicit padding.",
|
|
)
|
|
|
|
assert_true(
|
|
len(self.kernel_shape) == 2,
|
|
"The Average Pool operator currently supports only 2d kernels.",
|
|
)
|
|
|
|
assert_true(
|
|
self.count_include_pad == 1,
|
|
"Pad pixels must be included when calculating values on the edges. Please set "
|
|
"'count_include_pad' to 1.",
|
|
)
|
|
|
|
assert_true(
|
|
len(self.kernel_shape) == len(self.strides),
|
|
"The Average Pool operator requires the number of strides to be the same as the number "
|
|
"of kernel dimensions.",
|
|
)
|
|
|
|
assert_true(
|
|
len(self.pads) == 2 * len(self.kernel_shape),
|
|
"The Average Pool operator in Concrete ML requires padding to be specified as "
|
|
" (pad_left_dim1, pad_right_dim1, pad_left_dim2, pad_right_dim2, ...), following ONNX"
|
|
" standard.",
|
|
)
|
|
|
|
self.kernel: Union[numpy.ndarray, None] = None
|
|
self.norm_const: Union[float, None] = None
|
|
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
calibrate_rounding: bool = False,
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
|
|
# Retrieve the quantized inputs
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=True
|
|
)
|
|
q_input: QuantizedArray = prepared_inputs[0]
|
|
|
|
n_in_channels = q_input.qvalues.shape[1]
|
|
kernel = numpy.zeros(
|
|
(n_in_channels, n_in_channels, self.kernel_shape[0], self.kernel_shape[1]),
|
|
dtype=numpy.int64,
|
|
)
|
|
for i in range(n_in_channels):
|
|
kernel[i, i, ::] = 1
|
|
|
|
norm_const = 1.0 / onnx_avgpool_compute_norm_const(
|
|
q_input.qvalues.shape,
|
|
self.kernel_shape,
|
|
self.pads,
|
|
self.strides,
|
|
self.ceil_mode,
|
|
)
|
|
|
|
# for mypy: The Quantized ops can only run on QuantizedArray that have quantization
|
|
# parameters (i.e., were fully constructed). This should always be the case, except
|
|
# during the UniformQuantizer initialization when the zero_point can exist as None
|
|
assert q_input.quantizer.zero_point is not None
|
|
|
|
# Compute padding with floor and apply it to the input, pad with the input zero-point
|
|
pool_pads = compute_onnx_pool_padding(
|
|
q_input.qvalues.shape, self.kernel_shape, self.pads, self.strides, 0
|
|
)
|
|
|
|
# Can only pad with scalar zero-points, but zero-points can be float in special cases
|
|
# for output layers
|
|
_check_op_input_zero_point(q_input.quantizer.zero_point, self.op_instance_name)
|
|
pad_value = int(q_input.quantizer.zero_point)
|
|
q_input_pad = numpy_onnx_pad(q_input.qvalues, pool_pads, pad_value, int_only=True)
|
|
|
|
if self.ceil_mode == 1:
|
|
# Padding for TensorFlow style
|
|
|
|
# Compute padding with ceil and apply it to the input, pad with zeros, the zeros
|
|
# will be ignored in the computation
|
|
pool_pads_ceil = compute_onnx_pool_padding(
|
|
q_input.qvalues.shape, self.kernel_shape, self.pads, self.strides, 1
|
|
)
|
|
|
|
# Can only pad with scalar zero-points, but zero-points can be float in special cases
|
|
# for output layers
|
|
q_input_pad_ceil = numpy_onnx_pad(q_input.qvalues, pool_pads_ceil, 0, True)
|
|
|
|
# Copy the PyTorch style padded input to the larger 0 padded tensor
|
|
q_input_pad_ceil[:, :, 0 : q_input_pad.shape[2], 0 : q_input_pad.shape[3]] = q_input_pad
|
|
q_input_pad = q_input_pad_ceil
|
|
|
|
# Remark that here, we are _not_ using Concrete pad, since it would pad with
|
|
# 0's while we want to pad with zero-point's. So, instead, he have done the padding
|
|
# on our side, with q_input_pad
|
|
fake_pads = [0] * len(self.pads)
|
|
with tag(self.op_instance_name + ".avgpool"):
|
|
sum_result = fhe_conv(q_input_pad, kernel, None, fake_pads, self.strides)
|
|
|
|
with tag(self.op_instance_name + ".avgpool_rounding"):
|
|
# Apply Concrete rounding (if relevant)
|
|
sum_result = self.cnp_round(sum_result, calibrate_rounding)
|
|
|
|
if self.debug_value_tracker is not None:
|
|
# pylint: disable-next=unsubscriptable-object
|
|
self.debug_value_tracker[self.op_instance_name]["output"] = sum_result
|
|
|
|
result = (
|
|
sum_result.astype(numpy.float64) * norm_const - q_input.quantizer.zero_point
|
|
) * q_input.quantizer.scale
|
|
|
|
return QuantizedArray(
|
|
self.n_bits,
|
|
result,
|
|
value_is_float=True,
|
|
options=self._get_output_quant_opts(),
|
|
stats=self.output_quant_stats,
|
|
params=self.output_quant_params,
|
|
)
|
|
|
|
|
|
class QuantizedMaxPool(QuantizedOp):
|
|
"""Quantized Max Pooling op."""
|
|
|
|
_impl_for_op_named: str = "MaxPool"
|
|
|
|
# Since this op takes a single input, we can set int_input_names to a single default id
|
|
def __init__(
|
|
self,
|
|
n_bits_output: int,
|
|
op_instance_name: str,
|
|
int_input_names: Set[str] = None,
|
|
constant_inputs: Optional[Union[Dict[str, Any], Dict[int, Any]]] = None,
|
|
input_quant_opts: QuantizationOptions = None,
|
|
**attrs,
|
|
) -> None:
|
|
|
|
super().__init__(
|
|
n_bits_output,
|
|
op_instance_name,
|
|
int_input_names,
|
|
constant_inputs,
|
|
input_quant_opts,
|
|
**attrs,
|
|
)
|
|
|
|
# Get the ONNX parameters
|
|
self.auto_pad = attrs.get("auto_pad", "NOTSET")
|
|
self.ceil_mode = attrs.get("ceil_mode", 0)
|
|
self.kernel_shape = attrs.get("kernel_shape", None)
|
|
self.storage_order = attrs.get("storage_order", 0)
|
|
self.pads = attrs.get("pads", tuple([0] * 2 * (len(self.kernel_shape) - 2)))
|
|
self.dilations = attrs.get("dilations", tuple([1] * len(self.kernel_shape)))
|
|
self.strides = attrs.get("strides", tuple([1] * len(self.kernel_shape)))
|
|
|
|
# Validate the parameters
|
|
assert_true(self.ceil_mode == 0, "Only ceil_mode = 0 is supported by Concrete for now")
|
|
|
|
# Validate the parameters
|
|
assert_true(
|
|
len(self.kernel_shape) == 2,
|
|
"The Max Pool operator currently supports only 2d",
|
|
)
|
|
assert_true(
|
|
len(self.kernel_shape) == len(self.strides),
|
|
"The Max Pool operator requires the number of strides to "
|
|
"be the same as the number of kernel dimensions",
|
|
)
|
|
assert_true(
|
|
len(self.pads) == 2 * len(self.kernel_shape),
|
|
"The Max Pool operator in Concrete ML requires padding to be specified as "
|
|
" (pad_left_dim1, pad_right_dim1, pad_left_dim2, pad_right_dim2, ...), following ONNX"
|
|
" standard",
|
|
)
|
|
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
|
|
# Retrieve the quantized inputs
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=True
|
|
)
|
|
q_input: QuantizedArray = prepared_inputs[0]
|
|
|
|
# for mypy: The Quantized ops can only run on QuantizedArray that have quantization
|
|
# parameters (i.e., were fully constructed). This should always be the case, except
|
|
# during the UniformQuantizer initialization when the zero_point can exist as None
|
|
assert q_input.quantizer.zero_point is not None
|
|
|
|
assert_true(
|
|
self.ceil_mode == 0,
|
|
"Only ceil_mode = 0 is supported by Concrete for now",
|
|
)
|
|
|
|
# Simple padding for PyTorch style
|
|
pool_pads = compute_onnx_pool_padding(
|
|
q_input.qvalues.shape,
|
|
self.kernel_shape,
|
|
self.pads,
|
|
self.strides,
|
|
self.ceil_mode,
|
|
)
|
|
|
|
q_input_pad = numpy_onnx_pad(
|
|
q_input.qvalues, pool_pads, q_input.quantizer.zero_point, int_only=True
|
|
)
|
|
|
|
# Remark that here, we are _not_ using Concrete pad, since it would pad with
|
|
# 0's while we want to pad with zero-point's. So, instead, he have done the padding
|
|
# on our side, with q_input_pad
|
|
fake_pads = [0] * len(self.pads)
|
|
sum_result = fhe_maxpool(
|
|
q_input_pad,
|
|
kernel_shape=self.kernel_shape,
|
|
strides=self.strides,
|
|
auto_pad=self.auto_pad,
|
|
pads=fake_pads,
|
|
dilations=self.dilations,
|
|
ceil_mode=self.ceil_mode,
|
|
storage_order=self.storage_order,
|
|
)
|
|
|
|
result = (
|
|
sum_result.astype(numpy.float64) - q_input.quantizer.zero_point
|
|
) * q_input.quantizer.scale
|
|
|
|
return QuantizedArray(
|
|
self.n_bits,
|
|
result,
|
|
value_is_float=True,
|
|
options=self._get_output_quant_opts(),
|
|
stats=self.output_quant_stats,
|
|
params=self.output_quant_params,
|
|
)
|
|
|
|
def can_fuse(self) -> bool:
|
|
"""Determine if this op can be fused.
|
|
|
|
Max Pooling operation can not be fused since it must be performed over integer tensors and
|
|
it combines different elements of the input tensors.
|
|
|
|
Returns:
|
|
bool: False, this operation can not be fused as it adds different encrypted integers
|
|
"""
|
|
return False
|
|
|
|
|
|
class QuantizedPad(QuantizedOp):
|
|
"""Quantized Padding op."""
|
|
|
|
_impl_for_op_named: str = "Pad"
|
|
|
|
def __init__(
|
|
self,
|
|
n_bits_output: int,
|
|
op_instance_name: str,
|
|
int_input_names: Set[str] = None,
|
|
constant_inputs: Optional[Union[Dict[str, Any], Dict[int, Any]]] = None,
|
|
input_quant_opts: QuantizationOptions = None,
|
|
**attrs,
|
|
) -> None:
|
|
|
|
super().__init__(
|
|
n_bits_output,
|
|
op_instance_name,
|
|
int_input_names,
|
|
constant_inputs,
|
|
input_quant_opts,
|
|
**attrs,
|
|
)
|
|
|
|
# Get the ONNX parameters
|
|
self.mode = attrs.get("mode", None)
|
|
assert_true(
|
|
self.mode == "constant",
|
|
"Padding operator only supports padding with a constant",
|
|
)
|
|
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
calibrate_rounding: bool = False, # pylint: disable=unused-argument
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
# Retrieve the quantized inputs
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=True
|
|
)
|
|
q_input: QuantizedArray = prepared_inputs[0]
|
|
pads = prepared_inputs[1]
|
|
|
|
assert_true(
|
|
all(pads[i] == 0 and pads[4 + i] == 0 for i in range(0, 2)),
|
|
"Concrete ML only supports padding along the width & height dimensions, padding"
|
|
f" requested was {pads}",
|
|
)
|
|
|
|
assert_true(pads.size % 2 == 0) # must be a multiple of 2
|
|
assert_true(pads.size in [4, 8], "Only supporting pads of size 4 or 8.")
|
|
if pads.size == 8:
|
|
pads = numpy.asarray([pads[2], pads[3], pads[6], pads[7]])
|
|
|
|
assert_true(pads.size == 4, "Not currently supporting padding of 3D tensors")
|
|
|
|
pad_value = 0 if prepared_inputs[2] is None else prepared_inputs[2]
|
|
assert_true(pad_value == 0, "Concrete ML only supports padding with constant zero values")
|
|
|
|
assert q_input.quantizer.zero_point is not None
|
|
q_input_pad = numpy_onnx_pad(q_input.qvalues, pads, q_input.quantizer.zero_point, True)
|
|
|
|
return QuantizedArray(
|
|
q_input.quantizer.n_bits,
|
|
q_input_pad,
|
|
value_is_float=False,
|
|
options=q_input.quantizer.quant_options,
|
|
stats=q_input.quantizer.quant_stats,
|
|
params=q_input.quantizer.quant_params,
|
|
)
|
|
|
|
def can_fuse(self) -> bool:
|
|
"""Determine if this op can be fused.
|
|
|
|
Pad operation cannot be fused since it must be performed over integer tensors.
|
|
|
|
Returns:
|
|
bool: False, this operation cannot be fused as it is manipulates integer tensors
|
|
"""
|
|
return False
|
|
|
|
|
|
class QuantizedWhere(QuantizedOp):
|
|
"""Where operator on quantized arrays.
|
|
|
|
Supports only constants for the results produced on the True/False branches.
|
|
"""
|
|
|
|
_impl_for_op_named: str = "Where"
|
|
|
|
# This op takes a single variable input, so we can set int_input_names to a default id
|
|
def __init__(
|
|
self,
|
|
n_bits_output: int,
|
|
op_instance_name: str,
|
|
int_input_names: Set[str] = None,
|
|
constant_inputs: Optional[Union[Dict[str, Any], Dict[int, Any]]] = None,
|
|
input_quant_opts: QuantizationOptions = None,
|
|
**attrs,
|
|
) -> None:
|
|
super().__init__(
|
|
n_bits_output,
|
|
op_instance_name,
|
|
int_input_names,
|
|
constant_inputs,
|
|
input_quant_opts,
|
|
**attrs,
|
|
)
|
|
|
|
# Remember that there are examples of where with more than 1 variable which are going to be
|
|
# well managed, thanks to fusing:
|
|
# Act(x) = (x < 3) ? x + 42 : x ^ 42 would perfectly fuse
|
|
# But for this kind of example, we have int_input_names = {'x'} since they all depend only
|
|
# on x
|
|
assert_true(
|
|
self._int_input_names is not None and len(self._int_input_names) == 1,
|
|
"internal issue: "
|
|
+ f"{self._int_input_names} "
|
|
+ "{len(self._int_input_names) if self._int_input_names is not None else -1}",
|
|
)
|
|
|
|
|
|
class QuantizedCast(QuantizedOp):
|
|
"""Cast the input to the required data type.
|
|
|
|
In FHE we only support a limited number of output types. Booleans are cast to integers.
|
|
"""
|
|
|
|
_impl_for_op_named: str = "Cast"
|
|
|
|
|
|
class QuantizedGreater(QuantizedOp):
|
|
"""Comparison operator >.
|
|
|
|
Only supports comparison with a constant.
|
|
"""
|
|
|
|
_impl_for_op_named: str = "Greater"
|
|
|
|
# Since this op takes a single variable input, we can set int_input_names to a single default id
|
|
def __init__(
|
|
self,
|
|
n_bits_output: int,
|
|
op_instance_name: str,
|
|
int_input_names: Set[str] = None,
|
|
constant_inputs: Optional[Union[Dict[str, Any], Dict[int, Any]]] = None,
|
|
input_quant_opts: QuantizationOptions = None,
|
|
**attrs,
|
|
) -> None:
|
|
super().__init__(
|
|
n_bits_output,
|
|
op_instance_name,
|
|
int_input_names,
|
|
constant_inputs,
|
|
input_quant_opts,
|
|
**attrs,
|
|
)
|
|
|
|
# We do not support testing a > b where a,b are encrypted
|
|
# only comparing to a constant is supported
|
|
assert_true(constant_inputs is not None and len(constant_inputs) >= 1)
|
|
|
|
|
|
class QuantizedGreaterOrEqual(QuantizedOp):
|
|
"""Comparison operator >=.
|
|
|
|
Only supports comparison with a constant.
|
|
"""
|
|
|
|
_impl_for_op_named: str = "GreaterOrEqual"
|
|
|
|
# Since this op takes a single variable input, we can set int_input_names to a single default id
|
|
def __init__(
|
|
self,
|
|
n_bits_output: int,
|
|
op_instance_name: str,
|
|
int_input_names: Set[str] = None,
|
|
constant_inputs: Optional[Union[Dict[str, Any], Dict[int, Any]]] = None,
|
|
input_quant_opts: QuantizationOptions = None,
|
|
**attrs,
|
|
) -> None:
|
|
super().__init__(
|
|
n_bits_output,
|
|
op_instance_name,
|
|
int_input_names,
|
|
constant_inputs,
|
|
input_quant_opts,
|
|
**attrs,
|
|
)
|
|
|
|
# We do not support testing a >= b where a,b are encrypted
|
|
# only comparing to a constant is supported
|
|
assert_true(constant_inputs is not None and len(constant_inputs) >= 1)
|
|
|
|
|
|
class QuantizedLess(QuantizedOp):
|
|
"""Comparison operator <.
|
|
|
|
Only supports comparison with a constant.
|
|
"""
|
|
|
|
_impl_for_op_named: str = "Less"
|
|
|
|
# Since this op takes a single variable input, we can set int_input_names to a single default id
|
|
def __init__(
|
|
self,
|
|
n_bits_output: int,
|
|
op_instance_name: str,
|
|
int_input_names: Set[str] = None,
|
|
constant_inputs: Optional[Union[Dict[str, Any], Dict[int, Any]]] = None,
|
|
input_quant_opts: QuantizationOptions = None,
|
|
**attrs,
|
|
) -> None:
|
|
super().__init__(
|
|
n_bits_output,
|
|
op_instance_name,
|
|
int_input_names,
|
|
constant_inputs,
|
|
input_quant_opts,
|
|
**attrs,
|
|
)
|
|
|
|
# We do not support testing a < b where a,b are encrypted
|
|
# only comparing to a constant is supported
|
|
assert_true(constant_inputs is not None and len(constant_inputs) >= 1)
|
|
|
|
|
|
class QuantizedLessOrEqual(QuantizedOp):
|
|
"""Comparison operator <=.
|
|
|
|
Only supports comparison with a constant.
|
|
"""
|
|
|
|
_impl_for_op_named: str = "LessOrEqual"
|
|
|
|
# Since this op takes a single variable input, we can set int_input_names to a single default id
|
|
def __init__(
|
|
self,
|
|
n_bits_output: int,
|
|
op_instance_name: str,
|
|
int_input_names: Set[str] = None,
|
|
constant_inputs: Optional[Union[Dict[str, Any], Dict[int, Any]]] = None,
|
|
input_quant_opts: QuantizationOptions = None,
|
|
**attrs,
|
|
) -> None:
|
|
super().__init__(
|
|
n_bits_output,
|
|
op_instance_name,
|
|
int_input_names,
|
|
constant_inputs,
|
|
input_quant_opts,
|
|
**attrs,
|
|
)
|
|
|
|
# We do not support testing a <= b where a,b are encrypted
|
|
# only comparing to a constant is supported
|
|
assert_true(constant_inputs is not None and len(constant_inputs) >= 1)
|
|
|
|
|
|
class QuantizedOr(QuantizedOpUnivariateOfEncrypted, QuantizedOp):
|
|
"""Or operator ||.
|
|
|
|
This operation is not really working as a quantized operation. It just works when things got
|
|
fused, as in e.g., Act(x) = x || (x + 42))
|
|
"""
|
|
|
|
_impl_for_op_named: str = "Or"
|
|
|
|
|
|
class QuantizedDiv(QuantizedMixingOp):
|
|
"""Quantized Division operator.
|
|
|
|
Can divide either two variables (both encrypted) or a variable and a constant
|
|
"""
|
|
|
|
_impl_for_op_named: str = "Div"
|
|
|
|
def __init__(
|
|
self,
|
|
*args,
|
|
rounding_threshold_bits: Union[None, int, Dict[str, Union[str, int]]] = None,
|
|
**kwargs,
|
|
) -> None:
|
|
super().__init__(*args, rounding_threshold_bits=rounding_threshold_bits, **kwargs)
|
|
self.divider_quantizer: Optional[UniformQuantizer] = None
|
|
self.min_non_zero_value: Optional[numpy.float64] = None
|
|
|
|
def calibrate(self, *inputs: Union[QuantizedArray, numpy.ndarray]) -> numpy.ndarray:
|
|
"""Create corresponding QuantizedArray for the output of the activation function.
|
|
|
|
Args:
|
|
*inputs (Union[QuantizedArray, numpy.ndarray]): Calibration sample inputs.
|
|
|
|
Returns:
|
|
numpy.ndarray: the output values for the provided calibration samples.
|
|
"""
|
|
|
|
# If the op can not be fused and that the two inputs are not constants
|
|
# we need to compute the quantizer of the divider since we are doing
|
|
# an encrypted division where both numerator and denominator are encrypted
|
|
if not self.can_fuse() and len(inputs) == 2:
|
|
assert isinstance(
|
|
inputs[0], numpy.ndarray
|
|
), "Div calibrate does not support analytical calibration for now"
|
|
assert isinstance(inputs[1], numpy.ndarray)
|
|
|
|
# FIXME https://github.com/luxfi/luxfhe-ml-internal/issues/4556
|
|
min_non_zero_index = numpy.abs(inputs[1]).argmin(axis=None)
|
|
min_non_zero_value = inputs[1].flat[min_non_zero_index]
|
|
|
|
# mypy
|
|
assert min_non_zero_value is not None and min_non_zero_value != 0
|
|
self.min_non_zero_value = min_non_zero_value
|
|
|
|
q_array_divider = QuantizedArray(self.n_bits, 1 / inputs[1])
|
|
|
|
# Store the quantizer of the divider
|
|
self.divider_quantizer = q_array_divider.quantizer
|
|
|
|
return super().calibrate(*inputs)
|
|
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
calibrate_rounding: bool = False,
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
|
|
# If the op can be fused we perform the op in the clear
|
|
if self.can_fuse():
|
|
return super().q_impl(*q_inputs, **attrs)
|
|
|
|
# For mypy
|
|
assert self.output_quant_params is not None
|
|
assert self.output_quant_params.scale is not None
|
|
assert self.output_quant_params.zero_point is not None
|
|
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=True
|
|
)
|
|
|
|
q_input_0: QuantizedArray = prepared_inputs[0]
|
|
q_input_1: QuantizedArray = prepared_inputs[1]
|
|
|
|
assert q_input_0.quantizer.scale is not None
|
|
assert q_input_0.quantizer.zero_point is not None
|
|
|
|
assert q_input_1.quantizer.scale is not None
|
|
assert q_input_1.quantizer.zero_point is not None
|
|
|
|
# Dequantize
|
|
input_1 = q_input_1.dequant()
|
|
|
|
# Replace input_1 with min_non_zero_value if input_1 is 0
|
|
# mypy
|
|
assert self.min_non_zero_value is not None
|
|
input_1 = numpy.where(input_1 == 0, self.min_non_zero_value, input_1)
|
|
|
|
# Compute the inverse of input_1
|
|
input_1_inv = 1.0 / input_1
|
|
|
|
# Re-quantize the inverse using the same quantization parameters as q_input_1
|
|
# mypy
|
|
assert self.divider_quantizer is not None
|
|
# FIXME https://github.com/luxfi/luxfhe-ml-internal/issues/4556
|
|
q_input_1_inv_rescaled = self.divider_quantizer.quant(input_1_inv)
|
|
|
|
# The product of quantized encrypted integer values
|
|
product_q_values = q_input_0.qvalues * q_input_1_inv_rescaled
|
|
|
|
# mypy
|
|
assert q_input_0.quantizer.zero_point is not None
|
|
assert q_input_1.quantizer.zero_point is not None
|
|
assert self.divider_quantizer.zero_point is not None
|
|
|
|
# Integer quantized multiplication need adjustment based on the zero points.
|
|
if q_input_0.quantizer.zero_point:
|
|
product_q_values -= q_input_0.quantizer.zero_point * (
|
|
q_input_1_inv_rescaled - self.divider_quantizer.zero_point
|
|
)
|
|
if self.divider_quantizer.zero_point:
|
|
product_q_values -= self.divider_quantizer.zero_point * (
|
|
q_input_0.qvalues - q_input_0.quantizer.zero_point
|
|
)
|
|
|
|
# mypy
|
|
assert self.divider_quantizer.scale is not None
|
|
assert self.divider_quantizer.zero_point is not None
|
|
|
|
# Compute the scale and zero point based on the scale and zero point
|
|
# of the two quantized values multiplied together
|
|
new_scale = q_input_0.quantizer.scale * self.divider_quantizer.scale
|
|
new_zero_point = q_input_0.quantizer.zero_point * self.divider_quantizer.zero_point
|
|
|
|
if self.produces_graph_output:
|
|
return self.make_output_quant_parameters(product_q_values, new_scale, new_zero_point)
|
|
|
|
with tag(self.op_instance_name + ".rounding"):
|
|
# Apply Concrete rounding (if relevant)
|
|
product_q_values = self.cnp_round(product_q_values, calibrate_rounding)
|
|
|
|
# FIXME: https://github.com/luxfi/luxfhe-ml-internal/issues/4546
|
|
# De-quantize the product
|
|
dequant_product = (product_q_values - new_zero_point) * new_scale
|
|
|
|
# Return the raw float values without re-quantizing them to the new scale, as any
|
|
# following Gemm/Add/Conv will quantize them with _prepare_inputs_with_constants(...)
|
|
return QuantizedArray(
|
|
self.n_bits,
|
|
dequant_product,
|
|
value_is_float=True,
|
|
options=self._get_output_quant_opts(),
|
|
stats=self.output_quant_stats,
|
|
params=self.output_quant_params,
|
|
)
|
|
|
|
def can_fuse(self) -> bool:
|
|
"""Determine if this op can be fused.
|
|
|
|
Div operation can be computed in float and fused if it operates over inputs produced
|
|
by a single integer tensor.
|
|
|
|
Returns:
|
|
bool: Whether the number of integer input tensors allows computing this op as a TLU
|
|
"""
|
|
|
|
return len(self._int_input_names) == 1
|
|
|
|
|
|
class QuantizedMul(QuantizedMixingOp):
|
|
"""Quantized Multiplication operator.
|
|
|
|
Can multiply either two variables (both encrypted) or a variable and a constant
|
|
"""
|
|
|
|
_impl_for_op_named: str = "Mul"
|
|
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
calibrate_rounding: bool = False,
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
|
|
# If either input is a RawOpOutput or if the op can be fused,
|
|
# perform the op in the TLU using FP32
|
|
if (
|
|
len(q_inputs) == 1
|
|
or isinstance(q_inputs[0], RawOpOutput)
|
|
or isinstance(q_inputs[1], RawOpOutput)
|
|
or self.can_fuse()
|
|
):
|
|
return super().q_impl(*q_inputs, **attrs)
|
|
|
|
# For mypy
|
|
assert self.output_quant_params is not None
|
|
assert self.output_quant_params.scale is not None
|
|
assert self.output_quant_params.zero_point is not None
|
|
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=True
|
|
)
|
|
|
|
q_input_0: QuantizedArray = prepared_inputs[0]
|
|
q_input_1: QuantizedArray = prepared_inputs[1]
|
|
|
|
assert q_input_0.quantizer.scale is not None
|
|
assert q_input_0.quantizer.zero_point is not None
|
|
|
|
assert q_input_1.quantizer.scale is not None
|
|
assert q_input_1.quantizer.zero_point is not None
|
|
|
|
# Remove the manual encrypted multiplication when we
|
|
# can handle input precision with rounding
|
|
# FIXME: https://github.com/luxfi/luxfhe-ml-internal/issues/4127
|
|
@univariate
|
|
def copy_function(x):
|
|
return x
|
|
|
|
input0_q_values = q_input_0.qvalues
|
|
input1_q_values = q_input_1.qvalues
|
|
|
|
with tag(self.op_instance_name + ".enc_mul_rounding_input0"):
|
|
input0_q_values = self.cnp_round(
|
|
input0_q_values, calibrate_rounding, rounding_operation_id="input0_q_values_copy"
|
|
)
|
|
|
|
with tag(self.op_instance_name + ".enc_mul_rounding_input1"):
|
|
input1_q_values = self.cnp_round(
|
|
input1_q_values, calibrate_rounding, rounding_operation_id="input1_q_values_copy"
|
|
)
|
|
|
|
input0_q_values = copy_function(input0_q_values)
|
|
input1_q_values = copy_function(input1_q_values)
|
|
|
|
# The product of quantized encrypted integer values
|
|
product_q_values = input0_q_values * input1_q_values
|
|
|
|
# Integer quantized multiplication need adjustment based on the zero points.
|
|
if q_input_0.quantizer.zero_point:
|
|
product_q_values -= q_input_0.quantizer.zero_point * input1_q_values
|
|
if q_input_1.quantizer.zero_point:
|
|
product_q_values -= q_input_1.quantizer.zero_point * input0_q_values
|
|
|
|
# Compute the scale and zero point based on the scale and zero point
|
|
# of the two quantized values multiplied together
|
|
new_scale = q_input_0.quantizer.scale * q_input_1.quantizer.scale
|
|
new_zero_point = q_input_0.quantizer.zero_point * q_input_1.quantizer.zero_point
|
|
|
|
if self.produces_graph_output:
|
|
return self.make_output_quant_parameters(product_q_values, new_scale, new_zero_point)
|
|
|
|
with tag(self.op_instance_name + ".rounding"):
|
|
# Apply Concrete rounding (if relevant)
|
|
product_q_values = self.cnp_round(
|
|
product_q_values, calibrate_rounding, rounding_operation_id="product_q_values"
|
|
)
|
|
|
|
# De-quantize the product
|
|
dequant_product = (product_q_values + new_zero_point) * new_scale
|
|
|
|
# Return the raw float values without re-quantizing them to the new scale, as any
|
|
# following Gemm/Add/Conv will quantize them with _prepare_inputs_with_constants(...)
|
|
return QuantizedArray(
|
|
self.n_bits,
|
|
dequant_product,
|
|
value_is_float=True,
|
|
options=self._get_output_quant_opts(),
|
|
stats=self.output_quant_stats,
|
|
params=self.output_quant_params,
|
|
)
|
|
|
|
def can_fuse(self) -> bool:
|
|
"""Determine if this op can be fused.
|
|
|
|
Mul operation can be computed in float and fused if it operates over inputs produced
|
|
by a single integer tensor.
|
|
|
|
Returns:
|
|
bool: Whether the number of integer input tensors allows computing this op as a TLU
|
|
"""
|
|
|
|
return len(self._int_input_names) == 1
|
|
|
|
|
|
class QuantizedSub(QuantizedAdd):
|
|
"""Subtraction operator.
|
|
|
|
This works the same as addition on both encrypted - encrypted and on encrypted - constant.
|
|
"""
|
|
|
|
_impl_for_op_named: str = "Sub"
|
|
b_sign: int = -1
|
|
|
|
|
|
class QuantizedBatchNormalization(QuantizedOp):
|
|
"""Quantized Batch normalization with encrypted input and in-the-clear normalization params."""
|
|
|
|
_impl_for_op_named: str = "BatchNormalization"
|
|
|
|
def calibrate(self, *inputs: Union[QuantizedArray, numpy.ndarray]) -> numpy.ndarray:
|
|
"""Create corresponding QuantizedArray for the output of the activation function.
|
|
|
|
Args:
|
|
*inputs (numpy.ndarray): Calibration sample inputs.
|
|
|
|
Returns:
|
|
numpy.ndarray: the output values for the provided calibration samples.
|
|
"""
|
|
|
|
assert all(
|
|
isinstance(inp, numpy.ndarray) for inp in inputs
|
|
), "Batch Normalization calibrate does not support analytical calibration"
|
|
|
|
# Here we need the actual values of the constants, we need to pass through
|
|
# the numpy.ndarrays in the computation graph
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*inputs, calibrate=True, quantize_actual_values=False
|
|
)
|
|
|
|
raw_result = self.call_impl(*prepared_inputs, **self.attrs)
|
|
|
|
if not isinstance(raw_result, RawOpOutput):
|
|
# Check if batch normalization is applied per channel
|
|
scale = self.constant_inputs[self._params_name_to_input_idx["scale"]].values
|
|
bias = self.constant_inputs[self._params_name_to_input_idx["bias"]].values
|
|
if scale.size > 1 or bias.size > 1:
|
|
# Per channel batchnorm struggles with low bit-width quantization.
|
|
# Batchnorm parameters (scale/bias/mean/var) can vary significantly
|
|
# between channels. Our tensor-based quantization, however, only supports
|
|
# a global scale and offset. Errors in quantized values can thus be severe.
|
|
# To mitigate this, we use percentiles to clip extreme values.
|
|
|
|
lower_bound = numpy.percentile(raw_result, 0.1)
|
|
upper_bound = numpy.percentile(raw_result, 99.9)
|
|
|
|
raw_result = numpy.clip(raw_result, lower_bound, upper_bound)
|
|
|
|
quantized_samples = QuantizedArray(self.n_bits, raw_result)
|
|
|
|
self.output_quant_params = quantized_samples.quantizer.quant_params
|
|
self.output_quant_stats = quantized_samples.quantizer.quant_stats
|
|
|
|
raw_result = quantized_samples.values
|
|
|
|
return raw_result
|
|
|
|
|
|
class QuantizedFlatten(QuantizedOp):
|
|
"""Quantized flatten for encrypted inputs."""
|
|
|
|
_impl_for_op_named: str = "Flatten"
|
|
quantize_inputs_with_model_outputs_precision = True
|
|
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
"""Flatten the input integer encrypted tensor.
|
|
|
|
Args:
|
|
q_inputs: an encrypted integer tensor at index 0
|
|
attrs: contains axis attribute
|
|
|
|
Returns:
|
|
result (QuantizedArray): reshaped encrypted integer tensor
|
|
"""
|
|
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=True
|
|
)
|
|
|
|
# This op takes only encrypted inputs in the form of QuantizedArray
|
|
assert isinstance(q_inputs[0], QuantizedArray)
|
|
|
|
assert_true(len(q_inputs) == 1, "Flatten operator only takes a single input")
|
|
|
|
axis = attrs["axis"]
|
|
newshape: Sequence[SupportsIndex]
|
|
newshape = (
|
|
*q_inputs[0].qvalues.shape[0:axis],
|
|
numpy.prod(q_inputs[0].qvalues.shape[axis:]),
|
|
)
|
|
|
|
# Return a new quantized array with the same quantization parameters
|
|
return QuantizedArray(
|
|
prepared_inputs[0].quantizer.n_bits,
|
|
numpy.reshape(prepared_inputs[0].qvalues, newshape),
|
|
value_is_float=False,
|
|
options=prepared_inputs[0].quantizer.quant_options,
|
|
stats=prepared_inputs[0].quantizer.quant_stats,
|
|
params=prepared_inputs[0].quantizer.quant_params,
|
|
)
|
|
|
|
def can_fuse(self) -> bool:
|
|
"""Determine if this op can be fused.
|
|
|
|
Flatten operation cannot be fused since it must be performed over integer tensors.
|
|
|
|
Returns:
|
|
bool: False, this operation cannot be fused as it is manipulates integer tensors.
|
|
"""
|
|
|
|
return False
|
|
|
|
|
|
class QuantizedReduceSum(QuantizedMixingOp):
|
|
"""ReduceSum with encrypted input."""
|
|
|
|
_impl_for_op_named: str = "ReduceSum"
|
|
|
|
def __init__(
|
|
self,
|
|
n_bits_output: int,
|
|
op_instance_name: str,
|
|
int_input_names: Set[str] = None,
|
|
constant_inputs: Optional[Union[Dict[str, Any], Dict[int, Any]]] = None,
|
|
input_quant_opts: Optional[QuantizationOptions] = None,
|
|
**attrs,
|
|
) -> None:
|
|
"""Construct the quantized ReduceSum operator and retrieve parameters.
|
|
|
|
Args:
|
|
n_bits_output (int): Number of bits for the operator's quantization of outputs.
|
|
op_instance_name (str): The name that should be assigned to this operation, used
|
|
to retrieve it later or get debugging information about this op (bit-width, value
|
|
range, integer intermediary values, op-specific error messages). Usually this name
|
|
is the same as the ONNX operation name for which this operation is constructed.
|
|
int_input_names (Optional[Set[str]]): Names of input integer tensors. Default to None.
|
|
constant_inputs (Optional[Dict]): Input constant tensor.
|
|
axes (Optional[numpy.ndarray]): Array of integers along which to reduce.
|
|
The default is to reduce over all the dimensions of the input tensor if
|
|
'noop_with_empty_axes' is false, else act as an Identity op when
|
|
'noop_with_empty_axes' is true. Accepted range is [-r, r-1] where
|
|
r = rank(data). Default to None.
|
|
input_quant_opts (Optional[QuantizationOptions]): Options for the input quantizer.
|
|
Default to None.
|
|
attrs (dict): RecuseSum options.
|
|
keepdims (int): Keep the reduced dimension or not, 1 means keeping the
|
|
input dimension, 0 will reduce it along the given axis. Default to 1.
|
|
noop_with_empty_axes (int): Defines behavior if 'axes' is empty or set to None.
|
|
Default behavior with 0 is to reduce all axes. When axes is empty and this
|
|
attribute is set to true 1, input tensor will not be reduced, and the output
|
|
tensor would be equivalent to input tensor. Default to 0.
|
|
"""
|
|
super().__init__(
|
|
n_bits_output,
|
|
op_instance_name,
|
|
int_input_names,
|
|
constant_inputs,
|
|
input_quant_opts,
|
|
**attrs,
|
|
)
|
|
|
|
# Retrieve and set the ONNX parameters
|
|
# Numpy's keepdims parameter is a boolean while ONNX's one is an int (0 or 1). Even though
|
|
# Python handles them equivalently, we need to manually convert it as mypy doesn't accept
|
|
# this type difference
|
|
self.keepdims = bool(attrs.get("keepdims", 1))
|
|
self.noop_with_empty_axes = attrs.get("noop_with_empty_axes", 0)
|
|
self.copy_inputs = False
|
|
|
|
def calibrate(self, *inputs: Union[QuantizedArray, numpy.ndarray]) -> numpy.ndarray:
|
|
"""Create corresponding QuantizedArray for the output of the activation function.
|
|
|
|
Args:
|
|
*inputs (numpy.ndarray): Calibration sample inputs.
|
|
|
|
Returns:
|
|
numpy.ndarray: The output values for the provided calibration samples.
|
|
"""
|
|
# Here we need the actual values of the constants, we need to pass through
|
|
# the numpy.ndarrays in the computation graph
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*inputs, calibrate=True, quantize_actual_values=False
|
|
)
|
|
|
|
quantized_samples = QuantizedArray(
|
|
self.n_bits, self.call_impl(*prepared_inputs, **self.attrs)
|
|
)
|
|
|
|
self.output_quant_params = quantized_samples.quantizer.quant_params
|
|
self.output_quant_stats = quantized_samples.quantizer.quant_stats
|
|
|
|
return quantized_samples.values
|
|
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
calibrate_rounding: bool = False,
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
"""Sum the encrypted tensor's values along the given axes.
|
|
|
|
Args:
|
|
q_inputs (QuantizedArray): An encrypted integer tensor at index 0.
|
|
calibrate_rounding (bool): Whether to calibrate rounding or not.
|
|
attrs (Dict): Options are handled in constructor.
|
|
|
|
Returns:
|
|
(QuantizedArray): The sum of all values along the given axes.
|
|
"""
|
|
# Retrieve the quantized inputs as well as the function's constant parameters
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=True
|
|
)
|
|
|
|
assert_true(
|
|
isinstance(prepared_inputs[0], (QuantizedArray)),
|
|
"Prepared inputs' values should be found in a QuantizedArray. "
|
|
f"Got {type(prepared_inputs[0])}",
|
|
)
|
|
|
|
assert_true(
|
|
isinstance(prepared_inputs[1], numpy.ndarray),
|
|
"ReduceSum axis parameter should be a Numpy array. " f"Got {type(prepared_inputs[1])}",
|
|
)
|
|
|
|
# Retrieve the quantized values and the axes to sum over
|
|
# Parameter "axis" Numpy's sum operator has to be a tuple (and not an array)
|
|
q_values = prepared_inputs[0].qvalues
|
|
axes = tuple(prepared_inputs[1])
|
|
|
|
assert_true(
|
|
0 not in axes,
|
|
"ReduceSum axis parameter should not contain axis 0 as it is used for batching "
|
|
f"the inference. Got {axes}",
|
|
)
|
|
|
|
with tag(self.op_instance_name):
|
|
|
|
# Need to copy to prevent the following sum to raise precision of the input
|
|
# FIXME: https://github.com/luxfi/luxfhe-ml-internal/issues/4127
|
|
@univariate
|
|
def copy_function(x):
|
|
return x
|
|
|
|
if self.copy_inputs:
|
|
q_values = copy_function(q_values)
|
|
|
|
# Sum all the quantized values
|
|
q_sum = numpy.sum(q_values, axis=axes, keepdims=self.keepdims)
|
|
|
|
# Determining the number of output zero_points to use for de-quantization with
|
|
# the total number of elements summed all together, which is the product of all
|
|
# the number of elements found within the given axes
|
|
n_elem = numpy.prod([q_values.shape[axis] for axis in axes])
|
|
|
|
# Determining the output scale and zero_point
|
|
input_quantizer = prepared_inputs[0].quantizer
|
|
scale = input_quantizer.scale
|
|
zero_point = n_elem * input_quantizer.zero_point
|
|
|
|
# If this operator is the graph's last operator,
|
|
# there's is no need to created additional TLUs
|
|
if self.produces_graph_output:
|
|
return self.make_output_quant_parameters(q_sum, scale, zero_point)
|
|
|
|
f_substract = q_sum - zero_point
|
|
|
|
with tag(self.op_instance_name + ".rounding"):
|
|
# Apply Concrete rounding (if relevant)
|
|
f_substract = self.cnp_round(f_substract, calibrate_rounding)
|
|
|
|
# De-quantize the sum
|
|
f_sum = scale * f_substract
|
|
|
|
sum_qarray = QuantizedArray(
|
|
self.n_bits,
|
|
f_sum,
|
|
value_is_float=True,
|
|
options=self._get_output_quant_opts(),
|
|
stats=self.output_quant_stats,
|
|
params=self.output_quant_params,
|
|
)
|
|
|
|
return sum_qarray
|
|
|
|
|
|
class QuantizedErf(QuantizedOp):
|
|
"""Quantized erf op."""
|
|
|
|
_impl_for_op_named: str = "Erf"
|
|
|
|
|
|
class QuantizedNot(QuantizedOp):
|
|
"""Quantized Not op."""
|
|
|
|
_impl_for_op_named: str = "Not"
|
|
|
|
|
|
class QuantizedBrevitasQuant(QuantizedOp):
|
|
"""Brevitas uniform quantization with encrypted input."""
|
|
|
|
_impl_for_op_named: str = "onnx.brevitas.Quant"
|
|
# Note that this should be reset when the correctness test that finds
|
|
# all mismatches between Concrete ML and Brevitas is fixed
|
|
# FIXME: https://github.com/luxfi/luxfhe-ml-internal/issues/2373
|
|
quantize_inputs_with_model_outputs_precision = True
|
|
output_quant_opts: QuantizationOptions
|
|
|
|
def __init__(
|
|
self,
|
|
n_bits_output: int,
|
|
op_instance_name: str,
|
|
int_input_names: Set[str] = None,
|
|
constant_inputs: Optional[Union[Dict[str, Any], Dict[int, Any]]] = None,
|
|
input_quant_opts: Optional[QuantizationOptions] = None,
|
|
**attrs,
|
|
) -> None:
|
|
"""Construct the Brevitas quantization operator.
|
|
|
|
Args:
|
|
n_bits_output (int): Number of bits for the operator's quantization of outputs.
|
|
Not used, will be overridden by the bit_width in ONNX
|
|
op_instance_name (str): The name that should be assigned to this operation, used
|
|
to retrieve it later or get debugging information about this op (bit-width, value
|
|
range, integer intermediary values, op-specific error messages). Usually this name
|
|
is the same as the ONNX operation name for which this operation is constructed.
|
|
int_input_names (Optional[Set[str]]): Names of input integer tensors. Default to None.
|
|
constant_inputs (Optional[Dict]): Input constant tensor.
|
|
scale (float): Quantizer scale
|
|
zero_point (float): Quantizer zero-point
|
|
bit_width (int): Number of bits of the integer representation
|
|
input_quant_opts (Optional[QuantizationOptions]): Options for the input quantizer.
|
|
Default to None.
|
|
attrs (dict):
|
|
rounding_mode (str): Rounding mode (default and only accepted option is "ROUND")
|
|
signed (int): Whether this op quantizes to signed integers (default 1),
|
|
narrow (int): Whether this op quantizes to a narrow range of integers
|
|
e.g., [-2**n_bits-1 .. 2**n_bits-1] (default 0),
|
|
"""
|
|
|
|
super().__init__(
|
|
n_bits_output,
|
|
op_instance_name,
|
|
int_input_names,
|
|
constant_inputs,
|
|
input_quant_opts,
|
|
**attrs,
|
|
)
|
|
|
|
def check_float(v, err_msg):
|
|
assert_true(
|
|
isinstance(v, float)
|
|
or (isinstance(v, numpy.ndarray) and numpy.issubdtype(v.dtype, numpy.floating)),
|
|
err_msg,
|
|
)
|
|
|
|
assert_true(
|
|
attrs["rounding_mode"] == "ROUND",
|
|
"Only rounding quantization is supported for Brevitas",
|
|
)
|
|
assert_true(
|
|
int(attrs["signed"]) in {0, 1},
|
|
"Signed flag in Brevitas quantizer must be 0/1",
|
|
)
|
|
assert_true(
|
|
int(attrs["narrow"]) in {0, 1},
|
|
"Narrow range flag in Brevitas quantizer must be 0/1",
|
|
)
|
|
|
|
self.is_signed = bool(attrs["signed"])
|
|
self.is_narrow = bool(attrs["narrow"])
|
|
|
|
# FIXME: https://github.com/luxfi/luxfhe-ml-internal/issues/4544
|
|
# Remove this workaround when brevitas export is fixed
|
|
if self.is_signed is False and self.is_narrow is True:
|
|
self.is_signed = True
|
|
self.is_narrow = False
|
|
|
|
assert_false(
|
|
not self.is_signed and self.is_narrow,
|
|
"Can not use narrow range for non-signed Brevitas quantizers",
|
|
)
|
|
|
|
# To ensure de-quantization produces floats, the following parameters must be float.
|
|
# This should be default export setting in Brevitas
|
|
check_float(
|
|
self.constant_inputs is not None
|
|
and self.constant_inputs[self._params_name_to_input_idx["scale"]],
|
|
"Scale of Brevitas Quant op must be float",
|
|
)
|
|
check_float(
|
|
self.constant_inputs is not None
|
|
and self.constant_inputs[self._params_name_to_input_idx["zero_point"]],
|
|
"Zero Point of Brevitas Quant op must be float",
|
|
)
|
|
|
|
# For mypy
|
|
assert self.constant_inputs is not None
|
|
|
|
# The constant inputs can have either int or str keys, here it is an int
|
|
n_bits = self.constant_inputs[3] # type: ignore
|
|
|
|
# Set the QAT flag on the output of this operation, so that the
|
|
# following operation in the graph is aware of this flag and can
|
|
# just copy the quantization
|
|
self.output_quant_opts = self._get_output_quant_opts()
|
|
self.output_quant_opts.n_bits = n_bits
|
|
self.output_quant_opts.is_qat = True
|
|
|
|
# Disable the QAT value checker as we have the true parameters in ONNX
|
|
# Brevitas quantization layers store the scale/zero_point in the ONNX file
|
|
# so we don't need to compute/infer them
|
|
self.output_quant_opts.is_precomputed_qat = True
|
|
self.output_quant_opts.is_narrow = self.is_narrow
|
|
self.output_quant_opts.is_signed = self.is_signed
|
|
|
|
def calibrate(self, *inputs: Union[QuantizedArray, numpy.ndarray]) -> numpy.ndarray:
|
|
"""Create corresponding QuantizedArray for the output of Quantization function.
|
|
|
|
Args:
|
|
*inputs (numpy.ndarray): Calibration sample inputs.
|
|
|
|
Returns:
|
|
numpy.ndarray: the output values for the provided calibration samples.
|
|
"""
|
|
|
|
result = super().calibrate(*inputs)
|
|
|
|
# Override the output quantization params with
|
|
# those stored in the ONNX model. This allows the quantized module to
|
|
# pass these parameters to the module's input quantizer
|
|
|
|
n_bits = int(self.constant_inputs[3])
|
|
|
|
self.output_quant_params = UniformQuantizationParameters(
|
|
scale=numpy.float64(self.constant_inputs[1]),
|
|
zero_point=int(self.constant_inputs[2]),
|
|
offset=2 ** (n_bits - 1) if self.is_signed else 0,
|
|
)
|
|
|
|
return result
|
|
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
"""Quantize values.
|
|
|
|
Args:
|
|
q_inputs: an encrypted integer tensor at index 0,
|
|
scale, zero_point, n_bits at indices 1,2,3
|
|
attrs: additional optional attributes
|
|
|
|
Returns:
|
|
result (QuantizedArray): reshaped encrypted integer tensor
|
|
"""
|
|
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=False
|
|
)
|
|
|
|
# This op takes only encrypted inputs in the form of QuantizedArray
|
|
assert isinstance(q_inputs[0], QuantizedArray)
|
|
|
|
# Impose the number of bits that this op quantizes its inputs, to the value that
|
|
# was provided by Brevitas. This number of bits is normally
|
|
# different from the op's n_bits. The op will usually have an output number of bits
|
|
# that is suitable for network output, as the op could be the last op in the graph
|
|
|
|
# Thus, we enforce the QAT number of bits on this op's output
|
|
n_bits = int(prepared_inputs[3])
|
|
|
|
assert len(q_inputs) >= 1
|
|
|
|
assert_true(
|
|
self.output_quant_params is not None,
|
|
"You need to calibrate this op before using it",
|
|
)
|
|
|
|
# For mypy
|
|
assert self.output_quant_params is not None
|
|
assert self.output_quant_params.scale is not None
|
|
assert self.output_quant_params.zero_point is not None
|
|
|
|
# Pass-through when the input is already quantized in the manner that this
|
|
# layer wants to quantize it. This is done to optimize out the model input TLU.
|
|
# Only works when the layer has been calibrated.
|
|
if (
|
|
q_inputs[0].quantizer.quant_options.is_equal(self.output_quant_opts)
|
|
and q_inputs[0].quantizer.quant_params.scale == self.output_quant_params.scale
|
|
and q_inputs[0].quantizer.quant_params.zero_point == self.output_quant_params.zero_point
|
|
):
|
|
return q_inputs[0]
|
|
|
|
f_outputs = self.call_impl(*prepared_inputs, **attrs)
|
|
res = QuantizedArray(
|
|
n_bits,
|
|
f_outputs,
|
|
value_is_float=True,
|
|
options=self.output_quant_opts,
|
|
stats=self.output_quant_stats,
|
|
params=self.output_quant_params,
|
|
)
|
|
return res
|
|
|
|
|
|
class QuantizedTranspose(QuantizedOp):
|
|
"""Transpose operator for quantized inputs.
|
|
|
|
This operator performs quantization and transposes the encrypted data.
|
|
When the inputs are pre-computed QAT the input is only quantized if needed.
|
|
"""
|
|
|
|
_impl_for_op_named: str = "Transpose"
|
|
quantize_inputs_with_model_outputs_precision = True
|
|
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
"""Transpose the input integer encrypted tensor.
|
|
|
|
Args:
|
|
q_inputs: an encrypted integer tensor at index 0 and one constant shape at index 1
|
|
attrs: additional optional reshape options
|
|
|
|
Returns:
|
|
result (QuantizedArray): transposed encrypted integer tensor
|
|
"""
|
|
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=True
|
|
)
|
|
|
|
# This op takes only encrypted inputs in the form of QuantizedArray
|
|
assert isinstance(q_inputs[0], QuantizedArray)
|
|
|
|
# Return a new quantized array with the same quantization parameters
|
|
return QuantizedArray(
|
|
q_inputs[0].quantizer.n_bits,
|
|
self.call_impl(prepared_inputs[0].qvalues, **attrs),
|
|
value_is_float=False,
|
|
options=prepared_inputs[0].quantizer.quant_options,
|
|
stats=prepared_inputs[0].quantizer.quant_stats,
|
|
params=prepared_inputs[0].quantizer.quant_params,
|
|
)
|
|
|
|
def can_fuse(self) -> bool:
|
|
"""Determine if this op can be fused.
|
|
|
|
Transpose can not be fused since it must be performed over integer tensors as
|
|
it moves around different elements of these input tensors.
|
|
|
|
Returns:
|
|
bool: False, this operation can not be fused as it copies encrypted integers
|
|
"""
|
|
return False
|
|
|
|
|
|
class QuantizedFloor(QuantizedOp):
|
|
"""Quantized Floor op."""
|
|
|
|
_impl_for_op_named: str = "Floor"
|
|
|
|
|
|
class QuantizedMax(QuantizedOpUnivariateOfEncrypted, QuantizedOp):
|
|
"""Quantized Max op."""
|
|
|
|
_impl_for_op_named: str = "Max"
|
|
|
|
|
|
class QuantizedMin(QuantizedOpUnivariateOfEncrypted, QuantizedOp):
|
|
"""Quantized Min op."""
|
|
|
|
_impl_for_op_named: str = "Min"
|
|
|
|
|
|
class QuantizedNeg(QuantizedOp):
|
|
"""Quantized Neg op."""
|
|
|
|
_impl_for_op_named: str = "Neg"
|
|
|
|
|
|
class QuantizedSign(QuantizedOp):
|
|
"""Quantized Neg op."""
|
|
|
|
_impl_for_op_named: str = "Sign"
|
|
|
|
|
|
class QuantizedUnsqueeze(QuantizedOp):
|
|
"""Unsqueeze operator."""
|
|
|
|
_impl_for_op_named: str = "Unsqueeze"
|
|
quantize_inputs_with_model_outputs_precision = True
|
|
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
"""Unsqueeze the input tensors on a given axis.
|
|
|
|
Args:
|
|
q_inputs: an encrypted integer tensor at index 0, axes at index 1
|
|
attrs: additional optional unsqueeze options
|
|
|
|
Returns:
|
|
result (QuantizedArray): unsqueezed encrypted integer tensor
|
|
"""
|
|
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=True
|
|
)
|
|
|
|
# This op takes only encrypted inputs in the form of QuantizedArray
|
|
assert isinstance(q_inputs[0], QuantizedArray)
|
|
|
|
axes = prepared_inputs[1]
|
|
|
|
# Return a new quantized array with the same quantization parameters
|
|
return QuantizedArray(
|
|
q_inputs[0].quantizer.n_bits,
|
|
self.call_impl(prepared_inputs[0].qvalues, axes, **attrs),
|
|
value_is_float=False,
|
|
options=self._get_output_quant_opts(),
|
|
stats=prepared_inputs[0].quantizer.quant_stats,
|
|
params=prepared_inputs[0].quantizer.quant_params,
|
|
)
|
|
|
|
def can_fuse(self) -> bool:
|
|
"""Determine if this op can be fused.
|
|
|
|
Unsqueeze can not be fused since it must be performed over integer tensors as
|
|
it reshapes an encrypted tensor.
|
|
|
|
Returns:
|
|
bool: False, this operation can not be fused as it operates on encrypted tensors
|
|
"""
|
|
return False
|
|
|
|
|
|
class QuantizedConcat(QuantizedOp):
|
|
"""Concatenate operator."""
|
|
|
|
_impl_for_op_named: str = "Concat"
|
|
quantize_inputs_with_model_outputs_precision = True
|
|
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
"""Concatenate the input tensors on a given axis.
|
|
|
|
Args:
|
|
q_inputs: an encrypted integer tensor
|
|
attrs: additional optional concatenate options
|
|
|
|
Returns:
|
|
result (QuantizedArray): concatenated encrypted integer tensor
|
|
"""
|
|
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=True
|
|
)
|
|
|
|
# This op takes only encrypted inputs in the form of QuantizedArray
|
|
assert isinstance(q_inputs[0], QuantizedArray)
|
|
|
|
# The input tensors must have the same quantization parameters to be concatenated.
|
|
scales = [x.quantizer.scale for x in prepared_inputs]
|
|
zero_points = [x.quantizer.zero_point for x in prepared_inputs]
|
|
assert_true(
|
|
all(x == scales[0] for x in scales) and all(x == zero_points[0] for x in zero_points),
|
|
"All inputs must have the same scale and zero_point to be concatenated.",
|
|
)
|
|
assert_true(
|
|
all(
|
|
prep_input.quantizer.quant_options.is_equal(
|
|
prepared_inputs[0].quantizer.quant_options
|
|
)
|
|
for prep_input in prepared_inputs[1:]
|
|
)
|
|
)
|
|
|
|
tensors_to_concat = [prepared_input.qvalues for prepared_input in prepared_inputs]
|
|
|
|
# Return a new quantized array with the same quantization parameters
|
|
return QuantizedArray(
|
|
q_inputs[0].quantizer.n_bits,
|
|
self.call_impl(*tensors_to_concat, **attrs),
|
|
value_is_float=False,
|
|
options=prepared_inputs[0].quantizer.quant_options,
|
|
stats=prepared_inputs[0].quantizer.quant_stats,
|
|
params=prepared_inputs[0].quantizer.quant_params,
|
|
)
|
|
|
|
def can_fuse(self) -> bool:
|
|
"""Determine if this op can be fused.
|
|
|
|
Concatenation can not be fused since it must be performed over integer tensors as
|
|
it copies encrypted integers from one tensor to another.
|
|
|
|
Returns:
|
|
bool: False, this operation can not be fused as it copies encrypted integers
|
|
"""
|
|
return False
|
|
|
|
|
|
class QuantizedSqueeze(QuantizedOp):
|
|
"""Squeeze operator."""
|
|
|
|
_impl_for_op_named: str = "Squeeze"
|
|
quantize_inputs_with_model_outputs_precision = True
|
|
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
"""Squeeze the input tensors on a given axis.
|
|
|
|
Args:
|
|
q_inputs: an encrypted integer tensor at index 0, axes at index 1
|
|
attrs: additional optional squeeze options
|
|
|
|
Returns:
|
|
result (QuantizedArray): squeezed encrypted integer tensor
|
|
"""
|
|
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=True
|
|
)
|
|
|
|
# This op takes only encrypted inputs in the form of QuantizedArray
|
|
assert isinstance(q_inputs[0], QuantizedArray)
|
|
|
|
axes = prepared_inputs[1]
|
|
|
|
# Return a new quantized array with the same quantization parameters
|
|
return QuantizedArray(
|
|
q_inputs[0].quantizer.n_bits,
|
|
self.call_impl(prepared_inputs[0].qvalues, axes, **attrs),
|
|
value_is_float=False,
|
|
options=self._get_output_quant_opts(),
|
|
stats=prepared_inputs[0].quantizer.quant_stats,
|
|
params=prepared_inputs[0].quantizer.quant_params,
|
|
)
|
|
|
|
def can_fuse(self) -> bool:
|
|
"""Determine if this op can be fused.
|
|
|
|
Squeeze can not be fused since it must be performed over integer tensors as
|
|
it reshapes encrypted tensors.
|
|
|
|
Returns:
|
|
bool: False, this operation can not be fused as it reshapes encrypted tensors
|
|
"""
|
|
return False
|
|
|
|
|
|
class ONNXShape(QuantizedOp):
|
|
"""Shape operator."""
|
|
|
|
_impl_for_op_named: str = "Shape"
|
|
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
# This op takes only encrypted inputs in the form of QuantizedArray
|
|
assert isinstance(q_inputs[0], QuantizedArray)
|
|
|
|
return numpy.asarray(q_inputs[0].qvalues.shape, numpy.int64).view(RawOpOutput)
|
|
|
|
def can_fuse(self) -> bool:
|
|
"""Determine if this op can be fused.
|
|
|
|
This operation returns the shape of the tensor and thus can not be fused into a
|
|
univariate TLU.
|
|
|
|
Returns:
|
|
bool: False, this operation can not be fused
|
|
"""
|
|
return False
|
|
|
|
|
|
class ONNXConstantOfShape(QuantizedOp):
|
|
"""ConstantOfShape operator."""
|
|
|
|
_impl_for_op_named: str = "ConstantOfShape"
|
|
|
|
def can_fuse(self) -> bool:
|
|
"""Determine if this op can be fused.
|
|
|
|
This operation returns a new encrypted tensor and thus can
|
|
not be fused.
|
|
|
|
Returns:
|
|
bool: False, this operation can not be fused
|
|
"""
|
|
return False
|
|
|
|
|
|
class ONNXGather(QuantizedOp):
|
|
"""Gather operator.
|
|
|
|
Returns values at requested indices from the input tensor.
|
|
"""
|
|
|
|
_impl_for_op_named: str = "Gather"
|
|
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=True
|
|
)
|
|
|
|
# The first parameter can be either an encrypted tensor or a shape (int64 array)
|
|
if isinstance(prepared_inputs[0], QuantizedArray):
|
|
inputs = (prepared_inputs[0].qvalues, *prepared_inputs[1:])
|
|
return QuantizedArray(
|
|
prepared_inputs[0].quantizer.n_bits,
|
|
self.call_impl(*inputs, **attrs),
|
|
value_is_float=False,
|
|
options=prepared_inputs[0].quantizer.quant_options,
|
|
stats=prepared_inputs[0].quantizer.quant_stats,
|
|
params=prepared_inputs[0].quantizer.quant_params,
|
|
)
|
|
|
|
assert_true(isinstance(prepared_inputs[0], numpy.ndarray))
|
|
return self.call_impl(*prepared_inputs, **attrs)
|
|
|
|
def can_fuse(self) -> bool:
|
|
"""Determine if this op can be fused.
|
|
|
|
This operation returns values from a tensor and thus can not be fused into a
|
|
univariate TLU.
|
|
|
|
Returns:
|
|
bool: False, this operation can not be fused
|
|
"""
|
|
return False
|
|
|
|
|
|
class ONNXSlice(QuantizedOp):
|
|
"""Slice operator."""
|
|
|
|
_impl_for_op_named: str = "Slice"
|
|
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=True
|
|
)
|
|
|
|
assert_true(
|
|
isinstance(prepared_inputs[0], QuantizedArray),
|
|
"Slice currently only supports QuantizedArray inputs (encrypted inputs)",
|
|
)
|
|
|
|
inputs = (prepared_inputs[0].qvalues, *prepared_inputs[1:])
|
|
return QuantizedArray(
|
|
prepared_inputs[0].quantizer.n_bits,
|
|
self.call_impl(*inputs, **attrs),
|
|
value_is_float=False,
|
|
options=prepared_inputs[0].quantizer.quant_options,
|
|
stats=prepared_inputs[0].quantizer.quant_stats,
|
|
params=prepared_inputs[0].quantizer.quant_params,
|
|
)
|
|
|
|
def can_fuse(self) -> bool:
|
|
"""Determine if this op can be fused.
|
|
|
|
This operation returns values from a tensor and thus can not be fused into a
|
|
univariate TLU.
|
|
|
|
Returns:
|
|
bool: False, this operation can not be fused
|
|
"""
|
|
return False
|
|
|
|
|
|
class QuantizedExpand(QuantizedOp):
|
|
"""Expand operator for quantized tensors."""
|
|
|
|
_impl_for_op_named: str = "Expand"
|
|
quantize_inputs_with_model_outputs_precision = True
|
|
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
"""Expand the input tensor to a specified shape.
|
|
|
|
Args:
|
|
q_inputs: an encrypted integer tensor at index 0, shape at index 1
|
|
attrs: additional optional expand options
|
|
|
|
Returns:
|
|
result (QuantizedArray): expanded encrypted integer tensor
|
|
"""
|
|
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=True
|
|
)
|
|
|
|
# Ensure the input is a quantized array
|
|
assert isinstance(q_inputs[0], QuantizedArray)
|
|
|
|
target_shape = prepared_inputs[1]
|
|
|
|
# Return a new quantized array with the same quantization parameters
|
|
return QuantizedArray(
|
|
q_inputs[0].quantizer.n_bits,
|
|
self.call_impl(prepared_inputs[0].qvalues, target_shape, **attrs),
|
|
value_is_float=False,
|
|
options=self._get_output_quant_opts(),
|
|
stats=prepared_inputs[0].quantizer.quant_stats,
|
|
params=prepared_inputs[0].quantizer.quant_params,
|
|
)
|
|
|
|
def can_fuse(self) -> bool:
|
|
"""Determine if this op can be fused.
|
|
|
|
Unsqueeze can not be fused since it must be performed over integer tensors as
|
|
it reshapes an encrypted tensor.
|
|
|
|
Returns:
|
|
bool: False, this operation can not be fused as it operates on encrypted tensors
|
|
"""
|
|
return False
|
|
|
|
|
|
class QuantizedEqual(QuantizedOp):
|
|
"""Comparison operator ==.
|
|
|
|
Only supports comparison with a constant.
|
|
"""
|
|
|
|
_impl_for_op_named: str = "Equal"
|
|
|
|
# Since this op takes a single variable input, we can set int_input_names to a single default id
|
|
def __init__(
|
|
self,
|
|
n_bits_output: int,
|
|
op_instance_name: str,
|
|
int_input_names: Set[str] = None,
|
|
constant_inputs: Optional[Union[Dict[str, Any], Dict[int, Any]]] = None,
|
|
input_quant_opts: QuantizationOptions = None,
|
|
**attrs,
|
|
) -> None:
|
|
super().__init__(
|
|
n_bits_output,
|
|
op_instance_name,
|
|
int_input_names,
|
|
constant_inputs,
|
|
input_quant_opts,
|
|
**attrs,
|
|
)
|
|
|
|
# We do not support testing a == b where a,b are encrypted
|
|
# only comparing to a constant is supported
|
|
assert_true(constant_inputs is not None and len(constant_inputs) >= 1)
|
|
|
|
|
|
class QuantizedUnfold(QuantizedMixingOp):
|
|
"""Quantized Unfold op."""
|
|
|
|
_impl_for_op_named: str = "Unfold"
|
|
|
|
# Since this op takes a single input, we can set int_input_names to a single default id
|
|
def __init__(
|
|
self,
|
|
n_bits_output: int,
|
|
op_instance_name: str,
|
|
int_input_names: Set[str] = None,
|
|
constant_inputs: Optional[Union[Dict[str, Any], Dict[int, Any]]] = None,
|
|
input_quant_opts: QuantizationOptions = None,
|
|
**attrs,
|
|
) -> None:
|
|
|
|
super().__init__(
|
|
n_bits_output,
|
|
op_instance_name,
|
|
int_input_names,
|
|
constant_inputs,
|
|
input_quant_opts,
|
|
**attrs,
|
|
)
|
|
|
|
# Get the ONNX parameters
|
|
self.kernel_shape = attrs.get("kernel_shape", None)
|
|
self.pads = attrs.get("pads", tuple([0] * 2 * (len(self.kernel_shape) - 2)))
|
|
self.dilations = attrs.get("dilations", tuple([1] * len(self.kernel_shape)))
|
|
self.strides = attrs.get("strides", tuple([1] * len(self.kernel_shape)))
|
|
|
|
# Validate the parameters
|
|
assert_true(
|
|
len(self.kernel_shape) == 2,
|
|
"The Unfold operator currently supports only 2d",
|
|
)
|
|
assert_true(
|
|
len(self.kernel_shape) == len(self.strides),
|
|
"The Unfold operator requires the number of strides to "
|
|
"be the same as the number of kernel dimensions",
|
|
)
|
|
assert_true(
|
|
len(self.pads) == 2 * len(self.kernel_shape),
|
|
"The Unfold operator in Concrete ML requires padding to be specified as "
|
|
" (pad_left_dim1, pad_right_dim1, pad_left_dim2, pad_right_dim2, ...), following ONNX"
|
|
" standard",
|
|
)
|
|
|
|
self.kernel: Union[numpy.ndarray, None] = None
|
|
self.norm_const: Union[float, None] = None
|
|
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
|
|
# Retrieve the quantized inputs
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=True
|
|
)
|
|
q_input: QuantizedArray = prepared_inputs[0]
|
|
|
|
n_in_channels = q_input.qvalues.shape[1]
|
|
kernels_list = []
|
|
for _ in range(n_in_channels):
|
|
for row in range(self.kernel_shape[0]):
|
|
for col in range(self.kernel_shape[1]):
|
|
kernel = numpy.zeros(
|
|
(1, 1, self.kernel_shape[0], self.kernel_shape[1]),
|
|
dtype=numpy.int64,
|
|
)
|
|
kernel[:, :, row, col] = 1
|
|
kernels_list.append(kernel)
|
|
kernels = numpy.concatenate(numpy.array(kernels_list), axis=0)
|
|
|
|
# for mypy: The Quantized ops can only run on QuantizedArray that have quantization
|
|
# parameters (i.e., were fully constructed). This should always be the case, except
|
|
# during the UniformQuantizer initialization when the zero_point can exist as None
|
|
assert q_input.quantizer.zero_point is not None
|
|
|
|
# Compute padding with floor and apply it to the input, pad with the input zero-point
|
|
pool_pads = compute_onnx_pool_padding(
|
|
q_input.qvalues.shape, self.kernel_shape, self.pads, self.strides, ceil_mode=0
|
|
)
|
|
|
|
# Can only pad with scalar zero-points, but zero-points can be float in special cases
|
|
# for output layers
|
|
_check_op_input_zero_point(q_input.quantizer.zero_point, self.op_instance_name)
|
|
pad_value = int(q_input.quantizer.zero_point)
|
|
q_input_pad = numpy_onnx_pad(q_input.qvalues, pool_pads, pad_value, int_only=True)
|
|
|
|
# Remark that here, we are _not_ using Concrete pad, since it would pad with
|
|
# 0's while we want to pad with zero-point's. So, instead, he have done the padding
|
|
# on our side, with q_input_pad
|
|
fake_pads = [0] * len(self.pads)
|
|
|
|
with tag(self.op_instance_name + ".unfold"):
|
|
sum_result = fhe_conv(
|
|
q_input_pad, kernels, None, fake_pads, self.strides, None, None, n_in_channels
|
|
)
|
|
|
|
if self.debug_value_tracker is not None:
|
|
# pylint: disable-next=unsubscriptable-object
|
|
self.debug_value_tracker[self.op_instance_name][
|
|
"output"
|
|
] = sum_result # pragma: no cover
|
|
|
|
result = (
|
|
sum_result.astype(numpy.float64) - q_input.quantizer.zero_point
|
|
) * q_input.quantizer.scale
|
|
|
|
# Reshape to fit the same shape output as unfold
|
|
result = result.reshape((result.shape[0], result.shape[1], -1))
|
|
|
|
return QuantizedArray(
|
|
self.n_bits,
|
|
result,
|
|
value_is_float=True,
|
|
options=self._get_output_quant_opts(),
|
|
stats=self.output_quant_stats,
|
|
params=self.output_quant_params,
|
|
)
|