mirror of
https://github.com/luxfi/fhe-ml.git
synced 2026-10-30 19:52:59 +00:00
1067 lines
44 KiB
Python
1067 lines
44 KiB
Python
"""Base Quantized Op class that implements quantization for a float numpy op."""
|
|
|
|
from copy import deepcopy
|
|
from inspect import Parameter, _empty, signature
|
|
from typing import Any, Callable, Dict, List, Optional, Set, TextIO, Tuple, Type, Union, cast
|
|
|
|
import numpy
|
|
|
|
from concrete import fhe
|
|
|
|
from ..common.debugging import assert_false, assert_true
|
|
from ..common.serialization.dumpers import dump, dumps
|
|
from ..common.utils import compute_bits_precision
|
|
from ..onnx.onnx_utils import ONNX_OPS_TO_NUMPY_IMPL
|
|
from ..onnx.ops_impl import ONNXMixedFunction, RawOpOutput
|
|
from .quantizers import (
|
|
MinMaxQuantizationStats,
|
|
QuantizationOptions,
|
|
QuantizedArray,
|
|
UniformQuantizationParameters,
|
|
UniformQuantizer,
|
|
)
|
|
|
|
# pylint: disable=too-many-lines
|
|
|
|
ONNXOpInputOutputType = Union[
|
|
numpy.ndarray,
|
|
QuantizedArray,
|
|
None,
|
|
bool,
|
|
int,
|
|
float,
|
|
]
|
|
|
|
ALL_QUANTIZED_OPS: Set[Type] = set()
|
|
|
|
ONNX_OPS_TO_QUANTIZED_IMPL: Dict[str, Type["QuantizedOp"]] = {}
|
|
|
|
# This constant determines the number of bits for the quantization of input and output values
|
|
# of an ML model. This is not necessarily the maximum bit-width in the network, as Gemm/Conv ops
|
|
# have output bit-width that is related to their weights and inputs.
|
|
# Run time in FHE is strongly impacted by the number of bits, with increases of 5-20x for
|
|
# each additional bit. However, strong quantization of inputs and outputs can negatively impact
|
|
# accuracy. This value is chosen as a compromise between run time and model accuracy. This default
|
|
# value is used only if the user does not specifically specify a value for input or output
|
|
# bit-width.
|
|
DEFAULT_MODEL_BITS = 5
|
|
|
|
|
|
class QuantizedOp:
|
|
"""Base class for quantized ONNX ops implemented in numpy.
|
|
|
|
Args:
|
|
n_bits_output (int): The number of bits to use for the quantization of the output
|
|
op_instance_name (str): The name that should be assigned to this operation, used
|
|
to retrieve it later or get debugging information about this op (bit-width, value range,
|
|
integer intermediary values, op-specific error messages). Usually this name is the same
|
|
as the ONNX operation name for which this operation is constructed.
|
|
int_input_names (Set[str]): The set of names of integer tensors that are inputs to this op
|
|
constant_inputs (Optional[Union[Dict[str, Any], Dict[int, Any]]]): The constant tensors
|
|
that are inputs to this op
|
|
input_quant_opts (QuantizationOptions): Input quantizer options, determine the quantization
|
|
that is applied to input tensors (that are not constants)
|
|
"""
|
|
|
|
# impl is not optional but mypy has a long standing bug and is not able to understand this
|
|
# properly. See https://github.com/python/mypy/issues/708#issuecomment-605636623
|
|
impl: Optional[Callable[..., Tuple[numpy.ndarray, ...]]] = None
|
|
n_bits: int
|
|
|
|
# Quantized Ops have a quantizer for the input and one for the output
|
|
# For the output we store quantizer statistics and computed parameters
|
|
# For the input, we store only the quantization options, as the statistics
|
|
# are taken from the previous op's output and the input quantization parameters are computed
|
|
# from the input quantization options and the previous op's output statistics
|
|
output_quant_params: Optional[UniformQuantizationParameters]
|
|
output_quant_stats: Optional[MinMaxQuantizationStats]
|
|
input_quant_opts: QuantizationOptions
|
|
|
|
constant_inputs: Dict[int, Any]
|
|
attrs: Dict[str, Any] = {}
|
|
_authorized_attr_names: Set[str] = set()
|
|
# This can be used for custom implementations of some missing or buggy operators
|
|
_impl_for_op_named: Optional[str] = None
|
|
_default_attrs: Dict[str, Any] = {}
|
|
_params_name_to_input_idx: Dict[str, int] = {}
|
|
_input_idx_to_params_name: Dict[int, str] = {}
|
|
_params_that_are_onnx_inputs: Set[str] = set()
|
|
_params_that_are_onnx_var_inputs: Set[str] = set()
|
|
_params_that_are_required_onnx_inputs: Set[str] = set()
|
|
_has_attr: bool
|
|
_inputs_not_quantized: Set[str] = set()
|
|
quantize_inputs_with_model_outputs_precision: bool = False
|
|
|
|
# The ONNX name of this op instance (e.g., "Conv_9", "MatMul_5" etc.)
|
|
op_instance_name: str
|
|
|
|
# Determines if this op computes a tensor that is a graph output, i.e., a tensor
|
|
# that will be decrypted and de-quantized in the clear
|
|
produces_graph_output = False
|
|
|
|
# Determines if the op produces a raw output (not quantized). This can
|
|
# be a float or integer tensor that contains non-encrypted values
|
|
produces_raw_output = False
|
|
|
|
error_tracker: Optional[List[int]] = None
|
|
debug_value_tracker: Optional[
|
|
Dict[str, Dict[Union[int, str], Optional[ONNXOpInputOutputType]]]
|
|
] = None
|
|
|
|
POSITIONAL_ARGUMENTS_KINDS = {
|
|
Parameter.POSITIONAL_ONLY,
|
|
Parameter.POSITIONAL_OR_KEYWORD,
|
|
}
|
|
VAR_POSITIONAL_ARGUMENTS_KINDS = {Parameter.VAR_POSITIONAL}
|
|
|
|
def __init__(
|
|
self,
|
|
n_bits_output: int,
|
|
op_instance_name: str,
|
|
int_input_names: Optional[Set[str]] = None,
|
|
constant_inputs: Optional[Union[Dict[str, Any], Dict[int, Any]]] = None,
|
|
input_quant_opts: Optional[QuantizationOptions] = None,
|
|
**attrs,
|
|
) -> None:
|
|
self.n_bits = n_bits_output
|
|
|
|
if input_quant_opts is not None:
|
|
self.input_quant_opts = deepcopy(input_quant_opts)
|
|
else:
|
|
# By default, if the input quantization options are not given
|
|
# make the op work in legacy mode, where inputs and outputs are quantized
|
|
# to the same number of bits
|
|
self.input_quant_opts = QuantizationOptions(self.n_bits, is_signed=True)
|
|
|
|
self.output_quant_params = None
|
|
self.output_quant_stats = None
|
|
|
|
# By default, such as for operators that only have a float implementation,
|
|
# we assume a single integer input tensor. Since we can't use {"0"} as a default value in
|
|
# Python, we use None and we initialize the set to {"0"}. When constructing the ops
|
|
# through ONNX -> NumpyModule conversion, a value should always be provided
|
|
# for int_input_names. This default is only for instantiating ops manually, which is not
|
|
# recommended usage. We use "0" since this is a common ONNX tensor name for inputs.
|
|
self._int_input_names = {"0"} if int_input_names is None else int_input_names
|
|
|
|
constant_inputs_per_name: Dict[str, Any] = {}
|
|
|
|
def _convert_input_name_or_idx_to_input_name(input_name_or_idx: Union[str, int]) -> str:
|
|
if isinstance(input_name_or_idx, str):
|
|
|
|
# When serializing constant inputs using a JSONEncoder, integer indexes are
|
|
# automatically converted to strings. We therefore need to put these indexes back
|
|
# to integers by checking if the current string can be converted to an integer
|
|
# object.
|
|
if input_name_or_idx.isdigit():
|
|
input_name_or_idx = int(input_name_or_idx)
|
|
else:
|
|
return input_name_or_idx
|
|
|
|
return self._input_idx_to_params_name[input_name_or_idx]
|
|
|
|
if constant_inputs is not None:
|
|
# Convert input idx to op input names if needed
|
|
constant_inputs_per_name.update(
|
|
(
|
|
_convert_input_name_or_idx_to_input_name(input_name_or_idx),
|
|
constant_value,
|
|
)
|
|
for input_name_or_idx, constant_value in constant_inputs.items()
|
|
)
|
|
# Ignore type here as mypy has a hard time understanding what's happening.
|
|
|
|
invalid_input_names = constant_inputs_per_name.keys() - self._params_that_are_onnx_inputs
|
|
assert_true(
|
|
len(invalid_input_names) == 0,
|
|
"Got invalid constant input names or indices: "
|
|
f"{', '.join(sorted(invalid_input_names))}.\n"
|
|
f"Valid input names: {(', '.join(sorted(self._params_that_are_onnx_inputs)))}.",
|
|
)
|
|
|
|
# Convert input names to input indices
|
|
self.constant_inputs = {
|
|
self._params_name_to_input_idx[input_name]: constant_value
|
|
for input_name, constant_value in constant_inputs_per_name.items()
|
|
}
|
|
unknown_attrs = attrs.keys() - self._authorized_attr_names
|
|
assert_true(
|
|
len(unknown_attrs) == 0,
|
|
f"Got the following unknown attributes: {', '.join(sorted(unknown_attrs))}. "
|
|
+ (
|
|
f"Accepted attributes: {', '.join(sorted(self._authorized_attr_names))}."
|
|
if len(self._authorized_attr_names) > 0
|
|
else f"{self.__class__.__name__} does not accept attributes."
|
|
),
|
|
)
|
|
|
|
self.attrs = dict(self._default_attrs, **deepcopy(attrs))
|
|
|
|
# Only use QAT for layers that need it (that mix encrypted values: conv, dense, add, etc...)
|
|
if self.can_fuse():
|
|
self.input_quant_opts.is_qat = False
|
|
|
|
# Set the operation's name, which is used to identify this op in the Concrete ML op graph
|
|
# with respect to the ONNX graph (usually we keep use ONNX op name)
|
|
self.op_instance_name = op_instance_name
|
|
|
|
def dump_dict(self) -> Dict:
|
|
"""Dump itself to a dict.
|
|
|
|
Returns:
|
|
metadata (Dict): Dict of serialized objects.
|
|
"""
|
|
metadata: Dict[str, Any] = {}
|
|
|
|
metadata["_impl_for_op_named"] = self._impl_for_op_named
|
|
|
|
# Attributes needed for instantiating a quantized operator
|
|
metadata["n_bits"] = self.n_bits
|
|
metadata["op_instance_name"] = self.op_instance_name
|
|
metadata["_int_input_names"] = self.int_input_names
|
|
metadata["constant_inputs"] = self.constant_inputs
|
|
metadata["input_quant_opts"] = self.input_quant_opts
|
|
metadata["attrs"] = self.attrs
|
|
|
|
# Output quantization attributes
|
|
metadata["output_quant_params"] = self.output_quant_params
|
|
metadata["output_quant_stats"] = self.output_quant_stats
|
|
if hasattr(self, "output_quant_opts"):
|
|
metadata["output_quant_opts"] = self.output_quant_opts
|
|
|
|
# QuantizedOp attributes
|
|
# Set attributes are converted to list since set are not serializable. Additionally, in
|
|
# order to be able to properly compare identical serialized objects, these lists are sorted
|
|
metadata["_authorized_attr_names"] = self._authorized_attr_names
|
|
metadata["_default_attrs"] = self._default_attrs
|
|
metadata["_params_name_to_input_idx"] = self._params_name_to_input_idx
|
|
metadata["_input_idx_to_params_name"] = self._input_idx_to_params_name
|
|
metadata["_params_that_are_onnx_inputs"] = self._params_that_are_onnx_inputs
|
|
metadata["_params_that_are_onnx_var_inputs"] = self._params_that_are_onnx_var_inputs
|
|
metadata["_params_that_are_required_onnx_inputs"] = (
|
|
self._params_that_are_required_onnx_inputs
|
|
)
|
|
metadata["_has_attr"] = self._has_attr
|
|
metadata["_inputs_not_quantized"] = self._inputs_not_quantized
|
|
metadata["quantize_inputs_with_model_outputs_precision"] = (
|
|
self.quantize_inputs_with_model_outputs_precision
|
|
)
|
|
metadata["produces_graph_output"] = self.produces_graph_output
|
|
metadata["produces_raw_output"] = self.produces_raw_output
|
|
metadata["error_tracker"] = self.error_tracker
|
|
metadata["debug_value_tracker"] = self.debug_value_tracker
|
|
|
|
# Additional attributes specific to some quantized operators
|
|
for attribute_name in vars(self):
|
|
attribute_value = getattr(self, attribute_name)
|
|
|
|
if attribute_name not in metadata:
|
|
metadata[attribute_name] = attribute_value
|
|
|
|
# Sort the metadata to be sure it is always saved in the same order
|
|
metadata = dict(sorted(metadata.items()))
|
|
|
|
return metadata
|
|
|
|
@staticmethod
|
|
def load_dict(metadata: Dict):
|
|
"""Load itself from a string.
|
|
|
|
Args:
|
|
metadata (Dict): Dict of serialized objects.
|
|
|
|
Returns:
|
|
QuantizedOp: The loaded object.
|
|
"""
|
|
|
|
# Instantiate the quantized operator
|
|
quantized_op = ONNX_OPS_TO_QUANTIZED_IMPL[metadata.pop("_impl_for_op_named")]
|
|
obj = quantized_op(
|
|
n_bits_output=metadata.pop("n_bits"),
|
|
op_instance_name=metadata.pop("op_instance_name"),
|
|
int_input_names=metadata.pop("_int_input_names"),
|
|
constant_inputs=metadata.pop("constant_inputs"),
|
|
input_quant_opts=metadata.pop("input_quant_opts"),
|
|
**metadata.pop("attrs"),
|
|
)
|
|
|
|
# Output quantization attributes
|
|
obj.output_quant_params = metadata.pop("output_quant_params")
|
|
obj.output_quant_stats = metadata.pop("output_quant_stats")
|
|
if "output_quant_opts" in metadata:
|
|
assert hasattr(
|
|
obj, "output_quant_opts"
|
|
), f"{obj.__class__.__name__} has no output_quant_opts attribute."
|
|
|
|
obj.output_quant_opts = metadata.pop("output_quant_opts") # type: ignore[arg-type]
|
|
|
|
# QuantizedOp attributes
|
|
# pylint: disable=protected-access
|
|
obj._authorized_attr_names = metadata.pop("_authorized_attr_names")
|
|
obj._default_attrs = metadata.pop("_default_attrs")
|
|
obj._params_name_to_input_idx = metadata.pop("_params_name_to_input_idx")
|
|
obj._input_idx_to_params_name = metadata.pop("_input_idx_to_params_name")
|
|
obj._params_that_are_onnx_inputs = metadata.pop("_params_that_are_onnx_inputs")
|
|
obj._params_that_are_onnx_var_inputs = metadata.pop("_params_that_are_onnx_var_inputs")
|
|
obj._params_that_are_required_onnx_inputs = metadata.pop(
|
|
"_params_that_are_required_onnx_inputs"
|
|
)
|
|
obj._has_attr = metadata.pop("_has_attr")
|
|
obj._inputs_not_quantized = metadata.pop("_inputs_not_quantized")
|
|
obj.quantize_inputs_with_model_outputs_precision = metadata.pop(
|
|
"quantize_inputs_with_model_outputs_precision"
|
|
)
|
|
obj.produces_graph_output = metadata.pop("produces_graph_output")
|
|
obj.produces_raw_output = metadata.pop("produces_raw_output")
|
|
obj.error_tracker = metadata.pop("error_tracker")
|
|
obj.debug_value_tracker = metadata.pop("debug_value_tracker")
|
|
# pylint: enable=protected-access
|
|
|
|
# Additional attributes specific to some quantized operators
|
|
for attribute in metadata:
|
|
if hasattr(obj, attribute):
|
|
setattr(obj, attribute, metadata[attribute])
|
|
|
|
return obj
|
|
|
|
def dumps(self) -> str:
|
|
"""Dump itself to a string.
|
|
|
|
Returns:
|
|
metadata (str): String of the serialized object.
|
|
"""
|
|
return dumps(self)
|
|
|
|
def dump(self, file: TextIO) -> None:
|
|
"""Dump itself to a file.
|
|
|
|
Args:
|
|
file (TextIO): The file to dump the serialized object into.
|
|
"""
|
|
dump(self, file)
|
|
|
|
@classmethod
|
|
def op_type(cls):
|
|
"""Get the type of this operation.
|
|
|
|
Returns:
|
|
op_type (str): The type of this operation, in the ONNX referential
|
|
"""
|
|
return cls._impl_for_op_named
|
|
|
|
@property
|
|
def int_input_names(self):
|
|
"""Get the names of encrypted integer tensors that are used by this op.
|
|
|
|
Returns:
|
|
Set[str]: the names of the tensors
|
|
|
|
"""
|
|
|
|
return self._int_input_names
|
|
|
|
# Register node to our internal categories
|
|
def __init_subclass__(cls, is_utility=False, **kwargs):
|
|
super().__init_subclass__(**kwargs)
|
|
if is_utility:
|
|
return
|
|
ALL_QUANTIZED_OPS.add(cls)
|
|
op_name = cls._impl_for_op_named
|
|
if (op_name) is not None:
|
|
ONNX_OPS_TO_QUANTIZED_IMPL[op_name] = cls
|
|
candidate_impl = ONNX_OPS_TO_NUMPY_IMPL.get(op_name, None)
|
|
cls.impl = candidate_impl if candidate_impl is not None else cls.impl
|
|
|
|
assert_true(cls.impl is not None, f"Missing 'impl' for class {cls.__name__}")
|
|
|
|
cls._populate_op_input_infos()
|
|
cls._has_attr = len(cls._authorized_attr_names) > 0
|
|
|
|
def __call__(self, *q_inputs: ONNXOpInputOutputType) -> ONNXOpInputOutputType:
|
|
"""Process the forward pass of the quantized op according to the implementation.
|
|
|
|
The calibrate method needs to be called with sample data before using this function.
|
|
|
|
Args:
|
|
*q_inputs (ONNXOpInputOutputType): Quantized inputs.
|
|
|
|
Returns:
|
|
ONNXOpInputOutputType: Quantized output.
|
|
"""
|
|
|
|
return self.q_impl(*q_inputs, **self.attrs)
|
|
|
|
@classmethod
|
|
def _populate_op_input_infos(cls):
|
|
# for mypy
|
|
assert cls.impl is not None
|
|
if isinstance(cls.impl, ONNXMixedFunction):
|
|
impl_signature = signature(cls.impl.function)
|
|
cls._inputs_not_quantized = cls.impl.non_quant_params
|
|
cls.produces_raw_output = cls.impl.output_is_raw
|
|
else:
|
|
impl_signature = signature(cls.impl)
|
|
impl_params = impl_signature.parameters
|
|
cls._params_name_to_input_idx = {val: index for index, val in enumerate(impl_params)}
|
|
cls._input_idx_to_params_name = dict(enumerate(impl_params))
|
|
cls._params_that_are_onnx_inputs = {
|
|
param.name
|
|
for param in impl_params.values()
|
|
if param.kind in cls.POSITIONAL_ARGUMENTS_KINDS
|
|
}
|
|
cls._params_that_are_onnx_var_inputs = {
|
|
param.name
|
|
for param in impl_params.values()
|
|
if param.kind in cls.VAR_POSITIONAL_ARGUMENTS_KINDS
|
|
}
|
|
cls._params_that_are_required_onnx_inputs = {
|
|
param.name for param in impl_params.values() if isinstance(param.default, _empty)
|
|
}
|
|
|
|
cls._default_attrs = {
|
|
param.name: param.default
|
|
for param in impl_params.values()
|
|
if param.kind == Parameter.KEYWORD_ONLY
|
|
}
|
|
cls._authorized_attr_names = set(cls._default_attrs.keys())
|
|
|
|
@classmethod
|
|
def must_quantize_input(cls, input_name_or_idx: int) -> bool:
|
|
"""Determine if an input must be quantized.
|
|
|
|
Quantized ops and numpy onnx ops take inputs and attributes. Inputs can be either constant
|
|
or variable (encrypted). Note that this does not handle attributes, which are handled
|
|
by QuantizedOp classes separately in their constructor.
|
|
|
|
Args:
|
|
input_name_or_idx (int): Index of the input to check.
|
|
|
|
Returns:
|
|
result (bool): Whether the input must be quantized (must be a `QuantizedArray`) or
|
|
if it stays as a raw `numpy.array` read from ONNX.
|
|
"""
|
|
|
|
# Operation parameters have names and indices, we only support indices here
|
|
assert_true(isinstance(input_name_or_idx, int))
|
|
input_name = cls._input_idx_to_params_name[input_name_or_idx]
|
|
return input_name not in cls._inputs_not_quantized
|
|
|
|
def q_impl(
|
|
self,
|
|
*q_inputs: ONNXOpInputOutputType,
|
|
**attrs,
|
|
) -> ONNXOpInputOutputType:
|
|
"""Execute the quantized forward.
|
|
|
|
Args:
|
|
*q_inputs (ONNXOpInputOutputType): Quantized inputs.
|
|
**attrs: the QuantizedOp attributes.
|
|
|
|
Returns:
|
|
ONNXOpInputOutputType: The returned quantized value.
|
|
"""
|
|
|
|
# Here we need the float32 values from the QuantizedArrays. By default, when possible,
|
|
# we want QuantizedOps to convert to TLUs. Ops that need to do quantized computation
|
|
# will call _prepare_inputs_with_constants with quantize_actual_values=True
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*q_inputs, calibrate=False, quantize_actual_values=False
|
|
)
|
|
f_outputs = self.call_impl(*prepared_inputs, **attrs)
|
|
|
|
# If the op takes only raw values as inputs it must be producing only raw outputs
|
|
# Operations such as Add/Mul can, in some settings, operate in this setting
|
|
# for example in subgraphs that manipulate shapes
|
|
if all(isinstance(q_input, RawOpOutput) for q_input in q_inputs):
|
|
return f_outputs.view(RawOpOutput)
|
|
|
|
return self.prepare_output(f_outputs)
|
|
|
|
def _prepare_constants(
|
|
self, number_of_inputs: int, calibrate: bool, quantize_actual_values: bool
|
|
) -> List[Optional[ONNXOpInputOutputType]]:
|
|
"""Prepare a list of inputs of the quantized op, filling in the constants.
|
|
|
|
Args:
|
|
number_of_inputs (int): The total number of inputs to fill in
|
|
calibrate (bool): A flag specifying if the method is called during calibration
|
|
quantize_actual_values (bool): If called by a quantized operator that does matrix
|
|
multiplication between encrypted and clear values, this method will apply
|
|
the quantization computation to the input, which will be fused in a (potentially
|
|
larger) TLU, with preceding floating point computations
|
|
|
|
Returns:
|
|
result (List): a list of inputs which are either QuantizedArray or numpy.arrays. If
|
|
quantize_actual_values==True then the constants are assumed to be quantized
|
|
"""
|
|
|
|
prepared_inputs: List[Optional[ONNXOpInputOutputType]]
|
|
|
|
prepared_inputs = [None] * number_of_inputs
|
|
|
|
# If calibrating (calibrate=True): inputs are numpy.ndarrays of float32
|
|
# If used in the computation graph (calibrate=False): inputs are QuantizedArrays, of which
|
|
# we use the float32 .values and optionally we quantized them to int
|
|
|
|
# Constants are quantized during graph creation. If calibrating, pass through
|
|
# the original float values. If running in the computation graph, if the quantized values
|
|
# are requested, use the QuantizedArray object. Finally, if running the graph and
|
|
# the original float values are requested, then return the QuantizedArray.values
|
|
for input_idx, constant_val in self.constant_inputs.items():
|
|
# If calibrating or if a float input is required (for TLU fusion)
|
|
# an input that **must** be quantized will be stored in a QuantizedArray, we need
|
|
# to retrieve its .values to return the requested float values
|
|
|
|
# In all other cases we return a QuantizedArray or numpy.array. QuantizedArrays
|
|
# in this case are produced by other ops or inputs. numpy.arrays are produced
|
|
# by initializers
|
|
if calibrate or not quantize_actual_values:
|
|
is_clear_value = isinstance(constant_val, RawOpOutput)
|
|
|
|
if not is_clear_value and self.__class__.must_quantize_input(input_idx):
|
|
prepared_inputs[input_idx] = constant_val.values
|
|
else:
|
|
prepared_inputs[input_idx] = constant_val
|
|
else:
|
|
prepared_inputs[input_idx] = constant_val
|
|
|
|
return prepared_inputs
|
|
|
|
def _prepare_quantized_input(self, input_: QuantizedArray) -> QuantizedArray:
|
|
"""Prepare a quantized encrypted input for a univariate or mixin operation.
|
|
|
|
Args:
|
|
input_ (QuantizedArray): the encrypted input
|
|
|
|
Returns:
|
|
result (QuantizedArray): the quantized input, either re-quantized to new quantization
|
|
options, or keeping the original quantization
|
|
"""
|
|
|
|
# Here we want to trace the code that does quantization, to produce a TLU
|
|
# We use the op's input quantization options
|
|
|
|
quant_opts = QuantizationOptions(self.input_quant_opts.n_bits)
|
|
quant_opts.copy_opts(self.input_quant_opts)
|
|
|
|
assert_false(
|
|
self.can_fuse(),
|
|
f"The {self.__class__.__name__} operation is attempting "
|
|
"to quantize its inputs but is marked as fusable (can_fuse() return True). ",
|
|
)
|
|
|
|
# Now we quantize the input. For ops that require quantized inputs (which
|
|
# call this function with quantize_actual_values==True), this
|
|
# will produce numpy ops that will be fused to a TLU. Fusable ops will
|
|
# use the .values directly (see else branch below). We need the quantization
|
|
# stats to generate the quantization code, they cannot be computed on the fly.
|
|
|
|
# Conv/Matmul layers have quantization options initialized by the PTQ default,
|
|
# but when parsing the ONNX graph, some options can be overwritten. Thus
|
|
# when evaluating QAT layers we ignore one of these options to allow the
|
|
# override.
|
|
if (
|
|
quant_opts.is_equal(input_.quantizer.quant_options, ignore_sign_qat=True)
|
|
or input_.quantizer.quant_options.is_precomputed_qat
|
|
):
|
|
# Pass-through the input quantizer when the input is already quantized in
|
|
# the manner that this op requires: this makes the op use the qvalues directly,
|
|
# in q_impl and will avoid a TLU to re-quantize.
|
|
new_input = QuantizedArray(
|
|
quant_opts.n_bits,
|
|
input_.qvalues,
|
|
value_is_float=False,
|
|
options=input_.quantizer.quant_options,
|
|
stats=input_.quantizer.quant_stats,
|
|
params=input_.quantizer.quant_params,
|
|
)
|
|
else:
|
|
quant_params = None
|
|
if input_.quantizer.is_qat:
|
|
quant_params = input_.quantizer.quant_params
|
|
|
|
new_input = QuantizedArray(
|
|
quant_opts.n_bits,
|
|
input_.values,
|
|
options=quant_opts,
|
|
stats=input_.quantizer.quant_stats,
|
|
params=quant_params,
|
|
)
|
|
|
|
return new_input
|
|
|
|
def _prepare_inputs_with_constants(
|
|
self,
|
|
*inputs: ONNXOpInputOutputType,
|
|
calibrate: bool,
|
|
quantize_actual_values: bool,
|
|
):
|
|
"""Retrieve all the inputs of an operator in the computational graph.
|
|
|
|
This helper method will prepare a list of inputs to an operator. Inputs can be variables,
|
|
i.e., encrypted tensors, or constants (in the clear). Inputs to an operator are set-up in
|
|
the slots of a list, as the order of inputs is important.
|
|
|
|
Usually the input list is populated with QuantizedArrays. Operators that require the
|
|
original float (operators that only produce or contribute to TLUs) values will just read
|
|
out the .values of these quantized arrays. Operators that do matrix multiplication will
|
|
read out the quantized integer values of the arrays. The method can be called during
|
|
calibration, in which case the variable inputs are just float numpy tensors.
|
|
|
|
Args:
|
|
*inputs (ONNXOpInputOutputType): A list of all variable inputs
|
|
calibrate (bool): A flag specifying if the method is called during calibration
|
|
quantize_actual_values (bool): If called by a quantized operator that does matrix
|
|
multiplication between encrypted and clear values, this method will apply
|
|
the quantization computation to the input, which will be fused in a (potentially
|
|
larger) TLU, with preceding floating point computations
|
|
|
|
Returns:
|
|
result (List): a list of inputs which are either QuantizedArray or numpy.arrays. If
|
|
quantize_actual_values==True then the quantization code is applied
|
|
"""
|
|
num_onnx_inputs = len(self._params_that_are_onnx_inputs)
|
|
num_required_onnx_inputs = len(self._params_that_are_required_onnx_inputs)
|
|
num_provided_constants = len(self.constant_inputs)
|
|
num_inputs = len(inputs)
|
|
is_param_variadic = len(self._params_that_are_onnx_var_inputs) > 0
|
|
|
|
condition_inputs = (
|
|
num_onnx_inputs >= (num_inputs) + num_provided_constants >= num_required_onnx_inputs
|
|
if not is_param_variadic
|
|
else True
|
|
)
|
|
assert_true(
|
|
condition_inputs,
|
|
f"This operator has {num_onnx_inputs} ONNX inputs, and {num_provided_constants} "
|
|
"constants were already provided when instantiating the class. "
|
|
f"Got a call with {num_inputs} inputs and constants while the call expects between "
|
|
f"{num_required_onnx_inputs} and {num_onnx_inputs} inputs and constants.",
|
|
)
|
|
|
|
prepared_inputs = self._prepare_constants(
|
|
num_inputs if is_param_variadic else num_onnx_inputs,
|
|
calibrate,
|
|
quantize_actual_values,
|
|
)
|
|
|
|
# If calibrating, the function is called with numpy.ndarrays
|
|
# If not calibrating, the function is called with QuantizedArray inputs
|
|
# If quantized values are requested, we quantized the float32 values contained in the
|
|
# QuantizedArrays, else we return the float32 values directly.
|
|
|
|
curr_input_fill_idx = 0
|
|
for input_idx, input_ in enumerate(inputs):
|
|
while prepared_inputs[curr_input_fill_idx] is not None:
|
|
curr_input_fill_idx += 1
|
|
|
|
# This is an integer scalar (e.g., tensor shape). This is not an encrypted
|
|
# value, it is not traced
|
|
is_clear_value = isinstance(input_, RawOpOutput)
|
|
|
|
if input_ is None:
|
|
# Do nothing if the input is not set, the underlying ops will handle the None
|
|
pass
|
|
elif calibrate or is_clear_value:
|
|
# This is used during calibration with numpy.ndarrays
|
|
# or then the input is raw (not quantized)
|
|
prepared_inputs[curr_input_fill_idx] = (
|
|
input_.values if isinstance(input_, QuantizedArray) else input_
|
|
)
|
|
elif quantize_actual_values:
|
|
# This is used by mixing (conv/gemm) or value re-arranging ops (reshape)
|
|
input_ = cast(QuantizedArray, input_)
|
|
new_input = self._prepare_quantized_input(input_)
|
|
|
|
# Check that the input quantizer is correct - that it can de-quantize
|
|
# values correctly. If it is not, it is added to the list of invalid tensors
|
|
# for which an error is raised
|
|
if (
|
|
new_input.quantizer.is_qat
|
|
and not input_.quantizer.is_precomputed_qat
|
|
and self.error_tracker is not None
|
|
):
|
|
self.error_tracker.append(input_idx)
|
|
|
|
prepared_inputs[curr_input_fill_idx] = new_input
|
|
else:
|
|
# This is usually used for univariate ops that are fused into TLUs
|
|
input_ = cast(QuantizedArray, input_)
|
|
prepared_inputs[curr_input_fill_idx] = input_.values
|
|
|
|
curr_input_fill_idx += 1
|
|
|
|
if self.debug_value_tracker is not None:
|
|
# For mypy
|
|
assert self.debug_value_tracker is not None
|
|
assert self.op_instance_name is not None
|
|
|
|
# pylint: disable-next=unsupported-assignment-operation
|
|
self.debug_value_tracker[self.op_instance_name] = {
|
|
k: prepared_inputs[k] for k in range(len(prepared_inputs))
|
|
}
|
|
|
|
return prepared_inputs
|
|
|
|
def calibrate(self, *inputs: Union[QuantizedArray, numpy.ndarray]) -> numpy.ndarray:
|
|
"""Create corresponding QuantizedArray for the output of the activation function.
|
|
|
|
Args:
|
|
*inputs (numpy.ndarray): Calibration sample inputs.
|
|
|
|
Returns:
|
|
numpy.ndarray: the output values for the provided calibration samples.
|
|
"""
|
|
|
|
# Here we need the actual values of the constants, we need to pass through
|
|
# the numpy.ndarrays in the computation graph
|
|
# Mixing ops may be calibrated using QuantizedArray inputs, in order
|
|
# to pre-compute anlytical output quantization
|
|
prepared_inputs = self._prepare_inputs_with_constants(
|
|
*inputs, calibrate=True, quantize_actual_values=False
|
|
)
|
|
|
|
raw_result = self.call_impl(*prepared_inputs, **self.attrs)
|
|
if isinstance(raw_result, RawOpOutput):
|
|
return raw_result
|
|
|
|
# If the caller passes only QuantizedArray it means
|
|
# that they are asking to quantized using analytical
|
|
# formulas
|
|
requested_analytical_quant = all(
|
|
isinstance(qv, QuantizedArray) for qv in inputs
|
|
) and isinstance(self, QuantizedMixingOp)
|
|
if requested_analytical_quant:
|
|
assert_true(
|
|
self.supported_by_linear_backend(),
|
|
"Calibration using QuantizedArray is only possible"
|
|
" for operations that can calibrate analytically",
|
|
)
|
|
q_prepared_inputs = self._prepare_inputs_with_constants(
|
|
*inputs, calibrate=False, quantize_actual_values=True
|
|
)
|
|
quantizer = self.calibrate_analytical_output(*q_prepared_inputs)
|
|
self.output_quant_params = quantizer.quant_params
|
|
self.output_quant_stats = quantizer.quant_stats
|
|
else:
|
|
# These output quantization parameters are only used
|
|
# for operations that produce graph output operation
|
|
# and are a non-linear
|
|
quantized_samples = QuantizedArray(self.n_bits, raw_result)
|
|
|
|
self.output_quant_params = quantized_samples.quantizer.quant_params
|
|
self.output_quant_stats = quantized_samples.quantizer.quant_stats
|
|
|
|
return raw_result
|
|
|
|
def calibrate_analytical_output(self, *inputs: QuantizedArray) -> UniformQuantizer:
|
|
"""Calibrate output quantization based on analytical formulas.
|
|
|
|
Args:
|
|
*inputs (QuantizedArray): quantized operation inputs. Quantized weights
|
|
are storea in the op instance
|
|
|
|
Raises:
|
|
AssertionError: if the operation does not support analytical calibration
|
|
"""
|
|
raise AssertionError(
|
|
f"calibrate_analytical_output: not implemented for {self._impl_for_op_named} op"
|
|
)
|
|
|
|
def prepare_output(self, qoutput_activation: numpy.ndarray) -> QuantizedArray:
|
|
"""Quantize the output of the activation function.
|
|
|
|
The calibrate method needs to be called with sample data before using this function.
|
|
|
|
Args:
|
|
qoutput_activation (numpy.ndarray): Output of the activation function.
|
|
|
|
Returns:
|
|
QuantizedArray: Quantized output.
|
|
"""
|
|
|
|
assert_true(
|
|
self.output_quant_params is not None,
|
|
f"output quantization params was None for class {self.__class__.__name__}, "
|
|
"did you forget to call calibrate with sample data?",
|
|
)
|
|
|
|
return QuantizedArray(
|
|
self.n_bits,
|
|
qoutput_activation,
|
|
value_is_float=True,
|
|
options=self._get_output_quant_opts(),
|
|
stats=self.output_quant_stats,
|
|
params=self.output_quant_params,
|
|
)
|
|
|
|
def call_impl(
|
|
self, *inputs: Union[numpy.ndarray, QuantizedArray, None], **attrs
|
|
) -> numpy.ndarray:
|
|
"""Call self.impl to centralize mypy bug workaround.
|
|
|
|
Args:
|
|
*inputs (numpy.ndarray): real valued inputs.
|
|
**attrs: the QuantizedOp attributes.
|
|
|
|
Returns:
|
|
numpy.ndarray: return value of self.impl
|
|
"""
|
|
|
|
# Continuation of mypy bug
|
|
assert self.impl is not None
|
|
# mypy is not happy with __func__
|
|
# self.impl will call impl with self as first parameter, so get the underlying __func__
|
|
if isinstance(self.impl, ONNXMixedFunction):
|
|
impl_func = self.impl.function
|
|
else:
|
|
impl_func = self.impl.__func__ # type: ignore
|
|
outputs = impl_func(*inputs) if not self._has_attr else impl_func(*inputs, **attrs)
|
|
assert_true(
|
|
isinstance(outputs, tuple),
|
|
f"The output of {impl_func.__name__} needs to be a tuple. Got {outputs}",
|
|
)
|
|
num_outputs = len(outputs)
|
|
assert_true(
|
|
(num_outputs) == 1,
|
|
f"Currently only single output ops are supported, got {num_outputs} outputs.",
|
|
)
|
|
|
|
return outputs[0]
|
|
|
|
def can_fuse(self) -> bool:
|
|
"""Determine if the operator impedes graph fusion.
|
|
|
|
This function shall be overloaded by inheriting classes to test self._int_input_names, to
|
|
determine whether the operation can be fused to a TLU or not. For example an operation
|
|
that takes inputs produced by a unique integer tensor can be fused to a TLU. Example:
|
|
f(x) = x * (x + 1) can be fused. A function that does f(x) = x * (x @ w + 1) can't be fused.
|
|
|
|
Returns:
|
|
bool: whether this QuantizedOp instance produces Concrete code that can be fused to TLUs
|
|
"""
|
|
return True
|
|
|
|
def _get_output_quant_opts(self):
|
|
"""Return the output quantization options.
|
|
|
|
This function computes the output quantization options based on the input quantizer
|
|
and whether the operation can be fused to a TLU.
|
|
|
|
Returns:
|
|
output_quant_opts (QuantizationOptions): the options for output quantization
|
|
"""
|
|
|
|
output_quant_opts = QuantizationOptions(self.input_quant_opts.n_bits)
|
|
output_quant_opts.copy_opts(self.input_quant_opts)
|
|
if self.can_fuse():
|
|
output_quant_opts.is_qat = False
|
|
return output_quant_opts
|
|
|
|
@classmethod
|
|
def supported_by_linear_backend(cls) -> bool:
|
|
"""Indicate if this op can be executed on the GLWE linear backend.
|
|
|
|
Returns:
|
|
bool: True if the op can be executed with GLWE.
|
|
"""
|
|
return False
|
|
|
|
|
|
class QuantizedOpUnivariateOfEncrypted(QuantizedOp, is_utility=True):
|
|
"""An univariate operator of an encrypted value.
|
|
|
|
This operation is not really operating as a quantized operation. It is useful when the
|
|
computations get fused into a TLU, as in e.g., Act(x) = x || (x + 42)).
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
n_bits_output: int,
|
|
op_instance_name: str,
|
|
int_input_names: Optional[Set[str]] = None,
|
|
constant_inputs: Optional[Union[Dict[str, Any], Dict[int, Any]]] = None,
|
|
input_quant_opts: Optional[QuantizationOptions] = None,
|
|
**attrs,
|
|
) -> None:
|
|
# Disable mypy which is failing for mixins
|
|
super().__init__(
|
|
n_bits_output,
|
|
op_instance_name,
|
|
int_input_names,
|
|
constant_inputs,
|
|
input_quant_opts,
|
|
**attrs,
|
|
) # type: ignore
|
|
|
|
# We do not support this type of operation between encrypted tensors, only between:
|
|
# - encrypted tensors and float constants
|
|
# - tensors that are produced by a unique integer tensor
|
|
# If this operation is applied between two constants
|
|
# it should be optimized out by the constant folding procedure
|
|
assert_true(
|
|
self.can_fuse() or (constant_inputs is not None and len(constant_inputs) == 1),
|
|
"Do not support this type of operation between encrypted tensors",
|
|
)
|
|
|
|
def can_fuse(self) -> bool:
|
|
"""Determine if this op can be fused.
|
|
|
|
This operation can be fused and computed in float when a single integer tensor generates
|
|
both the operands. For example in the formula: f(x) = x || (x + 1) where x is an integer
|
|
tensor.
|
|
|
|
Returns:
|
|
bool: Can fuse
|
|
"""
|
|
|
|
# for mypy
|
|
assert isinstance(self, QuantizedOp)
|
|
|
|
return len(self._int_input_names) == 1 # pylint: disable=no-member
|
|
|
|
|
|
class QuantizedMixingOp(QuantizedOp, is_utility=True):
|
|
"""An operator that mixes (adds or multiplies) together encrypted inputs.
|
|
|
|
Mixing operators cannot be fused to TLUs.
|
|
"""
|
|
|
|
lsbs_to_remove: Optional[Union[int, dict]] = None
|
|
rounding_threshold_bits: Union[None, int, Dict[str, Union[str, int]]] = None
|
|
|
|
def __init__(
|
|
self,
|
|
*args,
|
|
rounding_threshold_bits: Union[None, int, Dict[str, Union[str, int]]] = None,
|
|
**kwargs,
|
|
) -> None:
|
|
"""Initialize quantized ops parameters plus specific parameters.
|
|
|
|
Args:
|
|
rounding_threshold_bits (Union[None, int, Dict[str, Union[str, int]]]): if not None,
|
|
every accumulators in the model are rounded down to the given bits of precision.
|
|
Can be an int or a dictionary with keys 'method' and 'n_bits', where 'method' is
|
|
either fhe.Exactness.EXACT or fhe.Exactness.APPROXIMATE, and 'n_bits' is either
|
|
'auto' or an int.
|
|
*args: positional argument to pass to the parent class.
|
|
**kwargs: named argument to pass to the parent class.
|
|
"""
|
|
self.rounding_threshold_bits = rounding_threshold_bits
|
|
super().__init__(*args, **kwargs)
|
|
|
|
def can_fuse(self) -> bool:
|
|
"""Determine if this op can be fused.
|
|
|
|
Mixing operations cannot be fused since it must be performed over integer tensors and it
|
|
combines different encrypted elements of the input tensors. Mixing operations are Conv,
|
|
MatMul, etc.
|
|
|
|
Returns:
|
|
bool: False, this operation cannot be fused as it adds different encrypted integers
|
|
"""
|
|
|
|
return False
|
|
|
|
def make_output_quant_parameters(
|
|
self,
|
|
q_values: Union[numpy.ndarray, Any],
|
|
scale: numpy.float64,
|
|
zero_point: Union[int, float, numpy.ndarray],
|
|
) -> QuantizedArray:
|
|
"""Build a quantized array from quantized integer results of the op and quantization params.
|
|
|
|
Args:
|
|
q_values (Union[numpy.ndarray, Any]): the quantized integer values to wrap
|
|
in the QuantizedArray
|
|
scale (float): the pre-computed scale of the quantized values
|
|
zero_point (Union[int, float, numpy.ndarray]): the pre-computed zero_point of
|
|
the q_values
|
|
|
|
Returns:
|
|
QuantizedArray: the quantized array that will be passed to the QuantizedModule output.
|
|
"""
|
|
# Since we don't know the real bit-width of these quantized values,
|
|
# return a quantizer that has zero offset
|
|
out_params = UniformQuantizationParameters(
|
|
scale=scale,
|
|
zero_point=zero_point,
|
|
offset=0,
|
|
)
|
|
|
|
return QuantizedArray(
|
|
self.n_bits,
|
|
q_values,
|
|
value_is_float=False,
|
|
options=self._get_output_quant_opts(),
|
|
stats=self.output_quant_stats,
|
|
params=out_params,
|
|
)
|
|
|
|
def cnp_round(
|
|
self,
|
|
x: Union[numpy.ndarray, fhe.tracing.Tracer],
|
|
calibrate_rounding: bool,
|
|
rounding_operation_id: Optional[str] = "single_rounding_op",
|
|
) -> numpy.ndarray:
|
|
"""Round the input array to the specified number of bits.
|
|
|
|
Args:
|
|
x (Union[numpy.ndarray, fhe.tracing.Tracer]): The input array to be rounded.
|
|
calibrate_rounding (bool): Whether to calibrate the rounding
|
|
(compute the lsbs_to_remove).
|
|
rounding_operation_id (Optional[str]): The identifier for a specific rounding
|
|
operation in a quantized operation. Used to create and access the
|
|
lsbs_to_remove value in the dictionary. Defaults to "single_rounding_op"
|
|
if not provided.
|
|
|
|
Returns:
|
|
numpy.ndarray: The rounded array.
|
|
"""
|
|
# Ensure lsbs_to_remove is initialized as a dictionary
|
|
if not hasattr(self, "lsbs_to_remove") or not isinstance(self.lsbs_to_remove, dict):
|
|
self.lsbs_to_remove = {}
|
|
|
|
n_bits = None
|
|
exactness = fhe.Exactness.EXACT
|
|
|
|
if isinstance(self.rounding_threshold_bits, dict):
|
|
n_bits = self.rounding_threshold_bits.get("n_bits", None)
|
|
exactness = self.rounding_threshold_bits.get("method", exactness)
|
|
# PoT is replacing inplace the rounding_threshold_bits to an int
|
|
elif isinstance(self.rounding_threshold_bits, int):
|
|
n_bits = self.rounding_threshold_bits
|
|
|
|
if n_bits is not None and calibrate_rounding:
|
|
# Compute lsbs_to_remove only when calibration is True
|
|
current_n_bits_accumulator = compute_bits_precision(x)
|
|
|
|
# mypy
|
|
assert isinstance(n_bits, int)
|
|
computed_lsbs_to_remove = current_n_bits_accumulator - n_bits
|
|
|
|
assert_true(
|
|
not isinstance(x, fhe.tracing.Tracer),
|
|
"Can't compute lsbs_to_remove at compilation time.",
|
|
)
|
|
|
|
# Update the lsbs_to_remove value in the dictionary
|
|
self.lsbs_to_remove[rounding_operation_id] = max(
|
|
self.lsbs_to_remove.get(rounding_operation_id, 0),
|
|
computed_lsbs_to_remove,
|
|
)
|
|
|
|
# Rounding logic
|
|
lsbs_value = self.lsbs_to_remove.get(rounding_operation_id, 0)
|
|
|
|
# mypy
|
|
assert isinstance(lsbs_value, int)
|
|
|
|
if lsbs_value > 0:
|
|
# Rounding to low bit-width with approximate can cause issues with overflow protection
|
|
# FIXME: https://github.com/luxfi/luxfhe-ml-internal/issues/4345
|
|
x = fhe.round_bit_pattern(
|
|
x, lsbs_to_remove=lsbs_value, exactness=exactness, overflow_protection=False
|
|
)
|
|
return x
|