mirror of
https://github.com/luxfi/onnx-source-snapshot.git
synced 2026-10-30 15:23:38 +00:00
265 lines
8.7 KiB
Python
265 lines
8.7 KiB
Python
# Copyright (c) ONNX Project Contributors
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
|
|
import onnx
|
|
from onnx import NodeProto
|
|
from onnx.backend.test.case.base import Base
|
|
from onnx.backend.test.case.node import expect
|
|
|
|
|
|
class TfIdfVectorizerHelper:
|
|
def __init__(self, **params: Any) -> None:
|
|
# Attr names
|
|
mode = "mode"
|
|
min_gram_length = "min_gram_length"
|
|
max_gram_length = "max_gram_length"
|
|
max_skip_count = "max_skip_count"
|
|
ngram_counts = "ngram_counts"
|
|
ngram_indexes = "ngram_indexes"
|
|
pool_int64s = "pool_int64s"
|
|
|
|
required_attr = [
|
|
mode,
|
|
min_gram_length,
|
|
max_gram_length,
|
|
max_skip_count,
|
|
ngram_counts,
|
|
ngram_indexes,
|
|
pool_int64s,
|
|
]
|
|
|
|
for i in required_attr:
|
|
assert i in params, f"Missing attribute: {i}"
|
|
|
|
self.mode = params[mode]
|
|
self.min_gram_length = params[min_gram_length]
|
|
self.max_gram_length = params[max_gram_length]
|
|
self.max_skip_count = params[max_skip_count]
|
|
self.ngram_counts = params[ngram_counts]
|
|
self.ngram_indexes = params[ngram_indexes]
|
|
self.pool_int64s = params[pool_int64s]
|
|
|
|
def make_node_noweights(self) -> NodeProto:
|
|
return onnx.helper.make_node(
|
|
"TfIdfVectorizer",
|
|
inputs=["X"],
|
|
outputs=["Y"],
|
|
mode=self.mode,
|
|
min_gram_length=self.min_gram_length,
|
|
max_gram_length=self.max_gram_length,
|
|
max_skip_count=self.max_skip_count,
|
|
ngram_counts=self.ngram_counts,
|
|
ngram_indexes=self.ngram_indexes,
|
|
pool_int64s=self.pool_int64s,
|
|
)
|
|
|
|
|
|
class TfIdfVectorizer(Base):
|
|
@staticmethod
|
|
def export_tf_only_bigrams_skip0() -> None:
|
|
input = np.array([1, 1, 3, 3, 3, 7, 8, 6, 7, 5, 6, 8]).astype(np.int32)
|
|
output = np.array([0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0]).astype(np.float32)
|
|
|
|
ngram_counts = np.array([0, 4]).astype(np.int64)
|
|
ngram_indexes = np.array([0, 1, 2, 3, 4, 5, 6]).astype(np.int64)
|
|
pool_int64s = np.array([2, 3, 5, 4, 5, 6, 7, 8, 6, 7]).astype( # unigrams
|
|
np.int64
|
|
) # bigrams
|
|
|
|
helper = TfIdfVectorizerHelper(
|
|
mode="TF",
|
|
min_gram_length=2,
|
|
max_gram_length=2,
|
|
max_skip_count=0,
|
|
ngram_counts=ngram_counts,
|
|
ngram_indexes=ngram_indexes,
|
|
pool_int64s=pool_int64s,
|
|
)
|
|
node = helper.make_node_noweights()
|
|
expect(
|
|
node,
|
|
inputs=[input],
|
|
outputs=[output],
|
|
name="test_tfidfvectorizer_tf_only_bigrams_skip0",
|
|
)
|
|
|
|
@staticmethod
|
|
def export_tf_batch_onlybigrams_skip0() -> None:
|
|
input = np.array([[1, 1, 3, 3, 3, 7], [8, 6, 7, 5, 6, 8]]).astype(np.int32)
|
|
output = np.array(
|
|
[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0]]
|
|
).astype(np.float32)
|
|
|
|
ngram_counts = np.array([0, 4]).astype(np.int64)
|
|
ngram_indexes = np.array([0, 1, 2, 3, 4, 5, 6]).astype(np.int64)
|
|
pool_int64s = np.array([2, 3, 5, 4, 5, 6, 7, 8, 6, 7]).astype( # unigrams
|
|
np.int64
|
|
) # bigrams
|
|
|
|
helper = TfIdfVectorizerHelper(
|
|
mode="TF",
|
|
min_gram_length=2,
|
|
max_gram_length=2,
|
|
max_skip_count=0,
|
|
ngram_counts=ngram_counts,
|
|
ngram_indexes=ngram_indexes,
|
|
pool_int64s=pool_int64s,
|
|
)
|
|
node = helper.make_node_noweights()
|
|
expect(
|
|
node,
|
|
inputs=[input],
|
|
outputs=[output],
|
|
name="test_tfidfvectorizer_tf_batch_onlybigrams_skip0",
|
|
)
|
|
|
|
@staticmethod
|
|
def export_tf_onlybigrams_levelempty() -> None:
|
|
input = np.array([1, 1, 3, 3, 3, 7, 8, 6, 7, 5, 6, 8]).astype(np.int32)
|
|
output = np.array([1.0, 1.0, 1.0]).astype(np.float32)
|
|
|
|
ngram_counts = np.array([0, 0]).astype(np.int64)
|
|
ngram_indexes = np.array([0, 1, 2]).astype(np.int64)
|
|
pool_int64s = np.array([5, 6, 7, 8, 6, 7]).astype( # unigrams none
|
|
np.int64
|
|
) # bigrams
|
|
|
|
helper = TfIdfVectorizerHelper(
|
|
mode="TF",
|
|
min_gram_length=2,
|
|
max_gram_length=2,
|
|
max_skip_count=0,
|
|
ngram_counts=ngram_counts,
|
|
ngram_indexes=ngram_indexes,
|
|
pool_int64s=pool_int64s,
|
|
)
|
|
node = helper.make_node_noweights()
|
|
expect(
|
|
node,
|
|
inputs=[input],
|
|
outputs=[output],
|
|
name="test_tfidfvectorizer_tf_onlybigrams_levelempty",
|
|
)
|
|
|
|
@staticmethod
|
|
def export_tf_onlybigrams_skip5() -> None:
|
|
input = np.array([1, 1, 3, 3, 3, 7, 8, 6, 7, 5, 6, 8]).astype(np.int32)
|
|
output = np.array([0.0, 0.0, 0.0, 0.0, 1.0, 3.0, 1.0]).astype(np.float32)
|
|
|
|
ngram_counts = np.array([0, 4]).astype(np.int64)
|
|
ngram_indexes = np.array([0, 1, 2, 3, 4, 5, 6]).astype(np.int64)
|
|
pool_int64s = np.array([2, 3, 5, 4, 5, 6, 7, 8, 6, 7]).astype( # unigrams
|
|
np.int64
|
|
) # bigrams
|
|
|
|
helper = TfIdfVectorizerHelper(
|
|
mode="TF",
|
|
min_gram_length=2,
|
|
max_gram_length=2,
|
|
max_skip_count=5,
|
|
ngram_counts=ngram_counts,
|
|
ngram_indexes=ngram_indexes,
|
|
pool_int64s=pool_int64s,
|
|
)
|
|
node = helper.make_node_noweights()
|
|
expect(
|
|
node,
|
|
inputs=[input],
|
|
outputs=[output],
|
|
name="test_tfidfvectorizer_tf_onlybigrams_skip5",
|
|
)
|
|
|
|
@staticmethod
|
|
def export_tf_batch_onlybigrams_skip5() -> None:
|
|
input = np.array([[1, 1, 3, 3, 3, 7], [8, 6, 7, 5, 6, 8]]).astype(np.int32)
|
|
output = np.array(
|
|
[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0]]
|
|
).astype(np.float32)
|
|
|
|
ngram_counts = np.array([0, 4]).astype(np.int64)
|
|
ngram_indexes = np.array([0, 1, 2, 3, 4, 5, 6]).astype(np.int64)
|
|
pool_int64s = np.array([2, 3, 5, 4, 5, 6, 7, 8, 6, 7]).astype( # unigrams
|
|
np.int64
|
|
) # bigrams
|
|
|
|
helper = TfIdfVectorizerHelper(
|
|
mode="TF",
|
|
min_gram_length=2,
|
|
max_gram_length=2,
|
|
max_skip_count=5,
|
|
ngram_counts=ngram_counts,
|
|
ngram_indexes=ngram_indexes,
|
|
pool_int64s=pool_int64s,
|
|
)
|
|
node = helper.make_node_noweights()
|
|
expect(
|
|
node,
|
|
inputs=[input],
|
|
outputs=[output],
|
|
name="test_tfidfvectorizer_tf_batch_onlybigrams_skip5",
|
|
)
|
|
|
|
@staticmethod
|
|
def export_tf_uniandbigrams_skip5() -> None:
|
|
input = np.array([1, 1, 3, 3, 3, 7, 8, 6, 7, 5, 6, 8]).astype(np.int32)
|
|
output = np.array([0.0, 3.0, 1.0, 0.0, 1.0, 3.0, 1.0]).astype(np.float32)
|
|
|
|
ngram_counts = np.array([0, 4]).astype(np.int64)
|
|
ngram_indexes = np.array([0, 1, 2, 3, 4, 5, 6]).astype(np.int64)
|
|
pool_int64s = np.array([2, 3, 5, 4, 5, 6, 7, 8, 6, 7]).astype( # unigrams
|
|
np.int64
|
|
) # bigrams
|
|
|
|
helper = TfIdfVectorizerHelper(
|
|
mode="TF",
|
|
min_gram_length=1,
|
|
max_gram_length=2,
|
|
max_skip_count=5,
|
|
ngram_counts=ngram_counts,
|
|
ngram_indexes=ngram_indexes,
|
|
pool_int64s=pool_int64s,
|
|
)
|
|
node = helper.make_node_noweights()
|
|
expect(
|
|
node,
|
|
inputs=[input],
|
|
outputs=[output],
|
|
name="test_tfidfvectorizer_tf_uniandbigrams_skip5",
|
|
)
|
|
|
|
@staticmethod
|
|
def export_tf_batch_uniandbigrams_skip5() -> None:
|
|
input = np.array([[1, 1, 3, 3, 3, 7], [8, 6, 7, 5, 6, 8]]).astype(np.int32)
|
|
output = np.array(
|
|
[[0.0, 3.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 1.0]]
|
|
).astype(np.float32)
|
|
|
|
ngram_counts = np.array([0, 4]).astype(np.int64)
|
|
ngram_indexes = np.array([0, 1, 2, 3, 4, 5, 6]).astype(np.int64)
|
|
pool_int64s = np.array([2, 3, 5, 4, 5, 6, 7, 8, 6, 7]).astype( # unigrams
|
|
np.int64
|
|
) # bigrams
|
|
|
|
helper = TfIdfVectorizerHelper(
|
|
mode="TF",
|
|
min_gram_length=1,
|
|
max_gram_length=2,
|
|
max_skip_count=5,
|
|
ngram_counts=ngram_counts,
|
|
ngram_indexes=ngram_indexes,
|
|
pool_int64s=pool_int64s,
|
|
)
|
|
node = helper.make_node_noweights()
|
|
expect(
|
|
node,
|
|
inputs=[input],
|
|
outputs=[output],
|
|
name="test_tfidfvectorizer_tf_batch_uniandbigrams_skip5",
|
|
)
|