mirror of
https://github.com/hanzoai/utf-8-validate.git
synced 2026-09-24 17:03:57 +00:00
[major] Use the is_utf8 library (#109)
Use the is_utf8 [1] library to greatly improve performance. [1]: https://github.com/simdutf/is_utf8
This commit is contained in:
Luigi Pinca
@@ -1,3 +1,4 @@
|
||||
deps/is_utf8/.github/
|
||||
node_modules/
|
||||
prebuilds/
|
||||
build/
|
||||
|
||||
@@ -23,10 +23,29 @@ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
"""
|
||||
|
||||
This license applies to parts originating from
|
||||
https://www.cl.cam.ac.uk/~mgk25/ucs/utf8_check.c:
|
||||
This license applies to all parts of utf-8-validate that are not externally
|
||||
maintained libraries.
|
||||
|
||||
The externally maintained is_utf8 library used by utf-8-validate, located at
|
||||
deps/is_utf8, is licensed as follows:
|
||||
|
||||
"""
|
||||
Markus Kuhn <http://www.cl.cam.ac.uk/~mgk25/> -- 2005-03-30
|
||||
License: http://www.cl.cam.ac.uk/~mgk25/short-license.html
|
||||
Copyright 2022 The is_utf8 authors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
this software and associated documentation files (the "Software"), to deal in
|
||||
the Software without restriction, including without limitation the rights to
|
||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||
the Software, and to permit persons to whom the Software is furnished to do so,
|
||||
subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
"""
|
||||
|
||||
+5
-2
@@ -2,8 +2,11 @@
|
||||
'targets': [
|
||||
{
|
||||
'target_name': 'validation',
|
||||
'sources': ['src/validation.c'],
|
||||
'cflags': ['-std=c99'],
|
||||
'sources': [
|
||||
'src/validation.cc',
|
||||
'deps/is_utf8/src/is_utf8.cpp'
|
||||
],
|
||||
'cflags_cc': ['-std=gnu++11'],
|
||||
'conditions': [
|
||||
["OS=='mac'", {
|
||||
'xcode_settings': {
|
||||
|
||||
Vendored
+106
@@ -0,0 +1,106 @@
|
||||
cmake_minimum_required(VERSION 3.15)
|
||||
|
||||
project(is_utf8
|
||||
DESCRIPTION "Fast UTF-8 Validation"
|
||||
LANGUAGES CXX
|
||||
VERSION 1.2.1
|
||||
)
|
||||
|
||||
include(GNUInstallDirs)
|
||||
include(CTest)
|
||||
|
||||
option(IS_UTF8_SANITIZE "Sanitize addresses" OFF)
|
||||
|
||||
if (NOT CMAKE_BUILD_TYPE)
|
||||
message(STATUS "No build type selected, default to Release")
|
||||
if(IS_UTF8_SANITIZE)
|
||||
set(CMAKE_BUILD_TYPE Debug CACHE STRING "Choose the type of build." FORCE)
|
||||
else()
|
||||
set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
set(CMAKE_CXX_STANDARD 14)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
set(CMAKE_CXX_EXTENSIONS OFF)
|
||||
set(CMAKE_MACOSX_RPATH OFF)
|
||||
|
||||
set(IS_UTF8_LIB_VERSION "1.2.1" CACHE STRING "is_utf8 library version")
|
||||
set(IS_UTF8_LIB_SOVERSION "1" CACHE STRING "is_utf8 library soversion")
|
||||
|
||||
set(IS_UTF8_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src)
|
||||
add_subdirectory(src)
|
||||
|
||||
if (BUILD_TESTING)
|
||||
message(STATUS "The tests are enabled.")
|
||||
add_subdirectory(tests)
|
||||
else()
|
||||
message(STATUS "The tests are disabled.")
|
||||
endif(BUILD_TESTING)
|
||||
|
||||
|
||||
add_subdirectory(benchmarks)
|
||||
# ---- Install rules ----
|
||||
add_library(is_utf8::is_utf8 ALIAS is_utf8)
|
||||
|
||||
set_target_properties(
|
||||
is_utf8 PROPERTIES
|
||||
VERSION "${IS_UTF8_LIB_VERSION}"
|
||||
SOVERSION "${IS_UTF8_LIB_SOVERSION}"
|
||||
WINDOWS_EXPORT_ALL_SYMBOLS YES
|
||||
)
|
||||
|
||||
include(CMakePackageConfigHelpers)
|
||||
include(GNUInstallDirs)
|
||||
|
||||
install(
|
||||
FILES include/is_utf8.h
|
||||
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
|
||||
COMPONENT is_utf8_Development
|
||||
)
|
||||
|
||||
install(
|
||||
TARGETS is_utf8
|
||||
EXPORT is_utf8Targets
|
||||
RUNTIME COMPONENT is_utf8_Runtime
|
||||
LIBRARY COMPONENT is_utf8_Runtime
|
||||
NAMELINK_COMPONENT is_utf8_Development
|
||||
ARCHIVE COMPONENT is_utf8_Development
|
||||
INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
|
||||
)
|
||||
|
||||
configure_file(cmake/is_utf8-config.cmake.in is_utf8-config.cmake @ONLY)
|
||||
|
||||
write_basic_package_version_file(
|
||||
is_utf8-config-version.cmake
|
||||
COMPATIBILITY SameMinorVersion
|
||||
)
|
||||
|
||||
set(
|
||||
IS_UTF8_INSTALL_CMAKEDIR "${CMAKE_INSTALL_LIBDIR}/cmake/is_utf8"
|
||||
CACHE STRING "CMake package config location relative to the install prefix"
|
||||
)
|
||||
mark_as_advanced(IS_UTF8_INSTALL_CMAKEDIR)
|
||||
|
||||
install(
|
||||
FILES
|
||||
"${PROJECT_BINARY_DIR}/is_utf8-config.cmake"
|
||||
"${PROJECT_BINARY_DIR}/is_utf8-config-version.cmake"
|
||||
DESTINATION "${IS_UTF8_INSTALL_CMAKEDIR}"
|
||||
COMPONENT is_utf8_Development
|
||||
)
|
||||
|
||||
#
|
||||
# CPack
|
||||
#
|
||||
if(is_top_project)
|
||||
set(CPACK_PACKAGE_VENDOR "Daniel Lemire")
|
||||
set(CPACK_PACKAGE_CONTACT "[email protected]")
|
||||
set(CPACK_RESOURCE_FILE_LICENSE "${PROJECT_SOURCE_DIR}/LICENSE-MIT")
|
||||
set(CPACK_RPM_PACKAGE_LICENSE "${PROJECT_SOURCE_DIR}/LICENSE-MIT")
|
||||
set(CPACK_RESOURCE_FILE_README "${PROJECT_SOURCE_DIR}/README.md")
|
||||
set(CPACK_SOURCE_GENERATOR "TGZ;ZIP")
|
||||
include(CPack)
|
||||
endif()
|
||||
|
||||
# ----
|
||||
Vendored
+204
@@ -0,0 +1,204 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
NOTE: This software may be used as part of software released under the
|
||||
GNU General Public License (GPL) version 2.
|
||||
Vendored
+23
@@ -0,0 +1,23 @@
|
||||
Boost Software License - Version 1.0 - August 17th, 2003
|
||||
|
||||
Permission is hereby granted, free of charge, to any person or organization
|
||||
obtaining a copy of the software and accompanying documentation covered by
|
||||
this license (the "Software") to use, reproduce, display, distribute,
|
||||
execute, and transmit the Software, and to prepare derivative works of the
|
||||
Software, and to permit third-parties to whom the Software is furnished to
|
||||
do so, all subject to the following:
|
||||
|
||||
The copyright notices in the Software and this entire statement, including
|
||||
the above license grant, this restriction and the following disclaimer,
|
||||
must be included in all copies of the Software, in whole or in part, and
|
||||
all derivative works of the Software, unless such copies or derivative
|
||||
works are solely in the form of machine-executable object code generated by
|
||||
a source language processor.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
|
||||
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
|
||||
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
|
||||
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
Vendored
+18
@@ -0,0 +1,18 @@
|
||||
Copyright 2022 The is_utf8 authors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
this software and associated documentation files (the "Software"), to deal in
|
||||
the Software without restriction, including without limitation the rights to
|
||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||
the Software, and to permit persons to whom the Software is furnished to do so,
|
||||
subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
Vendored
+59
@@ -0,0 +1,59 @@
|
||||
# is_utf8
|
||||
|
||||
Most strings online are in unicode using the UTF-8 encoding. Validating strings
|
||||
quickly before accepting them is important.
|
||||
|
||||
|
||||
|
||||
|
||||
## How to use is_utf8
|
||||
|
||||
This is a simple one-source file library to validate UTF-8 strings at high speeds using SIMD instructions. It works on all platforms (ARM, x64).
|
||||
|
||||
Build and link `is_utf8.cpp` with your project. Code usage:
|
||||
|
||||
```C++
|
||||
#include "is_utf8.h"
|
||||
|
||||
char * mystring = ...
|
||||
bool is_it_valid = is_utf8(mystring, thestringlength);
|
||||
```
|
||||
|
||||
It should be able to validate strings using less than 1 cycle per input byte.
|
||||
|
||||
## Reference
|
||||
|
||||
- John Keiser, Daniel Lemire, [Validating UTF-8 In Less Than One Instruction Per Byte](https://arxiv.org/abs/2010.03090), Software: Practice & Experience 51 (5), 2021
|
||||
|
||||
### Want more?
|
||||
|
||||
If you want a wide range of fast Unicode function for production use, you can rely on the simdutf library. It is as simple as the following:
|
||||
|
||||
```C++
|
||||
#include "simdutf.cpp"
|
||||
#include "simdutf.h"
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
const char *source = "1234";
|
||||
// 4 == strlen(source)
|
||||
bool validutf8 = simdutf::validate_utf8(source, 4);
|
||||
if (validutf8) {
|
||||
std::cout << "valid UTF-8" << std::endl;
|
||||
} else {
|
||||
std::cerr << "invalid UTF-8" << std::endl;
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
See https://github.com/simdutf/
|
||||
|
||||
|
||||
## License
|
||||
|
||||
This library is distributed under the terms of any of the following
|
||||
licenses, at your option:
|
||||
|
||||
* Apache License (Version 2.0) [LICENSE-APACHE](LICENSE-APACHE),
|
||||
* Boost Software License [LICENSE-BOOST](LICENSE-BOOST), or
|
||||
* MIT License [LICENSE-MIT](LICENSE-MIT).
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
|
||||
include(${PROJECT_SOURCE_DIR}/cmake/import.cmake)
|
||||
set_off(SIMDUTF_TOOLS)
|
||||
set_off(SIMDUTF_BENCHMARKS)
|
||||
|
||||
import_dependency(simdutf simdutf/simdutf v2.0.9)
|
||||
set(BUILD_TESTING OFF)
|
||||
add_dependency(simdutf)
|
||||
unset(BUILD_TESTING)
|
||||
|
||||
add_executable(bench bench.cpp)
|
||||
target_link_libraries(bench PRIVATE is_utf8)
|
||||
target_link_libraries(bench PRIVATE simdutf)
|
||||
Vendored
+251
@@ -0,0 +1,251 @@
|
||||
#include "is_utf8.h"
|
||||
#include "simdutf.h"
|
||||
#include <assert.h>
|
||||
#include <chrono>
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
uint64_t nano() {
|
||||
return std::chrono::duration_cast<::std::chrono::nanoseconds>(
|
||||
std::chrono::steady_clock::now().time_since_epoch())
|
||||
.count();
|
||||
}
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#define never_inline __declspec(noinline)
|
||||
#else
|
||||
#define never_inline __attribute__((noinline))
|
||||
#endif
|
||||
|
||||
// generate a string having at least length N
|
||||
// can exceed by up to 3 chars, returns the actual length
|
||||
size_t populate_utf8(char *data, size_t N) {
|
||||
size_t i = 0;
|
||||
for (; i < N;) {
|
||||
int w = rand() & 0xFF;
|
||||
if (w < 0x80) {
|
||||
data[i++] = 0x20; // w;
|
||||
} else if (w < 0xE0) {
|
||||
data[i++] = 0xC2 + rand() % (0xDF - 0xC2 + 1);
|
||||
data[i++] = 0x80 + rand() % (0xBF - 0x80 + 1);
|
||||
} else if (w == 0xE0) {
|
||||
data[i++] = w;
|
||||
data[i++] = 0xA0 + rand() % (0xBF - 0xA0 + 1);
|
||||
data[i++] = 0x80 + rand() % (0xBF - 0x80 + 1);
|
||||
} else if (w <= 0xEC) {
|
||||
data[i++] = w;
|
||||
data[i++] = 0x80 + rand() % (0xBF - 0x80 + 1);
|
||||
data[i++] = 0x80 + rand() % (0xBF - 0x80 + 1);
|
||||
} else if (w == 0xED) {
|
||||
data[i++] = w;
|
||||
data[i++] = 0x80 + rand() % (0x9F - 0x80 + 1);
|
||||
data[i++] = 0x80 + rand() % (0xBF - 0x80 + 1);
|
||||
} else if (w <= 0xEF) {
|
||||
data[i++] = w;
|
||||
data[i++] = 0x80 + rand() % (0xBF - 0x80 + 1);
|
||||
data[i++] = 0x80 + rand() % (0xBF - 0x80 + 1);
|
||||
} else if (w < 0xF0) {
|
||||
data[i++] = 0xF1 + rand() % (0xF3 - 0xF1 + 1);
|
||||
data[i++] = 0x80 + rand() % (0xBF - 0x80 + 1);
|
||||
data[i++] = 0x80 + rand() % (0xBF - 0x80 + 1);
|
||||
data[i++] = 0x80 + rand() % (0xBF - 0x80 + 1);
|
||||
} else if (w == 0xF0) {
|
||||
data[i++] = w;
|
||||
data[i++] = 0x90 + rand() % (0xBF - 0x90 + 1);
|
||||
data[i++] = 0x80 + rand() % (0xBF - 0x80 + 1);
|
||||
data[i++] = 0x80 + rand() % (0xBF - 0x80 + 1);
|
||||
} else if (w <= 0xF3) {
|
||||
data[i++] = 0xF1 + rand() % (0xF3 - 0xF1 + 1);
|
||||
data[i++] = 0x80 + rand() % (0xBF - 0x80 + 1);
|
||||
data[i++] = 0x80 + rand() % (0xBF - 0x80 + 1);
|
||||
data[i++] = 0x80 + rand() % (0xBF - 0x80 + 1);
|
||||
} else if (w == 0xF4) {
|
||||
data[i++] = w;
|
||||
data[i++] = 0x80 + rand() % (0x8F - 0x80 + 1);
|
||||
data[i++] = 0x80 + rand() % (0xBF - 0x80 + 1);
|
||||
data[i++] = 0x80 + rand() % (0xBF - 0x80 + 1);
|
||||
}
|
||||
}
|
||||
data[i] = '\0';
|
||||
return i;
|
||||
}
|
||||
|
||||
// copied in part from Guava
|
||||
static never_inline bool basic_validate_utf8(const char *b, size_t length) {
|
||||
const unsigned char *bytes = (const unsigned char *)b;
|
||||
for (size_t index = 0;;) {
|
||||
unsigned char byte1;
|
||||
|
||||
do { // fast ASCII Path
|
||||
if (index >= length) {
|
||||
return true;
|
||||
}
|
||||
byte1 = bytes[index++];
|
||||
} while (byte1 < 0x80);
|
||||
if (byte1 < 0xE0) {
|
||||
// Two-byte form.
|
||||
if (index == length) {
|
||||
return false;
|
||||
}
|
||||
if (byte1 < 0xC2 || bytes[index++] > 0xBF) {
|
||||
return false;
|
||||
}
|
||||
} else if (byte1 < 0xF0) {
|
||||
// Three-byte form.
|
||||
if (index + 1 >= length) {
|
||||
return false;
|
||||
}
|
||||
unsigned char byte2 = bytes[index++];
|
||||
if (byte2 > 0xBF
|
||||
// Overlong? 5 most significant bits must not all be zero.
|
||||
|| (byte1 == 0xE0 && byte2 < 0xA0)
|
||||
// Check for illegal surrogate codepoints.
|
||||
|| (byte1 == 0xED && 0xA0 <= byte2)
|
||||
// Third byte trailing-byte test.
|
||||
|| bytes[index++] > 0xBF) {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
|
||||
// Four-byte form.
|
||||
if (index + 2 >= length) {
|
||||
return false;
|
||||
}
|
||||
int byte2 = bytes[index++];
|
||||
if (byte2 > 0xBF
|
||||
// Check that 1 <= plane <= 16. Tricky optimized form of:
|
||||
// if (byte1 > (byte) 0xF4
|
||||
// || byte1 == (byte) 0xF0 && byte2 < (byte) 0x90
|
||||
// || byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F)
|
||||
|| (((byte1 << 28) + (byte2 - 0x90)) >> 30) != 0
|
||||
// Third byte trailing-byte test
|
||||
|| bytes[index++] > 0xBF
|
||||
// Fourth byte trailing-byte test
|
||||
|| bytes[index++] > 0xBF) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
bool zerobuffer_bench(size_t N) {
|
||||
printf("zero buffer \n");
|
||||
printf("string size = %zu \n", N);
|
||||
char *input = new char[N]{};
|
||||
volatile bool isgood{true};
|
||||
|
||||
{
|
||||
uint64_t start = nano();
|
||||
uint64_t finish = start;
|
||||
size_t count{0};
|
||||
uint64_t threshold = 500000000;
|
||||
for (; finish - start < threshold;) {
|
||||
count++;
|
||||
isgood &= basic_validate_utf8(input, N);
|
||||
finish = nano();
|
||||
}
|
||||
double t = (N * count) / double(finish - start);
|
||||
|
||||
printf("basic_validate_utf8 %f GB/s\n", t);
|
||||
}
|
||||
|
||||
{
|
||||
uint64_t start = nano();
|
||||
uint64_t finish = start;
|
||||
size_t count{0};
|
||||
uint64_t threshold = 500000000;
|
||||
for (; finish - start < threshold;) {
|
||||
count++;
|
||||
isgood &= simdutf::validate_utf8(input, N);
|
||||
finish = nano();
|
||||
}
|
||||
double t = (N * count) / double(finish - start);
|
||||
|
||||
printf("simdutf %f GB/s\n", t);
|
||||
}
|
||||
|
||||
{
|
||||
uint64_t start = nano();
|
||||
uint64_t finish = start;
|
||||
size_t count{0};
|
||||
uint64_t threshold = 500000000;
|
||||
for (; finish - start < threshold;) {
|
||||
count++;
|
||||
isgood &= is_utf8(input, N);
|
||||
finish = nano();
|
||||
}
|
||||
double t = (N * count) / double(finish - start);
|
||||
|
||||
printf("is_utf8 %f GB/s\n", t);
|
||||
}
|
||||
delete[] input;
|
||||
printf("\n");
|
||||
return isgood;
|
||||
}
|
||||
|
||||
bool bench(size_t N) {
|
||||
printf("random UTF-8\n");
|
||||
printf("string size = %zu \n", N);
|
||||
char *input = new char[N];
|
||||
populate_utf8(input, N);
|
||||
volatile bool isgood{true};
|
||||
|
||||
{
|
||||
uint64_t start = nano();
|
||||
uint64_t finish = start;
|
||||
size_t count{0};
|
||||
uint64_t threshold = 500000000;
|
||||
for (; finish - start < threshold;) {
|
||||
count++;
|
||||
isgood &= basic_validate_utf8(input, N);
|
||||
finish = nano();
|
||||
}
|
||||
double t = (N * count) / double(finish - start);
|
||||
|
||||
printf("basic_validate_utf8 %f GB/s\n", t);
|
||||
}
|
||||
|
||||
{
|
||||
uint64_t start = nano();
|
||||
uint64_t finish = start;
|
||||
size_t count{0};
|
||||
uint64_t threshold = 500000000;
|
||||
for (; finish - start < threshold;) {
|
||||
count++;
|
||||
isgood &= simdutf::validate_utf8(input, N);
|
||||
finish = nano();
|
||||
}
|
||||
double t = (N * count) / double(finish - start);
|
||||
|
||||
printf("simdutf %f GB/s\n", t);
|
||||
}
|
||||
|
||||
{
|
||||
uint64_t start = nano();
|
||||
uint64_t finish = start;
|
||||
size_t count{0};
|
||||
uint64_t threshold = 500000000;
|
||||
for (; finish - start < threshold;) {
|
||||
count++;
|
||||
isgood &= is_utf8(input, N);
|
||||
finish = nano();
|
||||
}
|
||||
double t = (N * count) / double(finish - start);
|
||||
|
||||
printf("is_utf8 %f GB/s\n", t);
|
||||
}
|
||||
delete[] input;
|
||||
printf("\n");
|
||||
return isgood;
|
||||
}
|
||||
|
||||
int main() {
|
||||
return (bench(40096) & bench(100000) & bench(50000))
|
||||
& (zerobuffer_bench(40096) & zerobuffer_bench(100000) & zerobuffer_bench(50000))
|
||||
? EXIT_SUCCESS : EXIT_FAILURE;
|
||||
}
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
# Helper so we don't have to repeat ourselves so much
|
||||
# Usage: add_cpp_test(testname [COMPILE_ONLY] [SOURCES a.cpp b.cpp ...] [LABELS acceptance per_implementation ...])
|
||||
# SOURCES defaults to testname.cpp if not specified.
|
||||
function(add_cpp_test TEST_NAME)
|
||||
# Parse arguments
|
||||
cmake_parse_arguments(PARSE_ARGV 1 ARGS "COMPILE_ONLY;LIBRARY;WILL_FAIL" "" "SOURCES;LABELS;DEPENDENCY_OF")
|
||||
if (NOT ARGS_SOURCES)
|
||||
list(APPEND ARGS_SOURCES ${TEST_NAME}.cpp)
|
||||
endif()
|
||||
if (ARGS_COMPILE_ONLY)
|
||||
list(APPEND ${ARGS_LABELS} compile_only)
|
||||
endif()
|
||||
if (IS_UTF8_SANITIZE)
|
||||
add_compile_options(-fsanitize=address -fno-omit-frame-pointer -fno-sanitize-recover=all)
|
||||
add_compile_definitions(ASAN_OPTIONS=detect_leaks=1)
|
||||
endif()
|
||||
# Add the compile target
|
||||
if (ARGS_LIBRARY)
|
||||
add_library(${TEST_NAME} STATIC ${ARGS_SOURCES})
|
||||
else(ARGS_LIBRARY)
|
||||
add_executable(${TEST_NAME} ${ARGS_SOURCES})
|
||||
endif(ARGS_LIBRARY)
|
||||
|
||||
# Add test
|
||||
if (ARGS_COMPILE_ONLY OR ARGS_LIBRARY)
|
||||
add_test(
|
||||
NAME ${TEST_NAME}
|
||||
COMMAND ${CMAKE_COMMAND} --build . --target ${TEST_NAME} --config $<CONFIGURATION>
|
||||
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
|
||||
)
|
||||
set_target_properties(${TEST_NAME} PROPERTIES EXCLUDE_FROM_ALL TRUE EXCLUDE_FROM_DEFAULT_BUILD TRUE)
|
||||
else()
|
||||
add_test(${TEST_NAME} ${TEST_NAME})
|
||||
|
||||
# Add to <label>_tests make targets
|
||||
foreach(label ${ARGS_LABELS})
|
||||
list(APPEND ARGS_DEPENDENCY_OF ${label})
|
||||
endforeach(label ${ARGS_LABELS})
|
||||
endif()
|
||||
|
||||
# Add to test labels
|
||||
if (ARGS_LABELS)
|
||||
set_property(TEST ${TEST_NAME} APPEND PROPERTY LABELS ${ARGS_LABELS})
|
||||
endif()
|
||||
|
||||
# Add as a dependency of given targets
|
||||
foreach(dependency_of ${ARGS_DEPENDENCY_OF})
|
||||
if (NOT TARGET ${dependency_of}_tests)
|
||||
add_custom_target(${dependency_of}_tests)
|
||||
add_dependencies(all_tests ${dependency_of}_tests)
|
||||
endif(NOT TARGET ${dependency_of}_tests)
|
||||
add_dependencies(${dependency_of}_tests ${TEST_NAME})
|
||||
endforeach(dependency_of ${ARGS_DEPENDENCY_OF})
|
||||
|
||||
# If it will fail, mark the test as such
|
||||
if (ARGS_WILL_FAIL)
|
||||
set_property(TEST ${TEST_NAME} PROPERTY WILL_FAIL TRUE)
|
||||
endif()
|
||||
endfunction()
|
||||
Vendored
+48
@@ -0,0 +1,48 @@
|
||||
set(dep_root "${IS_UTF8_SOURCE_DIR}/dependencies/.cache")
|
||||
if(DEFINED ENV{IS_UTF8_DEPENDENCY_CACHE_DIR})
|
||||
set(dep_root "$ENV{IS_UTF8_DEPENDENCY_CACHE_DIR}")
|
||||
endif()
|
||||
|
||||
function(import_dependency NAME GITHUB_REPO COMMIT)
|
||||
message(STATUS "Importing ${NAME} (${GITHUB_REPO}@${COMMIT})")
|
||||
set(target "${dep_root}/${NAME}")
|
||||
|
||||
# If the folder exists in the cache, then we assume that everything is as
|
||||
# should be and do nothing
|
||||
if(EXISTS "${target}")
|
||||
set("${NAME}_SOURCE_DIR" "${target}" PARENT_SCOPE)
|
||||
return()
|
||||
endif()
|
||||
|
||||
set(zip_url "https://github.com/${GITHUB_REPO}/archive/${COMMIT}.zip")
|
||||
set(archive "${dep_root}/archive.zip")
|
||||
set(dest "${dep_root}/_extract")
|
||||
|
||||
file(DOWNLOAD "${zip_url}" "${archive}")
|
||||
file(MAKE_DIRECTORY "${dest}")
|
||||
execute_process(
|
||||
WORKING_DIRECTORY "${dest}"
|
||||
COMMAND "${CMAKE_COMMAND}" -E tar xf "${archive}")
|
||||
file(REMOVE "${archive}")
|
||||
|
||||
# GitHub archives only ever have one folder component at the root, so this
|
||||
# will always match that single folder
|
||||
file(GLOB dir LIST_DIRECTORIES YES "${dest}/*")
|
||||
|
||||
file(RENAME "${dir}" "${target}")
|
||||
|
||||
set("${NAME}_SOURCE_DIR" "${target}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
# Delegates to the dependency
|
||||
macro(add_dependency NAME)
|
||||
if(NOT DEFINED "${NAME}_SOURCE_DIR")
|
||||
message(FATAL_ERROR "Missing ${NAME}_SOURCE_DIR variable")
|
||||
endif()
|
||||
|
||||
add_subdirectory("${${NAME}_SOURCE_DIR}" "${PROJECT_BINARY_DIR}/_deps/${NAME}")
|
||||
endmacro()
|
||||
|
||||
function(set_off NAME)
|
||||
set("${NAME}" OFF CACHE INTERNAL "")
|
||||
endfunction()
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
|
||||
include("${CMAKE_CURRENT_LIST_DIR}/is_utf8Targets.cmake")
|
||||
Vendored
+10
@@ -0,0 +1,10 @@
|
||||
#ifndef IS_UTF8
|
||||
#include <stddef.h>
|
||||
|
||||
// Check whether the provided string is UTF-8.
|
||||
// The function is designed for use cases where
|
||||
// 99.99% of the inputs are valid UTF-8.
|
||||
// Thus the function unconditionally scans the
|
||||
// whole input.
|
||||
bool is_utf8(const char *src, size_t len);
|
||||
#endif // IS_UTF8
|
||||
Vendored
+38
@@ -0,0 +1,38 @@
|
||||
add_library(is_utf8-include-source INTERFACE)
|
||||
target_include_directories(is_utf8-include-source INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
|
||||
add_library(is_utf8-source INTERFACE)
|
||||
target_sources(is_utf8-source INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>/is_utf8.cpp)
|
||||
target_link_libraries(is_utf8-source INTERFACE is_utf8-include-source)
|
||||
add_library(is_utf8 STATIC is_utf8.cpp)
|
||||
target_include_directories(is_utf8 PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}> )
|
||||
target_include_directories(is_utf8 PUBLIC "$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>")
|
||||
|
||||
if(IS_UTF8_SANITIZE)
|
||||
target_compile_options(is_utf8 INTERFACE -fsanitize=address -fno-omit-frame-pointer -fno-sanitize-recover=all)
|
||||
target_compile_definitions(is_utf8 INTERFACE ASAN_OPTIONS=detect_leaks=1)
|
||||
target_link_libraries(is_utf8 INTERFACE -fsanitize=address -fno-omit-frame-pointer -fno-sanitize-recover=all)
|
||||
endif()
|
||||
|
||||
if(MSVC)
|
||||
if("${MSVC_TOOLSET_VERSION}" STREQUAL "140")
|
||||
target_compile_options(is_utf8 INTERFACE /W0 /sdl)
|
||||
set(is_utf8_LEGACY_VISUAL_STUDIO TRUE)
|
||||
else()
|
||||
target_compile_options(is_utf8 INTERFACE /WX /W3 /sdl /w34714) # https://docs.microsoft.com/en-us/cpp/error-messages/compiler-warnings/compiler-warning-level-4-c4714?view=vs-2019
|
||||
endif()
|
||||
else(MSVC)
|
||||
if(NOT WIN32)
|
||||
target_compile_options(is_utf8 INTERFACE -fPIC)
|
||||
endif()
|
||||
target_compile_options(is_utf8 PRIVATE -Wall -Wextra -Weffc++)
|
||||
target_compile_options(is_utf8 PRIVATE -Wfatal-errors -Wsign-compare -Wshadow -Wwrite-strings -Wpointer-arith -Winit-self -Wconversion -Wno-sign-conversion)
|
||||
endif(MSVC)
|
||||
|
||||
# workaround for GNU GCC poor AVX load/store code generation
|
||||
if ((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_SYSTEM_PROCESSOR MATCHES "^(i.86|x86(_64)?)$"))
|
||||
target_compile_options(is_utf8 PRIVATE -mno-avx256-split-unaligned-load -mno-avx256-split-unaligned-store)
|
||||
endif()
|
||||
|
||||
if(MSVC AND BUILD_SHARED_LIBS)
|
||||
set(is_utf8_WINDOWS_DLL TRUE)
|
||||
endif()
|
||||
Vendored
+6740
File diff suppressed because it is too large
Load Diff
Vendored
+8
@@ -0,0 +1,8 @@
|
||||
if(MSVC)
|
||||
add_compile_options("/Zi" "/EHsc" "/GR")
|
||||
endif()
|
||||
|
||||
include(${PROJECT_SOURCE_DIR}/cmake/add_cpp_test.cmake)
|
||||
link_libraries(is_utf8)
|
||||
|
||||
add_cpp_test(unit)
|
||||
Vendored
+338
@@ -0,0 +1,338 @@
|
||||
#include "is_utf8.h"
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <random>
|
||||
#include <vector>
|
||||
|
||||
bool hard_coded() {
|
||||
std::cout << "hard coded tests." << std::endl;
|
||||
// additional tests are from autobahn websocket testsuite
|
||||
// https://github.com/crossbario/autobahn-testsuite/tree/master/autobahntestsuite/autobahntestsuite/case
|
||||
const char *goodsequences[] = {"a", "abcde12345", "\x71", "\x75\x4c",
|
||||
"\x7f\x4c\x23\x3c\x3a\x6f\x5d\x44\x13\x70"};
|
||||
const char *badsequences[] = {
|
||||
"\xc3\x28", // 0
|
||||
"\xa0\xa1", // 1
|
||||
"\xe2\x28\xa1", // 2
|
||||
"\xe2\x82\x28", // 3
|
||||
"\xf0\x28\x8c\xbc", // 4
|
||||
"\xf0\x90\x28\xbc", // 5
|
||||
"\xf0\x28\x8c\x28", // 6
|
||||
"\xc0\x9f", // 7
|
||||
"\xf5\xff\xff\xff", // 8
|
||||
"\xed\xa0\x81", // 9
|
||||
"\xf8\x90\x80\x80\x80", // 10
|
||||
"123456789012345\xed",
|
||||
"123456789012345\xf1",
|
||||
"123456789012345\xc2",
|
||||
"\xC2\x7F",
|
||||
"\xce",
|
||||
"\xce\xba\xe1",
|
||||
"\xce\xba\xe1\xbd",
|
||||
"\xce\xba\xe1\xbd\xb9\xcf",
|
||||
"\xce\xba\xe1\xbd\xb9\xcf\x83\xce",
|
||||
"\xce\xba\xe1\xbd\xb9\xcf\x83\xce\xbc\xce",
|
||||
"\xdf",
|
||||
"\xef\xbf",
|
||||
"\x80",
|
||||
"\x91\x85\x95\x9e",
|
||||
"\x6c\x02\x8e\x18",
|
||||
"\x25\x5b\x6e\x2c\x32\x2c\x5b\x5b\x33\x2c\x34\x2c\x05\x29\x2c\x33\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b"
|
||||
"\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5d\x2c\x35\x2e\x33\x2c\x39\x2e\x33"
|
||||
"\x2c\x37\x2e\x33\x2c\x39\x2e\x34\x2c\x37\x2e\x33\x2c\x39\x2e\x33\x2c\x37"
|
||||
"\x2e\x33\x2c\x39\x2e\x34\x5d\x5d\x5d\x5d\x5d\x5d\x5d\x5d\x5d\x5d\x5d\x5d"
|
||||
"\x5d\x5d\x5d\x5d\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x20\x01\x01"
|
||||
"\x01\x01\x01\x02\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x23\x0a\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x7e\x7e\x0a\x0a\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x5b\x5b"
|
||||
"\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5d\x2c\x37"
|
||||
"\x2e\x33\x2c\x39\x2e\x33\x2c\x37\x2e\x33\x2c\x39\x2e\x34\x2c\x37\x2e\x33"
|
||||
"\x2c\x39\x2e\x33\x2c\x37\x2e\x33\x2c\x39\x2e\x34\x5d\x5d\x5d\x5d\x5d\x5d"
|
||||
"\x5d\x5d\x5d\x5d\x5d\x5d\x5d\x5d\x5d\x01\x01\x80\x01\x01\x01\x79\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01",
|
||||
"[[[[[[[[[[[[[[["
|
||||
"\x80\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x010\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01",
|
||||
"\x20\x0b\x01\x01\x01\x64\x3a\x64\x3a\x64\x3a\x5b\x5b\x5b\x5b\x5b\x5b\x5b"
|
||||
"\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b"
|
||||
"\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x5b\x30\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x80\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01"
|
||||
"\x01\x01\x01\x01\x01",
|
||||
"\x80",
|
||||
"\x90",
|
||||
"\xa1",
|
||||
"\xb2",
|
||||
"\xc3",
|
||||
"\xd4",
|
||||
"\xe5",
|
||||
"\xf6"
|
||||
"\xc3\xb1"};
|
||||
size_t number_good = sizeof(goodsequences) / sizeof(goodsequences[0]);
|
||||
printf("Testing %zu good sequences.\n", number_good);
|
||||
for (size_t i = 0; i < number_good; i++) {
|
||||
size_t len = std::strlen(goodsequences[i]);
|
||||
if (!is_utf8(goodsequences[i], len)) {
|
||||
printf("bug goodsequences[%zu]\n", i);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
size_t number_bad = sizeof(badsequences) / sizeof(badsequences[0]);
|
||||
printf("Testing %zu bad sequences.\n", number_bad);
|
||||
for (size_t i = 0; i < number_bad; i++) {
|
||||
size_t len = std::strlen(badsequences[i]);
|
||||
if (is_utf8(badsequences[i], len)) {
|
||||
printf("bug badsequences[%zu]\n", i);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
printf("Success.\n");
|
||||
return true;
|
||||
}
|
||||
|
||||
class random_utf8 final {
|
||||
public:
|
||||
random_utf8(uint32_t seed, int prob_1byte, int prob_2bytes, int prob_3bytes,
|
||||
int prob_4bytes);
|
||||
|
||||
std::vector<uint8_t> generate(size_t output_bytes);
|
||||
// include the number of code points
|
||||
std::pair<std::vector<uint8_t>, size_t> generate_counted(size_t output_bytes);
|
||||
std::vector<uint8_t> generate(size_t output_bytes, long seed);
|
||||
|
||||
private:
|
||||
uint32_t generate();
|
||||
|
||||
std::mt19937 gen;
|
||||
std::discrete_distribution<> bytes_count;
|
||||
std::uniform_int_distribution<int> val_7bit{0x00, 0x7f}; // 0b0xxxxxxx
|
||||
std::uniform_int_distribution<int> val_6bit{0x00, 0x3f}; // 0b10xxxxxx
|
||||
std::uniform_int_distribution<int> val_5bit{0x00, 0x1f}; // 0b110xxxxx
|
||||
std::uniform_int_distribution<int> val_4bit{0x00, 0x0f}; // 0b1110xxxx
|
||||
std::uniform_int_distribution<int> val_3bit{0x00, 0x07}; // 0b11110xxx
|
||||
};
|
||||
|
||||
random_utf8::random_utf8(uint32_t seed, int prob_1byte, int prob_2bytes,
|
||||
int prob_3bytes, int prob_4bytes)
|
||||
: gen(seed), bytes_count({double(prob_1byte), double(prob_2bytes),
|
||||
double(prob_3bytes), double(prob_4bytes)}) {}
|
||||
|
||||
std::pair<std::vector<uint8_t>, size_t>
|
||||
random_utf8::generate_counted(size_t output_bytes) {
|
||||
std::vector<uint8_t> result;
|
||||
result.reserve(output_bytes);
|
||||
uint8_t candidate, head;
|
||||
size_t count{0};
|
||||
while (result.size() < output_bytes) {
|
||||
count++;
|
||||
switch (bytes_count(gen)) {
|
||||
case 0: // 1 byte
|
||||
candidate = uint8_t(val_7bit(gen));
|
||||
while (candidate == 0) { // though strictly speaking, a stream of nulls is
|
||||
// UTF8, it tends to break some code
|
||||
candidate = uint8_t(val_7bit(gen));
|
||||
}
|
||||
result.push_back(candidate);
|
||||
break;
|
||||
case 1: // 2 bytes
|
||||
candidate = 0xc0 | uint8_t(val_5bit(gen));
|
||||
while (candidate < 0xC2) {
|
||||
candidate = 0xc0 | uint8_t(val_5bit(gen));
|
||||
}
|
||||
result.push_back(candidate);
|
||||
result.push_back(0x80 | uint8_t(val_6bit(gen)));
|
||||
break;
|
||||
case 2: // 3 bytes
|
||||
head = 0xe0 | uint8_t(val_4bit(gen));
|
||||
result.push_back(head);
|
||||
candidate = 0x80 | uint8_t(val_6bit(gen));
|
||||
if (head == 0xE0) {
|
||||
while (candidate < 0xA0) {
|
||||
candidate = 0x80 | uint8_t(val_6bit(gen));
|
||||
}
|
||||
} else if (head == 0xED) {
|
||||
while (candidate > 0x9F) {
|
||||
candidate = 0x80 | uint8_t(val_6bit(gen));
|
||||
}
|
||||
}
|
||||
result.push_back(candidate);
|
||||
result.push_back(0x80 | uint8_t(val_6bit(gen)));
|
||||
break;
|
||||
case 3: // 4 bytes
|
||||
head = 0xf0 | uint8_t(val_3bit(gen));
|
||||
while (head > 0xF4) {
|
||||
head = 0xf0 | uint8_t(val_3bit(gen));
|
||||
}
|
||||
result.push_back(head);
|
||||
candidate = 0x80 | uint8_t(val_6bit(gen));
|
||||
if (head == 0xF0) {
|
||||
while (candidate < 0x90) {
|
||||
candidate = 0x80 | uint8_t(val_6bit(gen));
|
||||
}
|
||||
} else if (head == 0xF4) {
|
||||
while (candidate > 0x8F) {
|
||||
candidate = 0x80 | uint8_t(val_6bit(gen));
|
||||
}
|
||||
}
|
||||
result.push_back(candidate);
|
||||
result.push_back(0x80 | uint8_t(val_6bit(gen)));
|
||||
result.push_back(0x80 | uint8_t(val_6bit(gen)));
|
||||
break;
|
||||
}
|
||||
}
|
||||
result.push_back(0); // EOS for scalar code
|
||||
|
||||
return make_pair(result, count);
|
||||
}
|
||||
|
||||
std::vector<uint8_t> random_utf8::generate(size_t output_bytes) {
|
||||
return generate_counted(output_bytes).first;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> random_utf8::generate(size_t output_bytes, long seed) {
|
||||
gen.seed(uint32_t(seed));
|
||||
return generate(output_bytes);
|
||||
}
|
||||
|
||||
// credit: based on code from Google Fuchsia (Apache Licensed)
|
||||
bool reference_validate_utf8(const char *buf, size_t len) noexcept {
|
||||
const uint8_t *data = (const uint8_t *)buf;
|
||||
uint64_t pos = 0;
|
||||
uint64_t next_pos = 0;
|
||||
uint32_t code_point = 0;
|
||||
while (pos < len) {
|
||||
unsigned char byte = data[pos];
|
||||
if (byte < 0b10000000) {
|
||||
pos++;
|
||||
continue;
|
||||
} else if ((byte & 0b11100000) == 0b11000000) {
|
||||
next_pos = pos + 2;
|
||||
if (next_pos > len) {
|
||||
return false;
|
||||
}
|
||||
if ((data[pos + 1] & 0b11000000) != 0b10000000) {
|
||||
return false;
|
||||
}
|
||||
// range check
|
||||
code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111);
|
||||
if (code_point < 0x80 || 0x7ff < code_point) {
|
||||
return false;
|
||||
}
|
||||
} else if ((byte & 0b11110000) == 0b11100000) {
|
||||
next_pos = pos + 3;
|
||||
if (next_pos > len) {
|
||||
return false;
|
||||
}
|
||||
if ((data[pos + 1] & 0b11000000) != 0b10000000) {
|
||||
return false;
|
||||
}
|
||||
if ((data[pos + 2] & 0b11000000) != 0b10000000) {
|
||||
return false;
|
||||
}
|
||||
// range check
|
||||
code_point = (byte & 0b00001111) << 12 |
|
||||
(data[pos + 1] & 0b00111111) << 6 |
|
||||
(data[pos + 2] & 0b00111111);
|
||||
if (code_point < 0x800 || 0xffff < code_point ||
|
||||
(0xd7ff < code_point && code_point < 0xe000)) {
|
||||
return false;
|
||||
}
|
||||
} else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000
|
||||
next_pos = pos + 4;
|
||||
if (next_pos > len) {
|
||||
return false;
|
||||
}
|
||||
if ((data[pos + 1] & 0b11000000) != 0b10000000) {
|
||||
return false;
|
||||
}
|
||||
if ((data[pos + 2] & 0b11000000) != 0b10000000) {
|
||||
return false;
|
||||
}
|
||||
if ((data[pos + 3] & 0b11000000) != 0b10000000) {
|
||||
return false;
|
||||
}
|
||||
// range check
|
||||
code_point =
|
||||
(byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 |
|
||||
(data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111);
|
||||
if (code_point < 0xffff || 0x10ffff < code_point) {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
// we may have a continuation
|
||||
return false;
|
||||
}
|
||||
pos = next_pos;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
bool brute_force() {
|
||||
|
||||
std::cout << "brute force tests." << std::endl;
|
||||
uint32_t seed{1234};
|
||||
random_utf8 gen_1_2_3_4(seed, 1, 1, 1, 1);
|
||||
size_t total = 10000;
|
||||
for (size_t i = 0; i < total; i++) {
|
||||
if ((i % 1000) == 0) {
|
||||
printf(".");
|
||||
fflush(NULL);
|
||||
}
|
||||
|
||||
auto UTF8 = gen_1_2_3_4.generate(rand() % 1024);
|
||||
if (!is_utf8((const char *)UTF8.data(), UTF8.size())) {
|
||||
std::cerr << "bug" << std::endl;
|
||||
return false;
|
||||
}
|
||||
for (size_t flip = 0; flip < 1000; ++flip) {
|
||||
// we are going to hack the string as long as it is UTF-8
|
||||
const int bitflip{1 << (rand() % 8)};
|
||||
UTF8[rand() % UTF8.size()] = uint8_t(bitflip); // we flip exactly one bit
|
||||
bool is_ok = is_utf8((const char *)UTF8.data(), UTF8.size());
|
||||
bool is_ok_basic =
|
||||
reference_validate_utf8((const char *)UTF8.data(), UTF8.size());
|
||||
if (is_ok != is_ok_basic) {
|
||||
std::cerr << "bug" << std::endl;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
printf("\n");
|
||||
printf("Success.\n");
|
||||
return true;
|
||||
}
|
||||
|
||||
int main() {
|
||||
bool results = hard_coded() & brute_force();
|
||||
return results ? EXIT_SUCCESS : EXIT_FAILURE;
|
||||
}
|
||||
@@ -15,6 +15,9 @@
|
||||
"files": [
|
||||
"prebuilds/",
|
||||
"src/",
|
||||
"deps/is_utf8/LICENSE-MIT",
|
||||
"deps/is_utf8/include/is_utf8.h",
|
||||
"deps/is_utf8/src/is_utf8.cpp",
|
||||
"binding.gyp",
|
||||
"fallback.js",
|
||||
"index.js"
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
#define NAPI_VERSION 1
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
#include <node_api.h>
|
||||
|
||||
napi_value IsValidUTF8(napi_env env, napi_callback_info info) {
|
||||
napi_status status;
|
||||
size_t argc = 1;
|
||||
napi_value argv[1];
|
||||
|
||||
status = napi_get_cb_info(env, info, &argc, argv, NULL, NULL);
|
||||
assert(status == napi_ok);
|
||||
|
||||
uint8_t *buf;
|
||||
size_t len;
|
||||
|
||||
status = napi_get_buffer_info(env, argv[0], (void **)&buf, &len);
|
||||
assert(status == napi_ok);
|
||||
|
||||
size_t i = 0;
|
||||
|
||||
//
|
||||
// This code has been taken from utf8_check.c which was developed by
|
||||
// Markus Kuhn <http://www.cl.cam.ac.uk/~mgk25/>.
|
||||
//
|
||||
// For original code / licensing please refer to
|
||||
// https://www.cl.cam.ac.uk/%7Emgk25/ucs/utf8_check.c
|
||||
//
|
||||
while (i < len) {
|
||||
size_t j = i + 8;
|
||||
|
||||
if (j <= len) {
|
||||
//
|
||||
// Read 8 bytes and check if they are ASCII.
|
||||
//
|
||||
uint64_t chunk;
|
||||
memcpy(&chunk, buf + i, 8);
|
||||
|
||||
if ((chunk & 0x8080808080808080) == 0x00) {
|
||||
i = j;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
while ((buf[i] & 0x80) == 0x00) { // 0xxxxxxx
|
||||
if (++i == len) {
|
||||
goto exit;
|
||||
}
|
||||
}
|
||||
|
||||
if ((buf[i] & 0xe0) == 0xc0) { // 110xxxxx 10xxxxxx
|
||||
if (
|
||||
i + 1 == len ||
|
||||
(buf[i + 1] & 0xc0) != 0x80 ||
|
||||
(buf[i] & 0xfe) == 0xc0 // overlong
|
||||
) {
|
||||
break;
|
||||
}
|
||||
|
||||
i += 2;
|
||||
} else if ((buf[i] & 0xf0) == 0xe0) { // 1110xxxx 10xxxxxx 10xxxxxx
|
||||
if (
|
||||
i + 2 >= len ||
|
||||
(buf[i + 1] & 0xc0) != 0x80 ||
|
||||
(buf[i + 2] & 0xc0) != 0x80 ||
|
||||
(buf[i] == 0xe0 && (buf[i + 1] & 0xe0) == 0x80) || // overlong
|
||||
(buf[i] == 0xed && (buf[i + 1] & 0xe0) == 0xa0) // surrogate (U+D800 - U+DFFF)
|
||||
) {
|
||||
break;
|
||||
}
|
||||
|
||||
i += 3;
|
||||
} else if ((buf[i] & 0xf8) == 0xf0) { // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
|
||||
if (
|
||||
i + 3 >= len ||
|
||||
(buf[i + 1] & 0xc0) != 0x80 ||
|
||||
(buf[i + 2] & 0xc0) != 0x80 ||
|
||||
(buf[i + 3] & 0xc0) != 0x80 ||
|
||||
(buf[i] == 0xf0 && (buf[i + 1] & 0xf0) == 0x80) || // overlong
|
||||
(buf[i] == 0xf4 && buf[i + 1] > 0x8f) || buf[i] > 0xf4 // > U+10FFFF
|
||||
) {
|
||||
break;
|
||||
}
|
||||
|
||||
i += 4;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
exit:;
|
||||
napi_value result;
|
||||
status = napi_get_boolean(env, i == len, &result);
|
||||
assert(status == napi_ok);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
napi_value Init(napi_env env, napi_value exports) {
|
||||
napi_status status;
|
||||
napi_value isValidUTF8;
|
||||
|
||||
status = napi_create_function(env, NULL, 0, IsValidUTF8, NULL, &isValidUTF8);
|
||||
assert(status == napi_ok);
|
||||
|
||||
return isValidUTF8;
|
||||
}
|
||||
|
||||
NAPI_MODULE(NODE_GYP_MODULE_NAME, Init)
|
||||
@@ -0,0 +1,40 @@
|
||||
#define NAPI_VERSION 1
|
||||
#include <assert.h>
|
||||
#include <node_api.h>
|
||||
|
||||
#include "../deps/is_utf8/include/is_utf8.h"
|
||||
|
||||
napi_value IsValidUTF8(napi_env env, napi_callback_info info) {
|
||||
napi_status status;
|
||||
size_t argc = 1;
|
||||
napi_value argv[1];
|
||||
|
||||
status = napi_get_cb_info(env, info, &argc, argv, NULL, NULL);
|
||||
assert(status == napi_ok);
|
||||
|
||||
char *buf;
|
||||
size_t len;
|
||||
|
||||
status = napi_get_buffer_info(env, argv[0], (void **)&buf, &len);
|
||||
assert(status == napi_ok);
|
||||
|
||||
bool is_valid = is_utf8(buf, len);
|
||||
|
||||
napi_value result;
|
||||
status = napi_get_boolean(env, is_valid, &result);
|
||||
assert(status == napi_ok);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
napi_value Init(napi_env env, napi_value exports) {
|
||||
napi_status status;
|
||||
napi_value isValidUTF8;
|
||||
|
||||
status = napi_create_function(env, NULL, 0, IsValidUTF8, NULL, &isValidUTF8);
|
||||
assert(status == napi_ok);
|
||||
|
||||
return isValidUTF8;
|
||||
}
|
||||
|
||||
NAPI_MODULE(NODE_GYP_MODULE_NAME, Init)
|
||||
Reference in New Issue
Block a user