mirror of
https://github.com/luxfi/pqclean.git
synced 2026-11-03 09:57:54 +00:00
1201 lines
40 KiB
PHP
1201 lines
40 KiB
PHP
|
|
/*
|
|
* We offer
|
|
* CC0 1.0 Universal or the following MIT License for this file.
|
|
* You may freely choose one of them that applies.
|
|
*
|
|
* MIT License
|
|
*
|
|
* Copyright (c) 2023: Hanno Becker, Vincent Hwang, Matthias J. Kannwischer, Bo-Yin Yang, and Shang-Yi Yang
|
|
* Copyright (c) 2023: Vincent Hwang
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
* of this software and associated documentation files (the "Software"), to deal
|
|
* in the Software without restriction, including without limitation the rights
|
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
* copies of the Software, and to permit persons to whom the Software is
|
|
* furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice shall be included in
|
|
* all copies or substantial portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
* SOFTWARE.
|
|
*/
|
|
|
|
#ifndef MACROS_COMMON
|
|
#define MACROS_COMMON
|
|
|
|
// for ABI
|
|
|
|
.macro push_all
|
|
|
|
sub sp, sp, #(9*16)
|
|
stp x19, x20, [sp, #0*16]
|
|
stp x21, x22, [sp, #1*16]
|
|
stp x23, x24, [sp, #2*16]
|
|
stp x25, x26, [sp, #3*16]
|
|
stp x27, x28, [sp, #4*16]
|
|
stp d8, d9, [sp, #5*16]
|
|
stp d10, d11, [sp, #6*16]
|
|
stp d12, d13, [sp, #7*16]
|
|
stp d14, d15, [sp, #8*16]
|
|
|
|
.endm
|
|
|
|
.macro pop_all
|
|
|
|
ldp x19, x20, [sp, #0*16]
|
|
ldp x21, x22, [sp, #1*16]
|
|
ldp x23, x24, [sp, #2*16]
|
|
ldp x25, x26, [sp, #3*16]
|
|
ldp x27, x28, [sp, #4*16]
|
|
ldp d8, d9, [sp, #5*16]
|
|
ldp d10, d11, [sp, #6*16]
|
|
ldp d12, d13, [sp, #7*16]
|
|
ldp d14, d15, [sp, #8*16]
|
|
add sp, sp, #(9*16)
|
|
|
|
.endm
|
|
|
|
.macro push_simd
|
|
|
|
sub sp, sp, #(4*16)
|
|
stp d8, d9, [sp, #0*16]
|
|
stp d10, d11, [sp, #1*16]
|
|
stp d12, d13, [sp, #2*16]
|
|
stp d14, d15, [sp, #3*16]
|
|
|
|
.endm
|
|
|
|
.macro pop_simd
|
|
|
|
ldp d8, d9, [sp, #0*16]
|
|
ldp d10, d11, [sp, #1*16]
|
|
ldp d12, d13, [sp, #2*16]
|
|
ldp d14, d15, [sp, #3*16]
|
|
add sp, sp, #(4*16)
|
|
|
|
.endm
|
|
|
|
// vector-scalar butterflies
|
|
|
|
.macro wrap_dX_butterfly_top a0, a1, b0, b1, t0, t1, mod, z0, l0, h0, z1, l1, h1, wX, nX
|
|
|
|
mul \t0\wX, \b0\wX, \z0\nX[\h0]
|
|
mul \t1\wX, \b1\wX, \z1\nX[\h1]
|
|
|
|
sqrdmulh \b0\wX, \b0\wX, \z0\nX[\l0]
|
|
sqrdmulh \b1\wX, \b1\wX, \z1\nX[\l1]
|
|
|
|
mls \t0\wX, \b0\wX, \mod\nX[0]
|
|
mls \t1\wX, \b1\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_dX_butterfly_topl4 a0, a1, b0, b1, t0, t1, mod, z0, l0, h0, z1, l1, h1, wX, nX, src_ptr, c0, c1, c2, c3, memc0, memc1, memc2, memc3
|
|
|
|
ldr \c0, [\src_ptr, \memc0]
|
|
mul \t0\wX, \b0\wX, \z0\nX[\h0]
|
|
ldr \c1, [\src_ptr, \memc1]
|
|
mul \t1\wX, \b1\wX, \z1\nX[\h1]
|
|
|
|
ldr \c2, [\src_ptr, \memc2]
|
|
sqrdmulh \b0\wX, \b0\wX, \z0\nX[\l0]
|
|
ldr \c3, [\src_ptr, \memc3]
|
|
sqrdmulh \b1\wX, \b1\wX, \z1\nX[\l1]
|
|
|
|
mls \t0\wX, \b0\wX, \mod\nX[0]
|
|
mls \t1\wX, \b1\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_dX_butterfly_top2l4s4 a0, a1, b0, b1, t0, t1, mod, z0, l0, h0, z1, l1, h1, wX, nX, srcc_ptr, c0, c1, memc0, memc1, srcd_ptr, d0, d1, memd0, memd1, srce_ptr, e0, e1, e2, e3, meme0, meme1, meme2, meme3
|
|
|
|
ldr \c0, [\srcc_ptr, \memc0]
|
|
str \e0, [\srce_ptr, \meme0]
|
|
mul \t0\wX, \b0\wX, \z0\nX[\h0]
|
|
ldr \c1, [\srcc_ptr, \memc1]
|
|
str \e1, [\srce_ptr, \meme1]
|
|
mul \t1\wX, \b1\wX, \z1\nX[\h1]
|
|
|
|
ldr \d0, [\srcd_ptr, \memd0]
|
|
str \e2, [\srce_ptr, \meme2]
|
|
sqrdmulh \b0\wX, \b0\wX, \z0\nX[\l0]
|
|
ldr \d1, [\srcd_ptr, \memd1]
|
|
str \e3, [\srce_ptr, \meme3]
|
|
sqrdmulh \b1\wX, \b1\wX, \z1\nX[\l1]
|
|
|
|
mls \t0\wX, \b0\wX, \mod\nX[0]
|
|
mls \t1\wX, \b1\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
|
|
.macro wrap_dX_butterfly_bot a0, a1, b0, b1, t0, t1, mod, z0, l0, h0, z1, l1, h1, wX, nX
|
|
|
|
sub \b0\wX, \a0\wX, \t0\wX
|
|
sub \b1\wX, \a1\wX, \t1\wX
|
|
|
|
add \a0\wX, \a0\wX, \t0\wX
|
|
add \a1\wX, \a1\wX, \t1\wX
|
|
|
|
.endm
|
|
|
|
.macro wrap_dX_butterfly_mix a0, a1, b0, b1, t0, t1, a2, a3, b2, b3, t2, t3, mod, z0, l0, h0, z1, l1, h1, z2, l2, h2, z3, l3, h3, wX, nX
|
|
|
|
sub \b0\wX, \a0\wX, \t0\wX
|
|
mul \t2\wX, \b2\wX, \z2\nX[\h2]
|
|
sub \b1\wX, \a1\wX, \t1\wX
|
|
mul \t3\wX, \b3\wX, \z3\nX[\h3]
|
|
|
|
add \a0\wX, \a0\wX, \t0\wX
|
|
sqrdmulh \b2\wX, \b2\wX, \z2\nX[\l2]
|
|
add \a1\wX, \a1\wX, \t1\wX
|
|
sqrdmulh \b3\wX, \b3\wX, \z3\nX[\l3]
|
|
|
|
mls \t2\wX, \b2\wX, \mod\nX[0]
|
|
mls \t3\wX, \b3\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_dX_butterfly_mixl6 a0, a1, b0, b1, t0, t1, a2, a3, b2, b3, t2, t3, mod, z0, l0, h0, z1, l1, h1, z2, l2, h2, z3, l3, h3, wX, nX, srcc_ptr, c0, c1, c2, c3, c4, c5, memc0, memc1, memc2, memc3, memc4, memc5
|
|
|
|
ldr \c0, [\srcc_ptr, \memc0]
|
|
sub \b0\wX, \a0\wX, \t0\wX
|
|
mul \t2\wX, \b2\wX, \z2\nX[\h2]
|
|
ldr \c1, [\srcc_ptr, \memc1]
|
|
sub \b1\wX, \a1\wX, \t1\wX
|
|
mul \t3\wX, \b3\wX, \z3\nX[\h3]
|
|
|
|
ldr \c2, [\srcc_ptr, \memc2]
|
|
add \a0\wX, \a0\wX, \t0\wX
|
|
sqrdmulh \b2\wX, \b2\wX, \z2\nX[\l2]
|
|
ldr \c3, [\srcc_ptr, \memc3]
|
|
add \a1\wX, \a1\wX, \t1\wX
|
|
sqrdmulh \b3\wX, \b3\wX, \z3\nX[\l3]
|
|
|
|
ldr \c4, [\srcc_ptr, \memc4]
|
|
mls \t2\wX, \b2\wX, \mod\nX[0]
|
|
ldr \c5, [\srcc_ptr, \memc5]
|
|
mls \t3\wX, \b3\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_dX_butterfly_mix_rev a0, a1, b0, b1, t0, t1, a2, a3, b2, b3, t2, t3, mod, z0, l0, h0, z1, l1, h1, z2, l2, h2, z3, l3, h3, wX, nX
|
|
|
|
mul \t0\wX, \b0\wX, \z0\nX[\h0]
|
|
sub \b2\wX, \a2\wX, \t2\wX
|
|
mul \t1\wX, \b1\wX, \z1\nX[\h1]
|
|
sub \b3\wX, \a3\wX, \t3\wX
|
|
|
|
sqrdmulh \b0\wX, \b0\wX, \z0\nX[\l0]
|
|
add \a2\wX, \a2\wX, \t2\wX
|
|
sqrdmulh \b1\wX, \b1\wX, \z1\nX[\l1]
|
|
add \a3\wX, \a3\wX, \t3\wX
|
|
|
|
mls \t0\wX, \b0\wX, \mod\nX[0]
|
|
mls \t1\wX, \b1\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_qX_butterfly_top a0, a1, a2, a3, b0, b1, b2, b3, t0, t1, t2, t3, mod, z0, l0, h0, z1, l1, h1, z2, l2, h2, z3, l3, h3, wX, nX
|
|
|
|
mul \t0\wX, \b0\wX, \z0\nX[\h0]
|
|
mul \t1\wX, \b1\wX, \z1\nX[\h1]
|
|
mul \t2\wX, \b2\wX, \z2\nX[\h2]
|
|
mul \t3\wX, \b3\wX, \z3\nX[\h3]
|
|
|
|
sqrdmulh \b0\wX, \b0\wX, \z0\nX[\l0]
|
|
sqrdmulh \b1\wX, \b1\wX, \z1\nX[\l1]
|
|
sqrdmulh \b2\wX, \b2\wX, \z2\nX[\l2]
|
|
sqrdmulh \b3\wX, \b3\wX, \z3\nX[\l3]
|
|
|
|
mls \t0\wX, \b0\wX, \mod\nX[0]
|
|
mls \t1\wX, \b1\wX, \mod\nX[0]
|
|
mls \t2\wX, \b2\wX, \mod\nX[0]
|
|
mls \t3\wX, \b3\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_qX_butterfly_topl b0, b1, b2, b3, t0, t1, t2, t3, mod, z0, l0, h0, z1, l1, h1, z2, l2, h2, z3, l3, h3, wX, nX, src_ptr, c0, c1, c2, c3, mem0, mem1, mem2, mem3
|
|
|
|
mul \t0\wX, \b0\wX, \z0\nX[\h0]
|
|
mul \t1\wX, \b1\wX, \z1\nX[\h1]
|
|
mul \t2\wX, \b2\wX, \z2\nX[\h2]
|
|
mul \t3\wX, \b3\wX, \z3\nX[\h3]
|
|
|
|
ldr \c0, [\src_ptr, \mem0]
|
|
sqrdmulh \b0\wX, \b0\wX, \z0\nX[\l0]
|
|
ldr \c1, [\src_ptr, \mem1]
|
|
sqrdmulh \b1\wX, \b1\wX, \z1\nX[\l1]
|
|
ldr \c2, [\src_ptr, \mem2]
|
|
sqrdmulh \b2\wX, \b2\wX, \z2\nX[\l2]
|
|
ldr \c3, [\src_ptr, \mem3]
|
|
sqrdmulh \b3\wX, \b3\wX, \z3\nX[\l3]
|
|
|
|
mls \t0\wX, \b0\wX, \mod\nX[0]
|
|
mls \t1\wX, \b1\wX, \mod\nX[0]
|
|
mls \t2\wX, \b2\wX, \mod\nX[0]
|
|
mls \t3\wX, \b3\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_qX_butterfly_tops b0, b1, b2, b3, t0, t1, t2, t3, mod, z0, l0, h0, z1, l1, h1, z2, l2, h2, z3, l3, h3, wX, nX, src_ptr, c0, c1, c2, c3, mem0, mem1, mem2, mem3
|
|
|
|
mul \t0\wX, \b0\wX, \z0\nX[\h0]
|
|
mul \t1\wX, \b1\wX, \z1\nX[\h1]
|
|
mul \t2\wX, \b2\wX, \z2\nX[\h2]
|
|
mul \t3\wX, \b3\wX, \z3\nX[\h3]
|
|
|
|
str \c0, [\src_ptr, \mem0]
|
|
sqrdmulh \b0\wX, \b0\wX, \z0\nX[\l0]
|
|
str \c1, [\src_ptr, \mem1]
|
|
sqrdmulh \b1\wX, \b1\wX, \z1\nX[\l1]
|
|
str \c2, [\src_ptr, \mem2]
|
|
sqrdmulh \b2\wX, \b2\wX, \z2\nX[\l2]
|
|
str \c3, [\src_ptr, \mem3]
|
|
sqrdmulh \b3\wX, \b3\wX, \z3\nX[\l3]
|
|
|
|
mls \t0\wX, \b0\wX, \mod\nX[0]
|
|
mls \t1\wX, \b1\wX, \mod\nX[0]
|
|
mls \t2\wX, \b2\wX, \mod\nX[0]
|
|
mls \t3\wX, \b3\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_qX_butterfly_topsl b0, b1, b2, b3, t0, t1, t2, t3, mod, z0, l0, h0, z1, l1, h1, z2, l2, h2, z3, l3, h3, wX, nX, srcc_ptr, c0, c1, c2, c3, memc0, memc1, memc2, memc3, srcd_ptr, d0, d1, d2, d3, memd0, memd1, memd2, memd3
|
|
|
|
mul \t0\wX, \b0\wX, \z0\nX[\h0]
|
|
mul \t1\wX, \b1\wX, \z1\nX[\h1]
|
|
mul \t2\wX, \b2\wX, \z2\nX[\h2]
|
|
mul \t3\wX, \b3\wX, \z3\nX[\h3]
|
|
|
|
str \c0, [\srcc_ptr, \memc0]
|
|
ldr \d0, [\srcd_ptr, \memd0]
|
|
sqrdmulh \b0\wX, \b0\wX, \z0\nX[\l0]
|
|
str \c1, [\srcc_ptr, \memc1]
|
|
ldr \d1, [\srcd_ptr, \memd1]
|
|
sqrdmulh \b1\wX, \b1\wX, \z1\nX[\l1]
|
|
str \c2, [\srcc_ptr, \memc2]
|
|
ldr \d2, [\srcd_ptr, \memd2]
|
|
sqrdmulh \b2\wX, \b2\wX, \z2\nX[\l2]
|
|
str \c3, [\srcc_ptr, \memc3]
|
|
ldr \d3, [\srcd_ptr, \memd3]
|
|
sqrdmulh \b3\wX, \b3\wX, \z3\nX[\l3]
|
|
|
|
mls \t0\wX, \b0\wX, \mod\nX[0]
|
|
mls \t1\wX, \b1\wX, \mod\nX[0]
|
|
mls \t2\wX, \b2\wX, \mod\nX[0]
|
|
mls \t3\wX, \b3\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_qX_butterfly_bot a0, a1, a2, a3, b0, b1, b2, b3, t0, t1, t2, t3, mod, z0, l0, h0, z1, l1, h1, z2, l2, h2, z3, l3, h3, wX, nX
|
|
|
|
sub \b0\wX, \a0\wX, \t0\wX
|
|
sub \b1\wX, \a1\wX, \t1\wX
|
|
sub \b2\wX, \a2\wX, \t2\wX
|
|
sub \b3\wX, \a3\wX, \t3\wX
|
|
|
|
add \a0\wX, \a0\wX, \t0\wX
|
|
add \a1\wX, \a1\wX, \t1\wX
|
|
add \a2\wX, \a2\wX, \t2\wX
|
|
add \a3\wX, \a3\wX, \t3\wX
|
|
|
|
.endm
|
|
|
|
.macro wrap_qX_butterfly_botll a0, a1, a2, a3, b0, b1, b2, b3, t0, t1, t2, t3, wX, nX, srcc_ptr, c0, c1, c2, c3, memc0, memc1, memc2, memc3, srcd_ptr, d0, d1, d2, d3, memd0, memd1, memd2, memd3
|
|
|
|
sub \b0\wX, \a0\wX, \t0\wX
|
|
ldr \c0, [\srcc_ptr, \memc0]
|
|
sub \b1\wX, \a1\wX, \t1\wX
|
|
ldr \c1, [\srcc_ptr, \memc1]
|
|
sub \b2\wX, \a2\wX, \t2\wX
|
|
ldr \c2, [\srcc_ptr, \memc2]
|
|
sub \b3\wX, \a3\wX, \t3\wX
|
|
ldr \c3, [\srcc_ptr, \memc3]
|
|
|
|
add \a0\wX, \a0\wX, \t0\wX
|
|
ldr \d0, [\srcd_ptr, \memd0]
|
|
add \a1\wX, \a1\wX, \t1\wX
|
|
ldr \d1, [\srcd_ptr, \memd1]
|
|
add \a2\wX, \a2\wX, \t2\wX
|
|
ldr \d2, [\srcd_ptr, \memd2]
|
|
add \a3\wX, \a3\wX, \t3\wX
|
|
ldr \d3, [\srcd_ptr, \memd3]
|
|
|
|
.endm
|
|
|
|
.macro wrap_qX_butterfly_botsl a0, a1, a2, a3, b0, b1, b2, b3, t0, t1, t2, t3, wX, nX, srcc_ptr, c0, c1, c2, c3, memc0, memc1, memc2, memc3, srcd_ptr, d0, d1, d2, d3, memd0, memd1, memd2, memd3
|
|
|
|
sub \b0\wX, \a0\wX, \t0\wX
|
|
str \c0, [\srcc_ptr, \memc0]
|
|
sub \b1\wX, \a1\wX, \t1\wX
|
|
str \c1, [\srcc_ptr, \memc1]
|
|
sub \b2\wX, \a2\wX, \t2\wX
|
|
str \c2, [\srcc_ptr, \memc2]
|
|
sub \b3\wX, \a3\wX, \t3\wX
|
|
str \c3, [\srcc_ptr, \memc3]
|
|
|
|
add \a0\wX, \a0\wX, \t0\wX
|
|
ldr \d0, [\srcd_ptr, \memd0]
|
|
add \a1\wX, \a1\wX, \t1\wX
|
|
ldr \d1, [\srcd_ptr, \memd1]
|
|
add \a2\wX, \a2\wX, \t2\wX
|
|
ldr \d2, [\srcd_ptr, \memd2]
|
|
add \a3\wX, \a3\wX, \t3\wX
|
|
ldr \d3, [\srcd_ptr, \memd3]
|
|
|
|
.endm
|
|
|
|
.macro wrap_qX_butterfly_botss a0, a1, a2, a3, b0, b1, b2, b3, t0, t1, t2, t3, wX, nX, srcc_ptr, c0, c1, c2, c3, memc0, memc1, memc2, memc3, srcd_ptr, d0, d1, d2, d3, memd0, memd1, memd2, memd3
|
|
|
|
sub \b0\wX, \a0\wX, \t0\wX
|
|
str \c0, [\srcc_ptr, \memc0]
|
|
sub \b1\wX, \a1\wX, \t1\wX
|
|
str \c1, [\srcc_ptr, \memc1]
|
|
sub \b2\wX, \a2\wX, \t2\wX
|
|
str \c2, [\srcc_ptr, \memc2]
|
|
sub \b3\wX, \a3\wX, \t3\wX
|
|
str \c3, [\srcc_ptr, \memc3]
|
|
|
|
add \a0\wX, \a0\wX, \t0\wX
|
|
str \d0, [\srcd_ptr, \memd0]
|
|
add \a1\wX, \a1\wX, \t1\wX
|
|
str \d1, [\srcd_ptr, \memd1]
|
|
add \a2\wX, \a2\wX, \t2\wX
|
|
str \d2, [\srcd_ptr, \memd2]
|
|
add \a3\wX, \a3\wX, \t3\wX
|
|
str \d3, [\srcd_ptr, \memd3]
|
|
|
|
.endm
|
|
|
|
.macro wrap_qX_butterfly_botsls a0, a1, a2, a3, b0, b1, b2, b3, t0, t1, t2, t3, wX, nX, srcc_ptr, c0, c1, c2, c3, memc0, memc1, memc2, memc3, srcd_ptr, d0, d1, d2, d3, memd0, memd1, memd2, memd3, srce_ptr, e0, e1, e2, e3, meme0, meme1, meme2, meme3
|
|
|
|
sub \b0\wX, \a0\wX, \t0\wX
|
|
str \c0, [\srcc_ptr, \memc0]
|
|
ldr \d0, [\srcd_ptr, \memd0]
|
|
sub \b1\wX, \a1\wX, \t1\wX
|
|
str \c1, [\srcc_ptr, \memc1]
|
|
ldr \d1, [\srcd_ptr, \memd1]
|
|
sub \b2\wX, \a2\wX, \t2\wX
|
|
str \c2, [\srcc_ptr, \memc2]
|
|
ldr \d2, [\srcd_ptr, \memd2]
|
|
sub \b3\wX, \a3\wX, \t3\wX
|
|
str \c3, [\srcc_ptr, \memc3]
|
|
ldr \d3, [\srcd_ptr, \memd3]
|
|
|
|
add \a0\wX, \a0\wX, \t0\wX
|
|
str \e0, [\srce_ptr, \meme0]
|
|
add \a1\wX, \a1\wX, \t1\wX
|
|
str \e1, [\srce_ptr, \meme1]
|
|
add \a2\wX, \a2\wX, \t2\wX
|
|
str \e2, [\srce_ptr, \meme2]
|
|
add \a3\wX, \a3\wX, \t3\wX
|
|
str \e3, [\srce_ptr, \meme3]
|
|
|
|
.endm
|
|
|
|
.macro wrap_qX_butterfly_botsl_mul a0, a1, a2, a3, b0, b1, b2, b3, t0, t1, t2, t3, wX, nX, srcc_ptr, c0, c1, c2, c3, memc0, memc1, memc2, memc3, srcd_ptr, d0, d1, d2, d3, memd0, memd1, memd2, memd3, a4, a5, a6, a7, t4, t5, t6, t7, mod, z4, l4, h4, z5, l5, h5, z6, l6, h6, z7, l7, h7
|
|
|
|
sqrdmulh \t4\wX, \a4\wX, \z4\nX[\h4]
|
|
sub \b0\wX, \a0\wX, \t0\wX
|
|
str \c0, [\srcc_ptr, \memc0]
|
|
sqrdmulh \t5\wX, \a5\wX, \z5\nX[\h5]
|
|
sub \b1\wX, \a1\wX, \t1\wX
|
|
str \c1, [\srcc_ptr, \memc1]
|
|
sqrdmulh \t6\wX, \a6\wX, \z6\nX[\h6]
|
|
sub \b2\wX, \a2\wX, \t2\wX
|
|
str \c2, [\srcc_ptr, \memc2]
|
|
sqrdmulh \t7\wX, \a7\wX, \z7\nX[\h7]
|
|
sub \b3\wX, \a3\wX, \t3\wX
|
|
str \c3, [\srcc_ptr, \memc3]
|
|
|
|
mul \a4\wX, \a4\wX, \z4\nX[\l4]
|
|
add \a0\wX, \a0\wX, \t0\wX
|
|
ldr \d0, [\srcd_ptr, \memd0]
|
|
mul \a5\wX, \a5\wX, \z5\nX[\l5]
|
|
add \a1\wX, \a1\wX, \t1\wX
|
|
ldr \d1, [\srcd_ptr, \memd1]
|
|
mul \a6\wX, \a6\wX, \z6\nX[\l6]
|
|
add \a2\wX, \a2\wX, \t2\wX
|
|
ldr \d2, [\srcd_ptr, \memd2]
|
|
mul \a7\wX, \a7\wX, \z7\nX[\l7]
|
|
add \a3\wX, \a3\wX, \t3\wX
|
|
ldr \d3, [\srcd_ptr, \memd3]
|
|
|
|
mls \a4\wX, \t4\wX, \mod\nX[0]
|
|
mls \a5\wX, \t5\wX, \mod\nX[0]
|
|
mls \a6\wX, \t6\wX, \mod\nX[0]
|
|
mls \a7\wX, \t7\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_qX_butterfly_mix a0, a1, a2, a3, b0, b1, b2, b3, t0, t1, t2, t3, a4, a5, a6, a7, b4, b5, b6, b7, t4, t5, t6, t7, mod, z0, l0, h0, z1, l1, h1, z2, l2, h2, z3, l3, h3, z4, l4, h4, z5, l5, h5, z6, l6, h6, z7, l7, h7, wX, nX
|
|
|
|
sub \b0\wX, \a0\wX, \t0\wX
|
|
mul \t4\wX, \b4\wX, \z4\nX[\h4]
|
|
sub \b1\wX, \a1\wX, \t1\wX
|
|
mul \t5\wX, \b5\wX, \z5\nX[\h5]
|
|
sub \b2\wX, \a2\wX, \t2\wX
|
|
mul \t6\wX, \b6\wX, \z6\nX[\h6]
|
|
sub \b3\wX, \a3\wX, \t3\wX
|
|
mul \t7\wX, \b7\wX, \z7\nX[\h7]
|
|
|
|
add \a0\wX, \a0\wX, \t0\wX
|
|
sqrdmulh \b4\wX, \b4\wX, \z4\nX[\l4]
|
|
add \a1\wX, \a1\wX, \t1\wX
|
|
sqrdmulh \b5\wX, \b5\wX, \z5\nX[\l5]
|
|
add \a2\wX, \a2\wX, \t2\wX
|
|
sqrdmulh \b6\wX, \b6\wX, \z6\nX[\l6]
|
|
add \a3\wX, \a3\wX, \t3\wX
|
|
sqrdmulh \b7\wX, \b7\wX, \z7\nX[\l7]
|
|
|
|
mls \t4\wX, \b4\wX, \mod\nX[0]
|
|
mls \t5\wX, \b5\wX, \mod\nX[0]
|
|
mls \t6\wX, \b6\wX, \mod\nX[0]
|
|
mls \t7\wX, \b7\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_qX_butterfly_mixl a0, a1, a2, a3, b0, b1, b2, b3, t0, t1, t2, t3, b4, b5, b6, b7, t4, t5, t6, t7, mod, z4, l4, h4, z5, l5, h5, z6, l6, h6, z7, l7, h7, wX, nX, srcc_ptr, c0, c1, c2, c3, memc0, memc1, memc2, memc3
|
|
|
|
ldr \c0, [\srcc_ptr, \memc0]
|
|
sub \b0\wX, \a0\wX, \t0\wX
|
|
mul \t4\wX, \b4\wX, \z4\nX[\h4]
|
|
ldr \c1, [\srcc_ptr, \memc1]
|
|
sub \b1\wX, \a1\wX, \t1\wX
|
|
mul \t5\wX, \b5\wX, \z5\nX[\h5]
|
|
ldr \c2, [\srcc_ptr, \memc2]
|
|
sub \b2\wX, \a2\wX, \t2\wX
|
|
mul \t6\wX, \b6\wX, \z6\nX[\h6]
|
|
ldr \c3, [\srcc_ptr, \memc3]
|
|
sub \b3\wX, \a3\wX, \t3\wX
|
|
mul \t7\wX, \b7\wX, \z7\nX[\h7]
|
|
|
|
add \a0\wX, \a0\wX, \t0\wX
|
|
sqrdmulh \b4\wX, \b4\wX, \z4\nX[\l4]
|
|
add \a1\wX, \a1\wX, \t1\wX
|
|
sqrdmulh \b5\wX, \b5\wX, \z5\nX[\l5]
|
|
add \a2\wX, \a2\wX, \t2\wX
|
|
sqrdmulh \b6\wX, \b6\wX, \z6\nX[\l6]
|
|
add \a3\wX, \a3\wX, \t3\wX
|
|
sqrdmulh \b7\wX, \b7\wX, \z7\nX[\l7]
|
|
|
|
mls \t4\wX, \b4\wX, \mod\nX[0]
|
|
mls \t5\wX, \b5\wX, \mod\nX[0]
|
|
mls \t6\wX, \b6\wX, \mod\nX[0]
|
|
mls \t7\wX, \b7\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_qX_butterfly_mixll a0, a1, a2, a3, b0, b1, b2, b3, t0, t1, t2, t3, b4, b5, b6, b7, t4, t5, t6, t7, mod, z4, l4, h4, z5, l5, h5, z6, l6, h6, z7, l7, h7, wX, nX, srcc_ptr, c0, c1, c2, c3, memc0, memc1, memc2, memc3, srcd_ptr, d0, d1, d2, d3, memd0, memd1, memd2, memd3
|
|
|
|
ldr \c0, [\srcc_ptr, \memc0]
|
|
sub \b0\wX, \a0\wX, \t0\wX
|
|
mul \t4\wX, \b4\wX, \z4\nX[\h4]
|
|
ldr \c1, [\srcc_ptr, \memc1]
|
|
sub \b1\wX, \a1\wX, \t1\wX
|
|
mul \t5\wX, \b5\wX, \z5\nX[\h5]
|
|
ldr \c2, [\srcc_ptr, \memc2]
|
|
sub \b2\wX, \a2\wX, \t2\wX
|
|
mul \t6\wX, \b6\wX, \z6\nX[\h6]
|
|
ldr \c3, [\srcc_ptr, \memc3]
|
|
sub \b3\wX, \a3\wX, \t3\wX
|
|
mul \t7\wX, \b7\wX, \z7\nX[\h7]
|
|
|
|
ldr \d0, [\srcd_ptr, \memd0]
|
|
add \a0\wX, \a0\wX, \t0\wX
|
|
sqrdmulh \b4\wX, \b4\wX, \z4\nX[\l4]
|
|
ldr \d1, [\srcd_ptr, \memd1]
|
|
add \a1\wX, \a1\wX, \t1\wX
|
|
sqrdmulh \b5\wX, \b5\wX, \z5\nX[\l5]
|
|
ldr \d2, [\srcd_ptr, \memd2]
|
|
add \a2\wX, \a2\wX, \t2\wX
|
|
sqrdmulh \b6\wX, \b6\wX, \z6\nX[\l6]
|
|
ldr \d3, [\srcd_ptr, \memd3]
|
|
add \a3\wX, \a3\wX, \t3\wX
|
|
sqrdmulh \b7\wX, \b7\wX, \z7\nX[\l7]
|
|
|
|
mls \t4\wX, \b4\wX, \mod\nX[0]
|
|
mls \t5\wX, \b5\wX, \mod\nX[0]
|
|
mls \t6\wX, \b6\wX, \mod\nX[0]
|
|
mls \t7\wX, \b7\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_qX_butterfly_mixss a0, a1, a2, a3, b0, b1, b2, b3, t0, t1, t2, t3, b4, b5, b6, b7, t4, t5, t6, t7, mod, z4, l4, h4, z5, l5, h5, z6, l6, h6, z7, l7, h7, wX, nX, srcc_ptr, c0, c1, c2, c3, memc0, memc1, memc2, memc3, srcd_ptr, d0, d1, d2, d3, memd0, memd1, memd2, memd3
|
|
|
|
mul \t4\wX, \b4\wX, \z4\nX[\h4]
|
|
sub \b0\wX, \a0\wX, \t0\wX
|
|
str \c0, [\srcc_ptr, \memc0]
|
|
mul \t5\wX, \b5\wX, \z5\nX[\h5]
|
|
sub \b1\wX, \a1\wX, \t1\wX
|
|
str \c1, [\srcc_ptr, \memc1]
|
|
mul \t6\wX, \b6\wX, \z6\nX[\h6]
|
|
sub \b2\wX, \a2\wX, \t2\wX
|
|
str \c2, [\srcc_ptr, \memc2]
|
|
mul \t7\wX, \b7\wX, \z7\nX[\h7]
|
|
sub \b3\wX, \a3\wX, \t3\wX
|
|
str \c3, [\srcc_ptr, \memc3]
|
|
|
|
sqrdmulh \b4\wX, \b4\wX, \z4\nX[\l4]
|
|
add \a0\wX, \a0\wX, \t0\wX
|
|
str \d0, [\srcd_ptr, \memd0]
|
|
sqrdmulh \b5\wX, \b5\wX, \z5\nX[\l5]
|
|
add \a1\wX, \a1\wX, \t1\wX
|
|
str \d1, [\srcd_ptr, \memd1]
|
|
sqrdmulh \b6\wX, \b6\wX, \z6\nX[\l6]
|
|
add \a2\wX, \a2\wX, \t2\wX
|
|
str \d2, [\srcd_ptr, \memd2]
|
|
sqrdmulh \b7\wX, \b7\wX, \z7\nX[\l7]
|
|
add \a3\wX, \a3\wX, \t3\wX
|
|
str \d3, [\srcd_ptr, \memd3]
|
|
|
|
mls \t4\wX, \b4\wX, \mod\nX[0]
|
|
mls \t5\wX, \b5\wX, \mod\nX[0]
|
|
mls \t6\wX, \b6\wX, \mod\nX[0]
|
|
mls \t7\wX, \b7\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_qX_butterfly_mixsls a0, a1, a2, a3, b0, b1, b2, b3, t0, t1, t2, t3, b4, b5, b6, b7, t4, t5, t6, t7, mod, z4, l4, h4, z5, l5, h5, z6, l6, h6, z7, l7, h7, wX, nX, srcc_ptr, c0, c1, c2, c3, memc0, memc1, memc2, memc3, srcd_ptr, d0, d1, d2, d3, memd0, memd1, memd2, memd3, srce_ptr, e0, e1, e2, e3, meme0, meme1, meme2, meme3
|
|
|
|
sub \b0\wX, \a0\wX, \t0\wX
|
|
str \c0, [\srcc_ptr, \memc0]
|
|
mul \t4\wX, \b4\wX, \z4\nX[\h4]
|
|
sub \b1\wX, \a1\wX, \t1\wX
|
|
str \c1, [\srcc_ptr, \memc1]
|
|
mul \t5\wX, \b5\wX, \z5\nX[\h5]
|
|
sub \b2\wX, \a2\wX, \t2\wX
|
|
str \c2, [\srcc_ptr, \memc2]
|
|
mul \t6\wX, \b6\wX, \z6\nX[\h6]
|
|
sub \b3\wX, \a3\wX, \t3\wX
|
|
str \c3, [\srcc_ptr, \memc3]
|
|
mul \t7\wX, \b7\wX, \z7\nX[\h7]
|
|
|
|
ldr \d0, [\srcd_ptr, \memd0]
|
|
add \a0\wX, \a0\wX, \t0\wX
|
|
sqrdmulh \b4\wX, \b4\wX, \z4\nX[\l4]
|
|
ldr \d1, [\srcd_ptr, \memd1]
|
|
add \a1\wX, \a1\wX, \t1\wX
|
|
sqrdmulh \b5\wX, \b5\wX, \z5\nX[\l5]
|
|
ldr \d2, [\srcd_ptr, \memd2]
|
|
add \a2\wX, \a2\wX, \t2\wX
|
|
sqrdmulh \b6\wX, \b6\wX, \z6\nX[\l6]
|
|
ldr \d3, [\srcd_ptr, \memd3]
|
|
add \a3\wX, \a3\wX, \t3\wX
|
|
sqrdmulh \b7\wX, \b7\wX, \z7\nX[\l7]
|
|
|
|
str \e0, [\srce_ptr, \meme0]
|
|
mls \t4\wX, \b4\wX, \mod\nX[0]
|
|
str \e1, [\srce_ptr, \meme1]
|
|
mls \t5\wX, \b5\wX, \mod\nX[0]
|
|
str \e2, [\srce_ptr, \meme2]
|
|
mls \t6\wX, \b6\wX, \mod\nX[0]
|
|
str \e3, [\srce_ptr, \meme3]
|
|
mls \t7\wX, \b7\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_qX_butterfly_mixssl a0, a1, a2, a3, b0, b1, b2, b3, t0, t1, t2, t3, b4, b5, b6, b7, t4, t5, t6, t7, mod, z4, l4, h4, z5, l5, h5, z6, l6, h6, z7, l7, h7, wX, nX, srcc_ptr, c0, c1, c2, c3, memc0, memc1, memc2, memc3, srcd_ptr, d0, d1, d2, d3, memd0, memd1, memd2, memd3, srce_ptr, e0, e1, e2, e3, meme0, meme1, meme2, meme3
|
|
|
|
mul \t4\wX, \b4\wX, \z4\nX[\h4]
|
|
sub \b0\wX, \a0\wX, \t0\wX
|
|
str \c0, [\srcc_ptr, \memc0]
|
|
mul \t5\wX, \b5\wX, \z5\nX[\h5]
|
|
sub \b1\wX, \a1\wX, \t1\wX
|
|
str \c1, [\srcc_ptr, \memc1]
|
|
mul \t6\wX, \b6\wX, \z6\nX[\h6]
|
|
sub \b2\wX, \a2\wX, \t2\wX
|
|
str \c2, [\srcc_ptr, \memc2]
|
|
mul \t7\wX, \b7\wX, \z7\nX[\h7]
|
|
sub \b3\wX, \a3\wX, \t3\wX
|
|
str \c3, [\srcc_ptr, \memc3]
|
|
|
|
sqrdmulh \b4\wX, \b4\wX, \z4\nX[\l4]
|
|
add \a0\wX, \a0\wX, \t0\wX
|
|
str \d0, [\srcd_ptr, \memd0]
|
|
sqrdmulh \b5\wX, \b5\wX, \z5\nX[\l5]
|
|
add \a1\wX, \a1\wX, \t1\wX
|
|
str \d1, [\srcd_ptr, \memd1]
|
|
sqrdmulh \b6\wX, \b6\wX, \z6\nX[\l6]
|
|
add \a2\wX, \a2\wX, \t2\wX
|
|
str \d2, [\srcd_ptr, \memd2]
|
|
sqrdmulh \b7\wX, \b7\wX, \z7\nX[\l7]
|
|
add \a3\wX, \a3\wX, \t3\wX
|
|
str \d3, [\srcd_ptr, \memd3]
|
|
|
|
mls \t4\wX, \b4\wX, \mod\nX[0]
|
|
ldr \e0, [\srce_ptr, \meme0]
|
|
mls \t5\wX, \b5\wX, \mod\nX[0]
|
|
ldr \e1, [\srce_ptr, \meme1]
|
|
mls \t6\wX, \b6\wX, \mod\nX[0]
|
|
ldr \e2, [\srce_ptr, \meme2]
|
|
mls \t7\wX, \b7\wX, \mod\nX[0]
|
|
ldr \e3, [\srce_ptr, \meme3]
|
|
|
|
.endm
|
|
|
|
.macro wrap_qX_butterfly_mix_rev a0, a1, a2, a3, b0, b1, b2, b3, t0, t1, t2, t3, a4, a5, a6, a7, b4, b5, b6, b7, t4, t5, t6, t7, mod, z0, l0, h0, z1, l1, h1, z2, l2, h2, z3, l3, h3, z4, l4, h4, z5, l5, h5, z6, l6, h6, z7, l7, h7, wX, nX
|
|
|
|
mul \t0\wX, \b0\wX, \z0\nX[\h0]
|
|
sub \b4\wX, \a4\wX, \t4\wX
|
|
mul \t1\wX, \b1\wX, \z1\nX[\h1]
|
|
sub \b5\wX, \a5\wX, \t5\wX
|
|
mul \t2\wX, \b2\wX, \z2\nX[\h2]
|
|
sub \b6\wX, \a6\wX, \t6\wX
|
|
mul \t3\wX, \b3\wX, \z3\nX[\h3]
|
|
sub \b7\wX, \a7\wX, \t7\wX
|
|
|
|
sqrdmulh \b0\wX, \b0\wX, \z0\nX[\l0]
|
|
add \a4\wX, \a4\wX, \t4\wX
|
|
sqrdmulh \b1\wX, \b1\wX, \z1\nX[\l1]
|
|
add \a5\wX, \a5\wX, \t5\wX
|
|
sqrdmulh \b2\wX, \b2\wX, \z2\nX[\l2]
|
|
add \a6\wX, \a6\wX, \t6\wX
|
|
sqrdmulh \b3\wX, \b3\wX, \z3\nX[\l3]
|
|
add \a7\wX, \a7\wX, \t7\wX
|
|
|
|
mls \t0\wX, \b0\wX, \mod\nX[0]
|
|
mls \t1\wX, \b1\wX, \mod\nX[0]
|
|
mls \t2\wX, \b2\wX, \mod\nX[0]
|
|
mls \t3\wX, \b3\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
// vector-vector butterflies
|
|
|
|
.macro wrap_dX_butterfly_vec_top a0, a1, b0, b1, t0, t1, mod, l0, h0, l1, h1, wX, nX
|
|
|
|
mul \t0\wX, \b0\wX, \h0\wX
|
|
mul \t1\wX, \b1\wX, \h1\wX
|
|
|
|
sqrdmulh \b0\wX, \b0\wX, \l0\wX
|
|
sqrdmulh \b1\wX, \b1\wX, \l1\wX
|
|
|
|
mls \t0\wX, \b0\wX, \mod\nX[0]
|
|
mls \t1\wX, \b1\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_dX_butterfly_vec_top_trn_4x4 a0, a1, b0, b1, t0, t1, mod, l0, h0, l1, h1, wX, nX, trns0, trns1, trns2, trns3, trnt0, trnt1, trnt2, trnt3
|
|
|
|
mul \t0\wX, \b0\wX, \h0\wX
|
|
trn1 \trnt0\().4S, \trns0\().4S, \trns1\().4S
|
|
mul \t1\wX, \b1\wX, \h1\wX
|
|
trn2 \trnt1\().4S, \trns0\().4S, \trns1\().4S
|
|
|
|
sqrdmulh \b0\wX, \b0\wX, \l0\wX
|
|
trn1 \trnt2\().4S, \trns2\().4S, \trns3\().4S
|
|
sqrdmulh \b1\wX, \b1\wX, \l1\wX
|
|
trn2 \trnt3\().4S, \trns2\().4S, \trns3\().4S
|
|
|
|
mls \t0\wX, \b0\wX, \mod\nX[0]
|
|
trn1 \trns0\().2D, \trnt0\().2D, \trnt2\().2D
|
|
mls \t1\wX, \b1\wX, \mod\nX[0]
|
|
trn2 \trns2\().2D, \trnt0\().2D, \trnt2\().2D
|
|
|
|
trn1 \trns1\().2D, \trnt1\().2D, \trnt3\().2D
|
|
trn2 \trns3\().2D, \trnt1\().2D, \trnt3\().2D
|
|
|
|
.endm
|
|
|
|
.macro wrap_dX_butterfly_vec_top_ltrn_4x4 b0, b1, t0, t1, mod, l0, h0, l1, h1, wX, nX, src0_ptr, c0, c1, c2, c3, memc0, memc1, memc2, memc3, trns0, trns1, trns2, trns3, trnt0, trnt1, trnt2, trnt3
|
|
|
|
ldr \c0, [\src0_ptr, \memc0]
|
|
mul \t0\wX, \b0\wX, \h0\wX
|
|
ldr \c1, [\src0_ptr, \memc1]
|
|
mul \t1\wX, \b1\wX, \h1\wX
|
|
|
|
ldr \c2, [\src0_ptr, \memc2]
|
|
trn1 \trnt0\().4S, \trns0\().4S, \trns1\().4S
|
|
trn2 \trnt1\().4S, \trns0\().4S, \trns1\().4S
|
|
ldr \c3, [\src0_ptr, \memc3]
|
|
trn1 \trnt2\().4S, \trns2\().4S, \trns3\().4S
|
|
trn2 \trnt3\().4S, \trns2\().4S, \trns3\().4S
|
|
|
|
sqrdmulh \b0\wX, \b0\wX, \l0\wX
|
|
trn1 \trns0\().2D, \trnt0\().2D, \trnt2\().2D
|
|
sqrdmulh \b1\wX, \b1\wX, \l1\wX
|
|
trn2 \trns2\().2D, \trnt0\().2D, \trnt2\().2D
|
|
|
|
mls \t0\wX, \b0\wX, \mod\nX[0]
|
|
trn1 \trns1\().2D, \trnt1\().2D, \trnt3\().2D
|
|
mls \t1\wX, \b1\wX, \mod\nX[0]
|
|
trn2 \trns3\().2D, \trnt1\().2D, \trnt3\().2D
|
|
|
|
.endm
|
|
|
|
.macro wrap_dX_butterfly_vec_top_2ltrn_4x4 b0, b1, t0, t1, mod, l0, h0, l1, h1, wX, nX, src0_ptr, src1_ptr, c0, c1, c2, c3, memc0, memc1, memc2, memc3, trns0, trns1, trns2, trns3, trnt0, trnt1, trnt2, trnt3
|
|
|
|
ldr \c0, [\src0_ptr, \memc0]
|
|
mul \t0\wX, \b0\wX, \h0\wX
|
|
ldr \c1, [\src1_ptr, \memc1]
|
|
mul \t1\wX, \b1\wX, \h1\wX
|
|
|
|
ldr \c2, [\src0_ptr, \memc2]
|
|
trn1 \trnt0\().4S, \trns0\().4S, \trns1\().4S
|
|
trn2 \trnt1\().4S, \trns0\().4S, \trns1\().4S
|
|
ldr \c3, [\src1_ptr, \memc3]
|
|
trn1 \trnt2\().4S, \trns2\().4S, \trns3\().4S
|
|
trn2 \trnt3\().4S, \trns2\().4S, \trns3\().4S
|
|
|
|
sqrdmulh \b0\wX, \b0\wX, \l0\wX
|
|
trn1 \trns0\().2D, \trnt0\().2D, \trnt2\().2D
|
|
sqrdmulh \b1\wX, \b1\wX, \l1\wX
|
|
trn2 \trns2\().2D, \trnt0\().2D, \trnt2\().2D
|
|
|
|
mls \t0\wX, \b0\wX, \mod\nX[0]
|
|
trn1 \trns1\().2D, \trnt1\().2D, \trnt3\().2D
|
|
mls \t1\wX, \b1\wX, \mod\nX[0]
|
|
trn2 \trns3\().2D, \trnt1\().2D, \trnt3\().2D
|
|
|
|
.endm
|
|
|
|
.macro wrap_dX_butterfly_vec_bot a0, a1, b0, b1, t0, t1, mod, l0, h0, l1, h1, wX, nX
|
|
|
|
sub \b0\wX, \a0\wX, \t0\wX
|
|
sub \b1\wX, \a1\wX, \t1\wX
|
|
|
|
add \a0\wX, \a0\wX, \t0\wX
|
|
add \a1\wX, \a1\wX, \t1\wX
|
|
|
|
.endm
|
|
|
|
.macro wrap_dX_butterfly_vec_bot_oo_barrett_trn_4x4 a0, a1, b0, b1, t0, t1, a4, a5, a6, a7, t4, t5, t6, t7, a8, a9, a10, a11, t8, t9, t10, t11, barrett_const, shrv, Q, wX, nX
|
|
|
|
sqdmulh \t4\wX, \a4\wX, \barrett_const\nX[1]
|
|
sub \b0\wX, \a0\wX, \t0\wX
|
|
sqdmulh \t5\wX, \a5\wX, \barrett_const\nX[1]
|
|
sub \b1\wX, \a1\wX, \t1\wX
|
|
sqdmulh \t6\wX, \a6\wX, \barrett_const\nX[1]
|
|
add \a0\wX, \a0\wX, \t0\wX
|
|
sqdmulh \t7\wX, \a7\wX, \barrett_const\nX[1]
|
|
add \a1\wX, \a1\wX, \t1\wX
|
|
|
|
sqdmulh \t8\wX, \a8\wX, \barrett_const\nX[1]
|
|
srshr \t4\wX, \t4\wX, \shrv
|
|
sqdmulh \t9\wX, \a9\wX, \barrett_const\nX[1]
|
|
srshr \t5\wX, \t5\wX, \shrv
|
|
sqdmulh \t10\wX, \a10\wX, \barrett_const\nX[1]
|
|
srshr \t6\wX, \t6\wX, \shrv
|
|
sqdmulh \t11\wX, \a11\wX, \barrett_const\nX[1]
|
|
srshr \t7\wX, \t7\wX, \shrv
|
|
|
|
mls \a4\wX, \t4\wX, \Q\nX[0]
|
|
srshr \t8\wX, \t8\wX, \shrv
|
|
mls \a5\wX, \t5\wX, \Q\nX[0]
|
|
srshr \t9\wX, \t9\wX, \shrv
|
|
mls \a6\wX, \t6\wX, \Q\nX[0]
|
|
srshr \t10\wX, \t10\wX, \shrv
|
|
mls \a7\wX, \t7\wX, \Q\nX[0]
|
|
srshr \t11\wX, \t11\wX, \shrv
|
|
|
|
mls \a8\wX, \t8\wX, \Q\nX[0]
|
|
trn1 \t4\().4S, \a4\().4S, \a5\().4S
|
|
mls \a9\wX, \t9\wX, \Q\nX[0]
|
|
trn2 \t5\().4S, \a4\().4S, \a5\().4S
|
|
mls \a10\wX, \t10\wX, \Q\nX[0]
|
|
trn1 \t6\().4S, \a6\().4S, \a7\().4S
|
|
mls \a11\wX, \t11\wX, \Q\nX[0]
|
|
|
|
trn2 \t7\().4S, \a6\().4S, \a7\().4S
|
|
|
|
trn1 \a4\().2D, \t4\().2D, \t6\().2D
|
|
trn2 \a6\().2D, \t4\().2D, \t6\().2D
|
|
trn1 \a5\().2D, \t5\().2D, \t7\().2D
|
|
trn2 \a7\().2D, \t5\().2D, \t7\().2D
|
|
|
|
.endm
|
|
|
|
.macro wrap_dX_butterfly_vec_mix a0, a1, b0, b1, t0, t1, a2, a3, b2, b3, t2, t3, mod, l0, h0, l1, h1, l2, h2, l3, h3, wX, nX
|
|
|
|
sub \b0\wX, \a0\wX, \t0\wX
|
|
mul \t2\wX, \b2\wX, \h2\wX
|
|
sub \b1\wX, \a1\wX, \t1\wX
|
|
mul \t3\wX, \b3\wX, \h3\wX
|
|
|
|
add \a0\wX, \a0\wX, \t0\wX
|
|
sqrdmulh \b2\wX, \b2\wX, \l2\wX
|
|
add \a1\wX, \a1\wX, \t1\wX
|
|
sqrdmulh \b3\wX, \b3\wX, \l3\wX
|
|
|
|
mls \t2\wX, \b2\wX, \mod\nX[0]
|
|
mls \t3\wX, \b3\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_dX_butterfly_vec_mixl a0, a1, b0, b1, t0, t1, b2, b3, t2, t3, mod, l2, h2, l3, h3, wX, nX, srcc_ptr, c0, c1, c2, c3, memc0, memc1, memc2, memc3
|
|
|
|
ldr \c0, [\srcc_ptr, \memc0]
|
|
sub \b0\wX, \a0\wX, \t0\wX
|
|
mul \t2\wX, \b2\wX, \h2\wX
|
|
ldr \c1, [\srcc_ptr, \memc1]
|
|
sub \b1\wX, \a1\wX, \t1\wX
|
|
mul \t3\wX, \b3\wX, \h3\wX
|
|
|
|
ldr \c2, [\srcc_ptr, \memc2]
|
|
add \a0\wX, \a0\wX, \t0\wX
|
|
sqrdmulh \b2\wX, \b2\wX, \l2\wX
|
|
ldr \c3, [\srcc_ptr, \memc3]
|
|
add \a1\wX, \a1\wX, \t1\wX
|
|
sqrdmulh \b3\wX, \b3\wX, \l3\wX
|
|
|
|
mls \t2\wX, \b2\wX, \mod\nX[0]
|
|
mls \t3\wX, \b3\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_dX_butterfly_vec_mix_rev a0, a1, b0, b1, t0, t1, a2, a3, b2, b3, t2, t3, mod, l0, h0, l1, h1, l2, h2, l3, h3, wX, nX
|
|
|
|
mul \t0\wX, \b0\wX, \h0\wX
|
|
sub \b2\wX, \a2\wX, \t2\wX
|
|
mul \t1\wX, \b1\wX, \h1\wX
|
|
sub \b3\wX, \a3\wX, \t3\wX
|
|
|
|
sqrdmulh \b0\wX, \b0\wX, \l0\wX
|
|
add \a2\wX, \a2\wX, \t2\wX
|
|
sqrdmulh \b1\wX, \b1\wX, \l1\wX
|
|
add \a3\wX, \a3\wX, \t3\wX
|
|
|
|
mls \t0\wX, \b0\wX, \mod\nX[0]
|
|
mls \t1\wX, \b1\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_dX_butterfly_vec_mix_rev_l4 b0, b1, t0, t1, a2, a3, b2, b3, t2, t3, mod, l0, h0, l1, h1, wX, nX, srcc_ptr, c0, c1, c2, c3, memc0, memc1, memc2, memc3
|
|
|
|
ldr \c0, [\srcc_ptr, \memc0]
|
|
mul \t0\wX, \b0\wX, \h0\wX
|
|
sub \b2\wX, \a2\wX, \t2\wX
|
|
ldr \c1, [\srcc_ptr, \memc1]
|
|
mul \t1\wX, \b1\wX, \h1\wX
|
|
sub \b3\wX, \a3\wX, \t3\wX
|
|
|
|
ldr \c2, [\srcc_ptr, \memc2]
|
|
sqrdmulh \b0\wX, \b0\wX, \l0\wX
|
|
add \a2\wX, \a2\wX, \t2\wX
|
|
ldr \c3, [\srcc_ptr, \memc3]
|
|
sqrdmulh \b1\wX, \b1\wX, \l1\wX
|
|
add \a3\wX, \a3\wX, \t3\wX
|
|
|
|
mls \t0\wX, \b0\wX, \mod\nX[0]
|
|
mls \t1\wX, \b1\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_dX_butterfly_vec_mix_rev_l3 b0, b1, t0, t1, a2, a3, b2, b3, t2, t3, mod, l0, h0, l1, h1, wX, nX, srcc_ptr, c1, c2, c3, memc1, memc2, memc3
|
|
|
|
mul \t0\wX, \b0\wX, \h0\wX
|
|
sub \b2\wX, \a2\wX, \t2\wX
|
|
ldr \c1, [\srcc_ptr, \memc1]
|
|
mul \t1\wX, \b1\wX, \h1\wX
|
|
sub \b3\wX, \a3\wX, \t3\wX
|
|
|
|
ldr \c2, [\srcc_ptr, \memc2]
|
|
sqrdmulh \b0\wX, \b0\wX, \l0\wX
|
|
add \a2\wX, \a2\wX, \t2\wX
|
|
ldr \c3, [\srcc_ptr, \memc3]
|
|
sqrdmulh \b1\wX, \b1\wX, \l1\wX
|
|
add \a3\wX, \a3\wX, \t3\wX
|
|
|
|
mls \t0\wX, \b0\wX, \mod\nX[0]
|
|
mls \t1\wX, \b1\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
// vector-scalar Barrett reduction
|
|
|
|
.macro wrap_qX_barrett a0, a1, a2, a3, t0, t1, t2, t3, barrett_const, shrv, Q, wX, nX
|
|
|
|
sqdmulh \t0\wX, \a0\wX, \barrett_const\nX[1]
|
|
sqdmulh \t1\wX, \a1\wX, \barrett_const\nX[1]
|
|
|
|
sqdmulh \t2\wX, \a2\wX, \barrett_const\nX[1]
|
|
srshr \t0\wX, \t0\wX, \shrv
|
|
sqdmulh \t3\wX, \a3\wX, \barrett_const\nX[1]
|
|
srshr \t1\wX, \t1\wX, \shrv
|
|
|
|
srshr \t2\wX, \t2\wX, \shrv
|
|
mls \a0\wX, \t0\wX, \Q\nX[0]
|
|
srshr \t3\wX, \t3\wX, \shrv
|
|
mls \a1\wX, \t1\wX, \Q\nX[0]
|
|
|
|
mls \a2\wX, \t2\wX, \Q\nX[0]
|
|
mls \a3\wX, \t3\wX, \Q\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_oX_barrett a0, a1, a2, a3, t0, t1, t2, t3, a4, a5, a6, a7, t4, t5, t6, t7, barrett_const, shrv, Q, wX, nX
|
|
|
|
sqdmulh \t0\wX, \a0\wX, \barrett_const\nX[1]
|
|
sqdmulh \t1\wX, \a1\wX, \barrett_const\nX[1]
|
|
sqdmulh \t2\wX, \a2\wX, \barrett_const\nX[1]
|
|
sqdmulh \t3\wX, \a3\wX, \barrett_const\nX[1]
|
|
|
|
srshr \t0\wX, \t0\wX, \shrv
|
|
sqdmulh \t4\wX, \a4\wX, \barrett_const\nX[1]
|
|
srshr \t1\wX, \t1\wX, \shrv
|
|
sqdmulh \t5\wX, \a5\wX, \barrett_const\nX[1]
|
|
srshr \t2\wX, \t2\wX, \shrv
|
|
sqdmulh \t6\wX, \a6\wX, \barrett_const\nX[1]
|
|
srshr \t3\wX, \t3\wX, \shrv
|
|
sqdmulh \t7\wX, \a7\wX, \barrett_const\nX[1]
|
|
|
|
mls \a0\wX, \t0\wX, \Q\nX[0]
|
|
srshr \t4\wX, \t4\wX, \shrv
|
|
mls \a1\wX, \t1\wX, \Q\nX[0]
|
|
srshr \t5\wX, \t5\wX, \shrv
|
|
mls \a2\wX, \t2\wX, \Q\nX[0]
|
|
srshr \t6\wX, \t6\wX, \shrv
|
|
mls \a3\wX, \t3\wX, \Q\nX[0]
|
|
srshr \t7\wX, \t7\wX, \shrv
|
|
|
|
mls \a4\wX, \t4\wX, \Q\nX[0]
|
|
mls \a5\wX, \t5\wX, \Q\nX[0]
|
|
mls \a6\wX, \t6\wX, \Q\nX[0]
|
|
mls \a7\wX, \t7\wX, \Q\nX[0]
|
|
|
|
.endm
|
|
|
|
// vector-vector Barrett reduction
|
|
|
|
.macro wrap_qo_barrett_vec a0, a1, a2, a3, t0, t1, t2, t3, barrett_const, shrv, Q, wX, nX
|
|
|
|
sqdmulh \t0\wX, \a0\wX, \barrett_const\wX
|
|
sqdmulh \t1\wX, \a1\wX, \barrett_const\wX
|
|
|
|
sqdmulh \t2\wX, \a2\wX, \barrett_const\wX
|
|
srshr \t0\wX, \t0\wX, \shrv
|
|
sqdmulh \t3\wX, \a3\wX, \barrett_const\wX
|
|
srshr \t1\wX, \t1\wX, \shrv
|
|
|
|
srshr \t2\wX, \t2\wX, \shrv
|
|
mls \a0\wX, \t0\wX, \Q\wX
|
|
srshr \t3\wX, \t3\wX, \shrv
|
|
mls \a1\wX, \t1\wX, \Q\wX
|
|
|
|
mls \a2\wX, \t2\wX, \Q\wX
|
|
mls \a3\wX, \t3\wX, \Q\wX
|
|
|
|
.endm
|
|
|
|
.macro wrap_oo_barrett_vec a0, a1, a2, a3, t0, t1, t2, t3, a4, a5, a6, a7, t4, t5, t6, t7, barrett_const, shrv, Q, wX, nX
|
|
|
|
sqdmulh \t0\wX, \a0\wX, \barrett_const\wX
|
|
sqdmulh \t1\wX, \a1\wX, \barrett_const\wX
|
|
sqdmulh \t2\wX, \a2\wX, \barrett_const\wX
|
|
sqdmulh \t3\wX, \a3\wX, \barrett_const\wX
|
|
|
|
srshr \t0\wX, \t0\wX, \shrv
|
|
sqdmulh \t4\wX, \a4\wX, \barrett_const\wX
|
|
srshr \t1\wX, \t1\wX, \shrv
|
|
sqdmulh \t5\wX, \a5\wX, \barrett_const\wX
|
|
srshr \t2\wX, \t2\wX, \shrv
|
|
sqdmulh \t6\wX, \a6\wX, \barrett_const\wX
|
|
srshr \t3\wX, \t3\wX, \shrv
|
|
sqdmulh \t7\wX, \a7\wX, \barrett_const\wX
|
|
|
|
mls \a0\wX, \t0\wX, \Q\wX
|
|
srshr \t4\wX, \t4\wX, \shrv
|
|
mls \a1\wX, \t1\wX, \Q\wX
|
|
srshr \t5\wX, \t5\wX, \shrv
|
|
mls \a2\wX, \t2\wX, \Q\wX
|
|
srshr \t6\wX, \t6\wX, \shrv
|
|
mls \a3\wX, \t3\wX, \Q\wX
|
|
srshr \t7\wX, \t7\wX, \shrv
|
|
|
|
mls \a4\wX, \t4\wX, \Q\wX
|
|
mls \a5\wX, \t5\wX, \Q\wX
|
|
mls \a6\wX, \t6\wX, \Q\wX
|
|
mls \a7\wX, \t7\wX, \Q\wX
|
|
|
|
.endm
|
|
|
|
// Montgomery multiplication
|
|
|
|
.macro wrap_qX_montgomery_mul b0, b1, b2, b3, t0, t1, t2, t3, mod, z0, l0, h0, z1, l1, h1, z2, l2, h2, z3, l3, h3, wX, nX
|
|
|
|
mul \b0\wX, \t0\wX, \z0\nX[\h0]
|
|
mul \b1\wX, \t1\wX, \z1\nX[\h1]
|
|
mul \b2\wX, \t2\wX, \z2\nX[\h2]
|
|
mul \b3\wX, \t3\wX, \z3\nX[\h3]
|
|
|
|
sqrdmulh \t0\wX, \t0\wX, \z0\nX[\l0]
|
|
sqrdmulh \t1\wX, \t1\wX, \z1\nX[\l1]
|
|
sqrdmulh \t2\wX, \t2\wX, \z2\nX[\l2]
|
|
sqrdmulh \t3\wX, \t3\wX, \z3\nX[\l3]
|
|
|
|
mls \b0\wX, \t0\wX, \mod\nX[0]
|
|
mls \b1\wX, \t1\wX, \mod\nX[0]
|
|
mls \b2\wX, \t2\wX, \mod\nX[0]
|
|
mls \b3\wX, \t3\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_qX_montgomery_mul_in b0, b1, b2, b3, t0, t1, t2, t3, mod, z0, l0, h0, z1, l1, h1, z2, l2, h2, z3, l3, h3, wX, nX
|
|
|
|
sqrdmulh \t0\wX, \b0\wX, \z0\nX[\l0]
|
|
sqrdmulh \t1\wX, \b1\wX, \z1\nX[\l1]
|
|
sqrdmulh \t2\wX, \b2\wX, \z2\nX[\l2]
|
|
sqrdmulh \t3\wX, \b3\wX, \z3\nX[\l3]
|
|
|
|
mul \b0\wX, \b0\wX, \z0\nX[\h0]
|
|
mul \b1\wX, \b1\wX, \z1\nX[\h1]
|
|
mul \b2\wX, \b2\wX, \z2\nX[\h2]
|
|
mul \b3\wX, \b3\wX, \z3\nX[\h3]
|
|
|
|
mls \b0\wX, \t0\wX, \mod\nX[0]
|
|
mls \b1\wX, \t1\wX, \mod\nX[0]
|
|
mls \b2\wX, \t2\wX, \mod\nX[0]
|
|
mls \b3\wX, \t3\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_qX_montgomery_mul_ins b0, b1, b2, b3, t0, t1, t2, t3, mod, z0, l0, h0, z1, l1, h1, z2, l2, h2, z3, l3, h3, wX, nX, srcc_ptr, c0, c1, c2, c3, memc0, memc1, memc2, memc3
|
|
|
|
str \c0, [\srcc_ptr, \memc0]
|
|
sqrdmulh \t0\wX, \b0\wX, \z0\nX[\h0]
|
|
str \c1, [\srcc_ptr, \memc1]
|
|
sqrdmulh \t1\wX, \b1\wX, \z1\nX[\h1]
|
|
str \c2, [\srcc_ptr, \memc2]
|
|
sqrdmulh \t2\wX, \b2\wX, \z2\nX[\h2]
|
|
str \c3, [\srcc_ptr, \memc3]
|
|
sqrdmulh \t3\wX, \b3\wX, \z3\nX[\h3]
|
|
|
|
mul \b0\wX, \b0\wX, \z0\nX[\l0]
|
|
mul \b1\wX, \b1\wX, \z1\nX[\l1]
|
|
mul \b2\wX, \b2\wX, \z2\nX[\l2]
|
|
mul \b3\wX, \b3\wX, \z3\nX[\l3]
|
|
|
|
mls \b0\wX, \t0\wX, \mod\nX[0]
|
|
mls \b1\wX, \t1\wX, \mod\nX[0]
|
|
mls \b2\wX, \t2\wX, \mod\nX[0]
|
|
mls \b3\wX, \t3\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_qX_montgomery_mul_inl b0, b1, b2, b3, t0, t1, t2, t3, mod, z0, l0, h0, z1, l1, h1, z2, l2, h2, z3, l3, h3, wX, nX, srcc_ptr, c0, c1, c2, c3, memc0, memc1, memc2, memc3
|
|
|
|
ldr \c0, [\srcc_ptr, \memc0]
|
|
sqrdmulh \t0\wX, \b0\wX, \z0\nX[\h0]
|
|
ldr \c1, [\srcc_ptr, \memc1]
|
|
sqrdmulh \t1\wX, \b1\wX, \z1\nX[\h1]
|
|
ldr \c2, [\srcc_ptr, \memc2]
|
|
sqrdmulh \t2\wX, \b2\wX, \z2\nX[\h2]
|
|
ldr \c3, [\srcc_ptr, \memc3]
|
|
sqrdmulh \t3\wX, \b3\wX, \z3\nX[\h3]
|
|
|
|
mul \b0\wX, \b0\wX, \z0\nX[\l0]
|
|
mul \b1\wX, \b1\wX, \z1\nX[\l1]
|
|
mul \b2\wX, \b2\wX, \z2\nX[\l2]
|
|
mul \b3\wX, \b3\wX, \z3\nX[\l3]
|
|
|
|
mls \b0\wX, \t0\wX, \mod\nX[0]
|
|
mls \b1\wX, \t1\wX, \mod\nX[0]
|
|
mls \b2\wX, \t2\wX, \mod\nX[0]
|
|
mls \b3\wX, \t3\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
.macro wrap_qX_montgomery_mul_insl b0, b1, b2, b3, t0, t1, t2, t3, mod, z0, l0, h0, z1, l1, h1, z2, l2, h2, z3, l3, h3, wX, nX, srcc_ptr, c0, c1, c2, c3, memc0, memc1, memc2, memc3, srcd_ptr, d0, d1, d2, d3, memd0, memd1, memd2, memd3
|
|
|
|
str \c0, [\srcc_ptr, \memc0]
|
|
ldr \d0, [\srcd_ptr, \memd0]
|
|
sqrdmulh \t0\wX, \b0\wX, \z0\nX[\h0]
|
|
str \c1, [\srcc_ptr, \memc1]
|
|
ldr \d1, [\srcd_ptr, \memd1]
|
|
sqrdmulh \t1\wX, \b1\wX, \z1\nX[\h1]
|
|
str \c2, [\srcc_ptr, \memc2]
|
|
ldr \d2, [\srcd_ptr, \memd2]
|
|
sqrdmulh \t2\wX, \b2\wX, \z2\nX[\h2]
|
|
str \c3, [\srcc_ptr, \memc3]
|
|
ldr \d3, [\srcd_ptr, \memd3]
|
|
sqrdmulh \t3\wX, \b3\wX, \z3\nX[\h3]
|
|
|
|
mul \b0\wX, \b0\wX, \z0\nX[\l0]
|
|
mul \b1\wX, \b1\wX, \z1\nX[\l1]
|
|
mul \b2\wX, \b2\wX, \z2\nX[\l2]
|
|
mul \b3\wX, \b3\wX, \z3\nX[\l3]
|
|
|
|
mls \b0\wX, \t0\wX, \mod\nX[0]
|
|
mls \b1\wX, \t1\wX, \mod\nX[0]
|
|
mls \b2\wX, \t2\wX, \mod\nX[0]
|
|
mls \b3\wX, \t3\wX, \mod\nX[0]
|
|
|
|
.endm
|
|
|
|
|
|
|
|
// Montgomery reduction with long
|
|
|
|
.macro wrap_qX_montgomery c0, c1, c2, c3, l0, l1, l2, l3, h0, h1, h2, h3, t0, t1, t2, t3, Qprime, Q, lX, wX, dwX
|
|
|
|
uzp1 \t0\wX, \l0\wX, \h0\wX
|
|
uzp1 \t1\wX, \l1\wX, \h1\wX
|
|
uzp1 \t2\wX, \l2\wX, \h2\wX
|
|
uzp1 \t3\wX, \l3\wX, \h3\wX
|
|
|
|
mul \t0\wX, \t0\wX, \Qprime\wX
|
|
mul \t1\wX, \t1\wX, \Qprime\wX
|
|
mul \t2\wX, \t2\wX, \Qprime\wX
|
|
mul \t3\wX, \t3\wX, \Qprime\wX
|
|
|
|
smlal \l0\dwX, \t0\lX, \Q\lX
|
|
smlal2 \h0\dwX, \t0\wX, \Q\wX
|
|
smlal \l1\dwX, \t1\lX, \Q\lX
|
|
smlal2 \h1\dwX, \t1\wX, \Q\wX
|
|
smlal \l2\dwX, \t2\lX, \Q\lX
|
|
smlal2 \h2\dwX, \t2\wX, \Q\wX
|
|
smlal \l3\dwX, \t3\lX, \Q\lX
|
|
smlal2 \h3\dwX, \t3\wX, \Q\wX
|
|
|
|
uzp2 \c0\wX, \l0\wX, \h0\wX
|
|
uzp2 \c1\wX, \l1\wX, \h1\wX
|
|
uzp2 \c2\wX, \l2\wX, \h2\wX
|
|
uzp2 \c3\wX, \l3\wX, \h3\wX
|
|
|
|
.endm
|
|
|
|
// add_sub, sub_add
|
|
|
|
.macro wrap_qX_add_sub s0, s1, s2, s3, t0, t1, t2, t3, a0, a1, a2, a3, b0, b1, b2, b3, wX
|
|
|
|
add \s0\wX, \a0\wX, \b0\wX
|
|
sub \t0\wX, \a0\wX, \b0\wX
|
|
add \s1\wX, \a1\wX, \b1\wX
|
|
sub \t1\wX, \a1\wX, \b1\wX
|
|
add \s2\wX, \a2\wX, \b2\wX
|
|
sub \t2\wX, \a2\wX, \b2\wX
|
|
add \s3\wX, \a3\wX, \b3\wX
|
|
sub \t3\wX, \a3\wX, \b3\wX
|
|
|
|
.endm
|
|
|
|
.macro wrap_qX_sub_add s0, s1, s2, s3, t0, t1, t2, t3, a0, a1, a2, a3, b0, b1, b2, b3, wX
|
|
|
|
sub \t0\wX, \a0\wX, \b0\wX
|
|
add \s0\wX, \a0\wX, \b0\wX
|
|
sub \t1\wX, \a1\wX, \b1\wX
|
|
add \s1\wX, \a1\wX, \b1\wX
|
|
sub \t2\wX, \a2\wX, \b2\wX
|
|
add \s2\wX, \a2\wX, \b2\wX
|
|
sub \t3\wX, \a3\wX, \b3\wX
|
|
add \s3\wX, \a3\wX, \b3\wX
|
|
|
|
.endm
|
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|