blst/build/coff/ct_inverse_mod_384-armv8.S

.text

.globl	ct_inverse_mod_384

.def	ct_inverse_mod_384;
.type	32;
.endef
.p2align	5
ct_inverse_mod_384:
	hint	#25
	stp	x29, x30, [sp,#-16*__SIZEOF_POINTER__]!
	add	x29, sp, #0
	stp	x19, x20, [sp,#2*__SIZEOF_POINTER__]
	stp	x21, x22, [sp,#4*__SIZEOF_POINTER__]
	stp	x23, x24, [sp,#6*__SIZEOF_POINTER__]
	stp	x25, x26, [sp,#8*__SIZEOF_POINTER__]
	stp	x27, x28, [sp,#10*__SIZEOF_POINTER__]
	sub	sp, sp, #1056

	ldp	x22,   x4, [x1,#8*0]
	ldp	x5, x6, [x1,#8*2]
	ldp	x7, x8, [x1,#8*4]

#ifdef	__CHERI_PURE_CAPABILITY__
	add	x1,sp,#32+511
	alignd	c1,c1,#9
	scbnds	c1,c1,#512
#else
	add	x1, sp, #32+511	// find closest 512-byte-aligned spot
	and	x1, x1, #-512	// in the frame...
#endif
	stp	x0, x3, [sp]		// offload out_ptr, nx_ptr

	ldp	x9, x10, [x2,#8*0]
	ldp	x11, x12, [x2,#8*2]
	ldp	x13, x14, [x2,#8*4]

	stp	x22,   x4, [x1,#8*0]	// copy input to |a|
	stp	x5, x6, [x1,#8*2]
	stp	x7, x8, [x1,#8*4]
	stp	x9, x10, [x1,#8*6]	// copy modulus to |b|
	stp	x11, x12, [x1,#8*8]
	stp	x13, x14, [x1,#8*10]

	////////////////////////////////////////// first iteration
	mov	x2, #62
	bl	.Lab_approximation_62_loaded

	eor	x0, x1, #256		// pointer to dst |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c0,c1,x0
#endif
	bl	__smul_384_n_shift_by_62
	str	x15,[x0,#8*12]		// initialize |u| with |f0|

	mov	x15, x17			// |f1|
	mov	x16, x19			// |g1|
	add	x0,x0,#8*6
	bl	__smul_384_n_shift_by_62
	str	x15, [x0,#8*14]		// initialize |v| with |f1|

	////////////////////////////////////////// second iteration
	eor	x1, x1, #256		// flip-flop src |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c1,c0,x1
#endif
	mov	x2, #62
	bl	__ab_approximation_62

	eor	x0, x1, #256		// pointer to dst |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c0,c1,x0
#endif
	bl	__smul_384_n_shift_by_62
	mov	x20, x15			// corrected |f0|
	mov	x21, x16			// corrected |g0|

	mov	x15, x17			// |f1|
	mov	x16, x19			// |g1|
	add	x0,x0,#8*6
	bl	__smul_384_n_shift_by_62

	ldr	x7, [x1,#8*12]	// |u|
	ldr	x8, [x1,#8*20]	// |v|
	mul	x3, x20, x7		// |u|*|f0|
	smulh	x4, x20, x7
	mul	x5, x21, x8		// |v|*|g0|
	smulh	x6, x21, x8
	adds	x3, x3, x5
	adc	x4, x4, x6
	stp	x3, x4, [x0,#8*6]
	asr	x5, x4, #63		// sign extension
	stp	x5, x5, [x0,#8*8]
	stp	x5, x5, [x0,#8*10]

	mul	x3, x15, x7		// |u|*|f1|
	smulh	x4, x15, x7
	mul	x5, x16, x8		// |v|*|g1|
	smulh	x6, x16, x8
	adds	x3, x3, x5
	adc	x4, x4, x6
	stp	x3, x4, [x0,#8*14]
	asr	x5, x4, #63		// sign extension
	stp	x5, x5, [x0,#8*16]
	stp	x5, x5, [x0,#8*18]
	eor	x1, x1, #256		// flip-flop src |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c1,c0,x1
#endif
	mov	x2, #62
	bl	__ab_approximation_62

	eor	x0, x1, #256		// pointer to dst |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c0,c1,x0
#endif
	bl	__smul_384_n_shift_by_62
	mov	x20, x15			// corrected |f0|
	mov	x21, x16			// corrected |g0|

	mov	x15, x17			// |f1|
	mov	x16, x19			// |g1|
	add	x0,x0,#8*6
	bl	__smul_384_n_shift_by_62

	add	x0,x0,#8*6
	bl	__smul_384x63
	mov	x20, x15			// corrected |f1|
	mov	x21, x16			// corrected |g1|
	add	x0,x0,#8*8
	bl	__smul_384x63
	eor	x1, x1, #256		// flip-flop src |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c1,c0,x1
#endif
	mov	x2, #62
	bl	__ab_approximation_62

	eor	x0, x1, #256		// pointer to dst |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c0,c1,x0
#endif
	bl	__smul_384_n_shift_by_62
	mov	x20, x15			// corrected |f0|
	mov	x21, x16			// corrected |g0|

	mov	x15, x17			// |f1|
	mov	x16, x19			// |g1|
	add	x0,x0,#8*6
	bl	__smul_384_n_shift_by_62

	add	x0,x0,#8*6
	bl	__smul_384x63
	mov	x20, x15			// corrected |f1|
	mov	x21, x16			// corrected |g1|
	add	x0,x0,#8*8
	bl	__smul_384x63
	eor	x1, x1, #256		// flip-flop src |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c1,c0,x1
#endif
	mov	x2, #62
	bl	__ab_approximation_62

	eor	x0, x1, #256		// pointer to dst |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c0,c1,x0
#endif
	bl	__smul_384_n_shift_by_62
	mov	x20, x15			// corrected |f0|
	mov	x21, x16			// corrected |g0|

	mov	x15, x17			// |f1|
	mov	x16, x19			// |g1|
	add	x0,x0,#8*6
	bl	__smul_384_n_shift_by_62

	add	x0,x0,#8*6
	bl	__smul_384x63
	mov	x20, x15			// corrected |f1|
	mov	x21, x16			// corrected |g1|
	add	x0,x0,#8*8
	bl	__smul_384x63
	eor	x1, x1, #256		// flip-flop src |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c1,c0,x1
#endif
	mov	x2, #62
	bl	__ab_approximation_62

	eor	x0, x1, #256		// pointer to dst |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c0,c1,x0
#endif
	bl	__smul_384_n_shift_by_62
	mov	x20, x15			// corrected |f0|
	mov	x21, x16			// corrected |g0|

	mov	x15, x17			// |f1|
	mov	x16, x19			// |g1|
	add	x0,x0,#8*6
	bl	__smul_384_n_shift_by_62

	add	x0,x0,#8*6
	bl	__smul_384x63
	asr	x27, x27, #63
	str	x27, [x0,#8*6]
	mov	x20, x15			// corrected |f1|
	mov	x21, x16			// corrected |g1|
	add	x0,x0,#8*8
	bl	__smul_384x63
	asr	x27, x27, #63		// sign extension
	stp	x27, x27, [x0,#8*6]
	stp	x27, x27, [x0,#8*8]
	stp	x27, x27, [x0,#8*10]
	eor	x1, x1, #256		// flip-flop src |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c1,c0,x1
#endif
	mov	x2, #62
	bl	__ab_approximation_62

	eor	x0, x1, #256		// pointer to dst |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c0,c1,x0
#endif
	bl	__smul_384_n_shift_by_62
	mov	x20, x15			// corrected |f0|
	mov	x21, x16			// corrected |g0|

	mov	x15, x17			// |f1|
	mov	x16, x19			// |g1|
	add	x0,x0,#8*6
	bl	__smul_384_n_shift_by_62

	add	x0,x0,#8*6
	bl	__smul_384x63
	adc	x25, x25, x26
	str	x25, [x0,#8*6]
	mov	x20, x15			// corrected |f1|
	mov	x21, x16			// corrected |g1|
	add	x0,x0,#8*8
	bl	__smul_384x63
	bl	__smul_768x63_tail
	eor	x1, x1, #256		// flip-flop src |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c1,c0,x1
#endif
	mov	x2, #62
	bl	__ab_approximation_62

	eor	x0, x1, #256		// pointer to dst |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c0,c1,x0
#endif
	bl	__smul_384_n_shift_by_62
	mov	x20, x15			// corrected |f0|
	mov	x21, x16			// corrected |g0|

	mov	x15, x17			// |f1|
	mov	x16, x19			// |g1|
	add	x0,x0,#8*6
	bl	__smul_384_n_shift_by_62

	add	x0,x0,#8*6
	bl	__smul_384x63
	adc	x25, x25, x26
	str	x25, [x0,#8*6]
	mov	x20, x15			// corrected |f1|
	mov	x21, x16			// corrected |g1|
	add	x0,x0,#8*8
	bl	__smul_384x63
	bl	__smul_768x63_tail
	eor	x1, x1, #256		// flip-flop src |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c1,c0,x1
#endif
	mov	x2, #62
	bl	__ab_approximation_62

	eor	x0, x1, #256		// pointer to dst |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c0,c1,x0
#endif
	bl	__smul_384_n_shift_by_62
	mov	x20, x15			// corrected |f0|
	mov	x21, x16			// corrected |g0|

	mov	x15, x17			// |f1|
	mov	x16, x19			// |g1|
	add	x0,x0,#8*6
	bl	__smul_384_n_shift_by_62

	add	x0,x0,#8*6
	bl	__smul_384x63
	adc	x25, x25, x26
	str	x25, [x0,#8*6]
	mov	x20, x15			// corrected |f1|
	mov	x21, x16			// corrected |g1|
	add	x0,x0,#8*8
	bl	__smul_384x63
	bl	__smul_768x63_tail
	eor	x1, x1, #256		// flip-flop src |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c1,c0,x1
#endif
	mov	x2, #62
	bl	__ab_approximation_62

	eor	x0, x1, #256		// pointer to dst |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c0,c1,x0
#endif
	bl	__smul_384_n_shift_by_62
	mov	x20, x15			// corrected |f0|
	mov	x21, x16			// corrected |g0|

	mov	x15, x17			// |f1|
	mov	x16, x19			// |g1|
	add	x0,x0,#8*6
	bl	__smul_384_n_shift_by_62

	add	x0,x0,#8*6
	bl	__smul_384x63
	adc	x25, x25, x26
	str	x25, [x0,#8*6]
	mov	x20, x15			// corrected |f1|
	mov	x21, x16			// corrected |g1|
	add	x0,x0,#8*8
	bl	__smul_384x63
	bl	__smul_768x63_tail
	eor	x1, x1, #256		// flip-flop src |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c1,c0,x1
#endif
	mov	x2, #62
	bl	__ab_approximation_62

	eor	x0, x1, #256		// pointer to dst |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c0,c1,x0
#endif
	bl	__smul_384_n_shift_by_62
	mov	x20, x15			// corrected |f0|
	mov	x21, x16			// corrected |g0|

	mov	x15, x17			// |f1|
	mov	x16, x19			// |g1|
	add	x0,x0,#8*6
	bl	__smul_384_n_shift_by_62

	add	x0,x0,#8*6
	bl	__smul_384x63
	adc	x25, x25, x26
	str	x25, [x0,#8*6]
	mov	x20, x15			// corrected |f1|
	mov	x21, x16			// corrected |g1|
	add	x0,x0,#8*8
	bl	__smul_384x63
	bl	__smul_768x63_tail
	////////////////////////////////////////// iteration before last
	eor	x1, x1, #256		// flip-flop src |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c1,c0,x1
#endif
	mov	x2, #62
	//bl	__ab_approximation_62		// |a| and |b| are exact,
	ldp	x3, x8, [x1,#8*0]	// just load
	ldp	x9, x14, [x1,#8*6]
	bl	__inner_loop_62

	eor	x0, x1, #256		// pointer to dst |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c0,c1,x0
#endif
	str	x3, [x0,#8*0]
	str	x9, [x0,#8*6]

	mov	x20, x15			// exact |f0|
	mov	x21, x16			// exact |g0|
	mov	x15, x17
	mov	x16, x19
	add	x0,x0,#8*12
	bl	__smul_384x63
	adc	x25, x25, x26
	str	x25, [x0,#8*6]

	mov	x20, x15			// exact |f1|
	mov	x21, x16			// exact |g1|
	add	x0,x0,#8*8
	bl	__smul_384x63
	bl	__smul_768x63_tail

	////////////////////////////////////////// last iteration
	eor	x1, x1, #256		// flip-flop src |a|b|u|v|
#ifdef	__CHERI_PURE_CAPABILITY__
	scvalue	c1,c0,x1
#endif
	mov	x2, #24			// 768 % 62
	//bl	__ab_approximation_62		// |a| and |b| are exact,
	ldr	x3, [x1,#8*0]		// just load
	eor	x8, x8, x8
	ldr	x9, [x1,#8*6]
	eor	x14, x14, x14
	bl	__inner_loop_62

	mov	x20, x17
	mov	x21, x19
	ldp	x0, x15, [sp]			// original out_ptr and n_ptr
	bl	__smul_384x63
	bl	__smul_768x63_tail
	ldr	x30, [x29,#__SIZEOF_POINTER__]

	smulh	x23, x8, x21		// figure out top-most limb
	adc	x26, x26, x28
	ldp	x9, x10, [x15,#8*0]	// load |mod|
	add	x23, x23, x26		// x23 is 1, 0 or -1
	ldp	x11, x12, [x15,#8*2]
	asr	x22, x23, #63		// sign as mask
	ldp	x13, x14, [x15,#8*4]

	and	x26,   x9, x22		// add mod<<384 conditionally
	and	x27,   x10, x22
	adds	x3, x3, x26
	and	x28,   x11, x22
	adcs	x4, x4, x27
	and	x2,   x12, x22
	adcs	x5, x5, x28
	and	x26,   x13, x22
	adcs	x6, x6, x2
	and	x27,   x14, x22
	adcs	x7, x7, x26
	adcs	x8, x25,   x27
	adc	x23, x23, xzr		// x23 is 1, 0 or -1

	neg	x22, x23
	orr	x23, x23, x22		// excess bit or sign as mask
	asr	x22, x22, #63		// excess bit as mask

	and	x9, x9, x23		// mask |mod|
	and	x10, x10, x23
	and	x11, x11, x23
	and	x12, x12, x23
	and	x13, x13, x23
	and	x14, x14, x23

	eor	x9,  x9, x22	// conditionally negate |mod|
	eor	x10,  x10, x22
	adds	x9,  x9, x22, lsr#63
	eor	x11,  x11, x22
	adcs	x10,  x10, xzr
	eor	x12,  x12, x22
	adcs	x11,  x11, xzr
	eor	x13, x13, x22
	adcs	x12,  x12, xzr
	eor	x14, x14, x22
	adcs	x13, x13, xzr
	adc	x14, x14, xzr

	adds	x3, x3, x9	// final adjustment for |mod|<<384
	adcs	x4, x4, x10
	adcs	x5, x5, x11
	adcs	x6, x6, x12
	stp	x3, x4, [x0,#8*6]
	adcs	x7, x7, x13
	stp	x5, x6, [x0,#8*8]
	adc	x8, x8, x14
	stp	x7, x8, [x0,#8*10]

	add	sp, sp, #1056
	ldp	x19, x20, [x29,#2*__SIZEOF_POINTER__]
	ldp	x21, x22, [x29,#4*__SIZEOF_POINTER__]
	ldp	x23, x24, [x29,#6*__SIZEOF_POINTER__]
	ldp	x25, x26, [x29,#8*__SIZEOF_POINTER__]
	ldp	x27, x28, [x29,#10*__SIZEOF_POINTER__]
	ldr	x29, [sp],#16*__SIZEOF_POINTER__
	hint	#29
	ret


////////////////////////////////////////////////////////////////////////
// see corresponding commentary in ctx_inverse_mod_384-x86_64...
.def	__smul_384x63;
.type	32;
.endef
.p2align	5
__smul_384x63:
	ldp	x3, x4, [x1,#8*0+96]	// load |u| (or |v|)
	asr	x17, x20, #63		// |f_|'s sign as mask (or |g_|'s)
	ldp	x5, x6, [x1,#8*2+96]
	eor	x20, x20, x17		// conditionally negate |f_| (or |g_|)
	ldp	x7, x8, [x1,#8*4+96]

	eor	x3, x3, x17	// conditionally negate |u| (or |v|)
	ldr	x25, [x1,#8*6+96]
	sub	x20, x20, x17
	eor	x4, x4, x17
	adds	x3, x3, x17, lsr#63
	eor	x5, x5, x17
	adcs	x4, x4, xzr
	eor	x6, x6, x17
	adcs	x5, x5, xzr
	eor	x7, x7, x17
	adcs	x6, x6, xzr
	umulh	x22, x3, x20
	eor	x8, x8, x17
	umulh	x23, x4, x20
	adcs	x7, x7, xzr
	umulh	x24, x5, x20
	eor	x25, x25, x17
	mul	x3, x3, x20
	adcs	x8, x8, xzr
	mul	x4, x4, x20
	adcs	x25, x25, xzr
	cmp	x20, #0
	mul	x5, x5, x20
	csel	x25, x25, xzr, ne
	adds	x4, x4, x22
	umulh	x22, x6, x20
	adcs	x5, x5, x23
	umulh	x23, x7, x20
	mul	x6, x6, x20
	mul	x7, x7, x20
	adcs	x6, x6, x24
	mul	x27,x8, x20
	adcs	x7, x7, x22
	adcs	x27,x27,x23
	adc	x2, xzr, xzr
	ldp	x9, x10, [x1,#8*0+160]	// load |u| (or |v|)
	asr	x17, x21, #63		// |f_|'s sign as mask (or |g_|'s)
	ldp	x11, x12, [x1,#8*2+160]
	eor	x21, x21, x17		// conditionally negate |f_| (or |g_|)
	ldp	x13, x14, [x1,#8*4+160]

	eor	x9, x9, x17	// conditionally negate |u| (or |v|)
	ldr	x26, [x1,#8*6+160]
	sub	x21, x21, x17
	eor	x10, x10, x17
	adds	x9, x9, x17, lsr#63
	eor	x11, x11, x17
	adcs	x10, x10, xzr
	eor	x12, x12, x17
	adcs	x11, x11, xzr
	eor	x13, x13, x17
	adcs	x12, x12, xzr
	umulh	x22, x9, x21
	eor	x14, x14, x17
	umulh	x23, x10, x21
	adcs	x13, x13, xzr
	umulh	x24, x11, x21
	eor	x26, x26, x17
	mul	x9, x9, x21
	adcs	x14, x14, xzr
	mul	x10, x10, x21
	adcs	x26, x26, xzr
	adc	x19, xzr, xzr		// used in __smul_768x63_tail
	cmp	x21, #0
	mul	x11, x11, x21
	csel	x26, x26, xzr, ne
	adds	x10, x10, x22
	umulh	x22, x12, x21
	adcs	x11, x11, x23
	umulh	x23, x13, x21
	mul	x12, x12, x21
	mul	x13, x13, x21
	adcs	x12, x12, x24
	mul	x28,x14, x21
	adcs	x13, x13, x22
	adcs	x28,x28,x23
	adc	x2, x2, xzr

	adds	x3, x3, x9
	adcs	x4, x4, x10
	adcs	x5, x5, x11
	adcs	x6, x6, x12
	stp	x3, x4, [x0,#8*0]
	adcs	x7, x7, x13
	stp	x5, x6, [x0,#8*2]
	adcs	x27,   x27,   x28
	stp	x7, x27,   [x0,#8*4]

	ret


.def	__smul_768x63_tail;
.type	32;
.endef
.p2align	5
__smul_768x63_tail:
	umulh	x27, x8, x20
	ldr	x4, [x1,#8*27]// load rest of |v|
	adc	x2, x2, xzr
	ldp	x5, x6, [x1,#8*28]
	and	x25, x25, x20
	ldp	x7, x8, [x1,#8*30]
	sub	x27, x27, x25	// tie up |u|*|f1| chain

	umulh	x14, x14, x21	// resume |v|*|g1| chain
	eor	x4, x4, x17	// conditionally negate rest of |v|
	eor	x5, x5, x17
	eor	x6, x6, x17
	adds	x4, x4, x19
	eor	x7, x7, x17
	adcs	x5, x5, xzr
	eor	x8, x8, x17
	adcs	x6, x6, xzr
	umulh	x22, x26,   x21
	adcs	x7, x7, xzr
	umulh	x23, x4, x21
	adc	x8, x8, xzr

	umulh	x24, x5, x21
	add	x14, x14, x2
	umulh	x25, x6, x21
	asr	x28, x27, #63
	umulh	x2, x7, x21
	mul	x3, x26,   x21
	mul	x4, x4, x21
	mul	x5, x5, x21
	adds	x3, x3, x14
	mul	x6, x6, x21
	adcs	x4, x4, x22
	mul	x7, x7, x21
	adcs	x5, x5, x23
	mul	x22,   x8, x21
	adcs	x6, x6, x24
	adcs	x7, x7, x25
	adcs	x25,   x22, x2
	adc	x26, xzr, xzr		// used in the final step

	adds	x3, x3, x27
	adcs	x4, x4, x28
	adcs	x5, x5, x28
	adcs	x6, x6, x28
	stp	x3, x4, [x0,#8*6]
	adcs	x7, x7, x28
	stp	x5, x6, [x0,#8*8]
	adcs	x25,   x25,   x28	// carry is used in the final step
	stp	x7, x25,   [x0,#8*10]

	ret


.def	__smul_384_n_shift_by_62;
.type	32;
.endef
.p2align	5
__smul_384_n_shift_by_62:
	ldp	x3, x4, [x1,#8*0+0]	// load |a| (or |b|)
	asr	x28, x15, #63		// |f0|'s sign as mask (or |g0|'s)
	ldp	x5, x6, [x1,#8*2+0]
	eor	x2, x15, x28	// conditionally negate |f0| (or |g0|)
	ldp	x7, x8, [x1,#8*4+0]

	eor	x3, x3, x28	// conditionally negate |a| (or |b|)
	sub	x2, x2, x28
	eor	x4, x4, x28
	adds	x3, x3, x28, lsr#63
	eor	x5, x5, x28
	adcs	x4, x4, xzr
	eor	x6, x6, x28
	adcs	x5, x5, xzr
	eor	x7, x7, x28
	umulh	x22, x3, x2
	adcs	x6, x6, xzr
	umulh	x23, x4, x2
	eor	x8, x8, x28
	mul	x3, x3, x2
	adcs	x7, x7, xzr
	mul	x4, x4, x2
	adc	x8, x8, xzr

	umulh	x24, x5, x2
	and	x28, x28, x2
	umulh	x25, x6, x2
	adds	x4, x4, x22
	mul	x5, x5, x2
	umulh	x22, x7, x2
	neg	x28, x28
	mul	x6, x6, x2
	adcs	x5, x5, x23
	umulh	x23, x8, x2
	mul	x7, x7, x2
	adcs	x6, x6, x24
	mul	x8, x8, x2
	adcs	x7, x7, x25
	adcs	x8, x8, x22
	adc	x27, x23, x28
	ldp	x9, x10, [x1,#8*0+48]	// load |a| (or |b|)
	asr	x28, x16, #63		// |f0|'s sign as mask (or |g0|'s)
	ldp	x11, x12, [x1,#8*2+48]
	eor	x2, x16, x28	// conditionally negate |f0| (or |g0|)
	ldp	x13, x14, [x1,#8*4+48]

	eor	x9, x9, x28	// conditionally negate |a| (or |b|)
	sub	x2, x2, x28
	eor	x10, x10, x28
	adds	x9, x9, x28, lsr#63
	eor	x11, x11, x28
	adcs	x10, x10, xzr
	eor	x12, x12, x28
	adcs	x11, x11, xzr
	eor	x13, x13, x28
	umulh	x22, x9, x2
	adcs	x12, x12, xzr
	umulh	x23, x10, x2
	eor	x14, x14, x28
	mul	x9, x9, x2
	adcs	x13, x13, xzr
	mul	x10, x10, x2
	adc	x14, x14, xzr

	umulh	x24, x11, x2
	and	x28, x28, x2
	umulh	x25, x12, x2
	adds	x10, x10, x22
	mul	x11, x11, x2
	umulh	x22, x13, x2
	neg	x28, x28
	mul	x12, x12, x2
	adcs	x11, x11, x23
	umulh	x23, x14, x2
	mul	x13, x13, x2
	adcs	x12, x12, x24
	mul	x14, x14, x2
	adcs	x13, x13, x25
	adcs	x14, x14, x22
	adc	x28, x23, x28
	adds	x3, x3, x9
	adcs	x4, x4, x10
	adcs	x5, x5, x11
	adcs	x6, x6, x12
	adcs	x7, x7, x13
	adcs	x8, x8, x14
	adc	x9, x27,   x28

	extr	x3, x4, x3, #62
	extr	x4, x5, x4, #62
	extr	x5, x6, x5, #62
	asr	x28, x9, #63
	extr	x6, x7, x6, #62
	extr	x7, x8, x7, #62
	extr	x8, x9, x8, #62

	eor	x3, x3, x28
	eor	x4, x4, x28
	adds	x3, x3, x28, lsr#63
	eor	x5, x5, x28
	adcs	x4, x4, xzr
	eor	x6, x6, x28
	adcs	x5, x5, xzr
	eor	x7, x7, x28
	adcs	x6, x6, xzr
	eor	x8, x8, x28
	stp	x3, x4, [x0,#8*0]
	adcs	x7, x7, xzr
	stp	x5, x6, [x0,#8*2]
	adc	x8, x8, xzr
	stp	x7, x8, [x0,#8*4]

	eor	x15, x15, x28
	eor	x16, x16, x28
	sub	x15, x15, x28
	sub	x16, x16, x28

	ret

.def	__ab_approximation_62;
.type	32;
.endef
.p2align	4
__ab_approximation_62:
	ldp	x7, x8, [x1,#8*4]
	ldp	x13, x14, [x1,#8*10]
	ldp	x5, x6, [x1,#8*2]
	ldp	x11, x12, [x1,#8*8]

.Lab_approximation_62_loaded:
	orr	x22, x8, x14	// check top-most limbs, ...
	cmp	x22, #0
	csel	x8, x8, x7, ne
	csel	x14, x14, x13, ne
	csel	x7, x7, x6, ne
	orr	x22, x8, x14	// ... ones before top-most, ...
	csel	x13, x13, x12, ne

	ldp	x3, x4, [x1,#8*0]
	ldp	x9, x10, [x1,#8*6]

	cmp	x22, #0
	csel	x8, x8, x7, ne
	csel	x14, x14, x13, ne
	csel	x7, x7, x5, ne
	orr	x22, x8, x14	// ... and ones before that ...
	csel	x13, x13, x11, ne

	cmp	x22, #0
	csel	x8, x8, x7, ne
	csel	x14, x14, x13, ne
	csel	x7, x7, x4, ne
	orr	x22, x8, x14
	csel	x13, x13, x10, ne

	clz	x22, x22
	cmp	x22, #64
	csel	x22, x22, xzr, ne
	csel	x8, x8, x7, ne
	csel	x14, x14, x13, ne
	neg	x23, x22

	lslv	x8, x8, x22	// align high limbs to the left
	lslv	x14, x14, x22
	lsrv	x7, x7, x23
	lsrv	x13, x13, x23
	and	x7, x7, x23, asr#6
	and	x13, x13, x23, asr#6
	orr	x8, x8, x7
	orr	x14, x14, x13

	b	__inner_loop_62
	ret

.def	__inner_loop_62;
.type	32;
.endef
.p2align	4
__inner_loop_62:
	mov	x15, #1		// |f0|=1
	mov	x16, #0		// |g0|=0
	mov	x17, #0		// |f1|=0
	mov	x19, #1		// |g1|=1

.Loop_62:
	sbfx	x28, x3, #0, #1	// if |a_| is odd, then we'll be subtracting
	sub	x2, x2, #1
	subs	x24, x9, x3	// |b_|-|a_|
	and	x22, x9, x28
	sbc	x25, x14, x8
	and	x23, x14, x28
	subs	x26, x3, x22	// |a_|-|b_| (or |a_|-0 if |a_| was even)
	mov	x22, x15
	sbcs	x27, x8, x23
	mov	x23, x16
	csel	x9, x9, x3, hs	// |b_| = |a_|
	csel	x14, x14, x8, hs
	csel	x3, x26, x24, hs	// borrow means |a_|<|b_|, replace with |b_|-|a_|
	csel	x8, x27, x25, hs
	csel	x15, x15, x17,       hs	// exchange |f0| and |f1|
	csel	x17, x17, x22,     hs
	csel	x16, x16, x19,       hs	// exchange |g0| and |g1|
	csel	x19, x19, x23,     hs
	extr	x3, x8, x3, #1
	lsr	x8, x8, #1
	and	x22, x17, x28
	and	x23, x19, x28
	add	x17, x17, x17		// |f1|<<=1
	add	x19, x19, x19		// |g1|<<=1
	sub	x15, x15, x22		// |f0|-=|f1| (or |f0-=0| if |a_| was even)
	sub	x16, x16, x23		// |g0|-=|g1| (or |g0-=0| ...)
	cbnz	x2, .Loop_62

	ret