#include "arm_asm.h" #include "arm_arch.h" #ifndef __KERNEL__ .hidden OPENSSL_armv8_rsa_neonized #endif .text .globl bn_mul_mont .type bn_mul_mont,%function .align 5 bn_mul_mont: AARCH64_SIGN_LINK_REGISTER .Lbn_mul_mont: tst x5,#3 b.ne .Lmul_mont cmp x5,#32 b.le .Lscalar_impl #ifndef __KERNEL__ #ifndef __AARCH64EB__ adrp x17,OPENSSL_armv8_rsa_neonized ldr w17,[x17,#:lo12:OPENSSL_armv8_rsa_neonized] cbnz w17, bn_mul8x_mont_neon #endif #endif .Lscalar_impl: tst x5,#7 b.eq __bn_sqr8x_mont tst x5,#3 b.eq __bn_mul4x_mont .Lmul_mont: stp x29,x30,[sp,#-64]! add x29,sp,#0 stp x19,x20,[sp,#16] stp x21,x22,[sp,#32] stp x23,x24,[sp,#48] ldr x9,[x2],#8 // bp[0] sub x22,sp,x5,lsl#3 ldp x7,x8,[x1],#16 // ap[0..1] lsl x5,x5,#3 ldr x4,[x4] // *n0 and x22,x22,#-16 // ABI says so ldp x13,x14,[x3],#16 // np[0..1] mul x6,x7,x9 // ap[0]*bp[0] sub x21,x5,#16 // j=num-2 umulh x7,x7,x9 mul x10,x8,x9 // ap[1]*bp[0] umulh x11,x8,x9 mul x15,x6,x4 // "tp[0]"*n0 mov sp,x22 // alloca // (*) mul x12,x13,x15 // np[0]*m1 umulh x13,x13,x15 mul x16,x14,x15 // np[1]*m1 // (*) adds x12,x12,x6 // discarded // (*) As for removal of first multiplication and addition // instructions. The outcome of first addition is // guaranteed to be zero, which leaves two computationally // significant outcomes: it either carries or not. Then // question is when does it carry? Is there alternative // way to deduce it? If you follow operations, you can // observe that condition for carry is quite simple: // x6 being non-zero. So that carry can be calculated // by adding -1 to x6. That's what next instruction does. subs xzr,x6,#1 // (*) umulh x17,x14,x15 adc x13,x13,xzr cbz x21,.L1st_skip .L1st: ldr x8,[x1],#8 adds x6,x10,x7 sub x21,x21,#8 // j-- adc x7,x11,xzr ldr x14,[x3],#8 adds x12,x16,x13 mul x10,x8,x9 // ap[j]*bp[0] adc x13,x17,xzr umulh x11,x8,x9 adds x12,x12,x6 mul x16,x14,x15 // np[j]*m1 adc x13,x13,xzr umulh x17,x14,x15 str x12,[x22],#8 // tp[j-1] cbnz x21,.L1st .L1st_skip: adds x6,x10,x7 sub x1,x1,x5 // rewind x1 adc x7,x11,xzr adds x12,x16,x13 sub x3,x3,x5 // rewind x3 adc x13,x17,xzr adds x12,x12,x6 sub x20,x5,#8 // i=num-1 adcs x13,x13,x7 adc x19,xzr,xzr // upmost overflow bit stp x12,x13,[x22] .Louter: ldr x9,[x2],#8 // bp[i] ldp x7,x8,[x1],#16 ldr x23,[sp] // tp[0] add x22,sp,#8 mul x6,x7,x9 // ap[0]*bp[i] sub x21,x5,#16 // j=num-2 umulh x7,x7,x9 ldp x13,x14,[x3],#16 mul x10,x8,x9 // ap[1]*bp[i] adds x6,x6,x23 umulh x11,x8,x9 adc x7,x7,xzr mul x15,x6,x4 sub x20,x20,#8 // i-- // (*) mul x12,x13,x15 // np[0]*m1 umulh x13,x13,x15 mul x16,x14,x15 // np[1]*m1 // (*) adds x12,x12,x6 subs xzr,x6,#1 // (*) umulh x17,x14,x15 cbz x21,.Linner_skip .Linner: ldr x8,[x1],#8 adc x13,x13,xzr ldr x23,[x22],#8 // tp[j] adds x6,x10,x7 sub x21,x21,#8 // j-- adc x7,x11,xzr adds x12,x16,x13 ldr x14,[x3],#8 adc x13,x17,xzr mul x10,x8,x9 // ap[j]*bp[i] adds x6,x6,x23 umulh x11,x8,x9 adc x7,x7,xzr mul x16,x14,x15 // np[j]*m1 adds x12,x12,x6 umulh x17,x14,x15 stur x12,[x22,#-16] // tp[j-1] cbnz x21,.Linner .Linner_skip: ldr x23,[x22],#8 // tp[j] adc x13,x13,xzr adds x6,x10,x7 sub x1,x1,x5 // rewind x1 adc x7,x11,xzr adds x12,x16,x13 sub x3,x3,x5 // rewind x3 adcs x13,x17,x19 adc x19,xzr,xzr adds x6,x6,x23 adc x7,x7,xzr adds x12,x12,x6 adcs x13,x13,x7 adc x19,x19,xzr // upmost overflow bit stp x12,x13,[x22,#-16] cbnz x20,.Louter // Final step. We see if result is larger than modulus, and // if it is, subtract the modulus. But comparison implies // subtraction. So we subtract modulus, see if it borrowed, // and conditionally copy original value. ldr x23,[sp] // tp[0] add x22,sp,#8 ldr x14,[x3],#8 // np[0] subs x21,x5,#8 // j=num-1 and clear borrow mov x1,x0 .Lsub: sbcs x8,x23,x14 // tp[j]-np[j] ldr x23,[x22],#8 sub x21,x21,#8 // j-- ldr x14,[x3],#8 str x8,[x1],#8 // rp[j]=tp[j]-np[j] cbnz x21,.Lsub sbcs x8,x23,x14 sbcs x19,x19,xzr // did it borrow? str x8,[x1],#8 // rp[num-1] ldr x23,[sp] // tp[0] add x22,sp,#8 ldr x8,[x0],#8 // rp[0] sub x5,x5,#8 // num-- nop .Lcond_copy: sub x5,x5,#8 // num-- csel x14,x23,x8,lo // did it borrow? ldr x23,[x22],#8 ldr x8,[x0],#8 stur xzr,[x22,#-16] // wipe tp stur x14,[x0,#-16] cbnz x5,.Lcond_copy csel x14,x23,x8,lo stur xzr,[x22,#-8] // wipe tp stur x14,[x0,#-8] ldp x19,x20,[x29,#16] mov sp,x29 ldp x21,x22,[x29,#32] mov x0,#1 ldp x23,x24,[x29,#48] ldr x29,[sp],#64 AARCH64_VALIDATE_LINK_REGISTER ret .size bn_mul_mont,.-bn_mul_mont .type bn_mul8x_mont_neon,%function .align 5 bn_mul8x_mont_neon: // Not adding AARCH64_SIGN_LINK_REGISTER here because bn_mul8x_mont_neon is jumped to // only from bn_mul_mont which has already signed the return address. stp x29,x30,[sp,#-80]! mov x16,sp stp d8,d9,[sp,#16] stp d10,d11,[sp,#32] stp d12,d13,[sp,#48] stp d14,d15,[sp,#64] lsl x5,x5,#1 eor v14.16b,v14.16b,v14.16b .align 4 .LNEON_8n: eor v6.16b,v6.16b,v6.16b sub x7,sp,#128 eor v7.16b,v7.16b,v7.16b sub x7,x7,x5,lsl#4 eor v8.16b,v8.16b,v8.16b and x7,x7,#-64 eor v9.16b,v9.16b,v9.16b mov sp,x7 // alloca eor v10.16b,v10.16b,v10.16b add x7,x7,#256 eor v11.16b,v11.16b,v11.16b sub x8,x5,#8 eor v12.16b,v12.16b,v12.16b eor v13.16b,v13.16b,v13.16b .LNEON_8n_init: st1 {v6.2d,v7.2d},[x7],#32 subs x8,x8,#8 st1 {v8.2d,v9.2d},[x7],#32 st1 {v10.2d,v11.2d},[x7],#32 st1 {v12.2d,v13.2d},[x7],#32 bne .LNEON_8n_init add x6,sp,#256 ld1 {v0.4s,v1.4s},[x1],#32 add x10,sp,#8 ldr s30,[x4],#4 mov x9,x5 b .LNEON_8n_outer .align 4 .LNEON_8n_outer: ldr s28,[x2],#4 // *b++ uxtl v28.4s,v28.4h add x7,sp,#128 ld1 {v2.4s,v3.4s},[x3],#32 umlal v6.2d,v28.2s,v0.s[0] umlal v7.2d,v28.2s,v0.s[1] umlal v8.2d,v28.2s,v0.s[2] shl v29.2d,v6.2d,#16 ext v29.16b,v29.16b,v29.16b,#8 umlal v9.2d,v28.2s,v0.s[3] add v29.2d,v29.2d,v6.2d umlal v10.2d,v28.2s,v1.s[0] mul v29.2s,v29.2s,v30.2s umlal v11.2d,v28.2s,v1.s[1] st1 {v28.2s},[sp] // put aside smashed b[8*i+0] umlal v12.2d,v28.2s,v1.s[2] uxtl v29.4s,v29.4h umlal v13.2d,v28.2s,v1.s[3] ldr s28,[x2],#4 // *b++ umlal v6.2d,v29.2s,v2.s[0] umlal v7.2d,v29.2s,v2.s[1] uxtl v28.4s,v28.4h umlal v8.2d,v29.2s,v2.s[2] ushr v15.2d,v6.2d,#16 umlal v9.2d,v29.2s,v2.s[3] umlal v10.2d,v29.2s,v3.s[0] ext v6.16b,v6.16b,v6.16b,#8 add v6.2d,v6.2d,v15.2d umlal v11.2d,v29.2s,v3.s[1] ushr v6.2d,v6.2d,#16 umlal v12.2d,v29.2s,v3.s[2] umlal v13.2d,v29.2s,v3.s[3] add v16.2d,v7.2d,v6.2d ins v7.d[0],v16.d[0] st1 {v29.2s},[x10],#8 // put aside smashed m[8*i+0] umlal v7.2d,v28.2s,v0.s[0] ld1 {v6.2d},[x6],#16 umlal v8.2d,v28.2s,v0.s[1] umlal v9.2d,v28.2s,v0.s[2] shl v29.2d,v7.2d,#16 ext v29.16b,v29.16b,v29.16b,#8 umlal v10.2d,v28.2s,v0.s[3] add v29.2d,v29.2d,v7.2d umlal v11.2d,v28.2s,v1.s[0] mul v29.2s,v29.2s,v30.2s umlal v12.2d,v28.2s,v1.s[1] st1 {v28.2s},[x10],#8 // put aside smashed b[8*i+1] umlal v13.2d,v28.2s,v1.s[2] uxtl v29.4s,v29.4h umlal v6.2d,v28.2s,v1.s[3] ldr s28,[x2],#4 // *b++ umlal v7.2d,v29.2s,v2.s[0] umlal v8.2d,v29.2s,v2.s[1] uxtl v28.4s,v28.4h umlal v9.2d,v29.2s,v2.s[2] ushr v15.2d,v7.2d,#16 umlal v10.2d,v29.2s,v2.s[3] umlal v11.2d,v29.2s,v3.s[0] ext v7.16b,v7.16b,v7.16b,#8 add v7.2d,v7.2d,v15.2d umlal v12.2d,v29.2s,v3.s[1] ushr v7.2d,v7.2d,#16 umlal v13.2d,v29.2s,v3.s[2] umlal v6.2d,v29.2s,v3.s[3] add v16.2d,v8.2d,v7.2d ins v8.d[0],v16.d[0] st1 {v29.2s},[x10],#8 // put aside smashed m[8*i+1] umlal v8.2d,v28.2s,v0.s[0] ld1 {v7.2d},[x6],#16 umlal v9.2d,v28.2s,v0.s[1] umlal v10.2d,v28.2s,v0.s[2] shl v29.2d,v8.2d,#16 ext v29.16b,v29.16b,v29.16b,#8 umlal v11.2d,v28.2s,v0.s[3] add v29.2d,v29.2d,v8.2d umlal v12.2d,v28.2s,v1.s[0] mul v29.2s,v29.2s,v30.2s umlal v13.2d,v28.2s,v1.s[1] st1 {v28.2s},[x10],#8 // put aside smashed b[8*i+2] umlal v6.2d,v28.2s,v1.s[2] uxtl v29.4s,v29.4h umlal v7.2d,v28.2s,v1.s[3] ldr s28,[x2],#4 // *b++ umlal v8.2d,v29.2s,v2.s[0] umlal v9.2d,v29.2s,v2.s[1] uxtl v28.4s,v28.4h umlal v10.2d,v29.2s,v2.s[2] ushr v15.2d,v8.2d,#16 umlal v11.2d,v29.2s,v2.s[3] umlal v12.2d,v29.2s,v3.s[0] ext v8.16b,v8.16b,v8.16b,#8 add v8.2d,v8.2d,v15.2d umlal v13.2d,v29.2s,v3.s[1] ushr v8.2d,v8.2d,#16 umlal v6.2d,v29.2s,v3.s[2] umlal v7.2d,v29.2s,v3.s[3] add v16.2d,v9.2d,v8.2d ins v9.d[0],v16.d[0] st1 {v29.2s},[x10],#8 // put aside smashed m[8*i+2] umlal v9.2d,v28.2s,v0.s[0] ld1 {v8.2d},[x6],#16 umlal v10.2d,v28.2s,v0.s[1] umlal v11.2d,v28.2s,v0.s[2] shl v29.2d,v9.2d,#16 ext v29.16b,v29.16b,v29.16b,#8 umlal v12.2d,v28.2s,v0.s[3] add v29.2d,v29.2d,v9.2d umlal v13.2d,v28.2s,v1.s[0] mul v29.2s,v29.2s,v30.2s umlal v6.2d,v28.2s,v1.s[1] st1 {v28.2s},[x10],#8 // put aside smashed b[8*i+3] umlal v7.2d,v28.2s,v1.s[2] uxtl v29.4s,v29.4h umlal v8.2d,v28.2s,v1.s[3] ldr s28,[x2],#4 // *b++ umlal v9.2d,v29.2s,v2.s[0] umlal v10.2d,v29.2s,v2.s[1] uxtl v28.4s,v28.4h umlal v11.2d,v29.2s,v2.s[2] ushr v15.2d,v9.2d,#16 umlal v12.2d,v29.2s,v2.s[3] umlal v13.2d,v29.2s,v3.s[0] ext v9.16b,v9.16b,v9.16b,#8 add v9.2d,v9.2d,v15.2d umlal v6.2d,v29.2s,v3.s[1] ushr v9.2d,v9.2d,#16 umlal v7.2d,v29.2s,v3.s[2] umlal v8.2d,v29.2s,v3.s[3] add v16.2d,v10.2d,v9.2d ins v10.d[0],v16.d[0] st1 {v29.2s},[x10],#8 // put aside smashed m[8*i+3] umlal v10.2d,v28.2s,v0.s[0] ld1 {v9.2d},[x6],#16 umlal v11.2d,v28.2s,v0.s[1] umlal v12.2d,v28.2s,v0.s[2] shl v29.2d,v10.2d,#16 ext v29.16b,v29.16b,v29.16b,#8 umlal v13.2d,v28.2s,v0.s[3] add v29.2d,v29.2d,v10.2d umlal v6.2d,v28.2s,v1.s[0] mul v29.2s,v29.2s,v30.2s umlal v7.2d,v28.2s,v1.s[1] st1 {v28.2s},[x10],#8 // put aside smashed b[8*i+4] umlal v8.2d,v28.2s,v1.s[2] uxtl v29.4s,v29.4h umlal v9.2d,v28.2s,v1.s[3] ldr s28,[x2],#4 // *b++ umlal v10.2d,v29.2s,v2.s[0] umlal v11.2d,v29.2s,v2.s[1] uxtl v28.4s,v28.4h umlal v12.2d,v29.2s,v2.s[2] ushr v15.2d,v10.2d,#16 umlal v13.2d,v29.2s,v2.s[3] umlal v6.2d,v29.2s,v3.s[0] ext v10.16b,v10.16b,v10.16b,#8 add v10.2d,v10.2d,v15.2d umlal v7.2d,v29.2s,v3.s[1] ushr v10.2d,v10.2d,#16 umlal v8.2d,v29.2s,v3.s[2] umlal v9.2d,v29.2s,v3.s[3] add v16.2d,v11.2d,v10.2d ins v11.d[0],v16.d[0] st1 {v29.2s},[x10],#8 // put aside smashed m[8*i+4] umlal v11.2d,v28.2s,v0.s[0] ld1 {v10.2d},[x6],#16 umlal v12.2d,v28.2s,v0.s[1] umlal v13.2d,v28.2s,v0.s[2] shl v29.2d,v11.2d,#16 ext v29.16b,v29.16b,v29.16b,#8 umlal v6.2d,v28.2s,v0.s[3] add v29.2d,v29.2d,v11.2d umlal v7.2d,v28.2s,v1.s[0] mul v29.2s,v29.2s,v30.2s umlal v8.2d,v28.2s,v1.s[1] st1 {v28.2s},[x10],#8 // put aside smashed b[8*i+5] umlal v9.2d,v28.2s,v1.s[2] uxtl v29.4s,v29.4h umlal v10.2d,v28.2s,v1.s[3] ldr s28,[x2],#4 // *b++ umlal v11.2d,v29.2s,v2.s[0] umlal v12.2d,v29.2s,v2.s[1] uxtl v28.4s,v28.4h umlal v13.2d,v29.2s,v2.s[2] ushr v15.2d,v11.2d,#16 umlal v6.2d,v29.2s,v2.s[3] umlal v7.2d,v29.2s,v3.s[0] ext v11.16b,v11.16b,v11.16b,#8 add v11.2d,v11.2d,v15.2d umlal v8.2d,v29.2s,v3.s[1] ushr v11.2d,v11.2d,#16 umlal v9.2d,v29.2s,v3.s[2] umlal v10.2d,v29.2s,v3.s[3] add v16.2d,v12.2d,v11.2d ins v12.d[0],v16.d[0] st1 {v29.2s},[x10],#8 // put aside smashed m[8*i+5] umlal v12.2d,v28.2s,v0.s[0] ld1 {v11.2d},[x6],#16 umlal v13.2d,v28.2s,v0.s[1] umlal v6.2d,v28.2s,v0.s[2] shl v29.2d,v12.2d,#16 ext v29.16b,v29.16b,v29.16b,#8 umlal v7.2d,v28.2s,v0.s[3] add v29.2d,v29.2d,v12.2d umlal v8.2d,v28.2s,v1.s[0] mul v29.2s,v29.2s,v30.2s umlal v9.2d,v28.2s,v1.s[1] st1 {v28.2s},[x10],#8 // put aside smashed b[8*i+6] umlal v10.2d,v28.2s,v1.s[2] uxtl v29.4s,v29.4h umlal v11.2d,v28.2s,v1.s[3] ldr s28,[x2],#4 // *b++ umlal v12.2d,v29.2s,v2.s[0] umlal v13.2d,v29.2s,v2.s[1] uxtl v28.4s,v28.4h umlal v6.2d,v29.2s,v2.s[2] ushr v15.2d,v12.2d,#16 umlal v7.2d,v29.2s,v2.s[3] umlal v8.2d,v29.2s,v3.s[0] ext v12.16b,v12.16b,v12.16b,#8 add v12.2d,v12.2d,v15.2d umlal v9.2d,v29.2s,v3.s[1] ushr v12.2d,v12.2d,#16 umlal v10.2d,v29.2s,v3.s[2] umlal v11.2d,v29.2s,v3.s[3] add v16.2d,v13.2d,v12.2d ins v13.d[0],v16.d[0] st1 {v29.2s},[x10],#8 // put aside smashed m[8*i+6] umlal v13.2d,v28.2s,v0.s[0] ld1 {v12.2d},[x6],#16 umlal v6.2d,v28.2s,v0.s[1] umlal v7.2d,v28.2s,v0.s[2] shl v29.2d,v13.2d,#16 ext v29.16b,v29.16b,v29.16b,#8 umlal v8.2d,v28.2s,v0.s[3] add v29.2d,v29.2d,v13.2d umlal v9.2d,v28.2s,v1.s[0] mul v29.2s,v29.2s,v30.2s umlal v10.2d,v28.2s,v1.s[1] st1 {v28.2s},[x10],#8 // put aside smashed b[8*i+7] umlal v11.2d,v28.2s,v1.s[2] uxtl v29.4s,v29.4h umlal v12.2d,v28.2s,v1.s[3] ld1 {v28.2s},[sp] // pull smashed b[8*i+0] umlal v13.2d,v29.2s,v2.s[0] ld1 {v0.4s,v1.4s},[x1],#32 umlal v6.2d,v29.2s,v2.s[1] umlal v7.2d,v29.2s,v2.s[2] mov v5.16b,v13.16b ushr v5.2d,v5.2d,#16 ext v13.16b,v13.16b,v13.16b,#8 umlal v8.2d,v29.2s,v2.s[3] umlal v9.2d,v29.2s,v3.s[0] add v13.2d,v13.2d,v5.2d umlal v10.2d,v29.2s,v3.s[1] ushr v13.2d,v13.2d,#16 eor v15.16b,v15.16b,v15.16b ins v13.d[1],v15.d[0] umlal v11.2d,v29.2s,v3.s[2] umlal v12.2d,v29.2s,v3.s[3] add v6.2d,v6.2d,v13.2d st1 {v29.2s},[x10],#8 // put aside smashed m[8*i+7] add x10,sp,#8 // rewind sub x8,x5,#8 b .LNEON_8n_inner .align 4 .LNEON_8n_inner: subs x8,x8,#8 umlal v6.2d,v28.2s,v0.s[0] ld1 {v13.2d},[x6] umlal v7.2d,v28.2s,v0.s[1] ld1 {v29.2s},[x10],#8 // pull smashed m[8*i+0] umlal v8.2d,v28.2s,v0.s[2] ld1 {v2.4s,v3.4s},[x3],#32 umlal v9.2d,v28.2s,v0.s[3] b.eq .LInner_jump add x6,x6,#16 // don't advance in last iteration .LInner_jump: umlal v10.2d,v28.2s,v1.s[0] umlal v11.2d,v28.2s,v1.s[1] umlal v12.2d,v28.2s,v1.s[2] umlal v13.2d,v28.2s,v1.s[3] ld1 {v28.2s},[x10],#8 // pull smashed b[8*i+1] umlal v6.2d,v29.2s,v2.s[0] umlal v7.2d,v29.2s,v2.s[1] umlal v8.2d,v29.2s,v2.s[2] umlal v9.2d,v29.2s,v2.s[3] umlal v10.2d,v29.2s,v3.s[0] umlal v11.2d,v29.2s,v3.s[1] umlal v12.2d,v29.2s,v3.s[2] umlal v13.2d,v29.2s,v3.s[3] st1 {v6.2d},[x7],#16 umlal v7.2d,v28.2s,v0.s[0] ld1 {v6.2d},[x6] umlal v8.2d,v28.2s,v0.s[1] ld1 {v29.2s},[x10],#8 // pull smashed m[8*i+1] umlal v9.2d,v28.2s,v0.s[2] b.eq .LInner_jump1 add x6,x6,#16 // don't advance in last iteration .LInner_jump1: umlal v10.2d,v28.2s,v0.s[3] umlal v11.2d,v28.2s,v1.s[0] umlal v12.2d,v28.2s,v1.s[1] umlal v13.2d,v28.2s,v1.s[2] umlal v6.2d,v28.2s,v1.s[3] ld1 {v28.2s},[x10],#8 // pull smashed b[8*i+2] umlal v7.2d,v29.2s,v2.s[0] umlal v8.2d,v29.2s,v2.s[1] umlal v9.2d,v29.2s,v2.s[2] umlal v10.2d,v29.2s,v2.s[3] umlal v11.2d,v29.2s,v3.s[0] umlal v12.2d,v29.2s,v3.s[1] umlal v13.2d,v29.2s,v3.s[2] umlal v6.2d,v29.2s,v3.s[3] st1 {v7.2d},[x7],#16 umlal v8.2d,v28.2s,v0.s[0] ld1 {v7.2d},[x6] umlal v9.2d,v28.2s,v0.s[1] ld1 {v29.2s},[x10],#8 // pull smashed m[8*i+2] umlal v10.2d,v28.2s,v0.s[2] b.eq .LInner_jump2 add x6,x6,#16 // don't advance in last iteration .LInner_jump2: umlal v11.2d,v28.2s,v0.s[3] umlal v12.2d,v28.2s,v1.s[0] umlal v13.2d,v28.2s,v1.s[1] umlal v6.2d,v28.2s,v1.s[2] umlal v7.2d,v28.2s,v1.s[3] ld1 {v28.2s},[x10],#8 // pull smashed b[8*i+3] umlal v8.2d,v29.2s,v2.s[0] umlal v9.2d,v29.2s,v2.s[1] umlal v10.2d,v29.2s,v2.s[2] umlal v11.2d,v29.2s,v2.s[3] umlal v12.2d,v29.2s,v3.s[0] umlal v13.2d,v29.2s,v3.s[1] umlal v6.2d,v29.2s,v3.s[2] umlal v7.2d,v29.2s,v3.s[3] st1 {v8.2d},[x7],#16 umlal v9.2d,v28.2s,v0.s[0] ld1 {v8.2d},[x6] umlal v10.2d,v28.2s,v0.s[1] ld1 {v29.2s},[x10],#8 // pull smashed m[8*i+3] umlal v11.2d,v28.2s,v0.s[2] b.eq .LInner_jump3 add x6,x6,#16 // don't advance in last iteration .LInner_jump3: umlal v12.2d,v28.2s,v0.s[3] umlal v13.2d,v28.2s,v1.s[0] umlal v6.2d,v28.2s,v1.s[1] umlal v7.2d,v28.2s,v1.s[2] umlal v8.2d,v28.2s,v1.s[3] ld1 {v28.2s},[x10],#8 // pull smashed b[8*i+4] umlal v9.2d,v29.2s,v2.s[0] umlal v10.2d,v29.2s,v2.s[1] umlal v11.2d,v29.2s,v2.s[2] umlal v12.2d,v29.2s,v2.s[3] umlal v13.2d,v29.2s,v3.s[0] umlal v6.2d,v29.2s,v3.s[1] umlal v7.2d,v29.2s,v3.s[2] umlal v8.2d,v29.2s,v3.s[3] st1 {v9.2d},[x7],#16 umlal v10.2d,v28.2s,v0.s[0] ld1 {v9.2d},[x6] umlal v11.2d,v28.2s,v0.s[1] ld1 {v29.2s},[x10],#8 // pull smashed m[8*i+4] umlal v12.2d,v28.2s,v0.s[2] b.eq .LInner_jump4 add x6,x6,#16 // don't advance in last iteration .LInner_jump4: umlal v13.2d,v28.2s,v0.s[3] umlal v6.2d,v28.2s,v1.s[0] umlal v7.2d,v28.2s,v1.s[1] umlal v8.2d,v28.2s,v1.s[2] umlal v9.2d,v28.2s,v1.s[3] ld1 {v28.2s},[x10],#8 // pull smashed b[8*i+5] umlal v10.2d,v29.2s,v2.s[0] umlal v11.2d,v29.2s,v2.s[1] umlal v12.2d,v29.2s,v2.s[2] umlal v13.2d,v29.2s,v2.s[3] umlal v6.2d,v29.2s,v3.s[0] umlal v7.2d,v29.2s,v3.s[1] umlal v8.2d,v29.2s,v3.s[2] umlal v9.2d,v29.2s,v3.s[3] st1 {v10.2d},[x7],#16 umlal v11.2d,v28.2s,v0.s[0] ld1 {v10.2d},[x6] umlal v12.2d,v28.2s,v0.s[1] ld1 {v29.2s},[x10],#8 // pull smashed m[8*i+5] umlal v13.2d,v28.2s,v0.s[2] b.eq .LInner_jump5 add x6,x6,#16 // don't advance in last iteration .LInner_jump5: umlal v6.2d,v28.2s,v0.s[3] umlal v7.2d,v28.2s,v1.s[0] umlal v8.2d,v28.2s,v1.s[1] umlal v9.2d,v28.2s,v1.s[2] umlal v10.2d,v28.2s,v1.s[3] ld1 {v28.2s},[x10],#8 // pull smashed b[8*i+6] umlal v11.2d,v29.2s,v2.s[0] umlal v12.2d,v29.2s,v2.s[1] umlal v13.2d,v29.2s,v2.s[2] umlal v6.2d,v29.2s,v2.s[3] umlal v7.2d,v29.2s,v3.s[0] umlal v8.2d,v29.2s,v3.s[1] umlal v9.2d,v29.2s,v3.s[2] umlal v10.2d,v29.2s,v3.s[3] st1 {v11.2d},[x7],#16 umlal v12.2d,v28.2s,v0.s[0] ld1 {v11.2d},[x6] umlal v13.2d,v28.2s,v0.s[1] ld1 {v29.2s},[x10],#8 // pull smashed m[8*i+6] umlal v6.2d,v28.2s,v0.s[2] b.eq .LInner_jump6 add x6,x6,#16 // don't advance in last iteration .LInner_jump6: umlal v7.2d,v28.2s,v0.s[3] umlal v8.2d,v28.2s,v1.s[0] umlal v9.2d,v28.2s,v1.s[1] umlal v10.2d,v28.2s,v1.s[2] umlal v11.2d,v28.2s,v1.s[3] ld1 {v28.2s},[x10],#8 // pull smashed b[8*i+7] umlal v12.2d,v29.2s,v2.s[0] umlal v13.2d,v29.2s,v2.s[1] umlal v6.2d,v29.2s,v2.s[2] umlal v7.2d,v29.2s,v2.s[3] umlal v8.2d,v29.2s,v3.s[0] umlal v9.2d,v29.2s,v3.s[1] umlal v10.2d,v29.2s,v3.s[2] umlal v11.2d,v29.2s,v3.s[3] st1 {v12.2d},[x7],#16 umlal v13.2d,v28.2s,v0.s[0] ld1 {v12.2d},[x6] umlal v6.2d,v28.2s,v0.s[1] ld1 {v29.2s},[x10],#8 // pull smashed m[8*i+7] umlal v7.2d,v28.2s,v0.s[2] b.eq .LInner_jump7 add x6,x6,#16 // don't advance in last iteration .LInner_jump7: umlal v8.2d,v28.2s,v0.s[3] umlal v9.2d,v28.2s,v1.s[0] umlal v10.2d,v28.2s,v1.s[1] umlal v11.2d,v28.2s,v1.s[2] umlal v12.2d,v28.2s,v1.s[3] b.ne .LInner_after_rewind8 sub x1,x1,x5,lsl#2 // rewind .LInner_after_rewind8: umlal v13.2d,v29.2s,v2.s[0] ld1 {v28.2s},[sp] // pull smashed b[8*i+0] umlal v6.2d,v29.2s,v2.s[1] ld1 {v0.4s,v1.4s},[x1],#32 umlal v7.2d,v29.2s,v2.s[2] add x10,sp,#8 // rewind umlal v8.2d,v29.2s,v2.s[3] umlal v9.2d,v29.2s,v3.s[0] umlal v10.2d,v29.2s,v3.s[1] umlal v11.2d,v29.2s,v3.s[2] st1 {v13.2d},[x7],#16 umlal v12.2d,v29.2s,v3.s[3] bne .LNEON_8n_inner add x6,sp,#128 st1 {v6.2d,v7.2d},[x7],#32 eor v2.16b,v2.16b,v2.16b // v2 st1 {v8.2d,v9.2d},[x7],#32 eor v3.16b,v3.16b,v3.16b // v3 st1 {v10.2d,v11.2d},[x7],#32 st1 {v12.2d},[x7] subs x9,x9,#8 ld1 {v6.2d,v7.2d},[x6],#32 ld1 {v8.2d,v9.2d},[x6],#32 ld1 {v10.2d,v11.2d},[x6],#32 ld1 {v12.2d,v13.2d},[x6],#32 b.eq .LInner_8n_jump_2steps sub x3,x3,x5,lsl#2 // rewind b .LNEON_8n_outer .LInner_8n_jump_2steps: add x7,sp,#128 st1 {v2.2d,v3.2d}, [sp],#32 // start wiping stack frame mov v5.16b,v6.16b ushr v15.2d,v6.2d,#16 ext v6.16b,v6.16b,v6.16b,#8 st1 {v2.2d,v3.2d}, [sp],#32 add v6.2d,v6.2d,v15.2d st1 {v2.2d,v3.2d}, [sp],#32 ushr v15.2d,v6.2d,#16 st1 {v2.2d,v3.2d}, [sp],#32 zip1 v6.4h,v5.4h,v6.4h ins v15.d[1],v14.d[0] mov x8,x5 b .LNEON_tail_entry .align 4 .LNEON_tail: add v6.2d,v6.2d,v15.2d mov v5.16b,v6.16b ushr v15.2d,v6.2d,#16 ext v6.16b,v6.16b,v6.16b,#8 ld1 {v8.2d,v9.2d}, [x6],#32 add v6.2d,v6.2d,v15.2d ld1 {v10.2d,v11.2d}, [x6],#32 ushr v15.2d,v6.2d,#16 ld1 {v12.2d,v13.2d}, [x6],#32 zip1 v6.4h,v5.4h,v6.4h ins v15.d[1],v14.d[0] .LNEON_tail_entry: add v7.2d,v7.2d,v15.2d st1 {v6.s}[0], [x7],#4 ushr v15.2d,v7.2d,#16 mov v5.16b,v7.16b ext v7.16b,v7.16b,v7.16b,#8 add v7.2d,v7.2d,v15.2d ushr v15.2d,v7.2d,#16 zip1 v7.4h,v5.4h,v7.4h ins v15.d[1],v14.d[0] add v8.2d,v8.2d,v15.2d st1 {v7.s}[0], [x7],#4 ushr v15.2d,v8.2d,#16 mov v5.16b,v8.16b ext v8.16b,v8.16b,v8.16b,#8 add v8.2d,v8.2d,v15.2d ushr v15.2d,v8.2d,#16 zip1 v8.4h,v5.4h,v8.4h ins v15.d[1],v14.d[0] add v9.2d,v9.2d,v15.2d st1 {v8.s}[0], [x7],#4 ushr v15.2d,v9.2d,#16 mov v5.16b,v9.16b ext v9.16b,v9.16b,v9.16b,#8 add v9.2d,v9.2d,v15.2d ushr v15.2d,v9.2d,#16 zip1 v9.4h,v5.4h,v9.4h ins v15.d[1],v14.d[0] add v10.2d,v10.2d,v15.2d st1 {v9.s}[0], [x7],#4 ushr v15.2d,v10.2d,#16 mov v5.16b,v10.16b ext v10.16b,v10.16b,v10.16b,#8 add v10.2d,v10.2d,v15.2d ushr v15.2d,v10.2d,#16 zip1 v10.4h,v5.4h,v10.4h ins v15.d[1],v14.d[0] add v11.2d,v11.2d,v15.2d st1 {v10.s}[0], [x7],#4 ushr v15.2d,v11.2d,#16 mov v5.16b,v11.16b ext v11.16b,v11.16b,v11.16b,#8 add v11.2d,v11.2d,v15.2d ushr v15.2d,v11.2d,#16 zip1 v11.4h,v5.4h,v11.4h ins v15.d[1],v14.d[0] add v12.2d,v12.2d,v15.2d st1 {v11.s}[0], [x7],#4 ushr v15.2d,v12.2d,#16 mov v5.16b,v12.16b ext v12.16b,v12.16b,v12.16b,#8 add v12.2d,v12.2d,v15.2d ushr v15.2d,v12.2d,#16 zip1 v12.4h,v5.4h,v12.4h ins v15.d[1],v14.d[0] add v13.2d,v13.2d,v15.2d st1 {v12.s}[0], [x7],#4 ushr v15.2d,v13.2d,#16 mov v5.16b,v13.16b ext v13.16b,v13.16b,v13.16b,#8 add v13.2d,v13.2d,v15.2d ushr v15.2d,v13.2d,#16 zip1 v13.4h,v5.4h,v13.4h ins v15.d[1],v14.d[0] ld1 {v6.2d,v7.2d}, [x6],#32 subs x8,x8,#8 st1 {v13.s}[0], [x7],#4 bne .LNEON_tail st1 {v15.s}[0], [x7],#4 // top-most bit sub x3,x3,x5,lsl#2 // rewind x3 subs x1,sp,#0 // clear carry flag add x2,sp,x5,lsl#2 .LNEON_sub: ldp w4,w5,[x1],#8 ldp w6,w7,[x1],#8 ldp w8,w9,[x3],#8 ldp w10,w11,[x3],#8 sbcs w8,w4,w8 sbcs w9,w5,w9 sbcs w10,w6,w10 sbcs w11,w7,w11 sub x17,x2,x1 stp w8,w9,[x0],#8 stp w10,w11,[x0],#8 cbnz x17,.LNEON_sub ldr w10, [x1] // load top-most bit mov x11,sp eor v0.16b,v0.16b,v0.16b sub x11,x2,x11 // this is num*4 eor v1.16b,v1.16b,v1.16b mov x1,sp sub x0,x0,x11 // rewind x0 mov x3,x2 // second 3/4th of frame sbcs w10,w10,wzr // result is carry flag .LNEON_copy_n_zap: ldp w4,w5,[x1],#8 ldp w6,w7,[x1],#8 ldp w8,w9,[x0],#8 ldp w10,w11,[x0] sub x0,x0,#8 b.cs .LCopy_1 mov w8,w4 mov w9,w5 mov w10,w6 mov w11,w7 .LCopy_1: st1 {v0.2d,v1.2d}, [x3],#32 // wipe st1 {v0.2d,v1.2d}, [x3],#32 // wipe ldp w4,w5,[x1],#8 ldp w6,w7,[x1],#8 stp w8,w9,[x0],#8 stp w10,w11,[x0],#8 sub x1,x1,#32 ldp w8,w9,[x0],#8 ldp w10,w11,[x0] sub x0,x0,#8 b.cs .LCopy_2 mov w8, w4 mov w9, w5 mov w10, w6 mov w11, w7 .LCopy_2: st1 {v0.2d,v1.2d}, [x1],#32 // wipe st1 {v0.2d,v1.2d}, [x3],#32 // wipe sub x17,x2,x1 // preserves carry stp w8,w9,[x0],#8 stp w10,w11,[x0],#8 cbnz x17,.LNEON_copy_n_zap mov sp,x16 ldp d14,d15,[sp,#64] ldp d12,d13,[sp,#48] ldp d10,d11,[sp,#32] ldp d8,d9,[sp,#16] ldr x29,[sp],#80 AARCH64_VALIDATE_LINK_REGISTER ret // RET .size bn_mul8x_mont_neon,.-bn_mul8x_mont_neon .type __bn_sqr8x_mont,%function .align 5 __bn_sqr8x_mont: cmp x1,x2 b.ne __bn_mul4x_mont .Lsqr8x_mont: // Not adding AARCH64_SIGN_LINK_REGISTER here because __bn_sqr8x_mont is jumped to // only from bn_mul_mont which has already signed the return address. stp x29,x30,[sp,#-128]! add x29,sp,#0 stp x19,x20,[sp,#16] stp x21,x22,[sp,#32] stp x23,x24,[sp,#48] stp x25,x26,[sp,#64] stp x27,x28,[sp,#80] stp x0,x3,[sp,#96] // offload rp and np ldp x6,x7,[x1,#8*0] ldp x8,x9,[x1,#8*2] ldp x10,x11,[x1,#8*4] ldp x12,x13,[x1,#8*6] sub x2,sp,x5,lsl#4 lsl x5,x5,#3 ldr x4,[x4] // *n0 mov sp,x2 // alloca sub x27,x5,#8*8 b .Lsqr8x_zero_start .Lsqr8x_zero: sub x27,x27,#8*8 stp xzr,xzr,[x2,#8*0] stp xzr,xzr,[x2,#8*2] stp xzr,xzr,[x2,#8*4] stp xzr,xzr,[x2,#8*6] .Lsqr8x_zero_start: stp xzr,xzr,[x2,#8*8] stp xzr,xzr,[x2,#8*10] stp xzr,xzr,[x2,#8*12] stp xzr,xzr,[x2,#8*14] add x2,x2,#8*16 cbnz x27,.Lsqr8x_zero add x3,x1,x5 add x1,x1,#8*8 mov x19,xzr mov x20,xzr mov x21,xzr mov x22,xzr mov x23,xzr mov x24,xzr mov x25,xzr mov x26,xzr mov x2,sp str x4,[x29,#112] // offload n0 // Multiply everything but a[i]*a[i] .align 4 .Lsqr8x_outer_loop: // a[1]a[0] (i) // a[2]a[0] // a[3]a[0] // a[4]a[0] // a[5]a[0] // a[6]a[0] // a[7]a[0] // a[2]a[1] (ii) // a[3]a[1] // a[4]a[1] // a[5]a[1] // a[6]a[1] // a[7]a[1] // a[3]a[2] (iii) // a[4]a[2] // a[5]a[2] // a[6]a[2] // a[7]a[2] // a[4]a[3] (iv) // a[5]a[3] // a[6]a[3] // a[7]a[3] // a[5]a[4] (v) // a[6]a[4] // a[7]a[4] // a[6]a[5] (vi) // a[7]a[5] // a[7]a[6] (vii) mul x14,x7,x6 // lo(a[1..7]*a[0]) (i) mul x15,x8,x6 mul x16,x9,x6 mul x17,x10,x6 adds x20,x20,x14 // t[1]+lo(a[1]*a[0]) mul x14,x11,x6 adcs x21,x21,x15 mul x15,x12,x6 adcs x22,x22,x16 mul x16,x13,x6 adcs x23,x23,x17 umulh x17,x7,x6 // hi(a[1..7]*a[0]) adcs x24,x24,x14 umulh x14,x8,x6 adcs x25,x25,x15 umulh x15,x9,x6 adcs x26,x26,x16 umulh x16,x10,x6 stp x19,x20,[x2],#8*2 // t[0..1] adc x19,xzr,xzr // t[8] adds x21,x21,x17 // t[2]+lo(a[1]*a[0]) umulh x17,x11,x6 adcs x22,x22,x14 umulh x14,x12,x6 adcs x23,x23,x15 umulh x15,x13,x6 adcs x24,x24,x16 mul x16,x8,x7 // lo(a[2..7]*a[1]) (ii) adcs x25,x25,x17 mul x17,x9,x7 adcs x26,x26,x14 mul x14,x10,x7 adc x19,x19,x15 mul x15,x11,x7 adds x22,x22,x16 mul x16,x12,x7 adcs x23,x23,x17 mul x17,x13,x7 adcs x24,x24,x14 umulh x14,x8,x7 // hi(a[2..7]*a[1]) adcs x25,x25,x15 umulh x15,x9,x7 adcs x26,x26,x16 umulh x16,x10,x7 adcs x19,x19,x17 umulh x17,x11,x7 stp x21,x22,[x2],#8*2 // t[2..3] adc x20,xzr,xzr // t[9] adds x23,x23,x14 umulh x14,x12,x7 adcs x24,x24,x15 umulh x15,x13,x7 adcs x25,x25,x16 mul x16,x9,x8 // lo(a[3..7]*a[2]) (iii) adcs x26,x26,x17 mul x17,x10,x8 adcs x19,x19,x14 mul x14,x11,x8 adc x20,x20,x15 mul x15,x12,x8 adds x24,x24,x16 mul x16,x13,x8 adcs x25,x25,x17 umulh x17,x9,x8 // hi(a[3..7]*a[2]) adcs x26,x26,x14 umulh x14,x10,x8 adcs x19,x19,x15 umulh x15,x11,x8 adcs x20,x20,x16 umulh x16,x12,x8 stp x23,x24,[x2],#8*2 // t[4..5] adc x21,xzr,xzr // t[10] adds x25,x25,x17 umulh x17,x13,x8 adcs x26,x26,x14 mul x14,x10,x9 // lo(a[4..7]*a[3]) (iv) adcs x19,x19,x15 mul x15,x11,x9 adcs x20,x20,x16 mul x16,x12,x9 adc x21,x21,x17 mul x17,x13,x9 adds x26,x26,x14 umulh x14,x10,x9 // hi(a[4..7]*a[3]) adcs x19,x19,x15 umulh x15,x11,x9 adcs x20,x20,x16 umulh x16,x12,x9 adcs x21,x21,x17 umulh x17,x13,x9 stp x25,x26,[x2],#8*2 // t[6..7] adc x22,xzr,xzr // t[11] adds x19,x19,x14 mul x14,x11,x10 // lo(a[5..7]*a[4]) (v) adcs x20,x20,x15 mul x15,x12,x10 adcs x21,x21,x16 mul x16,x13,x10 adc x22,x22,x17 umulh x17,x11,x10 // hi(a[5..7]*a[4]) adds x20,x20,x14 umulh x14,x12,x10 adcs x21,x21,x15 umulh x15,x13,x10 adcs x22,x22,x16 mul x16,x12,x11 // lo(a[6..7]*a[5]) (vi) adc x23,xzr,xzr // t[12] adds x21,x21,x17 mul x17,x13,x11 adcs x22,x22,x14 umulh x14,x12,x11 // hi(a[6..7]*a[5]) adc x23,x23,x15 umulh x15,x13,x11 adds x22,x22,x16 mul x16,x13,x12 // lo(a[7]*a[6]) (vii) adcs x23,x23,x17 umulh x17,x13,x12 // hi(a[7]*a[6]) adc x24,xzr,xzr // t[13] adds x23,x23,x14 sub x27,x3,x1 // done yet? adc x24,x24,x15 adds x24,x24,x16 sub x14,x3,x5 // rewinded ap adc x25,xzr,xzr // t[14] add x25,x25,x17 cbz x27,.Lsqr8x_outer_break mov x4,x6 ldp x6,x7,[x2,#8*0] ldp x8,x9,[x2,#8*2] ldp x10,x11,[x2,#8*4] ldp x12,x13,[x2,#8*6] adds x19,x19,x6 adcs x20,x20,x7 ldp x6,x7,[x1,#8*0] adcs x21,x21,x8 adcs x22,x22,x9 ldp x8,x9,[x1,#8*2] adcs x23,x23,x10 adcs x24,x24,x11 ldp x10,x11,[x1,#8*4] adcs x25,x25,x12 mov x0,x1 adcs x26,xzr,x13 ldp x12,x13,[x1,#8*6] add x1,x1,#8*8 //adc x28,xzr,xzr // moved below mov x27,#-8*8 // a[8]a[0] // a[9]a[0] // a[a]a[0] // a[b]a[0] // a[c]a[0] // a[d]a[0] // a[e]a[0] // a[f]a[0] // a[8]a[1] // a[f]a[1]........................ // a[8]a[2] // a[f]a[2]........................ // a[8]a[3] // a[f]a[3]........................ // a[8]a[4] // a[f]a[4]........................ // a[8]a[5] // a[f]a[5]........................ // a[8]a[6] // a[f]a[6]........................ // a[8]a[7] // a[f]a[7]........................ .Lsqr8x_mul: mul x14,x6,x4 adc x28,xzr,xzr // carry bit, modulo-scheduled mul x15,x7,x4 add x27,x27,#8 mul x16,x8,x4 mul x17,x9,x4 adds x19,x19,x14 mul x14,x10,x4 adcs x20,x20,x15 mul x15,x11,x4 adcs x21,x21,x16 mul x16,x12,x4 adcs x22,x22,x17 mul x17,x13,x4 adcs x23,x23,x14 umulh x14,x6,x4 adcs x24,x24,x15 umulh x15,x7,x4 adcs x25,x25,x16 umulh x16,x8,x4 adcs x26,x26,x17 umulh x17,x9,x4 adc x28,x28,xzr str x19,[x2],#8 adds x19,x20,x14 umulh x14,x10,x4 adcs x20,x21,x15 umulh x15,x11,x4 adcs x21,x22,x16 umulh x16,x12,x4 adcs x22,x23,x17 umulh x17,x13,x4 ldr x4,[x0,x27] adcs x23,x24,x14 adcs x24,x25,x15 adcs x25,x26,x16 adcs x26,x28,x17 //adc x28,xzr,xzr // moved above cbnz x27,.Lsqr8x_mul // note that carry flag is guaranteed // to be zero at this point cmp x1,x3 // done yet? b.eq .Lsqr8x_break ldp x6,x7,[x2,#8*0] ldp x8,x9,[x2,#8*2] ldp x10,x11,[x2,#8*4] ldp x12,x13,[x2,#8*6] adds x19,x19,x6 ldur x4,[x0,#-8*8] adcs x20,x20,x7 ldp x6,x7,[x1,#8*0] adcs x21,x21,x8 adcs x22,x22,x9 ldp x8,x9,[x1,#8*2] adcs x23,x23,x10 adcs x24,x24,x11 ldp x10,x11,[x1,#8*4] adcs x25,x25,x12 mov x27,#-8*8 adcs x26,x26,x13 ldp x12,x13,[x1,#8*6] add x1,x1,#8*8 //adc x28,xzr,xzr // moved above b .Lsqr8x_mul .align 4 .Lsqr8x_break: ldp x6,x7,[x0,#8*0] add x1,x0,#8*8 ldp x8,x9,[x0,#8*2] sub x14,x3,x1 // is it last iteration? ldp x10,x11,[x0,#8*4] sub x15,x2,x14 ldp x12,x13,[x0,#8*6] cbz x14,.Lsqr8x_outer_loop stp x19,x20,[x2,#8*0] ldp x19,x20,[x15,#8*0] stp x21,x22,[x2,#8*2] ldp x21,x22,[x15,#8*2] stp x23,x24,[x2,#8*4] ldp x23,x24,[x15,#8*4] stp x25,x26,[x2,#8*6] mov x2,x15 ldp x25,x26,[x15,#8*6] b .Lsqr8x_outer_loop .align 4 .Lsqr8x_outer_break: // Now multiply above result by 2 and add a[n-1]*a[n-1]|...|a[0]*a[0] ldp x7,x9,[x14,#8*0] // recall that x14 is &a[0] ldp x15,x16,[sp,#8*1] ldp x11,x13,[x14,#8*2] add x1,x14,#8*4 ldp x17,x14,[sp,#8*3] stp x19,x20,[x2,#8*0] mul x19,x7,x7 stp x21,x22,[x2,#8*2] umulh x7,x7,x7 stp x23,x24,[x2,#8*4] mul x8,x9,x9 stp x25,x26,[x2,#8*6] mov x2,sp umulh x9,x9,x9 adds x20,x7,x15,lsl#1 extr x15,x16,x15,#63 sub x27,x5,#8*4 .Lsqr4x_shift_n_add: adcs x21,x8,x15 extr x16,x17,x16,#63 sub x27,x27,#8*4 adcs x22,x9,x16 ldp x15,x16,[x2,#8*5] mul x10,x11,x11 ldp x7,x9,[x1],#8*2 umulh x11,x11,x11 mul x12,x13,x13 umulh x13,x13,x13 extr x17,x14,x17,#63 stp x19,x20,[x2,#8*0] adcs x23,x10,x17 extr x14,x15,x14,#63 stp x21,x22,[x2,#8*2] adcs x24,x11,x14 ldp x17,x14,[x2,#8*7] extr x15,x16,x15,#63 adcs x25,x12,x15 extr x16,x17,x16,#63 adcs x26,x13,x16 ldp x15,x16,[x2,#8*9] mul x6,x7,x7 ldp x11,x13,[x1],#8*2 umulh x7,x7,x7 mul x8,x9,x9 umulh x9,x9,x9 stp x23,x24,[x2,#8*4] extr x17,x14,x17,#63 stp x25,x26,[x2,#8*6] add x2,x2,#8*8 adcs x19,x6,x17 extr x14,x15,x14,#63 adcs x20,x7,x14 ldp x17,x14,[x2,#8*3] extr x15,x16,x15,#63 cbnz x27,.Lsqr4x_shift_n_add ldp x1,x4,[x29,#104] // pull np and n0 adcs x21,x8,x15 extr x16,x17,x16,#63 adcs x22,x9,x16 ldp x15,x16,[x2,#8*5] mul x10,x11,x11 umulh x11,x11,x11 stp x19,x20,[x2,#8*0] mul x12,x13,x13 umulh x13,x13,x13 stp x21,x22,[x2,#8*2] extr x17,x14,x17,#63 adcs x23,x10,x17 extr x14,x15,x14,#63 ldp x19,x20,[sp,#8*0] adcs x24,x11,x14 extr x15,x16,x15,#63 ldp x6,x7,[x1,#8*0] adcs x25,x12,x15 extr x16,xzr,x16,#63 ldp x8,x9,[x1,#8*2] adc x26,x13,x16 ldp x10,x11,[x1,#8*4] // Reduce by 512 bits per iteration mul x28,x4,x19 // t[0]*n0 ldp x12,x13,[x1,#8*6] add x3,x1,x5 ldp x21,x22,[sp,#8*2] stp x23,x24,[x2,#8*4] ldp x23,x24,[sp,#8*4] stp x25,x26,[x2,#8*6] ldp x25,x26,[sp,#8*6] add x1,x1,#8*8 mov x30,xzr // initial top-most carry mov x2,sp mov x27,#8 .Lsqr8x_reduction: // (*) mul x14,x6,x28 // lo(n[0-7])*lo(t[0]*n0) mul x15,x7,x28 sub x27,x27,#1 mul x16,x8,x28 str x28,[x2],#8 // put aside t[0]*n0 for tail processing mul x17,x9,x28 // (*) adds xzr,x19,x14 subs xzr,x19,#1 // (*) mul x14,x10,x28 adcs x19,x20,x15 mul x15,x11,x28 adcs x20,x21,x16 mul x16,x12,x28 adcs x21,x22,x17 mul x17,x13,x28 adcs x22,x23,x14 umulh x14,x6,x28 // hi(n[0-7])*lo(t[0]*n0) adcs x23,x24,x15 umulh x15,x7,x28 adcs x24,x25,x16 umulh x16,x8,x28 adcs x25,x26,x17 umulh x17,x9,x28 adc x26,xzr,xzr adds x19,x19,x14 umulh x14,x10,x28 adcs x20,x20,x15 umulh x15,x11,x28 adcs x21,x21,x16 umulh x16,x12,x28 adcs x22,x22,x17 umulh x17,x13,x28 mul x28,x4,x19 // next t[0]*n0 adcs x23,x23,x14 adcs x24,x24,x15 adcs x25,x25,x16 adc x26,x26,x17 cbnz x27,.Lsqr8x_reduction ldp x14,x15,[x2,#8*0] ldp x16,x17,[x2,#8*2] mov x0,x2 sub x27,x3,x1 // done yet? adds x19,x19,x14 adcs x20,x20,x15 ldp x14,x15,[x2,#8*4] adcs x21,x21,x16 adcs x22,x22,x17 ldp x16,x17,[x2,#8*6] adcs x23,x23,x14 adcs x24,x24,x15 adcs x25,x25,x16 adcs x26,x26,x17 //adc x28,xzr,xzr // moved below cbz x27,.Lsqr8x8_post_condition ldur x4,[x2,#-8*8] ldp x6,x7,[x1,#8*0] ldp x8,x9,[x1,#8*2] ldp x10,x11,[x1,#8*4] mov x27,#-8*8 ldp x12,x13,[x1,#8*6] add x1,x1,#8*8 .Lsqr8x_tail: mul x14,x6,x4 adc x28,xzr,xzr // carry bit, modulo-scheduled mul x15,x7,x4 add x27,x27,#8 mul x16,x8,x4 mul x17,x9,x4 adds x19,x19,x14 mul x14,x10,x4 adcs x20,x20,x15 mul x15,x11,x4 adcs x21,x21,x16 mul x16,x12,x4 adcs x22,x22,x17 mul x17,x13,x4 adcs x23,x23,x14 umulh x14,x6,x4 adcs x24,x24,x15 umulh x15,x7,x4 adcs x25,x25,x16 umulh x16,x8,x4 adcs x26,x26,x17 umulh x17,x9,x4 adc x28,x28,xzr str x19,[x2],#8 adds x19,x20,x14 umulh x14,x10,x4 adcs x20,x21,x15 umulh x15,x11,x4 adcs x21,x22,x16 umulh x16,x12,x4 adcs x22,x23,x17 umulh x17,x13,x4 ldr x4,[x0,x27] adcs x23,x24,x14 adcs x24,x25,x15 adcs x25,x26,x16 adcs x26,x28,x17 //adc x28,xzr,xzr // moved above cbnz x27,.Lsqr8x_tail // note that carry flag is guaranteed // to be zero at this point ldp x6,x7,[x2,#8*0] sub x27,x3,x1 // done yet? sub x16,x3,x5 // rewinded np ldp x8,x9,[x2,#8*2] ldp x10,x11,[x2,#8*4] ldp x12,x13,[x2,#8*6] cbz x27,.Lsqr8x_tail_break ldur x4,[x0,#-8*8] adds x19,x19,x6 adcs x20,x20,x7 ldp x6,x7,[x1,#8*0] adcs x21,x21,x8 adcs x22,x22,x9 ldp x8,x9,[x1,#8*2] adcs x23,x23,x10 adcs x24,x24,x11 ldp x10,x11,[x1,#8*4] adcs x25,x25,x12 mov x27,#-8*8 adcs x26,x26,x13 ldp x12,x13,[x1,#8*6] add x1,x1,#8*8 //adc x28,xzr,xzr // moved above b .Lsqr8x_tail .align 4 .Lsqr8x_tail_break: ldr x4,[x29,#112] // pull n0 add x27,x2,#8*8 // end of current t[num] window subs xzr,x30,#1 // "move" top-most carry to carry bit adcs x14,x19,x6 adcs x15,x20,x7 ldp x19,x20,[x0,#8*0] adcs x21,x21,x8 ldp x6,x7,[x16,#8*0] // recall that x16 is &n[0] adcs x22,x22,x9 ldp x8,x9,[x16,#8*2] adcs x23,x23,x10 adcs x24,x24,x11 ldp x10,x11,[x16,#8*4] adcs x25,x25,x12 adcs x26,x26,x13 ldp x12,x13,[x16,#8*6] add x1,x16,#8*8 adc x30,xzr,xzr // top-most carry mul x28,x4,x19 stp x14,x15,[x2,#8*0] stp x21,x22,[x2,#8*2] ldp x21,x22,[x0,#8*2] stp x23,x24,[x2,#8*4] ldp x23,x24,[x0,#8*4] cmp x27,x29 // did we hit the bottom? stp x25,x26,[x2,#8*6] mov x2,x0 // slide the window ldp x25,x26,[x0,#8*6] mov x27,#8 b.ne .Lsqr8x_reduction // Final step. We see if result is larger than modulus, and // if it is, subtract the modulus. But comparison implies // subtraction. So we subtract modulus, see if it borrowed, // and conditionally copy original value. ldr x0,[x29,#96] // pull rp add x2,x2,#8*8 subs x14,x19,x6 sbcs x15,x20,x7 sub x27,x5,#8*8 mov x3,x0 // x0 copy .Lsqr8x_sub: sbcs x16,x21,x8 ldp x6,x7,[x1,#8*0] sbcs x17,x22,x9 stp x14,x15,[x0,#8*0] sbcs x14,x23,x10 ldp x8,x9,[x1,#8*2] sbcs x15,x24,x11 stp x16,x17,[x0,#8*2] sbcs x16,x25,x12 ldp x10,x11,[x1,#8*4] sbcs x17,x26,x13 ldp x12,x13,[x1,#8*6] add x1,x1,#8*8 ldp x19,x20,[x2,#8*0] sub x27,x27,#8*8 ldp x21,x22,[x2,#8*2] ldp x23,x24,[x2,#8*4] ldp x25,x26,[x2,#8*6] add x2,x2,#8*8 stp x14,x15,[x0,#8*4] sbcs x14,x19,x6 stp x16,x17,[x0,#8*6] add x0,x0,#8*8 sbcs x15,x20,x7 cbnz x27,.Lsqr8x_sub sbcs x16,x21,x8 mov x2,sp add x1,sp,x5 ldp x6,x7,[x3,#8*0] sbcs x17,x22,x9 stp x14,x15,[x0,#8*0] sbcs x14,x23,x10 ldp x8,x9,[x3,#8*2] sbcs x15,x24,x11 stp x16,x17,[x0,#8*2] sbcs x16,x25,x12 ldp x19,x20,[x1,#8*0] sbcs x17,x26,x13 ldp x21,x22,[x1,#8*2] sbcs xzr,x30,xzr // did it borrow? ldr x30,[x29,#8] // pull return address stp x14,x15,[x0,#8*4] stp x16,x17,[x0,#8*6] sub x27,x5,#8*4 .Lsqr4x_cond_copy: sub x27,x27,#8*4 csel x14,x19,x6,lo stp xzr,xzr,[x2,#8*0] csel x15,x20,x7,lo ldp x6,x7,[x3,#8*4] ldp x19,x20,[x1,#8*4] csel x16,x21,x8,lo stp xzr,xzr,[x2,#8*2] add x2,x2,#8*4 csel x17,x22,x9,lo ldp x8,x9,[x3,#8*6] ldp x21,x22,[x1,#8*6] add x1,x1,#8*4 stp x14,x15,[x3,#8*0] stp x16,x17,[x3,#8*2] add x3,x3,#8*4 stp xzr,xzr,[x1,#8*0] stp xzr,xzr,[x1,#8*2] cbnz x27,.Lsqr4x_cond_copy csel x14,x19,x6,lo stp xzr,xzr,[x2,#8*0] csel x15,x20,x7,lo stp xzr,xzr,[x2,#8*2] csel x16,x21,x8,lo csel x17,x22,x9,lo stp x14,x15,[x3,#8*0] stp x16,x17,[x3,#8*2] b .Lsqr8x_done .align 4 .Lsqr8x8_post_condition: adc x28,xzr,xzr ldr x30,[x29,#8] // pull return address // x19-7,x28 hold result, x6-7 hold modulus subs x6,x19,x6 ldr x1,[x29,#96] // pull rp sbcs x7,x20,x7 stp xzr,xzr,[sp,#8*0] sbcs x8,x21,x8 stp xzr,xzr,[sp,#8*2] sbcs x9,x22,x9 stp xzr,xzr,[sp,#8*4] sbcs x10,x23,x10 stp xzr,xzr,[sp,#8*6] sbcs x11,x24,x11 stp xzr,xzr,[sp,#8*8] sbcs x12,x25,x12 stp xzr,xzr,[sp,#8*10] sbcs x13,x26,x13 stp xzr,xzr,[sp,#8*12] sbcs x28,x28,xzr // did it borrow? stp xzr,xzr,[sp,#8*14] // x6-7 hold result-modulus csel x6,x19,x6,lo csel x7,x20,x7,lo csel x8,x21,x8,lo csel x9,x22,x9,lo stp x6,x7,[x1,#8*0] csel x10,x23,x10,lo csel x11,x24,x11,lo stp x8,x9,[x1,#8*2] csel x12,x25,x12,lo csel x13,x26,x13,lo stp x10,x11,[x1,#8*4] stp x12,x13,[x1,#8*6] .Lsqr8x_done: ldp x19,x20,[x29,#16] mov sp,x29 ldp x21,x22,[x29,#32] mov x0,#1 ldp x23,x24,[x29,#48] ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldr x29,[sp],#128 // x30 is loaded earlier AARCH64_VALIDATE_LINK_REGISTER ret .size __bn_sqr8x_mont,.-__bn_sqr8x_mont .type __bn_mul4x_mont,%function .align 5 __bn_mul4x_mont: // Not adding AARCH64_SIGN_LINK_REGISTER here because __bn_mul4x_mont is jumped to // only from bn_mul_mont (or __bn_sqr8x_mont from bn_mul_mont) which has already signed the return address. stp x29,x30,[sp,#-128]! add x29,sp,#0 stp x19,x20,[sp,#16] stp x21,x22,[sp,#32] stp x23,x24,[sp,#48] stp x25,x26,[sp,#64] stp x27,x28,[sp,#80] sub x26,sp,x5,lsl#3 lsl x5,x5,#3 ldr x4,[x4] // *n0 sub sp,x26,#8*4 // alloca add x10,x2,x5 add x27,x1,x5 stp x0,x10,[x29,#96] // offload rp and &b[num] ldr x24,[x2,#8*0] // b[0] ldp x6,x7,[x1,#8*0] // a[0..3] ldp x8,x9,[x1,#8*2] add x1,x1,#8*4 mov x19,xzr mov x20,xzr mov x21,xzr mov x22,xzr ldp x14,x15,[x3,#8*0] // n[0..3] ldp x16,x17,[x3,#8*2] adds x3,x3,#8*4 // clear carry bit mov x0,xzr mov x28,#0 mov x26,sp .Loop_mul4x_1st_reduction: mul x10,x6,x24 // lo(a[0..3]*b[0]) adc x0,x0,xzr // modulo-scheduled mul x11,x7,x24 add x28,x28,#8 mul x12,x8,x24 and x28,x28,#31 mul x13,x9,x24 adds x19,x19,x10 umulh x10,x6,x24 // hi(a[0..3]*b[0]) adcs x20,x20,x11 mul x25,x19,x4 // t[0]*n0 adcs x21,x21,x12 umulh x11,x7,x24 adcs x22,x22,x13 umulh x12,x8,x24 adc x23,xzr,xzr umulh x13,x9,x24 ldr x24,[x2,x28] // next b[i] (or b[0]) adds x20,x20,x10 // (*) mul x10,x14,x25 // lo(n[0..3]*t[0]*n0) str x25,[x26],#8 // put aside t[0]*n0 for tail processing adcs x21,x21,x11 mul x11,x15,x25 adcs x22,x22,x12 mul x12,x16,x25 adc x23,x23,x13 // can't overflow mul x13,x17,x25 // (*) adds xzr,x19,x10 subs xzr,x19,#1 // (*) umulh x10,x14,x25 // hi(n[0..3]*t[0]*n0) adcs x19,x20,x11 umulh x11,x15,x25 adcs x20,x21,x12 umulh x12,x16,x25 adcs x21,x22,x13 umulh x13,x17,x25 adcs x22,x23,x0 adc x0,xzr,xzr adds x19,x19,x10 sub x10,x27,x1 adcs x20,x20,x11 adcs x21,x21,x12 adcs x22,x22,x13 //adc x0,x0,xzr cbnz x28,.Loop_mul4x_1st_reduction cbz x10,.Lmul4x4_post_condition ldp x6,x7,[x1,#8*0] // a[4..7] ldp x8,x9,[x1,#8*2] add x1,x1,#8*4 ldr x25,[sp] // a[0]*n0 ldp x14,x15,[x3,#8*0] // n[4..7] ldp x16,x17,[x3,#8*2] add x3,x3,#8*4 .Loop_mul4x_1st_tail: mul x10,x6,x24 // lo(a[4..7]*b[i]) adc x0,x0,xzr // modulo-scheduled mul x11,x7,x24 add x28,x28,#8 mul x12,x8,x24 and x28,x28,#31 mul x13,x9,x24 adds x19,x19,x10 umulh x10,x6,x24 // hi(a[4..7]*b[i]) adcs x20,x20,x11 umulh x11,x7,x24 adcs x21,x21,x12 umulh x12,x8,x24 adcs x22,x22,x13 umulh x13,x9,x24 adc x23,xzr,xzr ldr x24,[x2,x28] // next b[i] (or b[0]) adds x20,x20,x10 mul x10,x14,x25 // lo(n[4..7]*a[0]*n0) adcs x21,x21,x11 mul x11,x15,x25 adcs x22,x22,x12 mul x12,x16,x25 adc x23,x23,x13 // can't overflow mul x13,x17,x25 adds x19,x19,x10 umulh x10,x14,x25 // hi(n[4..7]*a[0]*n0) adcs x20,x20,x11 umulh x11,x15,x25 adcs x21,x21,x12 umulh x12,x16,x25 adcs x22,x22,x13 adcs x23,x23,x0 umulh x13,x17,x25 adc x0,xzr,xzr ldr x25,[sp,x28] // next t[0]*n0 str x19,[x26],#8 // result!!! adds x19,x20,x10 sub x10,x27,x1 // done yet? adcs x20,x21,x11 adcs x21,x22,x12 adcs x22,x23,x13 //adc x0,x0,xzr cbnz x28,.Loop_mul4x_1st_tail sub x11,x27,x5 // rewinded x1 cbz x10,.Lmul4x_proceed ldp x6,x7,[x1,#8*0] ldp x8,x9,[x1,#8*2] add x1,x1,#8*4 ldp x14,x15,[x3,#8*0] ldp x16,x17,[x3,#8*2] add x3,x3,#8*4 b .Loop_mul4x_1st_tail .align 5 .Lmul4x_proceed: ldr x24,[x2,#8*4]! // *++b adc x30,x0,xzr ldp x6,x7,[x11,#8*0] // a[0..3] sub x3,x3,x5 // rewind np ldp x8,x9,[x11,#8*2] add x1,x11,#8*4 stp x19,x20,[x26,#8*0] // result!!! ldp x19,x20,[sp,#8*4] // t[0..3] stp x21,x22,[x26,#8*2] // result!!! ldp x21,x22,[sp,#8*6] ldp x14,x15,[x3,#8*0] // n[0..3] mov x26,sp ldp x16,x17,[x3,#8*2] adds x3,x3,#8*4 // clear carry bit mov x0,xzr .align 4 .Loop_mul4x_reduction: mul x10,x6,x24 // lo(a[0..3]*b[4]) adc x0,x0,xzr // modulo-scheduled mul x11,x7,x24 add x28,x28,#8 mul x12,x8,x24 and x28,x28,#31 mul x13,x9,x24 adds x19,x19,x10 umulh x10,x6,x24 // hi(a[0..3]*b[4]) adcs x20,x20,x11 mul x25,x19,x4 // t[0]*n0 adcs x21,x21,x12 umulh x11,x7,x24 adcs x22,x22,x13 umulh x12,x8,x24 adc x23,xzr,xzr umulh x13,x9,x24 ldr x24,[x2,x28] // next b[i] adds x20,x20,x10 // (*) mul x10,x14,x25 str x25,[x26],#8 // put aside t[0]*n0 for tail processing adcs x21,x21,x11 mul x11,x15,x25 // lo(n[0..3]*t[0]*n0 adcs x22,x22,x12 mul x12,x16,x25 adc x23,x23,x13 // can't overflow mul x13,x17,x25 // (*) adds xzr,x19,x10 subs xzr,x19,#1 // (*) umulh x10,x14,x25 // hi(n[0..3]*t[0]*n0 adcs x19,x20,x11 umulh x11,x15,x25 adcs x20,x21,x12 umulh x12,x16,x25 adcs x21,x22,x13 umulh x13,x17,x25 adcs x22,x23,x0 adc x0,xzr,xzr adds x19,x19,x10 adcs x20,x20,x11 adcs x21,x21,x12 adcs x22,x22,x13 //adc x0,x0,xzr cbnz x28,.Loop_mul4x_reduction adc x0,x0,xzr ldp x10,x11,[x26,#8*4] // t[4..7] ldp x12,x13,[x26,#8*6] ldp x6,x7,[x1,#8*0] // a[4..7] ldp x8,x9,[x1,#8*2] add x1,x1,#8*4 adds x19,x19,x10 adcs x20,x20,x11 adcs x21,x21,x12 adcs x22,x22,x13 //adc x0,x0,xzr ldr x25,[sp] // t[0]*n0 ldp x14,x15,[x3,#8*0] // n[4..7] ldp x16,x17,[x3,#8*2] add x3,x3,#8*4 .align 4 .Loop_mul4x_tail: mul x10,x6,x24 // lo(a[4..7]*b[4]) adc x0,x0,xzr // modulo-scheduled mul x11,x7,x24 add x28,x28,#8 mul x12,x8,x24 and x28,x28,#31 mul x13,x9,x24 adds x19,x19,x10 umulh x10,x6,x24 // hi(a[4..7]*b[4]) adcs x20,x20,x11 umulh x11,x7,x24 adcs x21,x21,x12 umulh x12,x8,x24 adcs x22,x22,x13 umulh x13,x9,x24 adc x23,xzr,xzr ldr x24,[x2,x28] // next b[i] adds x20,x20,x10 mul x10,x14,x25 // lo(n[4..7]*t[0]*n0) adcs x21,x21,x11 mul x11,x15,x25 adcs x22,x22,x12 mul x12,x16,x25 adc x23,x23,x13 // can't overflow mul x13,x17,x25 adds x19,x19,x10 umulh x10,x14,x25 // hi(n[4..7]*t[0]*n0) adcs x20,x20,x11 umulh x11,x15,x25 adcs x21,x21,x12 umulh x12,x16,x25 adcs x22,x22,x13 umulh x13,x17,x25 adcs x23,x23,x0 ldr x25,[sp,x28] // next a[0]*n0 adc x0,xzr,xzr str x19,[x26],#8 // result!!! adds x19,x20,x10 sub x10,x27,x1 // done yet? adcs x20,x21,x11 adcs x21,x22,x12 adcs x22,x23,x13 //adc x0,x0,xzr cbnz x28,.Loop_mul4x_tail sub x11,x3,x5 // rewinded np? adc x0,x0,xzr cbz x10,.Loop_mul4x_break ldp x10,x11,[x26,#8*4] ldp x12,x13,[x26,#8*6] ldp x6,x7,[x1,#8*0] ldp x8,x9,[x1,#8*2] add x1,x1,#8*4 adds x19,x19,x10 adcs x20,x20,x11 adcs x21,x21,x12 adcs x22,x22,x13 //adc x0,x0,xzr ldp x14,x15,[x3,#8*0] ldp x16,x17,[x3,#8*2] add x3,x3,#8*4 b .Loop_mul4x_tail .align 4 .Loop_mul4x_break: ldp x12,x13,[x29,#96] // pull rp and &b[num] adds x19,x19,x30 add x2,x2,#8*4 // bp++ adcs x20,x20,xzr sub x1,x1,x5 // rewind ap adcs x21,x21,xzr stp x19,x20,[x26,#8*0] // result!!! adcs x22,x22,xzr ldp x19,x20,[sp,#8*4] // t[0..3] adc x30,x0,xzr stp x21,x22,[x26,#8*2] // result!!! cmp x2,x13 // done yet? ldp x21,x22,[sp,#8*6] ldp x14,x15,[x11,#8*0] // n[0..3] ldp x16,x17,[x11,#8*2] add x3,x11,#8*4 b.eq .Lmul4x_post ldr x24,[x2] ldp x6,x7,[x1,#8*0] // a[0..3] ldp x8,x9,[x1,#8*2] adds x1,x1,#8*4 // clear carry bit mov x0,xzr mov x26,sp b .Loop_mul4x_reduction .align 4 .Lmul4x_post: // Final step. We see if result is larger than modulus, and // if it is, subtract the modulus. But comparison implies // subtraction. So we subtract modulus, see if it borrowed, // and conditionally copy original value. mov x0,x12 mov x27,x12 // x0 copy subs x10,x19,x14 add x26,sp,#8*8 sbcs x11,x20,x15 sub x28,x5,#8*4 .Lmul4x_sub: sbcs x12,x21,x16 ldp x14,x15,[x3,#8*0] sub x28,x28,#8*4 ldp x19,x20,[x26,#8*0] sbcs x13,x22,x17 ldp x16,x17,[x3,#8*2] add x3,x3,#8*4 ldp x21,x22,[x26,#8*2] add x26,x26,#8*4 stp x10,x11,[x0,#8*0] sbcs x10,x19,x14 stp x12,x13,[x0,#8*2] add x0,x0,#8*4 sbcs x11,x20,x15 cbnz x28,.Lmul4x_sub sbcs x12,x21,x16 mov x26,sp add x1,sp,#8*4 ldp x6,x7,[x27,#8*0] sbcs x13,x22,x17 stp x10,x11,[x0,#8*0] ldp x8,x9,[x27,#8*2] stp x12,x13,[x0,#8*2] ldp x19,x20,[x1,#8*0] ldp x21,x22,[x1,#8*2] sbcs xzr,x30,xzr // did it borrow? ldr x30,[x29,#8] // pull return address sub x28,x5,#8*4 .Lmul4x_cond_copy: sub x28,x28,#8*4 csel x10,x19,x6,lo stp xzr,xzr,[x26,#8*0] csel x11,x20,x7,lo ldp x6,x7,[x27,#8*4] ldp x19,x20,[x1,#8*4] csel x12,x21,x8,lo stp xzr,xzr,[x26,#8*2] add x26,x26,#8*4 csel x13,x22,x9,lo ldp x8,x9,[x27,#8*6] ldp x21,x22,[x1,#8*6] add x1,x1,#8*4 stp x10,x11,[x27,#8*0] stp x12,x13,[x27,#8*2] add x27,x27,#8*4 cbnz x28,.Lmul4x_cond_copy csel x10,x19,x6,lo stp xzr,xzr,[x26,#8*0] csel x11,x20,x7,lo stp xzr,xzr,[x26,#8*2] csel x12,x21,x8,lo stp xzr,xzr,[x26,#8*3] csel x13,x22,x9,lo stp xzr,xzr,[x26,#8*4] stp x10,x11,[x27,#8*0] stp x12,x13,[x27,#8*2] b .Lmul4x_done .align 4 .Lmul4x4_post_condition: adc x0,x0,xzr ldr x1,[x29,#96] // pull rp // x19-3,x0 hold result, x14-7 hold modulus subs x6,x19,x14 ldr x30,[x29,#8] // pull return address sbcs x7,x20,x15 stp xzr,xzr,[sp,#8*0] sbcs x8,x21,x16 stp xzr,xzr,[sp,#8*2] sbcs x9,x22,x17 stp xzr,xzr,[sp,#8*4] sbcs xzr,x0,xzr // did it borrow? stp xzr,xzr,[sp,#8*6] // x6-3 hold result-modulus csel x6,x19,x6,lo csel x7,x20,x7,lo csel x8,x21,x8,lo csel x9,x22,x9,lo stp x6,x7,[x1,#8*0] stp x8,x9,[x1,#8*2] .Lmul4x_done: ldp x19,x20,[x29,#16] mov sp,x29 ldp x21,x22,[x29,#32] mov x0,#1 ldp x23,x24,[x29,#48] ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldr x29,[sp],#128 // x30 loaded earlier AARCH64_VALIDATE_LINK_REGISTER ret .size __bn_mul4x_mont,.-__bn_mul4x_mont .section .rodata .byte 77,111,110,116,103,111,109,101,114,121,32,77,117,108,116,105,112,108,105,99,97,116,105,111,110,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 .align 2 .align 4