ntt_n256_l6_s32_bar.s

///
/// Copyright (c) 2021 Arm Limited
/// Copyright (c) 2022 Hanno Becker
/// Copyright (c) 2023 Amin Abdulrahman, Matthias Kannwischer
/// SPDX-License-Identifier: MIT
///
/// Permission is hereby granted, free of charge, to any person obtaining a copy
/// of this software and associated documentation files (the "Software"), to deal
/// in the Software without restriction, including without limitation the rights
/// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
/// copies of the Software, and to permit persons to whom the Software is
/// furnished to do so, subject to the following conditions:
///
/// The above copyright notice and this permission notice shall be included in all
/// copies or substantial portions of the Software.
///
/// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
/// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
/// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
/// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
/// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
/// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
/// SOFTWARE.
///



///
/// This assembly code has been auto-generated.
/// Don't modify it directly.
///

.data
roots:
#include "ntt_n256_l6_s32_twiddles.s"
.text

// Barrett multiplication
.macro mulmod dst, src, const, const_twisted
        vmul.s32       \dst,  \src, \const
        vqrdmulh.s32   \src,  \src, \const_twisted
        vmla.s32       \dst,  \src, modulus
.endm

.macro ct_butterfly a, b, root, root_twisted
        mulmod tmp, \b, \root, \root_twisted
        vsub.u32       \b,    \a, tmp
        vadd.u32       \a,    \a, tmp
.endm

.align 4
roots_addr: .word roots
.syntax unified
.type ntt_n256_u32_33556993_28678040_incomplete_manual, %function
.global ntt_n256_u32_33556993_28678040_incomplete_manual
ntt_n256_u32_33556993_28678040_incomplete_manual:

        push {r4-r11,lr}
        // Save MVE vector registers
        vpush {d8-d15}

        modulus  .req r12
        root_ptr .req r11

        .equ modulus_const, -33556993
        movw modulus, #:lower16:modulus_const
        movt modulus, #:upper16:modulus_const
        ldr  root_ptr, roots_addr

        in_low       .req r0
        in_high      .req r1

        add in_high, in_low, #(4*128)

        root0         .req r2
        root0_twisted .req r3
        root1         .req r4
        root1_twisted .req r5
        root2         .req r6
        root2_twisted .req r7

        data0 .req q0
        data1 .req q1
        data2 .req q2
        data3 .req q3

        tmp .req q4

        // Layers 1-2

        ldrd root0, root0_twisted, [root_ptr], #+8
        ldrd root1, root1_twisted, [root_ptr], #+8
        ldrd root2, root2_twisted, [root_ptr], #+8

        mov lr, #16
        vldrw.u32 q2, [in_high, #256]              // *.....
        vmul.s32 q0, q2, root0                     // .*....
        vldrw.u32 q5, [in_low, #256]               // ...*..
        vqrdmulh.s32 q2, q2, root0_twisted         // ..*...
        nop                                        // .....*
        vmla.s32 q0, q2, modulus                   // ....*.
        
        // original source code
        // vldrw.u32 q6, [in_high, #256]           // *.....
        // vmul.s32 q0, q6, root0                  // .*....
        // vqrdmulh.s32 q4, q6, root0_twisted      // ...*..
        // vldrw.u32 q5, [in_low, #256]            // ..*...
        // vmla.s32 q0, q4, modulus                // .....*
        // nop                                     // ....*.
        
        sub lr, lr, #1
layer12_loop:
        vsub.u32 q1, q5, q0                        // ............*...............
        vqrdmulh.s32 q4, q1, root2_twisted         // ....................*.......
        vldrw.u32 q7, [in_high]                    // ..*.........................
        vmul.s32 q2, q7, root0                     // ....*.......................
        vldrw.u32 q6, [in_high, #272]              // ...e........................
        vqrdmulh.s32 q7, q7, root0_twisted         // .....*......................
        vadd.u32 q0, q5, q0                        // .............*..............
        vmla.s32 q2, q7, modulus                   // ......*.....................
        vldrw.u32 q3, [in_low]                     // *...........................
        vmul.s32 q5, q1, root2                     // ...................*........
        vsub.u32 q7, q3, q2                        // .......*....................
        vmla.s32 q5, q4, modulus                   // .....................*......
        vadd.u32 q4, q3, q2                        // ........*...................
        vmul.s32 q3, q0, root1                     // ..............*.............
        vadd.u32 q2, q7, q5                        // .......................*....
        vqrdmulh.s32 q1, q0, root1_twisted         // ...............*............
        vstrw.u32 q2, [in_high] , #16              // ..........................*.
        vmla.s32 q3, q1, modulus                   // ................*...........
        vsub.u32 q2, q7, q5                        // ......................*.....
        vstrw.u32 q2, [in_high, #240]              // ...........................*
        vsub.u32 q5, q4, q3                        // .................*..........
        vmul.s32 q0, q6, root0                     // .........e..................
        vstrw.u32 q5, [in_low, #256]               // .........................*..
        vadd.u32 q1, q4, q3                        // ..................*.........
        vqrdmulh.s32 q4, q6, root0_twisted         // ..........e.................
        vldrw.u32 q5, [in_low, #272]               // .e..........................
        vmla.s32 q0, q4, modulus                   // ...........e................
        vstrw.u32 q1, [in_low] , #16               // ........................*...
        
        // original source code
        // vldrw.u32 data0, [in_low]                     // ................................*...................
        // vldrw.u32 data1, [in_low, #256]               // .....................e..............................
        // vldrw.u32 data2, [in_high]                    // ..........................*.........................
        // vldrw.u32 data3, [in_high, #256]              // e...................................................
        // vmul.s32 tmp, data2, root0                    // ...........................*........................
        // vqrdmulh.s32 data2, data2, root0_twisted      // .............................*......................
        // vmla.s32 tmp, data2, modulus                  // ...............................*....................
        // vsub.u32 data2, data0, tmp                    // ..................................*.................
        // vadd.u32 data0, data0, tmp                    // ....................................*...............
        // vmul.s32 tmp, data3, root0                    // .................e..................................
        // vqrdmulh.s32 data3, data3, root0_twisted      // ....................e...............................
        // vmla.s32 tmp, data3, modulus                  // ......................e.............................
        // vsub.u32 data3, data1, tmp                    // ........................*...........................
        // vadd.u32 data1, data1, tmp                    // ..............................*.....................
        // vmul.s32 tmp, data1, root1                    // .....................................*..............
        // vqrdmulh.s32 data1, data1, root1_twisted      // .......................................*............
        // vmla.s32 tmp, data1, modulus                  // .........................................*..........
        // vsub.u32 data1, data0, tmp                    // ............................................*.......
        // vadd.u32 data0, data0, tmp                    // ...............................................*....
        // vmul.s32 tmp, data3, root2                    // .................................*..................
        // vqrdmulh.s32 data3, data3, root2_twisted      // .........................*..........................
        // vmla.s32 tmp, data3, modulus                  // ...................................*................
        // vsub.u32 data3, data2, tmp                    // ..........................................*.........
        // vadd.u32 data2, data2, tmp                    // ......................................*.............
        // vstrw.u32 data0, [in_low] , #16               // ...................................................*
        // vstrw.u32 data1, [in_low, #240]               // ..............................................*.....
        // vstrw.u32 data2, [in_high] , #16              // ........................................*...........
        // vstrw.u32 data3, [in_high, #240]              // ...........................................*........
        
        le lr, layer12_loop
        vsub.u32 q1, q5, q0                        // *.......................
        vqrdmulh.s32 q7, q1, root2_twisted         // .*......................
        vldrw.u32 q3, [in_high]                    // ..*.....................
        vqrdmulh.s32 q6, q3, root0_twisted         // ....*...................
        vadd.u32 q0, q5, q0                        // .....*..................
        vmul.s32 q2, q3, root0                     // ...*....................
        vldrw.u32 q3, [in_low]                     // .......*................
        vmla.s32 q2, q6, modulus                   // ......*.................
        nop                                        // .......................*
        vmul.s32 q5, q1, root2                     // ........*...............
        vsub.u32 q1, q3, q2                        // .........*..............
        vmla.s32 q5, q7, modulus                   // ..........*.............
        vadd.u32 q2, q3, q2                        // ...........*............
        vmul.s32 q3, q0, root1                     // ............*...........
        vadd.u32 q6, q1, q5                        // .............*..........
        vstrw.u32 q6, [in_high] , #16              // ...............*........
        vqrdmulh.s32 q4, q0, root1_twisted         // ..............*.........
        vsub.u32 q7, q1, q5                        // .................*......
        vmla.s32 q3, q4, modulus                   // ................*.......
        vstrw.u32 q7, [in_high, #240]              // ..................*.....
        vsub.u32 q5, q2, q3                        // ...................*....
        vstrw.u32 q5, [in_low, #256]               // ....................*...
        vadd.u32 q2, q2, q3                        // .....................*..
        vstrw.u32 q2, [in_low] , #16               // ......................*.
        
        // original source code
        // vsub.u32 q1, q5, q0                     // *.......................
        // vqrdmulh.s32 q4, q1, root2_twisted      // .*......................
        // vldrw.u32 q7, [in_high]                 // ..*.....................
        // vmul.s32 q2, q7, root0                  // .....*..................
        // vqrdmulh.s32 q7, q7, root0_twisted      // ...*....................
        // vadd.u32 q0, q5, q0                     // ....*...................
        // vmla.s32 q2, q7, modulus                // .......*................
        // vldrw.u32 q3, [in_low]                  // ......*.................
        // vmul.s32 q5, q1, root2                  // .........*..............
        // vsub.u32 q7, q3, q2                     // ..........*.............
        // vmla.s32 q5, q4, modulus                // ...........*............
        // vadd.u32 q4, q3, q2                     // ............*...........
        // vmul.s32 q3, q0, root1                  // .............*..........
        // vadd.u32 q2, q7, q5                     // ..............*.........
        // vqrdmulh.s32 q1, q0, root1_twisted      // ................*.......
        // vstrw.u32 q2, [in_high] , #16           // ...............*........
        // vmla.s32 q3, q1, modulus                // ..................*.....
        // vsub.u32 q2, q7, q5                     // .................*......
        // vstrw.u32 q2, [in_high, #240]           // ...................*....
        // vsub.u32 q5, q4, q3                     // ....................*...
        // vstrw.u32 q5, [in_low, #256]            // .....................*..
        // vadd.u32 q1, q4, q3                     // ......................*.
        // vstrw.u32 q1, [in_low] , #16            // .......................*
        // nop                                     // ........*...............
        
layer12_loop_end:
        .unreq in_high
        .unreq in_low

        in .req r0
        sub in, in, #(64*4)

        // Layers 3,4

        // 4 butterfly blocks per root config, 4 root configs
        // loop over root configs

        count .req r1
        mov count, #4

out_start:
        ldrd root0, root0_twisted, [root_ptr], #+8
        ldrd root1, root1_twisted, [root_ptr], #+8
        ldrd root2, root2_twisted, [root_ptr], #+8

        mov lr, #4
        vldrw.u32 q2, [in, #192]                   // *.....
        vmul.s32 q0, q2, root0                     // .*....
        vldrw.u32 q5, [in, #64]                    // ...*..
        vqrdmulh.s32 q2, q2, root0_twisted         // ..*...
        nop                                        // .....*
        vmla.s32 q0, q2, modulus                   // ....*.
        
        // original source code
        // vldrw.u32 q6, [in, #192]                // *.....
        // vmul.s32 q0, q6, root0                  // .*....
        // vqrdmulh.s32 q4, q6, root0_twisted      // ...*..
        // vldrw.u32 q5, [in, #64]                 // ..*...
        // vmla.s32 q0, q4, modulus                // .....*
        // nop                                     // ....*.
        
        sub lr, lr, #1
layer34_loop:
        vsub.u32 q1, q5, q0                        // ............*...............
        vqrdmulh.s32 q4, q1, root2_twisted         // ....................*.......
        vldrw.u32 q7, [in, #128]                   // ..*.........................
        vmul.s32 q2, q7, root0                     // ....*.......................
        vldrw.u32 q6, [in, #208]                   // ...e........................
        vqrdmulh.s32 q7, q7, root0_twisted         // .....*......................
        vadd.u32 q0, q5, q0                        // .............*..............
        vmla.s32 q2, q7, modulus                   // ......*.....................
        vldrw.u32 q3, [in]                         // *...........................
        vmul.s32 q5, q1, root2                     // ...................*........
        vsub.u32 q7, q3, q2                        // .......*....................
        vmla.s32 q5, q4, modulus                   // .....................*......
        vadd.u32 q4, q3, q2                        // ........*...................
        vmul.s32 q3, q0, root1                     // ..............*.............
        vadd.u32 q2, q7, q5                        // .......................*....
        vqrdmulh.s32 q1, q0, root1_twisted         // ...............*............
        vstrw.u32 q2, [in, #128]                   // ..........................*.
        vmla.s32 q3, q1, modulus                   // ................*...........
        vsub.u32 q2, q7, q5                        // ......................*.....
        vstrw.u32 q2, [in, #192]                   // ...........................*
        vsub.u32 q5, q4, q3                        // .................*..........
        vmul.s32 q0, q6, root0                     // .........e..................
        vstrw.u32 q5, [in, #64]                    // .........................*..
        vadd.u32 q1, q4, q3                        // ..................*.........
        vqrdmulh.s32 q4, q6, root0_twisted         // ..........e.................
        vldrw.u32 q5, [in, #80]                    // .e..........................
        vmla.s32 q0, q4, modulus                   // ...........e................
        vstrw.u32 q1, [in] , #16                   // ........................*...
        
        // original source code
        // vldrw.u32 data0, [in]                         // ................................*...................
        // vldrw.u32 data1, [in, #64]                    // .....................e..............................
        // vldrw.u32 data2, [in, #128]                   // ..........................*.........................
        // vldrw.u32 data3, [in, #192]                   // e...................................................
        // vmul.s32 tmp, data2, root0                    // ...........................*........................
        // vqrdmulh.s32 data2, data2, root0_twisted      // .............................*......................
        // vmla.s32 tmp, data2, modulus                  // ...............................*....................
        // vsub.u32 data2, data0, tmp                    // ..................................*.................
        // vadd.u32 data0, data0, tmp                    // ....................................*...............
        // vmul.s32 tmp, data3, root0                    // .................e..................................
        // vqrdmulh.s32 data3, data3, root0_twisted      // ....................e...............................
        // vmla.s32 tmp, data3, modulus                  // ......................e.............................
        // vsub.u32 data3, data1, tmp                    // ........................*...........................
        // vadd.u32 data1, data1, tmp                    // ..............................*.....................
        // vmul.s32 tmp, data1, root1                    // .....................................*..............
        // vqrdmulh.s32 data1, data1, root1_twisted      // .......................................*............
        // vmla.s32 tmp, data1, modulus                  // .........................................*..........
        // vsub.u32 data1, data0, tmp                    // ............................................*.......
        // vadd.u32 data0, data0, tmp                    // ...............................................*....
        // vmul.s32 tmp, data3, root2                    // .................................*..................
        // vqrdmulh.s32 data3, data3, root2_twisted      // .........................*..........................
        // vmla.s32 tmp, data3, modulus                  // ...................................*................
        // vsub.u32 data3, data2, tmp                    // ..........................................*.........
        // vadd.u32 data2, data2, tmp                    // ......................................*.............
        // vstrw.u32 data0, [in] , #16                   // ...................................................*
        // vstrw.u32 data1, [in, #48]                    // ..............................................*.....
        // vstrw.u32 data2, [in, #112]                   // ........................................*...........
        // vstrw.u32 data3, [in, #176]                   // ...........................................*........
        
        le lr, layer34_loop
        vsub.u32 q1, q5, q0                        // *.......................
        vqrdmulh.s32 q7, q1, root2_twisted         // .*......................
        vldrw.u32 q3, [in, #128]                   // ..*.....................
        vqrdmulh.s32 q6, q3, root0_twisted         // ....*...................
        vadd.u32 q0, q5, q0                        // .....*..................
        vmul.s32 q2, q3, root0                     // ...*....................
        vldrw.u32 q3, [in]                         // .......*................
        vmla.s32 q2, q6, modulus                   // ......*.................
        nop                                        // .......................*
        vmul.s32 q5, q1, root2                     // ........*...............
        vsub.u32 q1, q3, q2                        // .........*..............
        vmla.s32 q5, q7, modulus                   // ..........*.............
        vadd.u32 q2, q3, q2                        // ...........*............
        vmul.s32 q3, q0, root1                     // ............*...........
        vadd.u32 q6, q1, q5                        // .............*..........
        vstrw.u32 q6, [in, #128]                   // ...............*........
        vqrdmulh.s32 q4, q0, root1_twisted         // ..............*.........
        vsub.u32 q7, q1, q5                        // .................*......
        vmla.s32 q3, q4, modulus                   // ................*.......
        vstrw.u32 q7, [in, #192]                   // ..................*.....
        vsub.u32 q5, q2, q3                        // ...................*....
        vstrw.u32 q5, [in, #64]                    // ....................*...
        vadd.u32 q2, q2, q3                        // .....................*..
        vstrw.u32 q2, [in] , #16                   // ......................*.
        
        // original source code
        // vsub.u32 q1, q5, q0                     // *.......................
        // vqrdmulh.s32 q4, q1, root2_twisted      // .*......................
        // vldrw.u32 q7, [in, #128]                // ..*.....................
        // vmul.s32 q2, q7, root0                  // .....*..................
        // vqrdmulh.s32 q7, q7, root0_twisted      // ...*....................
        // vadd.u32 q0, q5, q0                     // ....*...................
        // vmla.s32 q2, q7, modulus                // .......*................
        // vldrw.u32 q3, [in]                      // ......*.................
        // vmul.s32 q5, q1, root2                  // .........*..............
        // vsub.u32 q7, q3, q2                     // ..........*.............
        // vmla.s32 q5, q4, modulus                // ...........*............
        // vadd.u32 q4, q3, q2                     // ............*...........
        // vmul.s32 q3, q0, root1                  // .............*..........
        // vadd.u32 q2, q7, q5                     // ..............*.........
        // vqrdmulh.s32 q1, q0, root1_twisted      // ................*.......
        // vstrw.u32 q2, [in, #128]                // ...............*........
        // vmla.s32 q3, q1, modulus                // ..................*.....
        // vsub.u32 q2, q7, q5                     // .................*......
        // vstrw.u32 q2, [in, #192]                // ...................*....
        // vsub.u32 q5, q4, q3                     // ....................*...
        // vstrw.u32 q5, [in, #64]                 // .....................*..
        // vadd.u32 q1, q4, q3                     // ......................*.
        // vstrw.u32 q1, [in] , #16                // .......................*
        // nop                                     // ........*...............
        
layer34_loop_end:

        add in, in, #(4*64 - 4*16)
        subs count, count, #1
        bne out_start

        sub in, in, #(4*256)

        // Layers 5,6

        // 1 butterfly blocks per root config, 16 root configs
        // loop over root configs

        mov lr, #16
        ldrd r9, r7, [root_ptr] , #24         // ..*....
        vldrw.u32 q1, [in, #32]               // *......
        vmul.s32 q2, q1, r9                   // ...*...
        vldrw.u32 q4, [in, #48]               // .*.....
        vqrdmulh.s32 q0, q1, r7               // .....*.
        vldrw.u32 q1, [in]                    // ....*..
        vmla.s32 q2, q0, modulus              // ......*
        
        // original source code
        // vldrw.u32 q0, [in, #32]            // .*.....
        // vldrw.u32 q4, [in, #48]            // ...*...
        // ldrd r9, r7, [root_ptr] , #24      // *......
        // vmul.s32 q2, q0, r9                // ..*....
        // vldrw.u32 q1, [in]                 // .....*.
        // vqrdmulh.s32 q6, q0, r7            // ....*..
        // vmla.s32 q2, q6, modulus           // ......*
        
        sub lr, lr, #1
layer56_loop:
        vqrdmulh.s32 q5, q4, r7                // .............*.................
        vsub.u32 q7, q1, q2                    // ..........*....................
        vmul.s32 q0, q4, r9                    // ............*..................
        ldrd r2, r8, [root_ptr, #-16]          // .*.............................
        vmla.s32 q0, q5, modulus               // ..............*................
        vldrw.u32 q5, [in, #16]                // ....*..........................
        vadd.u32 q4, q5, q0                    // ................*..............
        vmul.s32 q3, q4, r2                    // .................*.............
        ldrd r1, r10, [root_ptr, #-8]          // ..*............................
        vqrdmulh.s32 q4, q4, r8                // ..................*............
        vsub.u32 q6, q5, q0                    // ...............*...............
        vmul.s32 q5, q6, r1                    // ......................*........
        vldrw.u32 q0, [in, #96]                // .....e.........................
        vmla.s32 q3, q4, modulus               // ...................*...........
        vadd.u32 q1, q1, q2                    // ...........*...................
        vqrdmulh.s32 q6, q6, r10               // .......................*.......
        vldrw.u32 q4, [in, #112]               // ......e........................
        vadd.u32 q2, q1, q3                    // .....................*.........
        ldrd r9, r7, [root_ptr] , #24          // e..............................
        vstrw.u32 q2, [in] , #64               // ...........................*...
        vmul.s32 q2, q0, r9                    // .......e.......................
        vsub.u32 q3, q1, q3                    // ....................*..........
        vmla.s32 q5, q6, modulus               // ........................*......
        vldrw.u32 q1, [in]                     // ...e...........................
        vqrdmulh.s32 q6, q0, r7                // ........e......................
        vstrw.u32 q3, [in, #-48]               // ............................*..
        vadd.u32 q3, q7, q5                    // ..........................*....
        vstrw.u32 q3, [in, #-32]               // .............................*.
        vmla.s32 q2, q6, modulus               // .........e.....................
        vsub.u32 q6, q7, q5                    // .........................*.....
        vstrw.u32 q6, [in, #-16]               // ..............................*
        
        // original source code
        // ldrd root0, root0_twisted, [root_ptr] , #24       // ......e...........................................
        // ldrd root1, root1_twisted, [root_ptr, #-16]       // ......................*...........................
        // ldrd root2, root2_twisted, [root_ptr, #-8]        // ...........................*......................
        // vldrw.u32 data0, [in]                             // ...........e......................................
        // vldrw.u32 data1, [in, #16]                        // ........................*.........................
        // vldrw.u32 data2, [in, #32]                        // e.................................................
        // vldrw.u32 data3, [in, #48]                        // ....e.............................................
        // vmul.s32 tmp, data2, root0                        // ........e.........................................
        // vqrdmulh.s32 data2, data2, root0_twisted          // ............e.....................................
        // vmla.s32 tmp, data2, modulus                      // ................e.................................
        // vsub.u32 data2, data0, tmp                        // ....................*.............................
        // vadd.u32 data0, data0, tmp                        // .................................*................
        // vmul.s32 tmp, data3, root0                        // .....................*............................
        // vqrdmulh.s32 data3, data3, root0_twisted          // ...................*..............................
        // vmla.s32 tmp, data3, modulus                      // .......................*..........................
        // vsub.u32 data3, data1, tmp                        // .............................*....................
        // vadd.u32 data1, data1, tmp                        // .........................*........................
        // vmul.s32 tmp, data1, root1                        // ..........................*.......................
        // vqrdmulh.s32 data1, data1, root1_twisted          // ............................*.....................
        // vmla.s32 tmp, data1, modulus                      // ................................*.................
        // vsub.u32 data1, data0, tmp                        // ........................................*.........
        // vadd.u32 data0, data0, tmp                        // ....................................*.............
        // vmul.s32 tmp, data3, root2                        // ..............................*...................
        // vqrdmulh.s32 data3, data3, root2_twisted          // ..................................*...............
        // vmla.s32 tmp, data3, modulus                      // .........................................*........
        // vsub.u32 data3, data2, tmp                        // ................................................*.
        // vadd.u32 data2, data2, tmp                        // .............................................*....
        // vstrw.u32 data0, [in] , #64                       // ......................................*...........
        // vstrw.u32 data1, [in, #-48]                       // ............................................*.....
        // vstrw.u32 data2, [in, #-32]                       // ..............................................*...
        // vstrw.u32 data3, [in, #-16]                       // .................................................*
        
        le lr, layer56_loop
        vqrdmulh.s32 q0, q4, r7                // *.......................
        vsub.u32 q7, q1, q2                    // .*......................
        vmul.s32 q6, q4, r9                    // ..*.....................
        ldrd r5, r2, [root_ptr, #-16]          // ...*....................
        vmla.s32 q6, q0, modulus               // ....*...................
        vldrw.u32 q5, [in, #16]                // .....*..................
        vadd.u32 q4, q5, q6                    // ......*.................
        vmul.s32 q0, q4, r5                    // .......*................
        vsub.u32 q3, q5, q6                    // ..........*.............
        vqrdmulh.s32 q4, q4, r2                // .........*..............
        vadd.u32 q1, q1, q2                    // .............*..........
        vmla.s32 q0, q4, modulus               // ............*...........
        ldrd r8, r1, [root_ptr, #-8]           // ........*...............
        vadd.u32 q2, q1, q0                    // ...............*........
        vqrdmulh.s32 q5, q3, r1                // ..............*.........
        vstrw.u32 q2, [in] , #64               // ................*.......
        vmul.s32 q6, q3, r8                    // ...........*............
        vsub.u32 q0, q1, q0                    // .................*......
        vmla.s32 q6, q5, modulus               // ..................*.....
        vstrw.u32 q0, [in, #-48]               // ...................*....
        vadd.u32 q0, q7, q6                    // ....................*...
        vstrw.u32 q0, [in, #-32]               // .....................*..
        vsub.u32 q0, q7, q6                    // ......................*.
        vstrw.u32 q0, [in, #-16]               // .......................*
        
        // original source code
        // vqrdmulh.s32 q5, q4, r7             // *.......................
        // vsub.u32 q7, q1, q2                 // .*......................
        // vmul.s32 q0, q4, r9                 // ..*.....................
        // ldrd r2, r8, [root_ptr, #-16]       // ...*....................
        // vmla.s32 q0, q5, modulus            // ....*...................
        // vldrw.u32 q5, [in, #16]             // .....*..................
        // vadd.u32 q4, q5, q0                 // ......*.................
        // vmul.s32 q3, q4, r2                 // .......*................
        // ldrd r1, r10, [root_ptr, #-8]       // ............*...........
        // vqrdmulh.s32 q4, q4, r8             // .........*..............
        // vsub.u32 q6, q5, q0                 // ........*...............
        // vmul.s32 q5, q6, r1                 // ................*.......
        // vmla.s32 q3, q4, modulus            // ...........*............
        // vadd.u32 q1, q1, q2                 // ..........*.............
        // vqrdmulh.s32 q6, q6, r10            // ..............*.........
        // vadd.u32 q2, q1, q3                 // .............*..........
        // vstrw.u32 q2, [in] , #64            // ...............*........
        // vsub.u32 q3, q1, q3                 // .................*......
        // vmla.s32 q5, q6, modulus            // ..................*.....
        // vstrw.u32 q3, [in, #-48]            // ...................*....
        // vadd.u32 q3, q7, q5                 // ....................*...
        // vstrw.u32 q3, [in, #-32]            // .....................*..
        // vsub.u32 q6, q7, q5                 // ......................*.
        // vstrw.u32 q6, [in, #-16]            // .......................*
        
layer56_loop_end:

        // Restore MVE vector registers
        vpop {d8-d15}
        // Restore GPRs
        pop {r4-r11,lr}
        bx lr