intt_n256_l6_s32_bar.s

///
/// Copyright (c) 2021 Arm Limited
/// Copyright (c) 2022 Hanno Becker
/// Copyright (c) 2023 Amin Abdulrahman, Matthias Kannwischer
/// SPDX-License-Identifier: MIT
///
/// Permission is hereby granted, free of charge, to any person obtaining a copy
/// of this software and associated documentation files (the "Software"), to deal
/// in the Software without restriction, including without limitation the rights
/// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
/// copies of the Software, and to permit persons to whom the Software is
/// furnished to do so, subject to the following conditions:
///
/// The above copyright notice and this permission notice shall be included in all
/// copies or substantial portions of the Software.
///
/// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
/// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
/// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
/// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
/// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
/// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
/// SOFTWARE.
///

.data
roots_inv:
#include "intt_n256_l6_s32_twiddles.s"
.text

// Barrett multiplication
.macro mulmod dst, src, const, const_twisted
        vmul.s32       \dst,  \src, \const
        vqrdmulh.s32   \src,  \src, \const_twisted
        vmla.s32       \dst,  \src, modulus
.endm

.macro gs_butterfly a, b, root, root_twisted
        vsub.u32       tmp, \a,  \b
        vadd.u32       \a,  \a,  \b
        mulmod         \b,  tmp, \root, \root_twisted
.endm

.align 4
roots_addr: .word roots_inv
.syntax unified
.type invntt_n256_u32_33556993_28678040_incomplete_manual, %function
.global invntt_n256_u32_33556993_28678040_incomplete_manual
invntt_n256_u32_33556993_28678040_incomplete_manual:

        push {r4-r11,lr}
        // Save MVE vector registers
        vpush {d8-d15}

        modulus     .req r12
        root_ptr    .req r11

        .equ modulus_const, -33556993
        movw modulus, #:lower16:modulus_const
        movt modulus, #:upper16:modulus_const
        ldr  root_ptr, roots_addr

        in .req r0

        root0         .req r2
        root0_twisted .req r3
        root1         .req r4
        root1_twisted .req r5
        root2         .req r6
        root2_twisted .req r7

        data0 .req q0
        data1 .req q1
        data2 .req q2
        data3 .req q3

        tmp .req q4

        // Layers 5,6

        mov lr, #16
        ldrd r1, r10, [root_ptr, #16]          // *...
        vldrw.u32 q5, [in, #32]                // ..*.
        nop                                    // ...*
        vldrw.u32 q7, [in, #48]                // .*..
        
        // original source code
        // ldrd r1, r10, [root_ptr, #16]       // *...
        // vldrw.u32 q7, [in, #48]             // ...*
        // vldrw.u32 q5, [in, #32]             // .*..
        // nop                                 // ..*.
        
        sub lr, lr, #1
layer56_loop:
        vsub.u32 q1, q5, q7                    // ............*..................
        vmul.s32 q0, q1, r1                    // ..............*................
        vldrw.u32 q2, [in]                     // ...*...........................
        vadd.u32 q7, q5, q7                    // .............*.................
        vldrw.u32 q5, [in, #16]                // ....*..........................
        vqrdmulh.s32 q3, q1, r10               // ...............*...............
        vadd.u32 q1, q2, q5                    // ........*......................
        ldrd r9, r7, [root_ptr] , #24          // *..............................
        vmla.s32 q0, q3, modulus               // ................*..............
        vsub.u32 q4, q1, q7                    // .................*.............
        vqrdmulh.s32 q6, q4, r7                // ....................*..........
        ldrd r2, r8, [root_ptr, #-16]          // .*.............................
        vmul.s32 q3, q4, r9                    // ...................*...........
        ldrd r1, r10, [root_ptr, #16]          // ..e............................
        vmla.s32 q3, q6, modulus               // .....................*.........
        vsub.u32 q2, q2, q5                    // .......*.......................
        vqrdmulh.s32 q4, q2, r8                // ..........*....................
        vstrw.u32 q3, [in, #32]                // .............................*.
        vmul.s32 q2, q2, r2                    // .........*.....................
        vadd.u32 q1, q1, q7                    // ..................*............
        vldrw.u32 q7, [in, #112]               // ......e........................
        vmla.s32 q2, q4, modulus               // ...........*...................
        vldrw.u32 q5, [in, #96]                // .....e.........................
        vsub.u32 q6, q2, q0                    // ......................*........
        vmul.s32 q3, q6, r9                    // ........................*......
        vadd.u32 q2, q2, q0                    // .......................*.......
        vstrw.u32 q2, [in, #16]                // ............................*..
        vqrdmulh.s32 q2, q6, r7                // .........................*.....
        vstrw.u32 q1, [in] , #64               // ...........................*...
        vmla.s32 q3, q2, modulus               // ..........................*....
        vstrw.u32 q3, [in, #-16]               // ..............................*
        
        // original source code
        // ldrd root0, root0_twisted, [root_ptr] , #24       // .........................*.......................
        // ldrd root1, root1_twisted, [root_ptr, #-16]       // .............................*...................
        // ldrd root2, root2_twisted, [root_ptr, #-8]        // e................................................
        // vldrw.u32 data0, [in]                             // ....................*............................
        // vldrw.u32 data1, [in, #16]                        // ......................*..........................
        // vldrw.u32 data2, [in, #32]                        // .........e.......................................
        // vldrw.u32 data3, [in, #48]                        // .......e.........................................
        // vsub.u32 tmp, data0, data1                        // .................................*...............
        // vadd.u32 data0, data0, data1                      // ........................*........................
        // vmul.s32 data1, tmp, root1                        // ....................................*............
        // vqrdmulh.s32 tmp, tmp, root1_twisted              // ..................................*..............
        // vmla.s32 data1, tmp, modulus                      // .......................................*.........
        // vsub.u32 tmp, data2, data3                        // ..................*..............................
        // vadd.u32 data2, data2, data3                      // .....................*...........................
        // vmul.s32 data3, tmp, root2                        // ...................*.............................
        // vqrdmulh.s32 tmp, tmp, root2_twisted              // .......................*.........................
        // vmla.s32 data3, tmp, modulus                      // ..........................*......................
        // vsub.u32 tmp, data0, data2                        // ...........................*.....................
        // vadd.u32 data0, data0, data2                      // .....................................*...........
        // vmul.s32 data2, tmp, root0                        // ..............................*..................
        // vqrdmulh.s32 tmp, tmp, root0_twisted              // ............................*....................
        // vmla.s32 data2, tmp, modulus                      // ................................*................
        // vsub.u32 tmp, data1, data3                        // .........................................*.......
        // vadd.u32 data1, data1, data3                      // ...........................................*.....
        // vmul.s32 data3, tmp, root0                        // ..........................................*......
        // vqrdmulh.s32 tmp, tmp, root0_twisted              // .............................................*...
        // vmla.s32 data3, tmp, modulus                      // ...............................................*.
        // vstrw.u32 data0, [in] , #64                       // ..............................................*..
        // vstrw.u32 data1, [in, #-48]                       // ............................................*....
        // vstrw.u32 data2, [in, #-32]                       // ...................................*.............
        // vstrw.u32 data3, [in, #-16]                       // ................................................*
        
        le lr, layer56_loop
        vsub.u32 q1, q5, q7                   // *...........................
        vmul.s32 q2, q1, r1                   // .*..........................
        vadd.u32 q7, q5, q7                   // ...*........................
        vldrw.u32 q5, [in, #16]               // ....*.......................
        vqrdmulh.s32 q6, q1, r10              // .....*......................
        ldrd r2, r8, [root_ptr, #8]           // ...........*................
        vldrw.u32 q0, [in]                    // ..*.........................
        vsub.u32 q1, q0, q5                   // ..............*.............
        vmul.s32 q3, q1, r2                   // .................*..........
        vadd.u32 q0, q0, q5                   // ......*.....................
        vqrdmulh.s32 q4, q1, r8               // ...............*............
        vadd.u32 q1, q0, q7                   // ..................*.........
        vmla.s32 q3, q4, modulus              // ...................*........
        ldrd r9, r7, [root_ptr] , #24         // .......*....................
        vmla.s32 q2, q6, modulus              // ........*...................
        vsub.u32 q0, q0, q7                   // .........*..................
        vqrdmulh.s32 q4, q0, r7               // ..........*.................
        vadd.u32 q6, q3, q2                   // ......................*.....
        vmul.s32 q0, q0, r9                   // ............*...............
        vsub.u32 q2, q3, q2                   // ....................*.......
        vmul.s32 q3, q2, r9                   // .....................*......
        vstrw.u32 q1, [in] , #64              // .........................*..
        vqrdmulh.s32 q1, q2, r7               // ........................*...
        vstrw.u32 q6, [in, #-48]              // .......................*....
        vmla.s32 q3, q1, modulus              // ..........................*.
        vstrw.u32 q3, [in, #-16]              // ...........................*
        vmla.s32 q0, q4, modulus              // .............*..............
        vstrw.u32 q0, [in, #-32]              // ................*...........
        
        // original source code
        // vsub.u32 q1, q5, q7                 // *...........................
        // vmul.s32 q0, q1, r1                 // .*..........................
        // vldrw.u32 q2, [in]                  // ......*.....................
        // vadd.u32 q7, q5, q7                 // ..*.........................
        // vldrw.u32 q5, [in, #16]             // ...*........................
        // vqrdmulh.s32 q3, q1, r10            // ....*.......................
        // vadd.u32 q1, q2, q5                 // .........*..................
        // ldrd r9, r7, [root_ptr] , #24       // .............*..............
        // vmla.s32 q0, q3, modulus            // ..............*.............
        // vsub.u32 q4, q1, q7                 // ...............*............
        // vqrdmulh.s32 q6, q4, r7             // ................*...........
        // ldrd r2, r8, [root_ptr, #-16]       // .....*......................
        // vmul.s32 q3, q4, r9                 // ..................*.........
        // vmla.s32 q3, q6, modulus            // ..........................*.
        // vsub.u32 q2, q2, q5                 // .......*....................
        // vqrdmulh.s32 q4, q2, r8             // ..........*.................
        // vstrw.u32 q3, [in, #32]             // ...........................*
        // vmul.s32 q2, q2, r2                 // ........*...................
        // vadd.u32 q1, q1, q7                 // ...........*................
        // vmla.s32 q2, q4, modulus            // ............*...............
        // vsub.u32 q6, q2, q0                 // ...................*........
        // vmul.s32 q3, q6, r9                 // ....................*.......
        // vadd.u32 q2, q2, q0                 // .................*..........
        // vstrw.u32 q2, [in, #16]             // .......................*....
        // vqrdmulh.s32 q2, q6, r7             // ......................*.....
        // vstrw.u32 q1, [in] , #64            // .....................*......
        // vmla.s32 q3, q2, modulus            // ........................*...
        // vstrw.u32 q3, [in, #-16]            // .........................*..
        
layer56_loop_end:

        sub in, in, #(4*256)

        // TEMPORARY: Barrett reduction
        barrett_const .req r1
        .equ const_barrett, 63
        movw barrett_const, #:lower16:const_barrett
        movt barrett_const, #:upper16:const_barrett
        mov lr, #64
1:
        vldrw.u32 data0, [in]
        vqrdmulh.s32 tmp, data0, barrett_const
        vmla.s32 data0, tmp, modulus
        vstrw.u32 data0, [in], #16
        le lr, 1b
2:
        sub in, in, #(4*256)
        .unreq barrett_const

        // Layers 3,4

        // 4 butterfly blocks per root config, 4 root configs
        // loop over root configs

        count .req r1
        mov count, #4

out_start:
        ldrd root0, root0_twisted, [root_ptr], #+24
        ldrd root1, root1_twisted, [root_ptr, #-16]
        ldrd root2, root2_twisted, [root_ptr, #-8]

        mov lr, #4
        vldrw.u32 q7, [in, #128]          // *..
        nop                               // ..*
        vldrw.u32 q0, [in, #192]          // .*.
        
        // original source code
        // vldrw.u32 q7, [in, #128]       // *..
        // vldrw.u32 q0, [in, #192]       // ..*
        // nop                            // .*.
        
        sub lr, lr, #1
layer34_loop:
        vsub.u32 q4, q7, q0                        // .........*..................
        vqrdmulh.s32 q1, q4, root2_twisted         // ............*...............
        vadd.u32 q7, q7, q0                        // ..........*.................
        vldrw.u32 q5, [in, #64]                    // .*..........................
        vmul.s32 q0, q4, root2                     // ...........*................
        vldrw.u32 q3, [in]                         // *...........................
        vmla.s32 q0, q1, modulus                   // .............*..............
        vsub.u32 q1, q3, q5                        // ....*.......................
        vmul.s32 q2, q1, root1                     // ......*.....................
        vadd.u32 q4, q3, q5                        // .....*......................
        vqrdmulh.s32 q3, q1, root1_twisted         // .......*....................
        vadd.u32 q6, q4, q7                        // ...............*............
        vmla.s32 q2, q3, modulus                   // ........*...................
        vstrw.u32 q6, [in] , #16                   // ........................*...
        vsub.u32 q5, q2, q0                        // ...................*........
        vqrdmulh.s32 q3, q5, root0_twisted         // ......................*.....
        vadd.u32 q1, q2, q0                        // ....................*.......
        vmul.s32 q2, q5, root0                     // .....................*......
        vstrw.u32 q1, [in, #48]                    // .........................*..
        vmla.s32 q2, q3, modulus                   // .......................*....
        vstrw.u32 q2, [in, #176]                   // ...........................*
        vsub.u32 q2, q4, q7                        // ..............*.............
        vqrdmulh.s32 q5, q2, root0_twisted         // .................*..........
        vldrw.u32 q7, [in, #128]                   // ..e.........................
        vmul.s32 q2, q2, root0                     // ................*...........
        vldrw.u32 q0, [in, #192]                   // ...e........................
        vmla.s32 q2, q5, modulus                   // ..................*.........
        vstrw.u32 q2, [in, #112]                   // ..........................*.
        
        // original source code
        // vldrw.u32 data0, [in]                     // ..........*......................
        // vldrw.u32 data1, [in, #64]                // ........*........................
        // vldrw.u32 data2, [in, #128]               // e................................
        // vldrw.u32 data3, [in, #192]               // ..e..............................
        // vsub.u32 tmp, data0, data1                // ............*....................
        // vadd.u32 data0, data0, data1              // ..............*..................
        // vmul.s32 data1, tmp, root1                // .............*...................
        // vqrdmulh.s32 tmp, tmp, root1_twisted      // ...............*.................
        // vmla.s32 data1, tmp, modulus              // .................*...............
        // vsub.u32 tmp, data2, data3                // .....*...........................
        // vadd.u32 data2, data2, data3              // .......*.........................
        // vmul.s32 data3, tmp, root2                // .........*.......................
        // vqrdmulh.s32 tmp, tmp, root2_twisted      // ......*..........................
        // vmla.s32 data3, tmp, modulus              // ...........*.....................
        // vsub.u32 tmp, data0, data2                // ..........................*......
        // vadd.u32 data0, data0, data2              // ................*................
        // vmul.s32 data2, tmp, root0                // .............................*...
        // vqrdmulh.s32 tmp, tmp, root0_twisted      // ...........................*.....
        // vmla.s32 data2, tmp, modulus              // ...............................*.
        // vsub.u32 tmp, data1, data3                // ...................*.............
        // vadd.u32 data1, data1, data3              // .....................*...........
        // vmul.s32 data3, tmp, root0                // ......................*..........
        // vqrdmulh.s32 tmp, tmp, root0_twisted      // ....................*............
        // vmla.s32 data3, tmp, modulus              // ........................*........
        // vstrw.u32 data0, [in] , #16               // ..................*..............
        // vstrw.u32 data1, [in, #48]                // .......................*.........
        // vstrw.u32 data2, [in, #112]               // ................................*
        // vstrw.u32 data3, [in, #176]               // .........................*.......
        
        le lr, layer34_loop
        vsub.u32 q3, q7, q0                        // *.........................
        vqrdmulh.s32 q5, q3, root2_twisted         // .*........................
        vadd.u32 q7, q7, q0                        // ..*.......................
        vmul.s32 q0, q3, root2                     // ....*.....................
        vldrw.u32 q1, [in, #64]                    // ...*......................
        vmla.s32 q0, q5, modulus                   // ......*...................
        vldrw.u32 q3, [in]                         // .....*....................
        vsub.u32 q5, q3, q1                        // .......*..................
        vmul.s32 q2, q5, root1                     // ........*.................
        vadd.u32 q3, q3, q1                        // .........*................
        vqrdmulh.s32 q5, q5, root1_twisted         // ..........*...............
        vadd.u32 q1, q3, q7                        // ...........*..............
        vmla.s32 q2, q5, modulus                   // ............*.............
        vsub.u32 q5, q3, q7                        // .....................*....
        vqrdmulh.s32 q3, q5, root0_twisted         // ......................*...
        vstrw.u32 q1, [in] , #16                   // .............*............
        vmul.s32 q5, q5, root0                     // .......................*..
        vsub.u32 q1, q2, q0                        // ..............*...........
        vmla.s32 q5, q3, modulus                   // ........................*.
        vstrw.u32 q5, [in, #112]                   // .........................*
        vmul.s32 q5, q1, root0                     // .................*........
        vadd.u32 q3, q2, q0                        // ................*.........
        vqrdmulh.s32 q1, q1, root0_twisted         // ...............*..........
        vstrw.u32 q3, [in, #48]                    // ..................*.......
        vmla.s32 q5, q1, modulus                   // ...................*......
        vstrw.u32 q5, [in, #176]                   // ....................*.....
        
        // original source code
        // vsub.u32 q4, q7, q0                     // *.........................
        // vqrdmulh.s32 q1, q4, root2_twisted      // .*........................
        // vadd.u32 q7, q7, q0                     // ..*.......................
        // vldrw.u32 q5, [in, #64]                 // ....*.....................
        // vmul.s32 q0, q4, root2                  // ...*......................
        // vldrw.u32 q3, [in]                      // ......*...................
        // vmla.s32 q0, q1, modulus                // .....*....................
        // vsub.u32 q1, q3, q5                     // .......*..................
        // vmul.s32 q2, q1, root1                  // ........*.................
        // vadd.u32 q4, q3, q5                     // .........*................
        // vqrdmulh.s32 q3, q1, root1_twisted      // ..........*...............
        // vadd.u32 q6, q4, q7                     // ...........*..............
        // vmla.s32 q2, q3, modulus                // ............*.............
        // vstrw.u32 q6, [in] , #16                // ...............*..........
        // vsub.u32 q5, q2, q0                     // .................*........
        // vqrdmulh.s32 q3, q5, root0_twisted      // ......................*...
        // vadd.u32 q1, q2, q0                     // .....................*....
        // vmul.s32 q2, q5, root0                  // ....................*.....
        // vstrw.u32 q1, [in, #48]                 // .......................*..
        // vmla.s32 q2, q3, modulus                // ........................*.
        // vstrw.u32 q2, [in, #176]                // .........................*
        // vsub.u32 q2, q4, q7                     // .............*............
        // vqrdmulh.s32 q5, q2, root0_twisted      // ..............*...........
        // vmul.s32 q2, q2, root0                  // ................*.........
        // vmla.s32 q2, q5, modulus                // ..................*.......
        // vstrw.u32 q2, [in, #112]                // ...................*......
        
layer34_loop_end:
        add in, in, #(4*64 - 4*16)

        subs count, count, #1
        bne out_start

        sub in, in, #(4*256)

        // TEMPORARY: Barrett reduction
        barrett_const .req r1
        movw barrett_const, #:lower16:const_barrett
        movt barrett_const, #:upper16:const_barrett
        mov lr, #64
1:
        vldrw.u32 data0, [in]
        vqrdmulh.s32 tmp, data0, barrett_const
        vmla.s32 data0, tmp, modulus
        vstrw.u32 data0, [in], #16
        le lr, 1b
2:
        sub in, in, #(4*256)
        .unreq barrett_const

        in_low       .req r0
        in_high      .req r1
        add in_high, in_low, #(4*128)

        // Layers 1,2

        ldrd root0, root0_twisted, [root_ptr], #+8
        ldrd root1, root1_twisted, [root_ptr], #+8
        ldrd root2, root2_twisted, [root_ptr], #+8

        mov lr, #16
        vldrw.u32 q7, [in_high]                // *..
        nop                                    // ..*
        vldrw.u32 q0, [in_high, #256]          // .*.
        
        // original source code
        // vldrw.u32 q7, [in_high]             // *..
        // vldrw.u32 q0, [in_high, #256]       // ..*
        // nop                                 // .*.
        
        sub lr, lr, #1
layer12_loop:
        vsub.u32 q4, q7, q0                        // .........*..................
        vqrdmulh.s32 q1, q4, root2_twisted         // ............*...............
        vadd.u32 q7, q7, q0                        // ..........*.................
        vldrw.u32 q5, [in_low, #256]               // .*..........................
        vmul.s32 q0, q4, root2                     // ...........*................
        vldrw.u32 q3, [in_low]                     // *...........................
        vmla.s32 q0, q1, modulus                   // .............*..............
        vsub.u32 q1, q3, q5                        // ....*.......................
        vmul.s32 q2, q1, root1                     // ......*.....................
        vadd.u32 q4, q3, q5                        // .....*......................
        vqrdmulh.s32 q3, q1, root1_twisted         // .......*....................
        vadd.u32 q6, q4, q7                        // ...............*............
        vmla.s32 q2, q3, modulus                   // ........*...................
        vstrw.u32 q6, [in_low] , #16               // ........................*...
        vsub.u32 q5, q2, q0                        // ...................*........
        vqrdmulh.s32 q3, q5, root0_twisted         // ......................*.....
        vadd.u32 q1, q2, q0                        // ....................*.......
        vmul.s32 q2, q5, root0                     // .....................*......
        vstrw.u32 q1, [in_low, #240]               // .........................*..
        vmla.s32 q2, q3, modulus                   // .......................*....
        vstrw.u32 q2, [in_high, #256]              // ...........................*
        vsub.u32 q2, q4, q7                        // ..............*.............
        vqrdmulh.s32 q5, q2, root0_twisted         // .................*..........
        vldrw.u32 q7, [in_high, #16]               // ..e.........................
        vmul.s32 q2, q2, root0                     // ................*...........
        vldrw.u32 q0, [in_high, #272]              // ...e........................
        vmla.s32 q2, q5, modulus                   // ..................*.........
        vstrw.u32 q2, [in_high] , #16              // ..........................*.
        
        // original source code
        // vldrw.u32 data0, [in_low]                 // ..........*......................
        // vldrw.u32 data1, [in_low, #256]           // ........*........................
        // vldrw.u32 data2, [in_high]                // e................................
        // vldrw.u32 data3, [in_high, #256]          // ..e..............................
        // vsub.u32 tmp, data0, data1                // ............*....................
        // vadd.u32 data0, data0, data1              // ..............*..................
        // vmul.s32 data1, tmp, root1                // .............*...................
        // vqrdmulh.s32 tmp, tmp, root1_twisted      // ...............*.................
        // vmla.s32 data1, tmp, modulus              // .................*...............
        // vsub.u32 tmp, data2, data3                // .....*...........................
        // vadd.u32 data2, data2, data3              // .......*.........................
        // vmul.s32 data3, tmp, root2                // .........*.......................
        // vqrdmulh.s32 tmp, tmp, root2_twisted      // ......*..........................
        // vmla.s32 data3, tmp, modulus              // ...........*.....................
        // vsub.u32 tmp, data0, data2                // ..........................*......
        // vadd.u32 data0, data0, data2              // ................*................
        // vmul.s32 data2, tmp, root0                // .............................*...
        // vqrdmulh.s32 tmp, tmp, root0_twisted      // ...........................*.....
        // vmla.s32 data2, tmp, modulus              // ...............................*.
        // vsub.u32 tmp, data1, data3                // ...................*.............
        // vadd.u32 data1, data1, data3              // .....................*...........
        // vmul.s32 data3, tmp, root0                // ......................*..........
        // vqrdmulh.s32 tmp, tmp, root0_twisted      // ....................*............
        // vmla.s32 data3, tmp, modulus              // ........................*........
        // vstrw.u32 data0, [in_low] , #16           // ..................*..............
        // vstrw.u32 data1, [in_low, #240]           // .......................*.........
        // vstrw.u32 data2, [in_high] , #16          // ................................*
        // vstrw.u32 data3, [in_high, #240]          // .........................*.......
        
        le lr, layer12_loop
        vsub.u32 q3, q7, q0                        // *.........................
        vqrdmulh.s32 q5, q3, root2_twisted         // .*........................
        vadd.u32 q7, q7, q0                        // ..*.......................
        vmul.s32 q0, q3, root2                     // ....*.....................
        vldrw.u32 q1, [in_low, #256]               // ...*......................
        vmla.s32 q0, q5, modulus                   // ......*...................
        vldrw.u32 q3, [in_low]                     // .....*....................
        vsub.u32 q5, q3, q1                        // .......*..................
        vmul.s32 q2, q5, root1                     // ........*.................
        vadd.u32 q3, q3, q1                        // .........*................
        vqrdmulh.s32 q5, q5, root1_twisted         // ..........*...............
        vadd.u32 q1, q3, q7                        // ...........*..............
        vmla.s32 q2, q5, modulus                   // ............*.............
        vsub.u32 q5, q3, q7                        // .....................*....
        vqrdmulh.s32 q3, q5, root0_twisted         // ......................*...
        vstrw.u32 q1, [in_low] , #16               // .............*............
        vmul.s32 q5, q5, root0                     // .......................*..
        vsub.u32 q1, q2, q0                        // ..............*...........
        vmla.s32 q5, q3, modulus                   // ........................*.
        vstrw.u32 q5, [in_high] , #16              // .........................*
        vmul.s32 q5, q1, root0                     // .................*........
        vadd.u32 q3, q2, q0                        // ................*.........
        vqrdmulh.s32 q1, q1, root0_twisted         // ...............*..........
        vstrw.u32 q3, [in_low, #240]               // ..................*.......
        vmla.s32 q5, q1, modulus                   // ...................*......
        vstrw.u32 q5, [in_high, #240]              // ....................*.....
        
        // original source code
        // vsub.u32 q4, q7, q0                     // *.........................
        // vqrdmulh.s32 q1, q4, root2_twisted      // .*........................
        // vadd.u32 q7, q7, q0                     // ..*.......................
        // vldrw.u32 q5, [in_low, #256]            // ....*.....................
        // vmul.s32 q0, q4, root2                  // ...*......................
        // vldrw.u32 q3, [in_low]                  // ......*...................
        // vmla.s32 q0, q1, modulus                // .....*....................
        // vsub.u32 q1, q3, q5                     // .......*..................
        // vmul.s32 q2, q1, root1                  // ........*.................
        // vadd.u32 q4, q3, q5                     // .........*................
        // vqrdmulh.s32 q3, q1, root1_twisted      // ..........*...............
        // vadd.u32 q6, q4, q7                     // ...........*..............
        // vmla.s32 q2, q3, modulus                // ............*.............
        // vstrw.u32 q6, [in_low] , #16            // ...............*..........
        // vsub.u32 q5, q2, q0                     // .................*........
        // vqrdmulh.s32 q3, q5, root0_twisted      // ......................*...
        // vadd.u32 q1, q2, q0                     // .....................*....
        // vmul.s32 q2, q5, root0                  // ....................*.....
        // vstrw.u32 q1, [in_low, #240]            // .......................*..
        // vmla.s32 q2, q3, modulus                // ........................*.
        // vstrw.u32 q2, [in_high, #256]           // .........................*
        // vsub.u32 q2, q4, q7                     // .............*............
        // vqrdmulh.s32 q5, q2, root0_twisted      // ..............*...........
        // vmul.s32 q2, q2, root0                  // ................*.........
        // vmla.s32 q2, q5, modulus                // ..................*.......
        // vstrw.u32 q2, [in_high] , #16           // ...................*......
        
layer12_loop_end:

        // Restore MVE vector registers
        vpop {d8-d15}
        // Restore GPRs
        pop {r4-r11,lr}
        bx lr